{"id":"arxiv:2011.14826","title":"Revisiting Rainbow: Promoting more Insightful and Inclusive Deep Reinforcement Learning Research","date":"2020-11-20","section":"Reinforcement learning","quality":-0.269176,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/","source_url":"https://arxiv.org/abs/2011.14826","text_available":true,"authors":["Johan S. Obando-Ceron","Pablo Samuel Castro"],"outgoing":[{"source":"arxiv:2011.14826","target":"arxiv:1509.06461","evidence":"id","id":"arxiv:1509.06461","title":"Deep Reinforcement Learning with Double Q-learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a2f115ac879cd2efb9548baf3ecdcb925d2492f03b4b0940720f6e5b0840715b/"},{"source":"arxiv:2011.14826","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1511.05952","title":"Prioritized Experience Replay","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/"},{"source":"arxiv:2011.14826","target":"arxiv:1511.06581","evidence":"title","id":"arxiv:1511.06581","title":"Dueling Network Architectures for Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a5f0593cf943c782ba38ae2fc386f953f859b9d3fc9722e8f17f67ca9cf50b5/"},{"source":"arxiv:2011.14826","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:1707.06887","title":"A Distributional Perspective on Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/"},{"source":"arxiv:2011.14826","target":"arxiv:1710.02298","evidence":"title","id":"arxiv:1710.02298","title":"Rainbow: Combining Improvements in Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4183a5aecdb6558733bd0b2028e6197904c8b46173181bf6c8270fb6cbcfc973/"}],"incoming":[{"source":"arxiv:2305.19452","target":"arxiv:2011.14826","evidence":"title","id":"arxiv:2305.19452","title":"Bigger, Better, Faster: Human-level Atari with human-level efficiency","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/727a7ee43fd005401d34d4591a2f49d85175f9366ebf2c9e7a862b340787614e/"},{"source":"arxiv:2411.03820","target":"arxiv:2011.14826","evidence":"title","id":"arxiv:2411.03820","title":"Beyond The Rainbow: High Performance Deep Reinforcement Learning on a Desktop PC","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8bcc96ef608c017d3ddb7ded7dc9e3b760390f7df441214eeee08a2586c981ea/"},{"source":"doi:10.1038/s41586-024-07711-7","target":"arxiv:2011.14826","evidence":"title","id":"doi:10.1038/s41586-024-07711-7","title":"Maintaining Plasticity in Deep Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5d46327415fe38f2b9160b874735530939961a71536f7f1ac4c0dc7a3af3224/"}],"text_parts":[{"start":0,"end":15747,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html"},{"start":15747,"end":31746,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html"},{"start":31746,"end":43723,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html"}],"contents":[{"title":"Revisiting Rainbow: Promoting more Insightful and Inclusive Deep Reinforcement Learning Research","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"Abstract","offset":100,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"Keywords:","offset":1059,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"1 Introduction","offset":1077,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"2 Preliminaries","offset":4555,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"2.1 DQN","offset":7934,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"2.2 Rainbow","offset":9203,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"3 The cost of Rainbow","offset":11658,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"4 Revisiting Rainbow","offset":13799,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"4.1 Methodology","offset":13824,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"4.2 Empirical evaluation","offset":15717,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-1.html#text"},{"title":"5 Beyond the Rainbow","offset":17610,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"5.1 Examining network architectures and batch sizes","offset":18142,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"5.2 Examining distribution parameterizations","offset":18928,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"5.3 Munchausen Reinforcement Learning","offset":21603,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"5.4 Reevaluating the Huber loss","offset":22960,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"6 Putting it all together","offset":25016,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"6.1 Rainbow flavours","offset":25046,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"6.2 Environment properties","offset":25414,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"7 Conclusion","offset":27369,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"8 Acknowledgements","offset":29127,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"Appendix A Environments","offset":29346,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"A.1 CartPole-v0","offset":29528,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"A.2 Acrobot-v1","offset":30094,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"A.3 LunarLander-v2","offset":30675,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-2.html#text"},{"title":"A.4 MountainCar-v0","offset":31746,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"A.5 MinAtar Environments","offset":32308,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix B Detailed description of the revisting rainbow components","offset":33135,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.1 Double Q-learning","offset":33311,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.2 Prioritized experience replay","offset":33951,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.3 Dueling networks","offset":34539,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.4 Multi-step learning","offset":35435,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.5 Distributional RL","offset":36062,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.6 Quantile Regression for Distributional RL","offset":36647,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.7 Implicit quantile networks","offset":37342,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.8 Noisy nets","offset":37855,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.9 Munchausen Reinforcement Learning","offset":38485,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"B.10 Loss functions","offset":39248,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix C Hyperparameters settings for classic control environments","offset":40351,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix D Hyperparameters settings for MinAtar environments","offset":40424,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix E Average runtime for environments used in the experiments","offset":40489,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix F Impact of algorithmic components","offset":40561,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix G Examining network architectures and batch sizes","offset":40896,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix H Reevaluating the Huber loss, complete results","offset":41298,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"H.1 Classic control and MinAtar results","offset":41519,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"H.2 Atari results","offset":41736,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Appendix I Rainbow flavours, full results","offset":42568,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"},{"title":"Instructions for reporting errors","offset":42759,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/part-3.html#text"}],"text_code_points":43723,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/source.md"}