{"id":"arxiv:1511.05952","title":"Prioritized Experience Replay","date":"2015-11-18","section":"Reinforcement learning","quality":-0.008072,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/","source_url":"https://arxiv.org/abs/1511.05952","text_available":true,"authors":["Tom Schaul","John Quan","Ioannis Antonoglou","David Silver"],"outgoing":[{"source":"arxiv:1511.05952","target":"arxiv:1509.06461","evidence":"id","id":"arxiv:1509.06461","title":"Deep Reinforcement Learning with Double Q-learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a2f115ac879cd2efb9548baf3ecdcb925d2492f03b4b0940720f6e5b0840715b/"},{"source":"arxiv:1511.05952","target":"arxiv:1511.06581","evidence":"id","id":"arxiv:1511.06581","title":"Dueling Network Architectures for Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a5f0593cf943c782ba38ae2fc386f953f859b9d3fc9722e8f17f67ca9cf50b5/"}],"incoming":[{"source":"arxiv:1511.06581","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1511.06581","title":"Dueling Network Architectures for Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a5f0593cf943c782ba38ae2fc386f953f859b9d3fc9722e8f17f67ca9cf50b5/"},{"source":"arxiv:1707.06887","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1707.06887","title":"A Distributional Perspective on Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/"},{"source":"arxiv:1710.02298","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1710.02298","title":"Rainbow: Combining Improvements in Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4183a5aecdb6558733bd0b2028e6197904c8b46173181bf6c8270fb6cbcfc973/"},{"source":"arxiv:1712.06567","target":"arxiv:1511.05952","evidence":"id","id":"arxiv:1712.06567","title":"Deep Neuroevolution: Genetic Algorithms Are a Competitive Alternative for Training Deep Neural Networks for Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7191dc11e224beb3dc9abb8cd4efc58c70b79939b61fd56e4416b4b5e1404312/"},{"source":"arxiv:1802.09477","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1802.09477","title":"Addressing Function Approximation Error in Actor-Critic Methods","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/"},{"source":"arxiv:1901.01753","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1901.01753","title":"Paired Open-Ended Trailblazer (POET): Endlessly Generating Increasingly Complex and Diverse Learning Environments and Their Solutions","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9790a1ede488fbbb2d7b4cdb97b884a5a1e21357b72c5e3419a83ea629d5a4b/"},{"source":"arxiv:2004.12919","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:2004.12919","title":"First return, then explore","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3cd790c501eaa633193918533947c58e30d1939bc08577606f1d88c56d368285/"},{"source":"arxiv:2011.14826","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:2011.14826","title":"Revisiting Rainbow: Promoting more Insightful and Inclusive Deep Reinforcement Learning Research","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/"},{"source":"arxiv:2205.07802","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:2205.07802","title":"The Primacy Bias in Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/"},{"source":"arxiv:2301.04104","target":"arxiv:1511.05952","evidence":"id","id":"arxiv:2301.04104","title":"Mastering Diverse Domains through World Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/266d59e63d2f705efdc2eb7ad93f5126b7092b4450efd0d8c17099c51574b176/"},{"source":"arxiv:2305.19452","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:2305.19452","title":"Bigger, Better, Faster: Human-level Atari with human-level efficiency","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/727a7ee43fd005401d34d4591a2f49d85175f9366ebf2c9e7a862b340787614e/"},{"source":"arxiv:2306.02451","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:2306.02451","title":"For SALE: State-Action Representation Learning for Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/952e0e96fff2857b76e6b62a24d57ca4f6ac86ad8e8a21cb4d31a4dc051f8885/"},{"source":"arxiv:2411.03820","target":"arxiv:1511.05952","evidence":"id","id":"arxiv:2411.03820","title":"Beyond The Rainbow: High Performance Deep Reinforcement Learning on a Desktop PC","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8bcc96ef608c017d3ddb7ded7dc9e3b760390f7df441214eeee08a2586c981ea/"},{"source":"arxiv:2509.13232","target":"arxiv:1511.05952","evidence":"id","id":"arxiv:2509.13232","title":"Single-stream Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d699d0e034b21f3b85c3f70ce4fb95064fbf59685ce752935e6d173dac2b71f3/"},{"source":"arxiv:2512.13961","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"}],"text_parts":[{"start":0,"end":15323,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html"},{"start":15323,"end":31098,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-2.html"},{"start":31098,"end":46325,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html"},{"start":46325,"end":54135,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-4.html"}],"contents":[{"title":"Prioritized Experience Replay","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"Abstract","offset":33,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"1 Introduction","offset":838,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"2 Background","offset":3847,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"3 Prioritized Replay","offset":6664,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"3.1 A Motivating Example","offset":7015,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"3.2 Prioritizing with TD-error","offset":8455,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"3.3 Stochastic Prioritization","offset":10808,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"3.4 Annealing the Bias","offset":14353,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-1.html#text"},{"title":"4 Atari Experiments","offset":17260,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-2.html#text"},{"title":"5 Discussion","offset":22870,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-2.html#text"},{"title":"6 Extensions","offset":25589,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-2.html#text"},{"title":"7 Conclusion","offset":31081,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-2.html#text"},{"title":"Acknowledgments","offset":31560,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"Appendix A Prioritization Variants","offset":31926,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"Appendix B Experimental Details","offset":37122,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.1 Blind Cliffwalk","offset":37158,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.2 Atari Experiments","offset":39490,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.2.1 Implementation Details","offset":39517,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"Rank-based prioritization","offset":39799,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"Proportional prioritization","offset":41135,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.2.2 Hyperparameters","offset":42551,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.2.3 Evaluation","offset":43488,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.3 Class-imbalanced MNIST","offset":45354,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.3.1 Dataset Setup","offset":45386,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"B.3.2 Training Setup","offset":46298,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-3.html#text"},{"title":"Instructions for reporting errors","offset":53171,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/part-4.html#text"}],"text_code_points":54135,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/source.md"}