{"id":"arxiv:2205.07802","title":"The Primacy Bias in Deep Reinforcement Learning","date":"2022-05-16","section":"Reinforcement learning","quality":0.266696,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/","source_url":"https://arxiv.org/abs/2205.07802","text_available":true,"authors":["Evgenii Nikishin","Max Schwarzer","Pierluca D'Oro","Pierre-Luc Bacon"],"outgoing":[{"source":"arxiv:2205.07802","target":"arxiv:1509.06461","evidence":"title","id":"arxiv:1509.06461","title":"Deep Reinforcement Learning with Double Q-learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a2f115ac879cd2efb9548baf3ecdcb925d2492f03b4b0940720f6e5b0840715b/"},{"source":"arxiv:2205.07802","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1511.05952","title":"Prioritized Experience Replay","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/"},{"source":"arxiv:2205.07802","target":"arxiv:1710.02298","evidence":"title","id":"arxiv:1710.02298","title":"Rainbow: Combining Improvements in Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4183a5aecdb6558733bd0b2028e6197904c8b46173181bf6c8270fb6cbcfc973/"},{"source":"arxiv:2205.07802","target":"arxiv:2108.06325","evidence":"id","id":"arxiv:2108.06325","title":"Continual Backprop: Stochastic Gradient Descent with Persistent Randomness","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/73c90c52bffd8b5b2485d93607350545d822f361c29d3c0063b8d3025576e793/"}],"incoming":[{"source":"arxiv:2305.19452","target":"arxiv:2205.07802","evidence":"title","id":"arxiv:2305.19452","title":"Bigger, Better, Faster: Human-level Atari with human-level efficiency","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/727a7ee43fd005401d34d4591a2f49d85175f9366ebf2c9e7a862b340787614e/"},{"source":"arxiv:2602.18037","target":"arxiv:2205.07802","evidence":"title","id":"arxiv:2602.18037","title":"Gradient Regularization Mitigates Reward Hacking in Reinforcement Learning from Human Feedback and Verifiable Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/"},{"source":"arxiv:2607.07847","target":"arxiv:2205.07802","evidence":"title","id":"arxiv:2607.07847","title":"When Does Continual Learning Require Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/86e88c6cf21a66879eab5c156913fc85d14eb69c394c78c33c0e81d144e528eb/"},{"source":"doi:10.1038/s41586-024-07711-7","target":"arxiv:2205.07802","evidence":"title","id":"doi:10.1038/s41586-024-07711-7","title":"Maintaining Plasticity in Deep Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5d46327415fe38f2b9160b874735530939961a71536f7f1ac4c0dc7a3af3224/"}],"text_parts":[{"start":0,"end":15806,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html"},{"start":15806,"end":31391,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html"},{"start":31391,"end":46704,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html"}],"contents":[{"title":"The Primacy Bias in Deep Reinforcement Learning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"Abstract","offset":51,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"Keywords:","offset":902,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"1 Introduction","offset":1052,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"2 Preliminaries","offset":5464,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"3 The Primacy Bias","offset":8258,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"3.1 Heavy Priming Causes Unrecoverable Overfitting","offset":9301,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"3.2 Experiences of Primed Agents are Sufficient","offset":11346,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"4 Have You Tried Resetting It?","offset":12423,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"5 Experiments","offset":12940,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"5.1 Setup","offset":13665,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-1.html#text"},{"title":"5.2 Resets Consistently Improve Performance","offset":16221,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"5.3 Learning Dynamics of Agents with Resets","offset":17318,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"5.4 Elements Behind the Success of Resets","offset":20349,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"Replay ratio","offset":20736,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"n n -step targets","offset":21915,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"TD failure modes","offset":22726,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"What and how to reset","offset":24133,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"5.5 Summary","offset":26276,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"6 Related Work","offset":26945,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"Overfitting in RL","offset":27130,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"Forgetting mechanisms","offset":29559,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"Cognitive science","offset":31367,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-2.html#text"},{"title":"7 Future Work and Limitations","offset":32780,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"8 Conclusion","offset":33886,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Acknowledgements","offset":35445,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Appendix A Experimental Details","offset":36306,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Appendix B Ablations","offset":37357,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Replay buffer","offset":37708,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Initialization","offset":38471,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Optimizer state","offset":39351,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Reset depth","offset":39907,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Which networks to reset","offset":40731,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Number of resets","offset":41549,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Other regularizers","offset":42185,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Appendix C Per-Environment and Additional Results","offset":44615,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"},{"title":"Instructions for reporting errors","offset":45740,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/part-3.html#text"}],"text_code_points":46704,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/source.md"}