{"id":"arxiv:1707.06887","title":"A Distributional Perspective on Reinforcement Learning","date":"2017-07-21","section":"Reinforcement learning","quality":0.349015,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/","source_url":"https://arxiv.org/abs/1707.06887","text_available":true,"authors":["Marc G. Bellemare","Will Dabney","Rémi Munos"],"outgoing":[{"source":"arxiv:1707.06887","target":"arxiv:1509.06461","evidence":"title","id":"arxiv:1509.06461","title":"Deep Reinforcement Learning with Double Q-learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a2f115ac879cd2efb9548baf3ecdcb925d2492f03b4b0940720f6e5b0840715b/"},{"source":"arxiv:1707.06887","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1511.05952","title":"Prioritized Experience Replay","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/"},{"source":"arxiv:1707.06887","target":"arxiv:1511.06581","evidence":"title","id":"arxiv:1511.06581","title":"Dueling Network Architectures for Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a5f0593cf943c782ba38ae2fc386f953f859b9d3fc9722e8f17f67ca9cf50b5/"}],"incoming":[{"source":"arxiv:1710.02298","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:1710.02298","title":"Rainbow: Combining Improvements in Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4183a5aecdb6558733bd0b2028e6197904c8b46173181bf6c8270fb6cbcfc973/"},{"source":"arxiv:1712.06567","target":"arxiv:1707.06887","evidence":"id","id":"arxiv:1712.06567","title":"Deep Neuroevolution: Genetic Algorithms Are a Competitive Alternative for Training Deep Neural Networks for Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7191dc11e224beb3dc9abb8cd4efc58c70b79939b61fd56e4416b4b5e1404312/"},{"source":"arxiv:1802.09477","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:1802.09477","title":"Addressing Function Approximation Error in Actor-Critic Methods","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/"},{"source":"arxiv:1807.02787","target":"arxiv:1707.06887","evidence":"id","id":"arxiv:1807.02787","title":"Financial Trading as a Game: A Deep Reinforcement Learning Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ffab71fb6c4e10245324b584ff3252a6fae8005a8220f84f1c44da083caec855/"},{"source":"arxiv:1910.01708","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:1910.01708","title":"Benchmarking Batch Deep Reinforcement Learning Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ce9cb9b764eb7fdaf1aa766cb24e6abaec20a00445ad5af5a6dacb5af3ad3ddc/"},{"source":"arxiv:2011.14826","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:2011.14826","title":"Revisiting Rainbow: Promoting more Insightful and Inclusive Deep Reinforcement Learning Research","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bf20803a5d9e686543fe5f3967e69b5aa481ba29ff91c9ca3fbeb147ab8c0aa/"},{"source":"arxiv:2301.04104","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:2301.04104","title":"Mastering Diverse Domains through World Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/266d59e63d2f705efdc2eb7ad93f5126b7092b4450efd0d8c17099c51574b176/"},{"source":"arxiv:2305.19452","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:2305.19452","title":"Bigger, Better, Faster: Human-level Atari with human-level efficiency","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/727a7ee43fd005401d34d4591a2f49d85175f9366ebf2c9e7a862b340787614e/"},{"source":"arxiv:2411.03820","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:2411.03820","title":"Beyond The Rainbow: High Performance Deep Reinforcement Learning on a Desktop PC","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8bcc96ef608c017d3ddb7ded7dc9e3b760390f7df441214eeee08a2586c981ea/"}],"text_parts":[{"start":0,"end":15049,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html"},{"start":15049,"end":29161,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html"},{"start":29161,"end":45137,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html"},{"start":45137,"end":61045,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html"},{"start":61045,"end":77024,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html"},{"start":77024,"end":82745,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-6.html"}],"contents":[{"title":"A Distributional Perspective on Reinforcement Learning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"Abstract","offset":58,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"1 Introduction","offset":1254,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"2 Setting","offset":5567,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"2.1 Bellman’s Equations","offset":6361,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"3 The Distributional Bellman Operators","offset":9021,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"3.1 Distributional Equations","offset":9645,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"3.2 The Wasserstein Metric","offset":11354,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"Lemma 1 (Partition lemma) .","offset":13530,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"Lemma 2 .","offset":14428,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"3.3 Policy Evaluation","offset":14503,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-1.html#text"},{"title":"3.3.1 Contraction in d ¯ p \\bar{d}_{p}","offset":16464,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Lemma 3 .","offset":16946,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"3.3.2 Contraction in Centered Moments","offset":17735,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"3.4 Control","offset":18720,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Definition 1 (Optimal value distribution) .","offset":19966,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Definition 2 .","offset":20373,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Lemma 4 .","offset":21640,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Definition 3 .","offset":22359,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Theorem 1 (Convergence in the control setting) .","offset":22566,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Proposition 1 .","offset":23794,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Proposition 2 .","offset":26033,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"Proposition 3 .","offset":26577,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"4 Approximate Distributional Learning","offset":27118,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"4.1 Parametric Distribution","offset":27534,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"4.2 Projected Bellman Update","offset":28512,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-2.html#text"},{"title":"5 Evaluation on Atari 2600 Games","offset":31316,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"5.1 Varying the Number of Atoms","offset":33713,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"5.2 State-of-the-Art Results","offset":35130,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"6 Discussion","offset":37623,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"6.1 Why does learning a distribution matter?","offset":37927,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"Acknowledgements","offset":41179,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"Erratum","offset":41799,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"Appendix A Related Work","offset":42593,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-3.html#text"},{"title":"Appendix B Proofs","offset":46123,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Lemma 1 (Partition lemma) .","offset":46145,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Proof.","offset":46585,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Lemma 2 .","offset":51682,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Proof.","offset":51757,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Lemma 3 .","offset":52683,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Proof.","offset":52808,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Proposition 1 (Sobel, 1982) .","offset":54533,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Proof.","offset":55252,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Lemma 4 .","offset":57118,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Proof.","offset":57448,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Theorem 1 (Convergence in the control setting) .","offset":57959,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Lemma 5 .","offset":60777,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Proof.","offset":61030,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-4.html#text"},{"title":"Lemma 6 .","offset":62353,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Proof.","offset":62544,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Proof of Theorem 1 .","offset":63412,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Proposition 4 .","offset":71596,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Lemma 7 (Sample Wasserstein distance) .","offset":73362,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Proof.","offset":73917,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Proposition 5 .","offset":76054,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Appendix C Algorithmic Details","offset":76989,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-5.html#text"},{"title":"Appendix D Comparison of Sampled Wasserstein Loss and Categorical Projection","offset":78122,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-6.html#text"},{"title":"Appendix E Supplemental Videos and Results","offset":81382,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-6.html#text"},{"title":"Instructions for reporting errors","offset":81781,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/part-6.html#text"}],"text_code_points":82745,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/source.md"}