{"id":"arxiv:1802.09477","title":"Addressing Function Approximation Error in Actor-Critic Methods","date":"2018-02-26","section":"Reinforcement learning","quality":-0.361532,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/","source_url":"https://arxiv.org/abs/1802.09477","text_available":true,"authors":["Scott Fujimoto","Herke van Hoof","David Meger"],"outgoing":[{"source":"arxiv:1802.09477","target":"arxiv:1509.06461","evidence":"title","id":"arxiv:1509.06461","title":"Deep Reinforcement Learning with Double Q-learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a2f115ac879cd2efb9548baf3ecdcb925d2492f03b4b0940720f6e5b0840715b/"},{"source":"arxiv:1802.09477","target":"arxiv:1511.05952","evidence":"title","id":"arxiv:1511.05952","title":"Prioritized Experience Replay","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c88bf0c11c1817dcc457c001dfef5cd120a0946c8a009a0fe40648dae03eb7d4/"},{"source":"arxiv:1802.09477","target":"arxiv:1707.06887","evidence":"title","id":"arxiv:1707.06887","title":"A Distributional Perspective on Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96b040bb771d28ca3254b5519a2191c3e958a35dfdd60075344ac86af6a590e1/"},{"source":"arxiv:1802.09477","target":"arxiv:1801.01290","evidence":"id","id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/"}],"incoming":[{"source":"arxiv:1801.01290","target":"arxiv:1802.09477","evidence":"id","id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/"},{"source":"arxiv:1910.01708","target":"arxiv:1802.09477","evidence":"title","id":"arxiv:1910.01708","title":"Benchmarking Batch Deep Reinforcement Learning Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ce9cb9b764eb7fdaf1aa766cb24e6abaec20a00445ad5af5a6dacb5af3ad3ddc/"},{"source":"arxiv:2106.06860","target":"arxiv:1802.09477","evidence":"title","id":"arxiv:2106.06860","title":"A Minimalist Approach to Offline Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/00a35c7a58d8775f23035efbd1cb0a5f540440d47b8a996be8a4fa0da2da3e0e/"}],"text_parts":[{"start":0,"end":14498,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html"},{"start":14498,"end":28666,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-2.html"},{"start":28666,"end":43525,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-3.html"},{"start":43525,"end":53143,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-4.html"}],"contents":[{"title":"Supplementary Material","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"Abstract","offset":26,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"Keywords:","offset":781,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"1 Introduction","offset":799,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"2 Related Work","offset":4928,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"3 Background","offset":7900,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"4 Overestimation Bias","offset":11299,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"4.1 Overestimation Bias in Actor-Critic","offset":12679,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-1.html#text"},{"title":"4.2 Clipped Double Q-Learning for Actor-Critic","offset":17472,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-2.html#text"},{"title":"5 Addressing Variance","offset":22521,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-2.html#text"},{"title":"5.1 Accumulating Error","offset":23151,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-2.html#text"},{"title":"5.2 Target Networks and Delayed Policy Updates","offset":25177,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-2.html#text"},{"title":"5.3 Target Policy Smoothing Regularization","offset":28618,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-2.html#text"},{"title":"6 Experiments","offset":31069,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-3.html#text"},{"title":"6.1 Evaluation","offset":32113,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-3.html#text"},{"title":"6.2 Ablation Studies","offset":35608,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-3.html#text"},{"title":"7 Conclusion","offset":37462,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-3.html#text"},{"title":"Appendix A Proof of Convergence of Clipped Double Q-Learning","offset":38959,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-3.html#text"},{"title":"Appendix B Overestimation Bias in Deterministic Policy Gradients","offset":47297,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-4.html#text"},{"title":"Appendix C DDPG Network and Hyper-parameter Comparison","offset":49191,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-4.html#text"},{"title":"Appendix D Additional Implementation Details","offset":49585,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-4.html#text"},{"title":"Appendix E Soft Actor-Critic Implementation Details","offset":51001,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-4.html#text"},{"title":"Appendix F Additional Learning Curves","offset":52137,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-4.html#text"},{"title":"Instructions for reporting errors","offset":52179,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/part-4.html#text"}],"text_code_points":53143,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/source.md"}