{"id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","date":"2018-01-04","section":"Reinforcement learning","quality":0.455513,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/","source_url":"https://arxiv.org/abs/1801.01290","text_available":true,"authors":["Tuomas Haarnoja","Aurick Zhou","Pieter Abbeel","Sergey Levine"],"outgoing":[{"source":"arxiv:1801.01290","target":"arxiv:1802.09477","evidence":"id","id":"arxiv:1802.09477","title":"Addressing Function Approximation Error in Actor-Critic Methods","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/"}],"incoming":[{"source":"arxiv:1802.09477","target":"arxiv:1801.01290","evidence":"id","id":"arxiv:1802.09477","title":"Addressing Function Approximation Error in Actor-Critic Methods","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d1a410c86d85da493735fae67eb1b2689350e5ac449d39717deafaf3904e7c1/"},{"source":"arxiv:2005.01643","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:2005.01643","title":"Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/"},{"source":"arxiv:2106.06860","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:2106.06860","title":"A Minimalist Approach to Offline Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/00a35c7a58d8775f23035efbd1cb0a5f540440d47b8a996be8a4fa0da2da3e0e/"},{"source":"arxiv:2301.04104","target":"arxiv:1801.01290","evidence":"id","id":"arxiv:2301.04104","title":"Mastering Diverse Domains through World Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/266d59e63d2f705efdc2eb7ad93f5126b7092b4450efd0d8c17099c51574b176/"},{"source":"arxiv:2505.15134","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:2505.15134","title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/"},{"source":"arxiv:2509.09675","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:2509.09675","title":"CDE: Curiosity-Driven Exploration for Efficient Reinforcement Learning in Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/"},{"source":"arxiv:2601.05053","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:2601.05053","title":"Reinforced Efficient Reasoning via Semantically Diverse Exploration","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/74918d32c56962f9c5b2063026f5e2243a84e20c284eb3e37c882bd4681d1a11/"},{"source":"arxiv:2601.20802","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"},{"source":"arxiv:2607.05609","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:2607.05609","title":"To Retain or to Adapt? Generalizing Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de2d99f43688431acd6fa456a50322ed0f00aa30d884a6767719f8a73b3b7c45/"},{"source":"arxiv:2608.17163","target":"arxiv:1801.01290","evidence":"id","id":"arxiv:2608.17163","title":"Q-Learning With World Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/"}],"text_parts":[{"start":0,"end":15512,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html"},{"start":15512,"end":31242,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html"},{"start":31242,"end":47106,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html"},{"start":47106,"end":50624,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-4.html"}],"contents":[{"title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"Abstract","offset":101,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"Keywords:","offset":1362,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"1 Introduction","offset":1380,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"2 Related Work","offset":6551,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"3 Preliminaries","offset":11909,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"3.1 Notation","offset":12040,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"3.2 Maximum Entropy Reinforcement Learning","offset":13141,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-1.html#text"},{"title":"4 From Soft Policy Iteration to Soft Actor-Critic","offset":16156,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"4.1 Derivation of Soft Policy Iteration","offset":16565,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"Lemma 1 (Soft Policy Evaluation) .","offset":18318,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"Proof.","offset":18729,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"Lemma 2 (Soft Policy Improvement) .","offset":20441,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"Proof.","offset":20937,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"Theorem 1 (Soft Policy Iteration) .","offset":21658,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"Proof.","offset":22146,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"4.2 Soft Actor-Critic","offset":22183,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"5 Experiments","offset":28357,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"5.1 Comparative Evaluation","offset":30831,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-2.html#text"},{"title":"5.2 Ablation Study","offset":32343,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Stochastic vs. deterministic policy.","offset":32812,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Policy evaluation.","offset":34350,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Reward scale.","offset":35036,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Target network update.","offset":36129,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"6 Conclusion","offset":37313,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Acknowledgments","offset":38371,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Appendix A Maximum Entropy Objective","offset":38744,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Appendix B Proofs","offset":40091,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"B.1 Lemma 1","offset":40113,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Proof.","offset":40532,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"B.2 Lemma 2","offset":41477,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Proof.","offset":41981,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"B.3 Theorem 1","offset":45543,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Proof.","offset":46039,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Appendix C Enforcing Action Bounds","offset":47067,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-3.html#text"},{"title":"Appendix D Hyperparameters","offset":48525,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-4.html#text"},{"title":"Appendix E Additional Baseline Results","offset":48737,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-4.html#text"},{"title":"Instructions for reporting errors","offset":49660,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/part-4.html#text"}],"text_code_points":50624,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/source.md"}