{"id":"arxiv:2608.17163","title":"Q-Learning With World Models","date":"2026-08-17","section":"Reinforcement learning","quality":0.08444,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/","source_url":"https://arxiv.org/abs/2608.17163","text_available":true,"authors":["Perry Dong","Yueru Jia","Chelsea Finn","Dorsa Sadigh"],"outgoing":[{"source":"arxiv:2608.17163","target":"arxiv:1801.01290","evidence":"id","id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/"},{"source":"arxiv:2608.17163","target":"arxiv:2106.06860","evidence":"id","id":"arxiv:2106.06860","title":"A Minimalist Approach to Offline Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/00a35c7a58d8775f23035efbd1cb0a5f540440d47b8a996be8a4fa0da2da3e0e/"},{"source":"arxiv:2608.17163","target":"arxiv:2203.11171","evidence":"id","id":"arxiv:2203.11171","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02a15b098d96a94d463bb470f59a34723abc34e59196f35c539f9796d8f3c3df/"},{"source":"arxiv:2608.17163","target":"arxiv:2301.04104","evidence":"id","id":"arxiv:2301.04104","title":"Mastering Diverse Domains through World Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/266d59e63d2f705efdc2eb7ad93f5126b7092b4450efd0d8c17099c51574b176/"}],"incoming":[],"text_parts":[{"start":0,"end":15595,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html"},{"start":15595,"end":31519,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html"},{"start":31519,"end":47366,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html"},{"start":47366,"end":50154,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-4.html"}],"contents":[{"title":"Q-Learning With World Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"Abstract","offset":32,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"1 Introduction","offset":1485,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"2 Related Work","offset":5123,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"World models for VLAs.","offset":8721,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"Test-time scaling and best-of- N N inference.","offset":9989,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"3 Preliminaries","offset":10895,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"4 Method","offset":15582,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-1.html#text"},{"title":"4.1 Tree Search With the World Model","offset":16158,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"4.2 Aggregating the Value","offset":17676,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"Root value.","offset":20537,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"4.3 Searching with the Q-function.","offset":21342,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"4.4 Practical Implementation","offset":23819,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"5 Experiments","offset":25542,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"5.1 How Does QWM Perform Compared to State-of-the-Art Model-Free and Model-Based RL Methods?","offset":26859,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"5.2 How Does QWM Perform Compared to the Base Method It Is Implemented on Top of?","offset":29336,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"5.3 Does QWM Scale to a Pixel-Based Setting?","offset":30362,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"5.4 What Components of QWM Are Most Important for Performance?","offset":31451,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-2.html#text"},{"title":"6 Discussion","offset":36614,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"7 Acknowledgments","offset":37698,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"Appendix A Author Contributions","offset":38220,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"Appendix B Additional Experiments","offset":38713,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"Appendix C Experiment Details","offset":39669,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"C.1 Hyperparameters","offset":39703,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"QWM -specific hyperparameters.","offset":40652,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"C.2 Environments","offset":42104,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"C.3 Additional World Model Details and Prediction Quality","offset":44367,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"C.4 Comparisons","offset":46440,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-3.html#text"},{"title":"Instructions for reporting errors","offset":49190,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/part-4.html#text"}],"text_code_points":50154,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/15ca8843297a713249e241e591505a44859dac258c54e299d79c62c81bd893c2/source.md"}