{"id":"arxiv:2005.01643","title":"Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems","date":"2020-05-04","section":"Reinforcement learning","quality":-0.419453,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/","source_url":"https://arxiv.org/abs/2005.01643","text_available":true,"authors":["Sergey Levine","Aviral Kumar","George Tucker","Justin Fu"],"outgoing":[{"source":"arxiv:2005.01643","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/"}],"incoming":[{"source":"arxiv:2106.06860","target":"arxiv:2005.01643","evidence":"id","id":"arxiv:2106.06860","title":"A Minimalist Approach to Offline Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/00a35c7a58d8775f23035efbd1cb0a5f540440d47b8a996be8a4fa0da2da3e0e/"},{"source":"arxiv:2211.10869","target":"arxiv:2005.01643","evidence":"id","id":"arxiv:2211.10869","title":"UniMASK: Unified Inference in Sequential Decision Problems","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/919bff55efff4ed3549bd106b843afa3d140b32938f5283c77821d7fd569452e/"},{"source":"arxiv:2305.19452","target":"arxiv:2005.01643","evidence":"id","id":"arxiv:2305.19452","title":"Bigger, Better, Faster: Human-level Atari with human-level efficiency","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/727a7ee43fd005401d34d4591a2f49d85175f9366ebf2c9e7a862b340787614e/"},{"source":"arxiv:2406.11741","target":"arxiv:2005.01643","evidence":"id","id":"arxiv:2406.11741","title":"Transcendence: Generative Models Can Outperform The Experts That Train Them","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/"},{"source":"arxiv:2508.05629","target":"arxiv:2005.01643","evidence":"id","id":"arxiv:2508.05629","title":"On the Generalization of SFT: A Reinforcement Learning Perspective with Reward Rectification","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/"}],"text_parts":[{"start":0,"end":14685,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html"},{"start":14685,"end":29399,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-2.html"},{"start":29399,"end":44871,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html"},{"start":44871,"end":59638,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-4.html"},{"start":59638,"end":72743,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-5.html"},{"start":72743,"end":88316,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-6.html"},{"start":88316,"end":102985,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-7.html"},{"start":102985,"end":117996,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-8.html"},{"start":117996,"end":133966,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-9.html"},{"start":133966,"end":149234,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-10.html"},{"start":149234,"end":164822,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html"},{"start":164822,"end":180098,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-12.html"}],"contents":[{"title":"Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"Abstract","offset":87,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"1 Introduction","offset":1216,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"2 Offline Reinforcement Learning Problem Statement and Overview","offset":7871,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"2.1 Reinforcement Learning Preliminaries","offset":8393,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"Definition 2.1 (Markov decision process) .","offset":8800,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"Definition 2.2 (Partially observed Markov decision process) .","offset":10348,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"Policy gradients.","offset":14661,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-1.html#text"},{"title":"Approximate dynamic programming.","offset":18304,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-2.html#text"},{"title":"Actor-critic algorithms.","offset":25575,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-2.html#text"},{"title":"Model-based reinforcement learning.","offset":29357,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-2.html#text"},{"title":"2.2 Offline Reinforcement Learning","offset":31374,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"2.3 Example Scenarios","offset":35155,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"Decision making in health care.","offset":35615,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"Learning goal-directed dialogue policies.","offset":36551,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"Learning robotic manipulation skills.","offset":37415,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"2.4 What Makes Offline Reinforcement Learning Difficult?","offset":38699,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"Theorem 2.1 (Behavioral cloning error bound) .","offset":43136,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"Proof.","offset":43632,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"Theorem 2.2 (DAgger error bound) .","offset":44166,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"Proof.","offset":44620,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-3.html#text"},{"title":"3 Offline Evaluation and Reinforcement Learning via Importance Sampling","offset":46508,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-4.html#text"},{"title":"3.1 Off-Policy Evaluation via Importance Sampling","offset":47432,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-4.html#text"},{"title":"3.2 The Off-Policy Policy Gradient","offset":53392,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-4.html#text"},{"title":"3.3 Approximate Off-Policy Policy Gradients","offset":59589,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-4.html#text"},{"title":"3.4 Marginalized Importance Sampling","offset":63266,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-5.html#text"},{"title":"Forward Bellman equation based approaches.","offset":64720,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-5.html#text"},{"title":"Backward Bellman equation based approaches via convex duality.","offset":71960,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-5.html#text"},{"title":"3.5 Challenges and Open Problems","offset":79582,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-6.html#text"},{"title":"4 Offline Reinforcement Learning via Dynamic Programming","offset":82490,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-6.html#text"},{"title":"4.1 Off-Policy Value Function Estimation with Linear Value Functions","offset":85419,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-6.html#text"},{"title":"Bellman residual minimization.","offset":88279,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-6.html#text"},{"title":"Least-squares fixed point approximation.","offset":90043,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-7.html#text"},{"title":"Least squares temporal difference Q-learning (LSTD-Q).","offset":94178,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-7.html#text"},{"title":"Least squares policy iteration (LSPI).","offset":95133,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-7.html#text"},{"title":"4.2 Distributional Shift in Offline Reinforcement Learning via Dynamic Programming","offset":96341,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-7.html#text"},{"title":"4.3 Policy Constraints for Off-Policy Evaluation and Improvement","offset":102915,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-7.html#text"},{"title":"Explicit f f -divergence constraints.","offset":109901,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-8.html#text"},{"title":"Implicit f f -divergence constraints.","offset":113358,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-8.html#text"},{"title":"Integral probability metrics (IPMs).","offset":115110,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-8.html#text"},{"title":"Tradeoffs between different constraint types.","offset":117944,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-8.html#text"},{"title":"4.4 Offline Approximate Dynamic Programming with Uncertainty Estimation","offset":123238,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-9.html#text"},{"title":"4.5 Conservative Q-Learning and Pessimistic Value Functions","offset":127321,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-9.html#text"},{"title":"4.6 Challenges and Open Problems","offset":133301,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-9.html#text"},{"title":"5 Offline Model-Based Reinforcement Learning","offset":141569,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-10.html#text"},{"title":"5.1 Model Exploitation and Distribution Shift","offset":142980,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-10.html#text"},{"title":"5.2 Brief Survey of Model-Based Offline Reinforcement Learning","offset":147743,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-10.html#text"},{"title":"Offline RL with standard model-based RL methods.","offset":149179,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-10.html#text"},{"title":"Off-policy evaluation with models.","offset":151288,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html#text"},{"title":"Distribution and safe region constraints.","offset":151874,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html#text"},{"title":"Conservative model-based RL algorithms.","offset":152909,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html#text"},{"title":"5.3 Challenges and Open Problems","offset":155424,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html#text"},{"title":"6 Applications and Evaluation","offset":158068,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html#text"},{"title":"6.1 Evaluation and Benchmarks","offset":158928,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html#text"},{"title":"6.2 Applications in Robotics","offset":163604,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-11.html#text"},{"title":"6.3 Applications in Healthcare","offset":166614,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-12.html#text"},{"title":"6.4 Applications in Autonomous Driving","offset":168416,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-12.html#text"},{"title":"6.5 Applications in Advertising and Recommender Systems","offset":170287,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-12.html#text"},{"title":"6.6 Applications in Language and Dialogue","offset":172089,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-12.html#text"},{"title":"7 Discussion and Perspectives","offset":173878,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/part-12.html#text"}],"text_code_points":180098,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/source.md"}