{"id":"arxiv:2412.14135","title":"Scaling of Search and Learning: A Roadmap to Reproduce o1 from Reinforcement Learning Perspective","date":"2024-12-18","section":"Reasoning and the \"physics\" of language models","quality":-0.653546,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/","source_url":"https://arxiv.org/abs/2412.14135","text_available":true,"authors":["Zhiyuan Zeng","Qinyuan Cheng","Zhangyue Yin","Bo Wang"],"outgoing":[{"source":"arxiv:2412.14135","target":"acl:2022.emnlp-main.340","evidence":"title","id":"acl:2022.emnlp-main.340","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e43a06571b7e8d89abd5f20f8f520c8bc3d3eb32ffb156cb7fc9e0469fc14b0a/"},{"source":"arxiv:2412.14135","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2412.14135","target":"arxiv:2203.11171","evidence":"title","id":"arxiv:2203.11171","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02a15b098d96a94d463bb470f59a34723abc34e59196f35c539f9796d8f3c3df/"},{"source":"arxiv:2412.14135","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2412.14135","target":"arxiv:2212.09196","evidence":"title","id":"arxiv:2212.09196","title":"Emergent Analogical Reasoning in Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f74e529a278203e2c144ab7400a343836d9549cf1ae4843c584408448ad3f5a9/"},{"source":"arxiv:2412.14135","target":"arxiv:2303.17651","evidence":"title","id":"arxiv:2303.17651","title":"Self-Refine: Iterative Refinement with Self-Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/accef2157d87cf33be93cb3a9af246528b0a3742a9e2e700a194956fac8f5cf4/"},{"source":"arxiv:2412.14135","target":"arxiv:2309.14316","evidence":"title","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2412.14135","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2412.14135","target":"arxiv:2406.11813","evidence":"title","id":"arxiv:2406.11813","title":"How Do Large Language Models Acquire Factual Knowledge During Pretraining?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a7b36ce24eb2182e7f0bba96c0cddcfe6efe610b830d5e8ad8aac2b7d14696c0/"},{"source":"arxiv:2412.14135","target":"arxiv:2407.21783","evidence":"title","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2412.14135","target":"url:minedojo.org","evidence":"title","id":"url:minedojo.org","title":"MineDojo: Building Open-Ended Embodied Agents with Internet-Scale Knowledge","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a1b263777b8aec5e510481b45b7b4e11d45d7c127fd41dc37b5ff908f6420d2/"},{"source":"arxiv:2412.14135","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[],"text_parts":[{"start":0,"end":15697,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html"},{"start":15697,"end":31269,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html"},{"start":31269,"end":46861,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html"},{"start":46861,"end":62257,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html"},{"start":62257,"end":78216,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-5.html"},{"start":78216,"end":93442,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html"},{"start":93442,"end":109441,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html"},{"start":109441,"end":116514,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html"}],"contents":[{"title":"Scaling of Search and Learning: A Roadmap to Reproduce o1 from Reinforcement Learning Perspective","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"Abstract","offset":101,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"1 Introduction","offset":2024,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"Policy Initialization","offset":5293,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"Reward Design","offset":5928,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"Search","offset":6698,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"Learning","offset":7524,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"2 Background","offset":8906,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"Agent","offset":9458,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"Environment","offset":10286,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"3 Policy Initialization","offset":11171,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"3.1 Pre-Training","offset":12885,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"3.1.1 Language Understanding and Generation","offset":13226,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"3.1.2 World Knowledge Acquisition and Storage","offset":14410,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"3.1.3 Basic Reasoning Abilities","offset":15659,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-1.html#text"},{"title":"3.2 Instruction Fine-Tuning","offset":16378,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"3.3 Human-like Reasoning Behaviours","offset":17913,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Problem Analysis","offset":18527,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Task Decomposition","offset":19505,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Task Completion","offset":20390,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Alternative Proposal","offset":21623,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Self-Evaluation","offset":22620,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Self-Correction","offset":23611,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"3.4 Speculation About the Policy Initialization of o1","offset":24968,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Long-Text Generation Capabilities","offset":25796,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Shaping Human-like Reasoning Behaviors Logically","offset":26598,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"Self-Reflection","offset":27656,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"3.5 Challenges of Policy Initialization for Reproducing o1","offset":28348,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"How to balance sampling-efficiency and sampling-diversity?","offset":28574,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"How to ensure domain generalization of reasoning behaviors?","offset":29235,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"4 Reward Design","offset":29976,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"4.1 Outcome Reward vs Process Reward","offset":31227,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-2.html#text"},{"title":"4.2 Reward Design Methods","offset":33673,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.2.1 Reward from Environment","offset":34148,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"From Realistic Environment","offset":34373,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"From Simulating Environment","offset":35471,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"From AI Judgment","offset":36861,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.2.2 Reward Modeling from Data","offset":37640,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"Learning Rewards from Preference Data","offset":38392,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"Learning Rewards from Expert Data","offset":39501,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.2.3 Reward Shaping","offset":40436,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.3 Speculation About the Reward Design of o1","offset":42791,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.4 Challenges of Reward Design for Reproducing o1","offset":43910,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"How to overcome distribution shift?","offset":43966,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"How to design fine-grained reward for language model?","offset":44728,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"How to choose data when modeling reward for complex tasks?","offset":45538,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.5 Generalization","offset":46154,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.5.1 Reward Ensemble","offset":46833,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-3.html#text"},{"title":"4.5.2 One for all: World Model","offset":47435,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5 Search","offset":48423,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.1 The role of Search in o1","offset":49779,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.2 Search Guidance","offset":52103,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.2.1 Internal Guidance","offset":52794,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"Model Uncertainty","offset":52824,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"Self-evaluation","offset":54197,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.2.2 External Guidance","offset":55830,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"Environmental Feedback","offset":55860,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"Heuristic Rules","offset":57207,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.2.3 Comparison of Internal and External Guidance","offset":57856,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.2.4 Combination of Internal and External Guidance","offset":58853,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.3 Search Strategies","offset":61827,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"5.3.1 Tree Search","offset":62233,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-4.html#text"},{"title":"Best-of-n Sampling","offset":63289,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-5.html#text"},{"title":"Beam Search","offset":65061,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-5.html#text"},{"title":"Monte Carlo Tree Search","offset":66541,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-5.html#text"},{"title":"Other Tree Search Algorithms.","offset":73098,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-5.html#text"},{"title":"5.3.2 Sequential Revisions","offset":73944,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-5.html#text"},{"title":"5.3.3 Comparison of Tree Search and Sequential Revisions","offset":78216,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"5.3.4 Combination of Tree Search and Sequential Revisions","offset":79214,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"5.4 Speculation About the Search of o1.","offset":79909,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"Train-time search.","offset":80100,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"Test-time search.","offset":80834,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"5.5 Scaling Law of Search","offset":81935,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"5.6 Challenges of Search for Reproducing o1","offset":83481,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"How to overcome Inverse Scaling","offset":83530,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"How to avoid over-thinking on simple tasks?","offset":83974,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"How to trade off tree search and sequential revisions?","offset":84505,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"How to improve efficiency of Search?","offset":84950,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"6 Learning","offset":85559,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"6.1 Learning Methods","offset":88366,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"6.1.1 Policy Gradient","offset":89151,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"REINFORCE","offset":89735,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"Actor-Critic","offset":90254,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"PPO","offset":91171,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"DPO","offset":93431,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-6.html#text"},{"title":"6.1.2 Behavior Cloning","offset":95677,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"6.1.3 Speculation about the learning of o1","offset":99004,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"Memory Cost","offset":99281,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"Data Utilization","offset":99876,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"6.2 Scaling Law of Reinforcement Learning","offset":101536,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"6.3 Challenges of Learning for Reproducing o1","offset":103623,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"How to improve Training Efficiency?","offset":103674,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"How to learn a Strong Question Generator.","offset":104630,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"How to Narrow the Distribution Shift in off-policy learning?","offset":105718,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"7 Open-source o1 Project","offset":107604,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"g1","offset":108330,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"Thinking Claude","offset":108560,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"Open-o1","offset":108897,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"o1 Journey","offset":109328,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-7.html#text"},{"title":"Part 1","offset":109441,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"Part 2","offset":110024,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"Open-Reasoner","offset":110617,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"Slow Thinking with LLMs","offset":111162,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"Part 1","offset":111332,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"Part 2","offset":111681,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"Marco-o1","offset":112166,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"o1-coder","offset":112476,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"8 Future Directions","offset":112885,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"How to adapt o1 to general domains?","offset":112909,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"How to introduce multiple modality to o1?","offset":113727,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"How to learn and search with a world model?","offset":114709,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"},{"title":"9 Conclusion","offset":116030,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/part-8.html#text"}],"text_code_points":116514,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/source.md"}