{"id":"arxiv:2509.03646","title":"Emergent Hierarchical Reasoning in LLMs through Reinforcement Learning","date":"2025-09-03","section":"Post-training","quality":-0.042926,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/","source_url":"https://arxiv.org/abs/2509.03646","text_available":true,"authors":["Haozhe Wang","Qixin Xu","Che Liu"],"outgoing":[{"source":"arxiv:2509.03646","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2509.03646","target":"arxiv:2503.14476","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2509.03646","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2509.03646","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2506.01939","title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/"},{"source":"arxiv:2509.03646","target":"arxiv:2506.21734","evidence":"id","id":"arxiv:2506.21734","title":"Hierarchical Reasoning Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3604312101ff09ab9903e7d6a32c7b2b8a9a7ece6a9d145d04ea01c9d13ffa3a/"}],"incoming":[{"source":"arxiv:2604.08690","target":"arxiv:2509.03646","evidence":"id","id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/"}],"text_parts":[{"start":0,"end":15906,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html"},{"start":15906,"end":31582,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html"},{"start":31582,"end":47576,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html"},{"start":47576,"end":49519,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-4.html"}],"contents":[{"title":"Emergent Hierarchical Reasoning in LLMs through Reinforcement Learning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"Abstract","offset":74,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"1 Introduction","offset":1399,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"2 The Emergent Reasoning Hierarchy","offset":7175,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"2.1 A Functional Proxy for the Reasoning Hierarchy","offset":7511,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"2.2 Emergence of the Reasoning Hierarchy","offset":11634,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"2.2.1 Forging Reliable Low-level Skills","offset":12607,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"2.2.2 Steering the Skills with Strategic Planning","offset":14553,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-1.html#text"},{"title":"3 HICRA: Hierarchy-Aware Credit Assignment","offset":19060,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"4 Experiments","offset":24035,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"4.1 Main Results","offset":25458,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"4.2 Analysis of RL’s Impact on Reasoning","offset":26178,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"4.2.1 Mastery of Strategic Planning Unlocks Improved Reasoning during RL","offset":26774,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"4.2.2 Justifying HICRA: Targeted vs. Indiscriminate Exploration","offset":28295,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"4.2.3 Semantic Entropy: A Compass for Strategic Exploration","offset":29861,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"4.2.4 Planning Tokens vs. High-Entropy ”Fork” Tokens","offset":31222,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-2.html#text"},{"title":"5 Related Work","offset":32343,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"6 Conclusions","offset":35625,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"Appendix A SG Construction","offset":36756,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"Appendix B Sensitivity Analysis of SGs","offset":36862,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"Appendix C Full Training Dynamics","offset":37821,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"Appendix D The Distribution Matching Perspective of Policy Gradients","offset":41446,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"Appendix E Extended Materials of Experiments","offset":45123,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"E.1 Experimental Setups","offset":45172,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"Training Datasets and Benchmarks","offset":45201,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"Baselines and Implementation.","offset":46441,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-3.html#text"},{"title":"E.2 HICRA Presumes a Dependency on a Procedural Foundation","offset":47576,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-4.html#text"},{"title":"Instructions for reporting errors","offset":48555,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/part-4.html#text"}],"text_code_points":49519,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/source.md"}