{"id":"arxiv:2609.30652","title":"Recursive Self-Improvement via On-Policy Distillation for Reasoning","date":"2026-09-25","section":"Post-training","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/","source_url":"https://arxiv.org/abs/2609.30652","text_available":true,"authors":["Shangjian Yin","Zehao Zhao","Kavosh Asadi","Rui Liu"],"outgoing":[{"source":"arxiv:2609.30652","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2609.30652","target":"arxiv:2501.19393","evidence":"title","id":"arxiv:2501.19393","title":"s1: Simple test-time scaling","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2a88818f7605b71cd0160831faebd51b3274c24ffd29626b4d519390b4bffffe/"},{"source":"arxiv:2609.30652","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2609.30652","target":"arxiv:2503.20783","evidence":"title","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2609.30652","target":"arxiv:2601.18734","evidence":"title","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"}],"incoming":[],"text_parts":[{"start":0,"end":15410,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html"},{"start":15410,"end":30948,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html"},{"start":30948,"end":46065,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html"},{"start":46065,"end":52286,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html"}],"contents":[{"title":"Recursive Self-Improvement via On-Policy Distillation for Reasoning","offset":33,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"Abstract","offset":104,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"1 Introduction","offset":1812,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"2 Related Work","offset":7990,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"2.1 Reflection and Verification in Reasoning Post-Training","offset":8009,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"2.2 On-Policy and Privileged Self-Distillation","offset":10619,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"2.3 Self-Refinement and Efficient Reasoning","offset":12607,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"3 Method","offset":14705,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"3.1 Dynamic Co-Evolution","offset":15380,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-1.html#text"},{"title":"3.2 Self-Refined Concise Learning","offset":19058,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"3.3 Joint Optimization","offset":21892,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"4 Experimental Setup","offset":22913,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"4.1 Models, Data, and Metrics","offset":22938,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"4.2 Baselines and Controls","offset":23869,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"4.3 Training Protocol","offset":24890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"5 Results","offset":26036,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"5.1 Main Experimental Results","offset":26050,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"5.2 Does Recursive Improvement Transfer Beyond Qwen3?","offset":27820,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"5.3 What Happens During Recursive Self-Improvement?","offset":28532,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"5.4 How Do Different Loss Weights Affect Performance?","offset":30889,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-2.html#text"},{"title":"5.5 How Does the Guidance Objective Influence Performance?","offset":32358,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"5.6 Does the Privileged Teacher Need to Co-Evolve?","offset":33774,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"5.7 Which Representation of the Verified Solution Works Better?","offset":34587,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"5.8 How Does the Method Perform under Tight Reasoning Budgets?","offset":35455,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"5.9 How Sensitive Is Performance to Decoding Hyperparameters?","offset":36907,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"6 Conclusion","offset":37567,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix A Does Recursive Improvement Scale to a Larger Model?","offset":38055,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix B Gemma-4-12B-IT Training Details and Additional Results","offset":38810,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix C Complete Qwen3-8B and Qwen3-4B Trajectories","offset":39762,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix D Teacher-Update Schedules","offset":40389,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix E Effect of Privileged-Context Placement","offset":41677,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Evolution of the privileged signal.","offset":42907,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix F Qwen3-1.7B Training Stability","offset":45175,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix G Fixed-Trace Probe Values","offset":45500,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Appendix H EOS-Penalty Ablation","offset":46012,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Objective.","offset":46048,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-3.html#text"},{"title":"Results.","offset":46972,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Appendix I Cross-Scale Evaluation of OPSD-TTS","offset":47305,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Appendix J Reproducibility Details","offset":47671,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Appendix K Additional SRCL-Weight Sensitivity","offset":47892,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Appendix L Results on Additional Benchmarks","offset":48341,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Appendix M SRCL Rewrite Filtering and Acceptance","offset":48844,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Filtering pipeline.","offset":49623,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Endpoint verification.","offset":50478,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Appendix N Prompt Templates","offset":50986,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"},{"title":"Instructions for reporting errors","offset":51322,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/part-4.html#text"}],"text_code_points":52286,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/source.md"}