{"id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","date":"2026-01-26","section":"Post-training","quality":0.122413,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/","source_url":"https://arxiv.org/abs/2601.18734","text_available":true,"authors":["Siyan Zhao","Zhihui Xie","Mengchen Liu","Jing Huang"],"outgoing":[{"source":"arxiv:2601.18734","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2601.18734","target":"arxiv:2402.13669","evidence":"title","id":"arxiv:2402.13669","title":"Self-Distillation Bridges Distribution Gap in Language Model Fine-Tuning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/"},{"source":"arxiv:2601.18734","target":"arxiv:2501.19393","evidence":"id","id":"arxiv:2501.19393","title":"s1: Simple test-time scaling","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2a88818f7605b71cd0160831faebd51b3274c24ffd29626b4d519390b4bffffe/"},{"source":"arxiv:2601.18734","target":"arxiv:2502.03387","evidence":"id","id":"arxiv:2502.03387","title":"LIMO: Less is More for Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/714392528bb842fb3a61cee5367942f697536f0131b0fcac85c6eb8d2212125c/"},{"source":"arxiv:2601.18734","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2601.18734","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2601.18734","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"},{"source":"arxiv:2601.18734","target":"arxiv:2601.19897","evidence":"id","id":"arxiv:2601.19897","title":"Self-Distillation Enables Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cc146df3e9ca40c6d111a5264e08710100c1b34bd0eabd9052e8d55d13589efe/"},{"source":"arxiv:2601.18734","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"}],"incoming":[{"source":"arxiv:2601.19897","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2601.19897","title":"Self-Distillation Enables Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cc146df3e9ca40c6d111a5264e08710100c1b34bd0eabd9052e8d55d13589efe/"},{"source":"arxiv:2601.20802","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"},{"source":"arxiv:2603.24472","target":"arxiv:2601.18734","evidence":"title","id":"arxiv:2603.24472","title":"Why Does Self-Distillation (Sometimes) Degrade the Reasoning Capability of LLMs?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d99edb6fac2f4ad6d63e98ec0a2952187ed2ae190604fbed3507b3ed56d57315/"},{"source":"arxiv:2603.25562","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2603.25562","title":"Revisiting On-Policy Distillation: Empirical Failure Modes and Simple Fixes","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/"},{"source":"arxiv:2604.02288","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2604.02288","title":"Unifying Group-Relative and Self-Distillation Policy Optimization via Sample Routing","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3d1160029d321f2069016f7702d4ebebee0de6cd31a8f296ee220abd6f69c99b/"},{"source":"arxiv:2604.13016","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2604.13016","title":"Rethinking On-Policy Distillation of Large Language Models: Phenomenology, Mechanism, and Recipe","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d6d7ff1e2e7b3b9c528f19f7144098920bf573f2cb411fcac5879d5d2cac93e0/"},{"source":"arxiv:2606.06021","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2606.06021","title":"OPRD: On-Policy Representation Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a16a9d7ea78c5748acc7f651c9a96663e0200d72e51d6a52c9528e1739a27d8d/"},{"source":"arxiv:2606.18810","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2606.18810","title":"Learning from Own Solutions: Self-Conditioned Credit Assignment for Reinforcement Learning with Verifiable Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9c11d77c6d08392f24aab79c8b45a3a3b0f4b743dd35e150a83ee7f6fa315248/"},{"source":"arxiv:2607.07847","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2607.07847","title":"When Does Continual Learning Require Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/86e88c6cf21a66879eab5c156913fc85d14eb69c394c78c33c0e81d144e528eb/"},{"source":"arxiv:2608.13040","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2608.13040","title":"Latent On-Policy Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2cfd504d6ca36ba88d75aba018620867d5fafd7433b03f3b2c9f51408422a02/"},{"source":"arxiv:2608.27448","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2608.27448","title":"TTPO: Test-Time Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/"},{"source":"arxiv:2609.30652","target":"arxiv:2601.18734","evidence":"title","id":"arxiv:2609.30652","title":"Recursive Self-Improvement via On-Policy Distillation for Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/"},{"source":"arxiv:2610.02140","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2610.02140","title":"Finetuning with Sampling: SFT Learns Better Than You Think","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/60f5edfae92f505bf86e8890135a44b4129825b0525b5ad8d242e8ba04182605/"}],"text_parts":[{"start":0,"end":15963,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html"},{"start":15963,"end":31396,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html"},{"start":31396,"end":45000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html"}],"contents":[{"title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"Abstract","offset":82,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"Keywords:","offset":1482,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"1 Introduction","offset":1500,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"2 Background","offset":6759,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"2.1 Knowledge Distillation for Autoregressive Large Language Models","offset":6776,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"2.2 Reinforcement Learning with Verifiable Rewards","offset":9319,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"3 Methods","offset":12328,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"3.1 Learning from Verifiable Reasoning Dataset","offset":12342,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"3.2 On-Policy Self-Distillation","offset":14159,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"Motivation: Learning by understanding solutions.","offset":14196,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"Teacher and student policies.","offset":14978,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-1.html#text"},{"title":"On-policy sampling from the student.","offset":15963,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"Training objective: Full-vocabulary logit distillation.","offset":16670,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"Per-Token Pointwise Divergence Clipping.","offset":18854,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"Alternative objective: Sampled-token distillation through policy gradient.","offset":19853,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"OPSD as dense-reward policy gradient and comparison to STaR.","offset":21448,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4 Experiments","offset":22125,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.1 Experimental Setup","offset":22615,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.2 Main Results","offset":23961,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.3 Ablation Studies & Discussions","offset":25630,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.3.1 Effect of Divergence Objective","offset":26060,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.3.2 Effect of Generation Styles and per-token KL Clipping","offset":26689,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.3.3 Effect of Per-Token Pointwise Clipping","offset":28205,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.3.4 Effect of Generation Length","offset":28666,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"4.3.5 Learning Objective Comparison: Full Vocabulary Logits Distillation vs. Sampled-Token Distillation","offset":29694,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"5 Related Work","offset":31351,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"LLM Self-Training.","offset":31370,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-2.html#text"},{"title":"6 Conclusion","offset":35228,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"7 Impact Statement","offset":35882,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Appendix A Limitations and Future Directions","offset":36215,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Appendix B Experimental Details","offset":37280,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Appendix C Token Category Definitions","offset":38156,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Style Tokens.","offset":38419,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Math Tokens.","offset":38933,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Appendix D Policy-Gradient Interpretation of OPSD and Comparison to STaR","offset":39379,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"D.1 STaR as Sequence-Level Policy-Gradient","offset":39820,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"D.2 OPSD as Dense-Reward Policy Gradient","offset":41565,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Comparison.","offset":43542,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"},{"title":"Instructions for reporting errors","offset":44036,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/part-3.html#text"}],"text_code_points":45000,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/source.md"}