{"id":"arxiv:2603.25562","title":"Revisiting On-Policy Distillation: Empirical Failure Modes and Simple Fixes","date":"2026-03-26","section":"Post-training","quality":-0.059566,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/","source_url":"https://arxiv.org/abs/2603.25562","text_available":true,"authors":["Yuqian Fu","Haohuan Huang","Kaiwen Jiang","Jiacai Liu"],"outgoing":[{"source":"arxiv:2603.25562","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2603.25562","target":"arxiv:2505.10978","evidence":"title","id":"arxiv:2505.10978","title":"Group-in-Group Policy Optimization for LLM Agent Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/"},{"source":"arxiv:2603.25562","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"},{"source":"arxiv:2603.25562","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"},{"source":"arxiv:2603.25562","target":"arxiv:2604.13016","evidence":"id","id":"arxiv:2604.13016","title":"Rethinking On-Policy Distillation of Large Language Models: Phenomenology, Mechanism, and Recipe","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d6d7ff1e2e7b3b9c528f19f7144098920bf573f2cb411fcac5879d5d2cac93e0/"}],"incoming":[{"source":"arxiv:2604.13016","target":"arxiv:2603.25562","evidence":"id","id":"arxiv:2604.13016","title":"Rethinking On-Policy Distillation of Large Language Models: Phenomenology, Mechanism, and Recipe","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d6d7ff1e2e7b3b9c528f19f7144098920bf573f2cb411fcac5879d5d2cac93e0/"},{"source":"arxiv:2606.06021","target":"arxiv:2603.25562","evidence":"id","id":"arxiv:2606.06021","title":"OPRD: On-Policy Representation Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a16a9d7ea78c5748acc7f651c9a96663e0200d72e51d6a52c9528e1739a27d8d/"},{"source":"arxiv:2608.13040","target":"arxiv:2603.25562","evidence":"id","id":"arxiv:2608.13040","title":"Latent On-Policy Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2cfd504d6ca36ba88d75aba018620867d5fafd7433b03f3b2c9f51408422a02/"}],"text_parts":[{"start":0,"end":15897,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html"},{"start":15897,"end":30748,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html"},{"start":30748,"end":46710,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html"},{"start":46710,"end":55031,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html"}],"contents":[{"title":"Revisiting On-Policy Distillation: Empirical Failure Modes and Simple Fixes","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"Abstract","offset":79,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"1 Introduction","offset":1618,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"2 Understanding Sampled-Token OPD: Trade-offs and Failure Modes","offset":6369,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"2.1 From reverse-KL to token-level OPD","offset":6437,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"2.2 Why sampled-token OPD is brittle in practice","offset":10423,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"A highly imbalanced sampled-token signal.","offset":11084,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"The teacher signal can become unreliable on student-generated prefixes.","offset":11778,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"Tokenizer and special-token mismatch.","offset":12746,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"3 Method","offset":13915,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"3.1 Teacher top- K K local support matching","offset":14415,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-1.html#text"},{"title":"3.2 Practical stabilization choices","offset":17287,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Support-set renormalization.","offset":17328,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Top- p p rollout sampling.","offset":17785,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Special-token masking.","offset":18153,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"4 Experiments","offset":18677,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"4.1 Setup","offset":19057,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"4.2 Single-task math reasoning","offset":20281,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"4.3 Multi-task agentic-plus-reasoning training","offset":21252,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"4.4 Ablations","offset":22102,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Top- K K support variants.","offset":22869,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"5 Conclusion","offset":24355,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Appendix A Discussion and Limitations","offset":25486,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Sampled-token augmentation of the teacher top- K K support.","offset":25528,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Support-set KL is a truncated objective.","offset":26542,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Training–inference mismatch.","offset":27174,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Teacher matching remains an imperfect proxy for task success.","offset":27515,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Appendix B Future Directions","offset":28308,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Top- p p truncation as an adaptive support.","offset":28341,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"OPD versus RL in multi-task settings.","offset":28650,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Continual learning as a testbed.","offset":29410,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Relation to other stabilization directions.","offset":30037,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"Appendix C Related Work","offset":30689,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"On-policy distillation.","offset":30717,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-2.html#text"},{"title":"KL computation for LLM post-training.","offset":32868,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"KL divergence in RL for LLMs.","offset":34383,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Appendix D Bias and variance analysis of token-level versus sequence-level OPD","offset":35987,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"D.1 Bias of the token-level estimator","offset":36070,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"D.2 Worst-case variance upper bounds","offset":37096,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"D.3 Discussion","offset":38331,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Appendix E Toy experiment details","offset":38711,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"E.1 Environment","offset":38749,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"E.2 Gradient variance estimation","offset":39579,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"E.3 Additional Results of Toy Experiments","offset":40288,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Appendix F Experiment Setups","offset":41365,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Appendix G Additional Results","offset":41747,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"G.1 Additional Evidence on Single-task WebShop","offset":41781,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"G.2 Training Dynamics and Alignment","offset":42534,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Better learning curves.","offset":42660,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"More stable optimization.","offset":42936,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Improved teacher–student alignment.","offset":43329,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"G.3 Additional Results of Top- K K Variants","offset":43593,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"G.3.1 Formulations of Five Top- K K Variants","offset":43906,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Shared notation.","offset":43957,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Renormalized local reverse-KL.","offset":45051,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Variant 1: teacher top- K K renormalization (our method).","offset":46168,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-3.html#text"},{"title":"Variant 2: student top- K K + sampled token + renormalization.","offset":46710,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"Variant 3: teacher top- K K + sampled token + renormalization.","offset":47520,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"EMA-PG-style local loss.","offset":48302,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"Variant 4: student top- K K + EMA-PG correction.","offset":50253,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"Variant 5: teacher top- K K + EMA-PG correction.","offset":50780,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"Summary.","offset":51303,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"G.3.2 Results under Single-task and Multi-task Settings","offset":51725,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"Appendix H Qualitative OPD reward-hacking case study","offset":52181,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"},{"title":"Instructions for reporting errors","offset":54067,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/part-4.html#text"}],"text_code_points":55031,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/source.md"}