{"id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","date":"2026-01-28","section":"Post-training","quality":0.277712,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/","source_url":"https://arxiv.org/abs/2601.20802","text_available":true,"authors":["Jonas Hübotter","Frederike Lübeck","Lejs Behric"],"outgoing":[{"source":"arxiv:2601.20802","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/"},{"source":"arxiv:2601.20802","target":"arxiv:2303.17651","evidence":"title","id":"arxiv:2303.17651","title":"Self-Refine: Iterative Refinement with Self-Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/accef2157d87cf33be93cb3a9af246528b0a3742a9e2e700a194956fac8f5cf4/"},{"source":"arxiv:2601.20802","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2601.20802","target":"arxiv:2402.13669","evidence":"title","id":"arxiv:2402.13669","title":"Self-Distillation Bridges Distribution Gap in Language Model Fine-Tuning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/"},{"source":"arxiv:2601.20802","target":"arxiv:2501.19393","evidence":"title","id":"arxiv:2501.19393","title":"s1: Simple test-time scaling","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2a88818f7605b71cd0160831faebd51b3274c24ffd29626b4d519390b4bffffe/"},{"source":"arxiv:2601.20802","target":"arxiv:2502.06807","evidence":"id","id":"arxiv:2502.06807","title":"Competitive Programming with Large Reasoning Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d1e6e9343805a99e3ba9032540f71dd4fc3436db1425ff988c9639e0c3641c8c/"},{"source":"arxiv:2601.20802","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2601.20802","target":"arxiv:2503.20783","evidence":"title","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2601.20802","target":"arxiv:2504.13837","evidence":"title","id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/"},{"source":"arxiv:2601.20802","target":"arxiv:2504.16084","evidence":"title","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2601.20802","target":"arxiv:2505.21493","evidence":"title","id":"arxiv:2505.21493","title":"Reinforcing General Reasoning without Verifiers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/"},{"source":"arxiv:2601.20802","target":"arxiv:2506.01939","evidence":"title","id":"arxiv:2506.01939","title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/"},{"source":"arxiv:2601.20802","target":"arxiv:2506.13585","evidence":"id","id":"arxiv:2506.13585","title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5beb2ec6c7e6362f9d957219d7a205dd1adcc53f739acc0c7fd5f1f2bded4d83/"},{"source":"arxiv:2601.20802","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"},{"source":"arxiv:2601.20802","target":"arxiv:2510.13786","evidence":"title","id":"arxiv:2510.13786","title":"The Art of Scaling Reinforcement Learning Compute for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/72580bcba3d9b32b2f3e83d074f626234b26dc18126edd22a5eacef927cf87da/"},{"source":"arxiv:2601.20802","target":"arxiv:2512.13961","evidence":"id","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"},{"source":"arxiv:2601.20802","target":"arxiv:2601.16175","evidence":"id","id":"arxiv:2601.16175","title":"Learning to Discover at Test Time","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d0b45cbbc72945765e689cbfb7ac7e4e92dfbb764861b27682ac0744ef05b70/"},{"source":"arxiv:2601.20802","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"},{"source":"arxiv:2601.20802","target":"arxiv:2601.19897","evidence":"id","id":"arxiv:2601.19897","title":"Self-Distillation Enables Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cc146df3e9ca40c6d111a5264e08710100c1b34bd0eabd9052e8d55d13589efe/"},{"source":"arxiv:2601.20802","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[{"source":"arxiv:2601.18734","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"},{"source":"arxiv:2601.19897","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2601.19897","title":"Self-Distillation Enables Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cc146df3e9ca40c6d111a5264e08710100c1b34bd0eabd9052e8d55d13589efe/"},{"source":"arxiv:2603.24472","target":"arxiv:2601.20802","evidence":"title","id":"arxiv:2603.24472","title":"Why Does Self-Distillation (Sometimes) Degrade the Reasoning Capability of LLMs?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d99edb6fac2f4ad6d63e98ec0a2952187ed2ae190604fbed3507b3ed56d57315/"},{"source":"arxiv:2603.25562","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2603.25562","title":"Revisiting On-Policy Distillation: Empirical Failure Modes and Simple Fixes","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/"},{"source":"arxiv:2604.02288","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2604.02288","title":"Unifying Group-Relative and Self-Distillation Policy Optimization via Sample Routing","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3d1160029d321f2069016f7702d4ebebee0de6cd31a8f296ee220abd6f69c99b/"},{"source":"arxiv:2604.13016","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2604.13016","title":"Rethinking On-Policy Distillation of Large Language Models: Phenomenology, Mechanism, and Recipe","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d6d7ff1e2e7b3b9c528f19f7144098920bf573f2cb411fcac5879d5d2cac93e0/"},{"source":"arxiv:2606.06021","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2606.06021","title":"OPRD: On-Policy Representation Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a16a9d7ea78c5748acc7f651c9a96663e0200d72e51d6a52c9528e1739a27d8d/"},{"source":"arxiv:2606.18810","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2606.18810","title":"Learning from Own Solutions: Self-Conditioned Credit Assignment for Reinforcement Learning with Verifiable Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9c11d77c6d08392f24aab79c8b45a3a3b0f4b743dd35e150a83ee7f6fa315248/"},{"source":"arxiv:2607.07847","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2607.07847","title":"When Does Continual Learning Require Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/86e88c6cf21a66879eab5c156913fc85d14eb69c394c78c33c0e81d144e528eb/"},{"source":"arxiv:2608.13040","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2608.13040","title":"Latent On-Policy Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2cfd504d6ca36ba88d75aba018620867d5fafd7433b03f3b2c9f51408422a02/"},{"source":"arxiv:2609.37633","target":"arxiv:2601.20802","evidence":"id","id":"arxiv:2609.37633","title":"RLTL;DR: Self-improvement by Internalizing Self-generated Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c7a4c1dbb0493059883d9a01c1f89d4db3644209b909e1195c5afa4d3a320999/"}],"text_parts":[{"start":0,"end":15762,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html"},{"start":15762,"end":31512,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html"},{"start":31512,"end":47174,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html"},{"start":47174,"end":62750,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html"},{"start":62750,"end":78711,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html"},{"start":78711,"end":83670,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html"}],"contents":[{"title":"Reinforcement Learning via Self-Distillation","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"Abstract","offset":48,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"1 Introduction","offset":1614,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"Summary of evaluation results.","offset":6728,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"2 SDPO: Self-Distillation Policy Optimization","offset":8533,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"Proposition 2.1 .","offset":11003,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"2.1 Comparison to RLVR","offset":11663,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"2.2 Compute time & memory","offset":13532,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"2.3 Stability improvements","offset":14642,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"3 Learning without Rich Environment Feedback","offset":15275,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"3.1 Experimental setting","offset":15732,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-1.html#text"},{"title":"Baselines.","offset":16409,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"3.2 Results","offset":17447,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"3.3 Self-distillation learns to reason concisely","offset":18402,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"4 Learning with Rich Environment Feedback","offset":20001,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"Results.","offset":21270,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"4.1 Self-distillation benefits from stronger models","offset":22076,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"4.2 Self-distillation performs dense credit assignment","offset":23133,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"4.3 The self-teacher improves during training","offset":24975,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"4.4 On-policy self-distillation avoids catastrophic forgetting","offset":26220,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"Off-policy self-distillation baseline.","offset":27322,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"4.5 Can GRPO and SDPO be combined?","offset":28242,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"4.6 Which feedback is most informative?","offset":29801,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"Sample solutions.","offset":30180,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"Environment output.","offset":30740,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"Student’s original attempt.","offset":31137,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-2.html#text"},{"title":"5 Solving Hard Questions via Test-Time Self-Distillation","offset":31871,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"Definition 5.1 (Discovery time) .","offset":32244,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"5.1 Experimental setting","offset":35652,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"5.2 Results","offset":37308,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"Question 3 is only solved by SDPO.","offset":38775,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"The initial self-teacher does not solve hard questions.","offset":39306,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"6 Related Work","offset":39771,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"6.1 Reinforcement Learning with LLMs","offset":39790,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"6.2 Learning from Rich Feedback and through Retrospection","offset":41764,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"6.3 Distillation","offset":44081,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"6.4 Self-Distillation","offset":44965,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"7 Conclusion, Limitations, and Future Work","offset":46518,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-3.html#text"},{"title":"Limitations.","offset":48012,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"Future Work.","offset":48727,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"Author Contributions","offset":49889,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"Acknowledgments","offset":52198,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"Appendix A Implementation of SDPO","offset":53242,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"A.1 Gradient Estimators","offset":53977,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"Per-token estimator.","offset":54222,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"Sequence-level estimator.","offset":55568,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"A.2 Regularized teacher","offset":57463,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"A.3 Approximate Logit Distillation","offset":59678,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"A.4 Off-Policy Training: Generalization to Logit-Level Losses","offset":61456,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-4.html#text"},{"title":"Appendix B Theoretical Analysis","offset":64302,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"B.1 Proof of Proposition 2.1 .","offset":64794,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Proof.","offset":64830,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"B.2 Trust-region Teacher","offset":68727,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Proposition B.1 .","offset":70382,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Proof.","offset":70789,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Appendix C Additional Related Work","offset":72570,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Value networks and Monte Carlo advantage estimation.","offset":72609,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Dense credit assignment with a reward model.","offset":73434,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Partial observability.","offset":73943,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Relation to test-time training.","offset":74598,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"C.1 SDPO as Maximum Entropy RL","offset":75312,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Maximum Entropy RL","offset":75507,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"SDPO optimizes an implicit reward defined by the teacher","offset":77517,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"Appendix D Additional Results & Ablations","offset":78224,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"D.1 Learning without rich environment feedback","offset":78494,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-5.html#text"},{"title":"D.2 Learning with rich environment feedback","offset":78711,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"D.2.1 Additional Results","offset":78760,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"D.2.2 Training Stability","offset":79433,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"D.2.3 Baselines","offset":79605,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"D.3 Test-time self-distillation","offset":79749,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"Appendix E Experiment Details","offset":80448,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"E.1 Technical setup","offset":80482,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"E.2 Hyperparameters","offset":81034,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"E.2.1 Details on Hyperparameter Selection ( Section 3 )","offset":81153,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"E.3 User Templates","offset":82038,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"Appendix F Qualitative Examples","offset":82234,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"F.1 Visualization of Advantages","offset":82270,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"F.2 Examples","offset":82388,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"F.3 Environment Feedback","offset":82480,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"F.4 Illustrative Example","offset":82595,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"},{"title":"Instructions for reporting errors","offset":82706,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/part-6.html#text"}],"text_code_points":83670,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/source.md"}