{"id":"arxiv:2402.13669","title":"Self-Distillation Bridges Distribution Gap in Language Model Fine-Tuning","date":"2024-02-21","section":"Post-training","quality":0.219878,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/","source_url":"https://arxiv.org/abs/2402.13669","text_available":true,"authors":["Zhaorui Yang","Tianyu Pang","Haozhe Feng","Han Wang"],"outgoing":[{"source":"arxiv:2402.13669","target":"arxiv:2303.17651","evidence":"title","id":"arxiv:2303.17651","title":"Self-Refine: Iterative Refinement with Self-Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/accef2157d87cf33be93cb3a9af246528b0a3742a9e2e700a194956fac8f5cf4/"},{"source":"arxiv:2402.13669","target":"doi:10.1073/pnas.1611835114","evidence":"title","id":"doi:10.1073/pnas.1611835114","title":"Overcoming catastrophic forgetting in neural networks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/849082368a44d06b93ef3ba95bcfddc6672b7a5e8064ad2b2d19601639266e8b/"},{"source":"arxiv:2402.13669","target":"url:ai.meta.com/research/publications/code-llama-open-foundation-models-for-code","evidence":"id","id":"url:ai.meta.com/research/publications/code-llama-open-foundation-models-for-code","title":"Code Llama: Open Foundation Models for Code","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0dc2d6030e514e45150f64f511c0f50fc010fdc1ddae0bdca7edcd189b084d41/"},{"source":"arxiv:2402.13669","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[{"source":"arxiv:2601.18734","target":"arxiv:2402.13669","evidence":"title","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"},{"source":"arxiv:2601.19897","target":"arxiv:2402.13669","evidence":"id","id":"arxiv:2601.19897","title":"Self-Distillation Enables Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cc146df3e9ca40c6d111a5264e08710100c1b34bd0eabd9052e8d55d13589efe/"},{"source":"arxiv:2601.20802","target":"arxiv:2402.13669","evidence":"title","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"}],"text_parts":[{"start":0,"end":15661,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html"},{"start":15661,"end":30722,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html"}],"contents":[{"title":"Self-Distillation Bridges Distribution Gap in Language Model Fine-Tuning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"Abstract","offset":76,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"1 Introduction","offset":1106,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"2 Related Work","offset":4982,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"Fine-Tuning","offset":5001,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"Continual Learning","offset":5727,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"Alignment","offset":7197,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"Prompting Based Learning","offset":8314,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"3 Method","offset":9006,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"3.1 Fine-tuning LLMs","offset":9201,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"3.2 Self-Distillation Fine-Tuning","offset":10590,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"3.3 Distillation Template","offset":12722,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"4 Experiments","offset":13389,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"4.1 Experimental Setup","offset":13838,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"4.2 Datasets for Fine-tuning and Evaluation","offset":14631,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-1.html#text"},{"title":"4.3 SDFT Achieves Better Results on Downstream Tasks","offset":17860,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"4.4 SDFT Preserves Alignment","offset":19071,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"4.5 General Knowledge Remains Intact","offset":20699,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"5 Analysis","offset":21136,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"5.1 Distribution Shift Correlates with Catastrophic Forgetting","offset":22341,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"5.2 Robustness among Distillation Templates","offset":24109,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"5.3 Efficacy of SDFT Across Model Scales and Architectures","offset":24634,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"6 Conclusions and Limitations","offset":25421,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"Ethics Statement","offset":26579,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"Acknowledgements","offset":27137,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"Appendix A Experimental Details","offset":27402,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"Appendix B Templates and Examples","offset":28906,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"},{"title":"Instructions for reporting errors","offset":29758,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/part-2.html#text"}],"text_code_points":30722,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1a8d11d48a58f35055e7f88f699cf42688a3698c63bd7026d5789f72a80581bf/source.md"}