{"id":"arxiv:2508.05629","title":"On the Generalization of SFT: A Reinforcement Learning Perspective with Reward Rectification","date":"2025-08-07","section":"Post-training","quality":0.432404,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/","source_url":"https://arxiv.org/abs/2508.05629","text_available":true,"authors":["Yongliang Wu","Yizhou Zhou","Zhou Ziheng"],"outgoing":[{"source":"arxiv:2508.05629","target":"arxiv:2005.01643","evidence":"id","id":"arxiv:2005.01643","title":"Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/"},{"source":"arxiv:2508.05629","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2508.05629","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2508.05629","target":"arxiv:2507.12856","evidence":"id","id":"arxiv:2507.12856","title":"Supervised Fine Tuning on Curated Data is Reinforcement Learning (and can be improved)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5f6c1906eaa67369d9ff15b974756b89141d1b4cf9d7c0f6554e0d56f26e79be/"},{"source":"arxiv:2508.05629","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"},{"source":"arxiv:2508.05629","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[{"source":"arxiv:2508.11408","target":"arxiv:2508.05629","evidence":"id","id":"arxiv:2508.11408","title":"On-Policy RL Meets Off-Policy Experts: Harmonizing Supervised Fine-Tuning and Reinforcement Learning via Dynamic Weighting","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48594da2e3f2cdfeb17195dd033d047235fc4b4ffb65030800f64c6faedc4fbe/"},{"source":"arxiv:2601.19897","target":"arxiv:2508.05629","evidence":"id","id":"arxiv:2601.19897","title":"Self-Distillation Enables Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cc146df3e9ca40c6d111a5264e08710100c1b34bd0eabd9052e8d55d13589efe/"},{"source":"arxiv:2606.23740","target":"arxiv:2508.05629","evidence":"id","id":"arxiv:2606.23740","title":"Weight-Space Geometry of Offline Reasoning Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e4479f0d0930797db9347855247f1c640a12d96f63d43b83ba9635561e3ff352/"},{"source":"arxiv:2610.02140","target":"arxiv:2508.05629","evidence":"id","id":"arxiv:2610.02140","title":"Finetuning with Sampling: SFT Learns Better Than You Think","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/60f5edfae92f505bf86e8890135a44b4129825b0525b5ad8d242e8ba04182605/"}],"text_parts":[{"start":0,"end":15312,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html"},{"start":15312,"end":31181,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html"},{"start":31181,"end":47002,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html"},{"start":47002,"end":51633,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-4.html"}],"contents":[{"title":"On the Generalization of SFT: A Reinforcement Learning Perspective with Reward Rectification","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"Abstract","offset":96,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"1 Introduction","offset":1233,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"2 Related Work","offset":6407,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"3 Method","offset":11669,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"3.1 Preliminaries","offset":11682,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"Supervised Fine-Tuning.","offset":11705,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"Reinforcement Learning.","offset":12431,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"3.2 Unify SFT and RL Gradient Expression","offset":13108,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"Rewriting SFT Gradient as Policy Gradient via Importance Sampling.","offset":13154,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-1.html#text"},{"title":"3.3 Proposed Method","offset":16043,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"Reward Rectification via Dynamic Reweighting.","offset":16068,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4 Experiments","offset":18784,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4.1 Main Experiment - Mathematical Reasoning Task","offset":19301,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"Implementation details.","offset":19507,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4.2 Exploratory Experiment - Offline RL Setting","offset":24238,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"Implementation details.","offset":24882,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4.3 Exploratory Experiment - Code Generation Task","offset":27273,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"Implementation details.","offset":27328,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4.4 Exploratory Experiment - Multi-Modal Reasoning","offset":28634,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"Implementation details.","offset":28690,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4.5 Limitations of DFT: A Case Study on Factual Knowledge","offset":29940,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4.6 An Empirical Comparison with Sentence-Level Weighting","offset":31118,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-2.html#text"},{"title":"4.7 Analysis of Probabilities","offset":32315,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"5 Conclusion","offset":33808,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"Limitations.","offset":34941,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"Acknowledgements","offset":35943,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"Ethics Statement","offset":36225,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"Reproducibility Statement","offset":37237,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"Appendix A Appendix","offset":38092,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"A.1 Usage of LLM","offset":38116,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"A.2 Detailed Derivation of Equation (5)","offset":38366,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"A.3 Discussions and Insights","offset":40972,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"Gradient Analysis of DFT.","offset":41006,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"Learning from Noisy Data.","offset":43908,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"A.4 Comparision with Concurrent work iw-SFT","offset":44800,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"A.5 Exploratory Experiment - OpenR1-Math Training Dataset","offset":46939,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-3.html#text"},{"title":"A.6 Exploratory Experiment - PEFT Training Setting","offset":48280,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-4.html#text"},{"title":"A.7 Training Hyper-Parameters Ablation","offset":49160,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-4.html#text"},{"title":"Instructions for reporting errors","offset":50669,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/part-4.html#text"}],"text_code_points":51633,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/source.md"}