{"id":"arxiv:2510.00977","title":"It Takes Two: Your GRPO Is Secretly DPO","date":"2025-10-01","section":"Post-training","quality":-0.095378,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/","source_url":"https://arxiv.org/abs/2510.00977","text_available":true,"authors":["Yihong Wu","Liheng Ma","Lei Ding"],"outgoing":[{"source":"arxiv:2510.00977","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2510.00977","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2510.00977","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2510.00977","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"}],"incoming":[{"source":"arxiv:2604.08690","target":"arxiv:2510.00977","evidence":"id","id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/"}],"text_parts":[{"start":0,"end":15319,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html"},{"start":15319,"end":30602,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html"},{"start":30602,"end":45755,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html"},{"start":45755,"end":61698,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html"},{"start":61698,"end":69080,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html"}],"contents":[{"title":"It Takes Two: Your GRPO Is Secretly DPO","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"Abstract","offset":43,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"1 Introduction","offset":1100,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"2 Preliminary","offset":4685,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"2.1 Problem Setting and Notation","offset":4703,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"2.2 The Story of Variance Reduction: VPG, PPO, and GRPO","offset":5965,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"3 A Tale of Two Algorithms: GRPO and DPO","offset":9064,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"3.1 Contrastive Objective for Sequences","offset":9494,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"Definition 3.1 (Contrastive Loss for Sequences) .","offset":9911,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"DPO is a 1-vs-1 contrastive learning","offset":11559,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"3.2 GRPO: N-vs-M Contrastive Learning","offset":12438,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"Proposition 3.2 .","offset":14846,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"3.3 Echoes of Contrastiveness: GRPO and DPO","offset":15270,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-1.html#text"},{"title":"4 Why Viewing GRPO From Contrastive Learning?","offset":18600,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"4.1 Variance Reduction via Contrastive Objective","offset":18650,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"Proposition 4.1 .","offset":18873,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"4.2 GRPO with Small Group Size: It Should Fail, But Doesn’t","offset":21008,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"4.3 Introducing 2-GRPO","offset":22251,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"5 Experiments","offset":23789,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"5.1 Math Reasoning","offset":24412,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"5.2 Ablation Study: The Effect of Group Size","offset":26997,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"6 Conclusion","offset":28252,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"Appendix","offset":29447,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"Appendix A Related Work","offset":29460,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"A.1 Contrastive Learning and LLM Alignment","offset":29488,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"A.2 Adaptive Rollouts in RLVR","offset":30567,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-2.html#text"},{"title":"Appendix B Theorems","offset":31512,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"B.1 Variance Reduction of Policy Gradient Estimate","offset":31536,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"B.2 Mean Estimation with Samples n = 2 {n=2}","offset":32890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"B.3 Reveal GRPO as Contrastive Learning","offset":33485,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"Proof of Proposition 3.2 .","offset":33530,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"B.4 Further Discussion on Importance Sampling and the Log-likelihood Term","offset":41945,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"B.5 Proof of Lemma B.1 : DPO is 1-vs-1 contrastive learning","offset":43986,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"Lemma B.1 .","offset":44051,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"Proof of Lemma B.1 .","offset":44131,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"B.6 GRPO v.s. DPO from Contrastive Learning","offset":45706,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-3.html#text"},{"title":"B.7 Proof of Proposition 4.1","offset":47872,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Proof.","offset":47906,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"B.8 The Correlation between The Positive and The Negative","offset":49193,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"B.9 Proof of Proposition C.1","offset":50289,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Proof.","offset":50323,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"B.10 Proof of Lemma C.3","offset":51990,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Proof of Lemma C.3 .","offset":52019,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Appendix C Theoretical Analysis of 2-GRPO","offset":52730,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"C.1 Implicit Weighting in Stochastic Optimization","offset":52776,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Proposition C.1 .","offset":53661,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"C.2 Key of Variance Reduction: the Training Batch Size, not the Group Size","offset":56415,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Definition C.2 (Variance of Gradient Estimate in Mini-Batch) .","offset":57384,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Lemma C.3 .","offset":58775,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"C.3 Exploration on Hard Questions","offset":60743,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Proposition C.4 .","offset":61346,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-4.html#text"},{"title":"Appendix D Experiments","offset":62896,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"D.1 Experiment Details","offset":62923,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"Dataset and Baselines","offset":62951,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"Hyper-parameters","offset":64389,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"D.2 The Connection Between Training Rollouts and Computational Cost","offset":65638,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"D.3 Sample Discard Rate of 2-GRPO","offset":66791,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"D.4 Ablation Study on Group Size","offset":67350,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"Appendix E Limitation","offset":67639,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"Appendix F The Use of Large Language Models (LLMs)","offset":67963,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"},{"title":"Instructions for reporting errors","offset":68116,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/part-5.html#text"}],"text_code_points":69080,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/source.md"}