{"id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","date":"2025-07-24","section":"Post-training","quality":0.050742,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/","source_url":"https://arxiv.org/abs/2507.18071","text_available":true,"authors":["Chujie Zheng","Shixuan Liu","Mingze Li","Xiong-Hui Chen"],"outgoing":[{"source":"arxiv:2507.18071","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2507.18071","target":"arxiv:2506.13585","evidence":"id","id":"arxiv:2506.13585","title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5beb2ec6c7e6362f9d957219d7a205dd1adcc53f739acc0c7fd5f1f2bded4d83/"}],"incoming":[{"source":"arxiv:2505.21444","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2505.21444","title":"Can Large Reasoning Models Self-Train?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/206224ca50098e0bdc230140722feec107cfc90c1cef2ed8d3ed52355ef56bca/"},{"source":"arxiv:2508.05629","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2508.05629","title":"On the Generalization of SFT: A Reinforcement Learning Perspective with Reward Rectification","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/"},{"source":"arxiv:2509.13232","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2509.13232","title":"Single-stream Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d699d0e034b21f3b85c3f70ce4fb95064fbf59685ce752935e6d173dac2b71f3/"},{"source":"arxiv:2510.00977","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2510.00977","title":"It Takes Two: Your GRPO Is Secretly DPO","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/"},{"source":"arxiv:2510.13786","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2510.13786","title":"The Art of Scaling Reinforcement Learning Compute for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/72580bcba3d9b32b2f3e83d074f626234b26dc18126edd22a5eacef927cf87da/"},{"source":"arxiv:2511.20347","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2511.20347","title":"Soft Adaptive Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/"},{"source":"arxiv:2512.00499","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2512.00499","title":"ESPO: Entropy Importance Sampling Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/342b84df2bf40f64c57ea7839579c1f337374ad9c28b16ce7d477c42e24c67b4/"},{"source":"arxiv:2601.05053","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2601.05053","title":"Reinforced Efficient Reasoning via Semantically Diverse Exploration","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/74918d32c56962f9c5b2063026f5e2243a84e20c284eb3e37c882bd4681d1a11/"},{"source":"arxiv:2601.07055","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2601.07055","title":"Dr. Zero: Self-Evolving Search Agents without Training Data","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3baa3e7fc5768b75e4ef3e64c3cc5297fa42ef19b03bc918ef5352e437e1c168/"},{"source":"arxiv:2601.18734","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"},{"source":"arxiv:2601.20802","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"},{"source":"arxiv:2602.09000","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2602.09000","title":"iGRPO: Self-Feedback-Driven LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/"},{"source":"arxiv:2604.02288","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2604.02288","title":"Unifying Group-Relative and Self-Distillation Policy Optimization via Sample Routing","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3d1160029d321f2069016f7702d4ebebee0de6cd31a8f296ee220abd6f69c99b/"},{"source":"arxiv:2604.08690","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/"},{"source":"arxiv:2604.20659","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2604.20659","title":"GRPO-VPS: Enhancing Group Relative Policy Optimization with Verifiable Process Supervision for Effective Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/42caf0f9c992f5df8bac4d0f7e45c3a5bf57879661f41febd01a047694d52628/"},{"source":"arxiv:2605.12969","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2605.12969","title":"Revisiting Reinforcement Learning with Verifiable Rewards from a Contrastive Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/30e87f5eb9beda8e09b3945d6ba1707dec761abed83c49d129a21adaed02ce45/"},{"source":"arxiv:2606.18810","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2606.18810","title":"Learning from Own Solutions: Self-Conditioned Credit Assignment for Reinforcement Learning with Verifiable Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9c11d77c6d08392f24aab79c8b45a3a3b0f4b743dd35e150a83ee7f6fa315248/"},{"source":"url:cameronrwolfe.substack.com/p/grpo-tricks","target":"arxiv:2507.18071","evidence":"id","id":"url:cameronrwolfe.substack.com/p/grpo-tricks","title":"GRPO++: Tricks for Making RL Actually Work (Cameron Wolfe)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d170bc3da5b8e6a055fd511719fcc3764635236bac638947bd0a018412245eb5/"}],"text_parts":[{"start":0,"end":13674,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html"},{"start":13674,"end":28061,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html"}],"contents":[{"title":"Group Sequence Policy Optimization","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"Abstract","offset":38,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"1 Introduction","offset":763,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"2 Preliminaries","offset":3226,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"Notation","offset":3246,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"Proximal Policy Optimization (PPO)","offset":3857,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"Group Relative Policy Optimization (GRPO)","offset":5590,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"3 Motivation","offset":7217,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"4 Algorithm","offset":10541,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"4.1 GSPO: Group Sequence Policy Optimization","offset":10557,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"4.2 Gradient Analysis","offset":13647,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-1.html#text"},{"title":"4.3 GSPO-token: A Token-level Objective Variant","offset":16961,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"5 Experiments and Discussion","offset":20580,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"5.1 Empirical Results","offset":20613,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"5.2 Curious Observation on Clipping Fractions","offset":22187,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"5.3 Benefit of GSPO for MoE Training","offset":23105,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"Background","offset":23147,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"Our Previous Approach","offset":24312,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"Benefit of GSPO","offset":25326,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"5.4 Benefit of GSPO for RL Infrastructure","offset":26494,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"},{"title":"6 Conclusion","offset":27320,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/part-2.html#text"}],"text_code_points":28061,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/source.md"}