{"id":"arxiv:2505.10978","title":"Group-in-Group Policy Optimization for LLM Agent Training","date":"2025-05-16","section":"Post-training","quality":0.31133,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/","source_url":"https://arxiv.org/abs/2505.10978","text_available":true,"authors":["Lang Feng","Zhenghai Xue","Tingcong Liu","Bo An"],"outgoing":[{"source":"arxiv:2505.10978","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2505.10978","target":"arxiv:2503.14476","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2505.10978","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"}],"incoming":[{"source":"arxiv:2603.25562","target":"arxiv:2505.10978","evidence":"title","id":"arxiv:2603.25562","title":"Revisiting On-Policy Distillation: Empirical Failure Modes and Simple Fixes","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a66707a05c8aa09328e4a1f322a57800a3e12262a85a9a1cf195b41dc1926b32/"}],"text_parts":[{"start":0,"end":15115,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html"},{"start":15115,"end":29606,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-2.html"},{"start":29606,"end":45223,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html"},{"start":45223,"end":52163,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html"}],"contents":[{"title":"Group-in-Group Policy Optimization for LLM Agent Training","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html#text"},{"title":"Abstract","offset":61,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html#text"},{"title":"1 Introduction","offset":2014,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html#text"},{"title":"2 Related Work","offset":6841,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html#text"},{"title":"3 Preliminaries","offset":10345,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html#text"},{"title":"4 Training LLM Agents with GiGPO","offset":13066,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html#text"},{"title":"4.1 Episode Relative Advantages","offset":15078,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-1.html#text"},{"title":"4.2 Step Relative Advantages","offset":18332,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-2.html#text"},{"title":"4.3 Group-in-Group Policy Optimization","offset":24183,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-2.html#text"},{"title":"5 Experiment","offset":26282,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-2.html#text"},{"title":"5.1 Experiment Setup","offset":26666,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-2.html#text"},{"title":"5.2 Performance on ALFWorld and WebShop","offset":29561,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-2.html#text"},{"title":"5.3 Performance on QA tasks","offset":31940,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"5.4 Ablation Study","offset":33165,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"5.5 Dynamics of Step-Level Group","offset":34655,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"5.6 Computational Budget","offset":36454,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"6 Conclusions and Limitations","offset":37720,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"Acknowledgements","offset":39151,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"Appendix A Open Source Codebase: verl-agent","offset":39292,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"Appendix B Broader Impacts","offset":40714,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"Appendix C Unbiasedness","offset":41870,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"Appendix D Pseudo Code","offset":43376,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"Appendix E Experiment Details","offset":44139,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"E.1 Details of Training","offset":44173,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-3.html#text"},{"title":"E.2 Prompts","offset":47137,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"E.3 Performance on Vision-Language Agents","offset":48228,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"E.4 Orthogonality to Single-Turn Group-Based RL","offset":49451,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"E.5 Sensitivity Analysis on ω \\omega","offset":50123,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"Appendix F Reasoning Behavior","offset":50923,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"F.1 ALFWorld","offset":51148,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"F.2 WebShop","offset":51166,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"F.3 Search","offset":51183,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"},{"title":"Instructions for reporting errors","offset":51199,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/part-4.html#text"}],"text_code_points":52163,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/source.md"}