{"id":"arxiv:2602.09000","title":"iGRPO: Self-Feedback-Driven LLM Reasoning","date":"2026-02-09","section":"Post-training","quality":-0.04492,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/","source_url":"https://arxiv.org/abs/2602.09000","text_available":true,"authors":["Ali Hatamizadeh","Shrimai Prabhumoye","Igor Gitman","Ximing Lu"],"outgoing":[{"source":"arxiv:2602.09000","target":"arxiv:2303.17651","evidence":"title","id":"arxiv:2303.17651","title":"Self-Refine: Iterative Refinement with Self-Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/accef2157d87cf33be93cb3a9af246528b0a3742a9e2e700a194956fac8f5cf4/"},{"source":"arxiv:2602.09000","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2602.09000","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2602.09000","target":"arxiv:2503.20783","evidence":"title","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2602.09000","target":"arxiv:2504.20571","evidence":"id","id":"arxiv:2504.20571","title":"Reinforcement Learning for Reasoning in Large Language Models with One Training Example","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b2b21c3df49619588c371411dba9467a41f81b3fd4da5441898b5145bb0de120/"},{"source":"arxiv:2602.09000","target":"arxiv:2506.03106","evidence":"id","id":"arxiv:2506.03106","title":"Critique-GRPO: Advancing LLM Reasoning with Natural Language and Numerical Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/"},{"source":"arxiv:2602.09000","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"}],"incoming":[{"source":"arxiv:2609.37633","target":"arxiv:2602.09000","evidence":"id","id":"arxiv:2609.37633","title":"RLTL;DR: Self-improvement by Internalizing Self-generated Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c7a4c1dbb0493059883d9a01c1f89d4db3644209b909e1195c5afa4d3a320999/"}],"text_parts":[{"start":0,"end":15815,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html"},{"start":15815,"end":31796,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html"},{"start":31796,"end":47078,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html"},{"start":47078,"end":61926,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html"}],"contents":[{"title":"iGRPO: Self - Feedback–Driven LLM Reasoning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"Abstract","offset":47,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"1 Introduction","offset":1756,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"2 Related Work","offset":5701,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"RL for Reasoning.","offset":5720,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"GRPO and Variants.","offset":6645,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"LLM Self-Learning and Self-Improvement.","offset":7684,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"3 Methodology","offset":9396,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"3.1 Background: Group Relative Policy Optimization","offset":9414,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"3.2 Iterative Group Relative Policy Optimization","offset":13498,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"3.2.1 Motivation: From Static Examples to Dynamic Self-Conditioning","offset":14167,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-1.html#text"},{"title":"3.2.2 Algorithmic Framework","offset":16700,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Stage 1: Exploratory Draft Generation.","offset":16972,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Stage 2: Conditioned Refinement.","offset":17688,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"3.2.3 Theoretical Analysis: Bootstrapped Policy Improvement","offset":18350,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Definition 1 (Self-Conditioned Prompt Construction) .","offset":18416,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Proposition 3.1 (Progressive Conditioning Quality for Binary Rewards) .","offset":19342,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Proof.","offset":20514,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"3.2.4 Mathematical Formulation","offset":21462,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Stage 1: Draft Selection.","offset":21769,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Stage 2: Conditioned Generation and Advantage Computation.","offset":22085,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Full Objective.","offset":23169,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Reward Function.","offset":25483,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"3.2.5 Computational Analysis","offset":26118,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Baseline GRPO Cost.","offset":26634,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"iGRPO Rollout Cost.","offset":26847,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"4 Experiments","offset":27705,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"4.1 Setup","offset":27723,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"4.2 Results","offset":28949,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Controlled study with matched sampling budget.","offset":29270,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Generalist 8B model: the largest gains from self-feedback.","offset":30051,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Stronger 7B distilled reasoner: consistent gains concentrated on multi-step tasks.","offset":30967,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Math-specialized 7B model: improvements persist when the base is already strong.","offset":31708,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-2.html#text"},{"title":"Scaling to 14B parameters: benefits persist on complex reasoning.","offset":32402,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"Competitiveness against critique-style objectives.","offset":33239,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"4.3 Generalization to a Stronger Base and Harder Dataset","offset":34003,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"5 Ablation","offset":34904,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"Beyond GRPO: self-feedback as a reusable refinement wrapper.","offset":34919,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"Generative judge study.","offset":35617,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"Entropy analysis.","offset":36233,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"6 Conclusion","offset":37691,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"Appendix","offset":39462,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"Appendix A Policy Gradient Derivation for iGRPO","offset":39475,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"A.1 Two-stage sampling and the induced self-conditioned prompt distribution","offset":40052,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"A.2 From the self-conditioned expected reward to a REINFORCE-style gradient","offset":41662,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"A.3 Group-relative advantage used in iGRPO","offset":43175,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"A.4 Off-policy sampling from π θ old \\pi_{\\theta_{\\mathrm{old}}} and PPO-style clipping","offset":44761,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"A.5 Including the per-token KL penalty","offset":47034,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-3.html#text"},{"title":"A.6 Final iGRPO surrogate objective and resulting policy gradient","offset":48697,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Interpretation.","offset":51234,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Appendix B Scaling OpenMath-Nemotron-14B with iGRPO","offset":51792,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"B.1 Analysis of Pass@N on AIME Benchmarks","offset":52278,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Appendix C Hyperparameter Setup","offset":53649,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Prompt:","offset":55157,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Appendix D Memory and Throughput Comparisons","offset":55232,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"D.1 Setup","offset":55281,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"D.2 Measurements","offset":57237,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Appendix E Additional Ablation Studies","offset":59094,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Training Dynamics and Response Length.","offset":59137,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Effect of KL Divergence Term.","offset":59888,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Effect of Number of Completions.","offset":60552,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"},{"title":"Instructions for reporting errors","offset":60962,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/part-4.html#text"}],"text_code_points":61926,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/source.md"}