{"id":"arxiv:2506.03106","title":"Critique-GRPO: Advancing LLM Reasoning with Natural Language and Numerical Feedback","date":"2025-06-03","section":"Post-training","quality":0.423423,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/","source_url":"https://arxiv.org/abs/2506.03106","text_available":true,"authors":["Xiaoying Zhang","Yipeng Zhang","Hao Sun","Kaituo Feng"],"outgoing":[{"source":"arxiv:2506.03106","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2506.03106","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2506.03106","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2506.03106","target":"arxiv:2503.14476","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2506.03106","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2506.03106","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/"},{"source":"arxiv:2506.03106","target":"url:openai.com/research/weak-to-strong-generalization","evidence":"id","id":"url:openai.com/research/weak-to-strong-generalization","title":"Weak-to-Strong Generalization: Eliciting Strong Capabilities With Weak Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fb7b23a617dce974dbb70db824c97c1de84afa11e1e290d5cb9f5bd88857e762/"},{"source":"arxiv:2506.03106","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[{"source":"arxiv:2602.09000","target":"arxiv:2506.03106","evidence":"id","id":"arxiv:2602.09000","title":"iGRPO: Self-Feedback-Driven LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/"},{"source":"arxiv:2604.08690","target":"arxiv:2506.03106","evidence":"id","id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/"}],"text_parts":[{"start":0,"end":15464,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html"},{"start":15464,"end":31323,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html"},{"start":31323,"end":46860,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html"},{"start":46860,"end":62667,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html"},{"start":62667,"end":78321,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html"},{"start":78321,"end":93861,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html"},{"start":93861,"end":104060,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html"}],"contents":[{"title":"Advancing LLM Reasoning with Natural Language and Numerical Feedback","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"Abstract","offset":72,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"Keywords:","offset":1407,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"1 Introduction","offset":1480,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"2 Related Work","offset":6679,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"Enhancing LLM Reasoning with RL.","offset":6698,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"3 Limitations of RL and the Promise of Natural Language Guidance","offset":7751,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"3.1 Three Limitations of Learning with Solely Numerical Feedback","offset":7820,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"3.2 Promise of Learning from Natural Language Feedback","offset":9575,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"4 Critique-GRPO","offset":11126,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"4.1 From GRPO to Critique-GRPO","offset":11369,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"4.2 Online RL with Critique-GRPO","offset":13698,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-1.html#text"},{"title":"Proposition 4.1 (Complexity Reduction via Critique-Guided Exploration) .","offset":17183,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5 Experiments","offset":22397,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.1 Experimental Setup","offset":22415,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.2 Main Results","offset":24814,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.3 Investigation on Math-Centric Backbone Models","offset":26175,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.4 Self-Improvement via Self-Critiquing","offset":27123,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.5 Exploration with Varying Models","offset":27792,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.6 Investigation of Policy Exploration","offset":28769,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.7 Fine-Grained Ablation Studies","offset":30026,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.8 Exploration of Weak-to-Strong Generalization","offset":30878,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-2.html#text"},{"title":"5.9 Online Joint Optimization vs. Sequential Baseline","offset":31697,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"6 Conclusion","offset":32622,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Acknowledgments","offset":33199,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Impact Statement","offset":33322,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Appendix","offset":33885,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Appendix A Additional Discussion on Related Work","offset":33898,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Learning from Natural Language Feedback.","offset":33951,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Enhancing LLM Reasoning with Reinforcement Learning.","offset":35367,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Appendix B Analysis of Training Computational Cost","offset":37122,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Appendix C Impact of Critique Length on Critique-GRPO","offset":37952,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Appendix D Scalability of Critique-GRPO to Large-Scale Models","offset":39011,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Appendix E Limitations of RL with Numerical Feedback and the Promise of Natural Language Guidance (Detailed Analyses)","offset":39280,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"E.1 Three Limitations of Learning with Numerical Feedback","offset":39402,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Setup.","offset":39882,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"Results.","offset":42288,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-3.html#text"},{"title":"E.2 Promise of Learning from Natural Language Feedback","offset":48047,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Setup.","offset":49150,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"E.3 Leveraging Textual Critiques for Refining LLM Responses","offset":52747,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"E.4 Investigation on Qwen3-8B-Base","offset":55721,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Appendix F More Details about Critique-GRPO","offset":56320,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"F.1 The Critique-GRPO Algorithm","offset":56368,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"F.2 The shaping Function in Critique-GRPO","offset":56465,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"F.3 Ratio of Responses per Prompt in Critique-GRPO","offset":57314,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"F.4 Theoretical Analysis of Sample Efficiency","offset":57743,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Preliminaries: Complexity Measures and Verifiers.","offset":58403,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Definition F.1 (Eluder Dimension ( Russo and Van Roy, 2013 ) ) .","offset":58745,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Assumption F.2 (Verifier ( Xu et al., 2025 ) ) .","offset":59763,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Definition F.3 (Transfer Eluder Dimension ( Xu et al., 2025 ) ) .","offset":60546,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Critique as Reward-Informative Feedback.","offset":61375,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Definition F.4 (Reward-Informative Feedback) .","offset":61714,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Proposition F.5 (Generalization Bound) .","offset":62618,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-4.html#text"},{"title":"Complexity Reduction Analysis.","offset":63279,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Proposition F.6 (Sample Efficiency of Critique-Guided Refinement) .","offset":63639,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Proof.","offset":66130,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"F.5 Theoretical Analysis of Convergence and Policy Improvement","offset":69638,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Preliminaries and Problem Setup.","offset":70193,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Definition F.7 (Sparse Reward Landscape) .","offset":70398,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Definition F.8 (GRPO Gradient Estimate) .","offset":70668,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"F.5.1 Inefficiency of Standard Exploration","offset":71238,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Proposition F.9 (Signal-to-Noise Ratio in Standard GRPO) .","offset":71632,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"F.5.2 Policy Improvement via Critique-Guided Refinement","offset":72219,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Lemma F.10 (Trajectory Quality Enhancement) .","offset":72595,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Theorem F.11 (Policy Improvement Lower Bound) .","offset":73346,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Proof.","offset":74444,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Theorem F.12 (Global Convergence via Policy Shaping) .","offset":76704,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Proof.","offset":77099,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"F.6 Theoretical Analysis: Gradient Efficiency and Manifold Preservation","offset":77790,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"F.6.1 Gradient Dynamics of the Shaping Function","offset":78267,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-5.html#text"},{"title":"Lemma F.13 (Gradient Rescaling) .","offset":78670,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Proof.","offset":79153,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Analysis of the Modulation Term Ψ ⁡ ( π t ) \\Psi(\\pi_{t}) .","offset":79909,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"F.6.2 The Insufficiency of Clipping for Off-Policy Data","offset":80908,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Proposition F.14 (Gradient Orthogonality) .","offset":81270,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"F.6.3 Manifold Preservation via The Anchor Hypothesis","offset":82363,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Theorem F.15 (The Anchor Condition) .","offset":82560,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Proof.","offset":82942,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Appendix G Implementation Details","offset":84402,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Appendix H Detailed Results and Analysis of Self-Critiquing Mechanisms","offset":89687,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Appendix I Detailed Investigation of Policy Exploration During RL Finetuning","offset":91695,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-6.html#text"},{"title":"Appendix J Qualitative Analysis","offset":95077,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Appendix K Limitations","offset":97486,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Appendix L Analysis of Cognitive Behaviors","offset":99480,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Appendix M Prompts","offset":101270,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Training Prompt.","offset":101293,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Prompt for Generating Chain-of-Thought Critique.","offset":101395,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Prompt for Generating Chain-of-Thought Critique with Internal Knowledge","offset":101819,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Refinement Prompt.","offset":102080,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Prompt for Qualitative Analysis.","offset":102346,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Appendix N An Example of Successful Refinement using a CoT Critique","offset":102491,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Appendix O Responses Utilized in Qualitative Analysis","offset":102811,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Appendix P An Example of Failed Refinement","offset":102869,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"},{"title":"Instructions for reporting errors","offset":103096,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/part-7.html#text"}],"text_code_points":104060,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/source.md"}