{"id":"arxiv:2505.21493","title":"Reinforcing General Reasoning without Verifiers","date":"2025-05-27","section":"Reasoning and the \"physics\" of language models","quality":0.501779,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/","source_url":"https://arxiv.org/abs/2505.21493","text_available":true,"authors":["Xiangxin Zhou","Zichen Liu","Anya Sims"],"outgoing":[{"source":"arxiv:2505.21493","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2505.21493","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2505.21493","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2505.21493","target":"arxiv:2503.14476","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2505.21493","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2505.21493","target":"arxiv:2504.16084","evidence":"id","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2505.21493","target":"url:oatllm.notion.site/oat-zero","evidence":"url","id":"url:oatllm.notion.site/oat-zero","title":"Oat-Zero: reproducing R1-Zero and analyzing self-reflection","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e82b07c066d5c0a7cd37ee24c59ffaab40a3659b37607a86a541c7154c465a62/"}],"incoming":[{"source":"arxiv:2506.08007","target":"arxiv:2505.21493","evidence":"id","id":"arxiv:2506.08007","title":"Reinforcement Pre-Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/"},{"source":"arxiv:2509.14234","target":"arxiv:2505.21493","evidence":"title","id":"arxiv:2509.14234","title":"Compute as Teacher: Turning Inference Compute Into Reference-Free Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a6f1443a7e55dd6421fdbe43e65e19c1962a19b135d737fd67f414d3324ce85e/"},{"source":"arxiv:2601.20802","target":"arxiv:2505.21493","evidence":"title","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"}],"text_parts":[{"start":0,"end":15350,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html"},{"start":15350,"end":30676,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-2.html"},{"start":30676,"end":46578,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html"},{"start":46578,"end":55849,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-4.html"}],"contents":[{"title":"Reinforcing General Reasoning without Verifiers","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"Abstract","offset":51,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"1 Introduction","offset":1633,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"2 Methodology","offset":5700,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"2.1 Preliminaries: Verifier-Based Reinforcement Learning","offset":5718,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"2.2 VeriFree Policy Optimization","offset":8908,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"Theorem 1 .","offset":12452,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"2.3 Comparison to Existing Approaches","offset":15307,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-1.html#text"},{"title":"2.4 How to handle the tokenization at patching point?","offset":20303,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-2.html#text"},{"title":"3 Experiments","offset":22004,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-2.html#text"},{"title":"3.1 Setup","offset":22022,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-2.html#text"},{"title":"3.2 Main Results","offset":27349,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-2.html#text"},{"title":"3.3 Ablation Study","offset":30481,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-2.html#text"},{"title":"4 Related Works","offset":33193,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"5 Conclusions","offset":35466,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"Appendix","offset":36277,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"Table of Contents","offset":36290,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"Appendix A Theoretical Analysis","offset":36312,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"A.1 Derivation of Gradient Estimators","offset":36348,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"Proof.","offset":36586,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"A.2 Proof of Lower Variance","offset":41186,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"Proof.","offset":41787,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-3.html#text"},{"title":"Appendix B Off-policy Gradient Estimators","offset":50926,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-4.html#text"},{"title":"Appendix C Dataset Details","offset":52569,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-4.html#text"},{"title":"Appendix D Extended Empirical Results","offset":52686,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-4.html#text"},{"title":"Appendix E Case Study","offset":53673,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-4.html#text"},{"title":"Appendix F Broader Impact","offset":53930,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-4.html#text"},{"title":"Instructions for reporting errors","offset":54885,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/part-4.html#text"}],"text_code_points":55849,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/source.md"}