{"id":"arxiv:2505.19590","title":"Learning to Reason without External Rewards","date":"2025-05-26","section":"Post-training","quality":0.010191,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/","source_url":"https://arxiv.org/abs/2505.19590","text_available":true,"authors":["Xuandong Zhao","Zhewei Kang","Aosong Feng","Sergey Levine"],"outgoing":[{"source":"arxiv:2505.19590","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2505.19590","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2505.19590","target":"arxiv:2407.00695","evidence":"title","id":"arxiv:2407.00695","title":"Learning Formal Mathematics From Intrinsic Motivation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/68b19af6f86f72a1a7cedd8c0e476588d44d21a891d79fc640768859eac9a909/"},{"source":"arxiv:2505.19590","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2501.00656","title":"2 OLMo 2 Furious","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/"},{"source":"arxiv:2505.19590","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2505.19590","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/"},{"source":"arxiv:2505.19590","target":"arxiv:2504.16084","evidence":"id","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2505.19590","target":"arxiv:2505.15134","evidence":"id","id":"arxiv:2505.15134","title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/"},{"source":"arxiv:2505.19590","target":"arxiv:2505.21444","evidence":"id","id":"arxiv:2505.21444","title":"Can Large Reasoning Models Self-Train?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/206224ca50098e0bdc230140722feec107cfc90c1cef2ed8d3ed52355ef56bca/"},{"source":"arxiv:2505.19590","target":"arxiv:2506.10947","evidence":"id","id":"arxiv:2506.10947","title":"Spurious Rewards: Rethinking Training Signals in RLVR","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f48e13a9656d8eff1882eb80988e07a820c4799401298947deac735c004e8592/"},{"source":"arxiv:2505.19590","target":"url:openai.com/research/weak-to-strong-generalization","evidence":"id","id":"url:openai.com/research/weak-to-strong-generalization","title":"Weak-to-Strong Generalization: Eliciting Strong Capabilities With Weak Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fb7b23a617dce974dbb70db824c97c1de84afa11e1e290d5cb9f5bd88857e762/"}],"incoming":[{"source":"arxiv:2505.21444","target":"arxiv:2505.19590","evidence":"id","id":"arxiv:2505.21444","title":"Can Large Reasoning Models Self-Train?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/206224ca50098e0bdc230140722feec107cfc90c1cef2ed8d3ed52355ef56bca/"},{"source":"arxiv:2506.10139","target":"arxiv:2505.19590","evidence":"id","id":"arxiv:2506.10139","title":"Unsupervised Elicitation of Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/"},{"source":"arxiv:2506.10947","target":"arxiv:2505.19590","evidence":"id","id":"arxiv:2506.10947","title":"Spurious Rewards: Rethinking Training Signals in RLVR","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f48e13a9656d8eff1882eb80988e07a820c4799401298947deac735c004e8592/"},{"source":"arxiv:2509.14234","target":"arxiv:2505.19590","evidence":"title","id":"arxiv:2509.14234","title":"Compute as Teacher: Turning Inference Compute Into Reference-Free Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a6f1443a7e55dd6421fdbe43e65e19c1962a19b135d737fd67f414d3324ce85e/"},{"source":"arxiv:2510.14901","target":"arxiv:2505.19590","evidence":"id","id":"arxiv:2510.14901","title":"Reasoning with Sampling: Your Base Model is Smarter Than You Think","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/46373e6f386387299bbade0a302219a611c3497c5e5b5970e3502f7a5f189687/"},{"source":"arxiv:2607.13104","target":"arxiv:2505.19590","evidence":"id","id":"arxiv:2607.13104","title":"Self-Improvements in Modern Agentic Systems: A Survey","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1452caf7b16ffdb8c2efd5093f46dc4df0f89c3d45a5566cd5fdafd69385b2a7/"}],"text_parts":[{"start":0,"end":15779,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html"},{"start":15779,"end":30738,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html"},{"start":30738,"end":46369,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html"},{"start":46369,"end":50809,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html"}],"contents":[{"title":"Learning to Reason without External Rewards","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"Abstract","offset":47,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"1 Introduction","offset":1184,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"2 Related Work","offset":6548,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"Reinforcement Learning from Human Feedback (RLHF).","offset":6567,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"Reinforcement Learning with Verifiable Rewards (RLVR).","offset":7333,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"Intrinsic Signals and Self-Play in LLM Optimization.","offset":8491,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"3 Method","offset":9549,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"3.1 Reinforcement Learning from Internal Feedback (RLIF)","offset":9562,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"3.2 Intuitor : Policy Optimization with Self-Certainty","offset":11585,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-1.html#text"},{"title":"4 Experimental Setup","offset":17291,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"5 Results and Analysis","offset":19879,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"5.1 Learning to Follow Instructions","offset":20853,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"5.2 Fostering Structured Reasoning","offset":21580,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"5.3 Understanding Emergent Long-Form Reasoning","offset":23353,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"5.4 Online Self-Certainty Prevents Reward Exploitation","offset":25403,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"5.5 Ablation Studies","offset":27402,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"6 Discussion and Future Research","offset":28485,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"7 Conclusion","offset":30721,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-2.html#text"},{"title":"Ethics Statement","offset":31992,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"Reproducibility Statement","offset":32573,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"Appendix A Additional Background","offset":33100,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"A.1 From External Supervision to Internal Feedback","offset":33137,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"Appendix B Additional Experimental Details","offset":35569,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.1 Influence of the KL Penalty","offset":35616,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.2 Scaling to Larger Models","offset":36457,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.3 Generalization Across Model Families","offset":37564,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.4 Comparison with Alternative Reward Signals","offset":38436,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.5 Ablation on Optimization Strategy: Policy Gradient vs. Direct Optimization","offset":42304,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.6 The Effect of Rollout Size","offset":43234,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.7 Attempts at Combining Golden Answers and Self-Certainty","offset":43755,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.8 Sequential Training Across Domains","offset":44967,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.9 Standard Deviation of Response Correctness","offset":45645,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.10 Failure Case Analysis","offset":46337,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-3.html#text"},{"title":"B.11 Training Hyperparameters","offset":47801,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"Appendix C Prompts and Model Completions","offset":47887,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"C.1 Training Prompts","offset":48126,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"C.2 Example from LiveCodeBench Code Generation","offset":48152,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"C.3 Example from LiveCodeBench Code Generation","offset":48377,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"C.4 Example from CRUXEval-O","offset":48639,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"C.5 Rollout Example of Intuitor During Training on Code Corpus","offset":48893,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"C.6 Evolution of Qwen2.5-3B’s Responses on LiveCodeBench Code Generation trained with Intuitor on MATH","offset":49437,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"},{"title":"Instructions for reporting errors","offset":49845,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/part-4.html#text"}],"text_code_points":50809,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/source.md"}