{"id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","date":"2025-04-18","section":"Post-training","quality":0.022969,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/","source_url":"https://arxiv.org/abs/2504.13837","text_available":true,"authors":["Yang Yue","Zhiqi Chen","Rui Lu","Andrew Zhao"],"outgoing":[{"source":"arxiv:2504.13837","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2504.13837","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2504.13837","target":"arxiv:2503.14476","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2504.13837","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2504.13837","target":"arxiv:2506.01939","evidence":"title","id":"arxiv:2506.01939","title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/"},{"source":"arxiv:2504.13837","target":"url:oatllm.notion.site/oat-zero","evidence":"url","id":"url:oatllm.notion.site/oat-zero","title":"Oat-Zero: reproducing R1-Zero and analyzing self-reflection","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e82b07c066d5c0a7cd37ee24c59ffaab40a3659b37607a86a541c7154c465a62/"}],"incoming":[{"source":"arxiv:2504.20571","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2504.20571","title":"Reinforcement Learning for Reasoning in Large Language Models with One Training Example","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b2b21c3df49619588c371411dba9467a41f81b3fd4da5441898b5145bb0de120/"},{"source":"arxiv:2505.19590","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2505.19590","title":"Learning to Reason without External Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/"},{"source":"arxiv:2506.01939","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2506.01939","title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/"},{"source":"arxiv:2506.03106","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2506.03106","title":"Critique-GRPO: Advancing LLM Reasoning with Natural Language and Numerical Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/"},{"source":"arxiv:2506.06632","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2506.06632","title":"Curriculum Reinforcement Learning from Easy to Hard Tasks Improves LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79fda9609f9b88bb71f71f531b63ea6b5237d1c889b7ae59e9ca07c12dbe70d1/"},{"source":"arxiv:2506.10139","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2506.10139","title":"Unsupervised Elicitation of Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/"},{"source":"arxiv:2506.10947","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2506.10947","title":"Spurious Rewards: Rethinking Training Signals in RLVR","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f48e13a9656d8eff1882eb80988e07a820c4799401298947deac735c004e8592/"},{"source":"arxiv:2508.11408","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2508.11408","title":"On-Policy RL Meets Off-Policy Experts: Harmonizing Supervised Fine-Tuning and Reinforcement Learning via Dynamic Weighting","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48594da2e3f2cdfeb17195dd033d047235fc4b4ffb65030800f64c6faedc4fbe/"},{"source":"arxiv:2509.14234","target":"arxiv:2504.13837","evidence":"title","id":"arxiv:2509.14234","title":"Compute as Teacher: Turning Inference Compute Into Reference-Free Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a6f1443a7e55dd6421fdbe43e65e19c1962a19b135d737fd67f414d3324ce85e/"},{"source":"arxiv:2509.25123","target":"arxiv:2504.13837","evidence":"title","id":"arxiv:2509.25123","title":"From $f(x)$ and $g(x)$ to $f(g(x))$: LLMs Learn New Skills in RL by Composing Old Ones","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fdfe7abe26ee324fe7abad3ce802148163129a7c157d77bdec859c990868e03/"},{"source":"arxiv:2510.11686","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2510.11686","title":"Representation-Based Exploration for Language Models: From Test-Time to Post-Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/89b1f9c1bcd283b28d316cb6cddafdd9361be5cd510b2be2a1e761c20a0fd62e/"},{"source":"arxiv:2510.14901","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2510.14901","title":"Reasoning with Sampling: Your Base Model is Smarter Than You Think","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/46373e6f386387299bbade0a302219a611c3497c5e5b5970e3502f7a5f189687/"},{"source":"arxiv:2512.13961","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"},{"source":"arxiv:2601.11061","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2601.11061","title":"Spurious Rewards Paradox: Mechanistically Understanding How RLVR Activates Memorization Shortcuts in LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96002027cf4c274573f7320398481a0cd73dbf72f18d7f37c783605f5158ea7b/"},{"source":"arxiv:2601.14525","target":"arxiv:2504.13837","evidence":"title","id":"arxiv:2601.14525","title":"Towards Execution-Grounded Automated AI Research","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/895006e818735c9a565dbe99ae31404366eefd6bdc2278612271c65a8493e4a5/"},{"source":"arxiv:2601.19897","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2601.19897","title":"Self-Distillation Enables Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cc146df3e9ca40c6d111a5264e08710100c1b34bd0eabd9052e8d55d13589efe/"},{"source":"arxiv:2601.20802","target":"arxiv:2504.13837","evidence":"title","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"},{"source":"arxiv:2605.06241","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2605.06241","title":"Rethinking RL for LLM Reasoning: It's Sparse Policy Selection, Not Capability Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9e4fdd6e0c24f5a635ff70c2bca19da19c4012a05f69ceb20fdc0a86fd8b775/"},{"source":"arxiv:2605.22074","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2605.22074","title":"From Reasoning Chains to Verifiable Subproblems: Curriculum Reinforcement Learning Enables Credit Assignment for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e51c03db071e17a4d5403433f1037e79c3cda795e0b20f6a490df35167411dd0/"},{"source":"arxiv:2608.17981","target":"arxiv:2504.13837","evidence":"title","id":"arxiv:2608.17981","title":"Recirculation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd91ffc957070136795d3728f662c997738234065b624eb720acd8547d2edaf6/"},{"source":"arxiv:2609.37633","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2609.37633","title":"RLTL;DR: Self-improvement by Internalizing Self-generated Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c7a4c1dbb0493059883d9a01c1f89d4db3644209b909e1195c5afa4d3a320999/"}],"text_parts":[{"start":0,"end":15899,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-1.html"},{"start":15899,"end":31364,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html"},{"start":31364,"end":46576,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html"},{"start":46576,"end":54005,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html"}],"contents":[{"title":"1 Introduction","offset":911,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-1.html#text"},{"title":"2 Preliminaries","offset":7646,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-1.html#text"},{"title":"2.1 Reinforcement Learning with Verifiable Rewards","offset":7867,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-1.html#text"},{"title":"2.2 Metrics for LLM Reasoning Capacity Boundary","offset":10588,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-1.html#text"},{"title":"3 RLVR’s Effect on Reasoning Capacity Boundary","offset":13259,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-1.html#text"},{"title":"3.1 RLVR for Mathematical Reasoning","offset":14871,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-1.html#text"},{"title":"3.2 RLVR for Code Generation","offset":18912,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"3.3 RLVR for Visual Reasoning","offset":20055,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"4 Deep Analysis","offset":21352,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"4.1 Reasoning Paths Already Present in Base Models","offset":21661,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"4.2 Distillation Expands the Reasoning Boundary","offset":25806,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"4.3 Effects of Different RL Algorithms","offset":27124,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"4.4 Effects of RL Training","offset":29444,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"4.5 Effects of Entropy","offset":30602,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"4.6 Effects of Model Size Scaling","offset":31325,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-2.html#text"},{"title":"5 Discussion","offset":33244,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"6 Related Work","offset":38154,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"7 Conclusion and Limitations","offset":39469,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"Author Contributions","offset":41130,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"Acknowledgements","offset":42454,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"Appendix","offset":42781,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"Appendix Contents","offset":42794,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"Appendix A Implementation Details","offset":42816,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"A.1 RLVR Algorithms","offset":42854,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"A.2 Low-Variance pass@ k k Estimation","offset":43576,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"Appendix B More Related Works","offset":44747,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-3.html#text"},{"title":"Appendix C Detailed Experimental Results","offset":47697,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.1 More Results on Mathematics and Coding","offset":47742,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.2 Validity of Chain-of-Thought on AIME24","offset":47790,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.3 Accuracy Distribution Visulization","offset":49363,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.4 Perplexity Analysis","offset":49407,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.5 Different RLVR Algorithms","offset":50035,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.6 Effects of KL and Rollout Number","offset":50957,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.7 Solvable Problem Coverage Analysis","offset":50999,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.8 Temperature and Entropy Analysis","offset":51743,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.9 Training Dynamics","offset":51785,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"C.10 CoT Case Analysis","offset":51812,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"Appendix D Prompt Templates","offset":51840,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"Appendix E Broader Impacts","offset":52776,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"},{"title":"Instructions for reporting errors","offset":53041,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/part-4.html#text"}],"text_code_points":54005,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/source.md"}