{"id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","date":"2024-02-05","section":"Post-training","quality":0.322538,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/","source_url":"https://arxiv.org/abs/2402.03300","text_available":true,"authors":["Zhihong Shao","Peiyi Wang","Qihao Zhu","Runxin Xu"],"outgoing":[{"source":"arxiv:2402.03300","target":"url:openai.com/research/weak-to-strong-generalization","evidence":"id","id":"url:openai.com/research/weak-to-strong-generalization","title":"Weak-to-Strong Generalization: Eliciting Strong Capabilities With Weak Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fb7b23a617dce974dbb70db824c97c1de84afa11e1e290d5cb9f5bd88857e762/"}],"incoming":[{"source":"arxiv:2407.21783","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2412.06769","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2412.06769","title":"Training Large Language Models to Reason in a Continuous Latent Space","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/68c71dc1a042dedd5e3c4d9c1ad32c9d046fd31b8dc79080bd6dff1c9800abc7/"},{"source":"arxiv:2412.14135","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2412.14135","title":"Scaling of Search and Learning: A Roadmap to Reproduce o1 from Reinforcement Learning Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/"},{"source":"arxiv:2501.00656","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2501.00656","title":"2 OLMo 2 Furious","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/"},{"source":"arxiv:2502.03387","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2502.03387","title":"LIMO: Less is More for Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/714392528bb842fb3a61cee5367942f697536f0131b0fcac85c6eb8d2212125c/"},{"source":"arxiv:2502.05171","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"},{"source":"arxiv:2503.00735","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2503.00735","title":"LADDER: Self-Improving LLMs Through Recursive Problem Decomposition","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/65b264030233bf2bcd17284eb9a521e6e38d4ee83eb7fec6587e4c047199f227/"},{"source":"arxiv:2503.14476","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2503.20783","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2504.13837","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/"},{"source":"arxiv:2504.16084","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2504.20571","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2504.20571","title":"Reinforcement Learning for Reasoning in Large Language Models with One Training Example","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b2b21c3df49619588c371411dba9467a41f81b3fd4da5441898b5145bb0de120/"},{"source":"arxiv:2505.10978","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2505.10978","title":"Group-in-Group Policy Optimization for LLM Agent Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2fa466eef15ebc0489217004e5ebb80efd37db9675a603331ccc3b7da055d147/"},{"source":"arxiv:2505.15134","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2505.15134","title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/"},{"source":"arxiv:2505.19590","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2505.19590","title":"Learning to Reason without External Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/"},{"source":"arxiv:2505.21444","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2505.21444","title":"Can Large Reasoning Models Self-Train?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/206224ca50098e0bdc230140722feec107cfc90c1cef2ed8d3ed52355ef56bca/"},{"source":"arxiv:2505.21493","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2505.21493","title":"Reinforcing General Reasoning without Verifiers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/"},{"source":"arxiv:2506.01939","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2506.01939","title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/"},{"source":"arxiv:2506.03106","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2506.03106","title":"Critique-GRPO: Advancing LLM Reasoning with Natural Language and Numerical Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/"},{"source":"arxiv:2506.06632","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2506.06632","title":"Curriculum Reinforcement Learning from Easy to Hard Tasks Improves LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79fda9609f9b88bb71f71f531b63ea6b5237d1c889b7ae59e9ca07c12dbe70d1/"},{"source":"arxiv:2506.10947","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2506.10947","title":"Spurious Rewards: Rethinking Training Signals in RLVR","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f48e13a9656d8eff1882eb80988e07a820c4799401298947deac735c004e8592/"},{"source":"arxiv:2506.13585","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2506.13585","title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5beb2ec6c7e6362f9d957219d7a205dd1adcc53f739acc0c7fd5f1f2bded4d83/"},{"source":"arxiv:2507.12856","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2507.12856","title":"Supervised Fine Tuning on Curated Data is Reinforcement Learning (and can be improved)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5f6c1906eaa67369d9ff15b974756b89141d1b4cf9d7c0f6554e0d56f26e79be/"},{"source":"arxiv:2507.18071","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"},{"source":"arxiv:2507.19457","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2507.19457","title":"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c1ef72d96fa3292590a6a3d112ab509840081b59aaa771cf42b8b0e8ef62ba08/"},{"source":"arxiv:2508.05629","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2508.05629","title":"On the Generalization of SFT: A Reinforcement Learning Perspective with Reward Rectification","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0a17fccfeffa8a04070bb16903a7217d8adee804cf42e0cfc8dd128d3390474f/"},{"source":"arxiv:2508.07629","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2508.07629","title":"Klear-Reasoner: Advancing Reasoning Capability via Gradient-Preserving Clipping Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a232533e4e78ad28d2ffe9fafe799b66cf626b18ad523d0088d9cc12f1ba1b76/"},{"source":"arxiv:2508.11408","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2508.11408","title":"On-Policy RL Meets Off-Policy Experts: Harmonizing Supervised Fine-Tuning and Reinforcement Learning via Dynamic Weighting","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48594da2e3f2cdfeb17195dd033d047235fc4b4ffb65030800f64c6faedc4fbe/"},{"source":"arxiv:2509.09675","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2509.09675","title":"CDE: Curiosity-Driven Exploration for Efficient Reinforcement Learning in Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/"},{"source":"arxiv:2509.13232","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2509.13232","title":"Single-stream Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d699d0e034b21f3b85c3f70ce4fb95064fbf59685ce752935e6d173dac2b71f3/"},{"source":"arxiv:2509.14234","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2509.14234","title":"Compute as Teacher: Turning Inference Compute Into Reference-Free Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a6f1443a7e55dd6421fdbe43e65e19c1962a19b135d737fd67f414d3324ce85e/"},{"source":"arxiv:2509.25123","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2509.25123","title":"From $f(x)$ and $g(x)$ to $f(g(x))$: LLMs Learn New Skills in RL by Composing Old Ones","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fdfe7abe26ee324fe7abad3ce802148163129a7c157d77bdec859c990868e03/"},{"source":"arxiv:2510.00977","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2510.00977","title":"It Takes Two: Your GRPO Is Secretly DPO","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/"},{"source":"arxiv:2510.11686","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2510.11686","title":"Representation-Based Exploration for Language Models: From Test-Time to Post-Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/89b1f9c1bcd283b28d316cb6cddafdd9361be5cd510b2be2a1e761c20a0fd62e/"},{"source":"arxiv:2510.13786","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2510.13786","title":"The Art of Scaling Reinforcement Learning Compute for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/72580bcba3d9b32b2f3e83d074f626234b26dc18126edd22a5eacef927cf87da/"},{"source":"arxiv:2511.07317","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2511.07317","title":"RLVE: Scaling Up Reinforcement Learning for Language Models with Adaptive Verifiable Environments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c0c18cb0ae7fe76a7239f38ce819c7ac588c1184dd27c5672f868ff4e2e26605/"},{"source":"arxiv:2511.20347","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2511.20347","title":"Soft Adaptive Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/"},{"source":"arxiv:2512.00499","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2512.00499","title":"ESPO: Entropy Importance Sampling Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/342b84df2bf40f64c57ea7839579c1f337374ad9c28b16ce7d477c42e24c67b4/"},{"source":"arxiv:2512.02807","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2512.02807","title":"SR-GRPO: Stable Rank as an Intrinsic Geometric Reward for Large Language Model Alignment","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ebb3b04baa11b5d0365ca4445da1c2f833108b680ea65272710ed66a57751c0e/"},{"source":"arxiv:2512.13961","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"},{"source":"arxiv:2512.18160","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2512.18160","title":"Propose, Solve, Verify: Self-Play Through Formal Verification","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6b68bf20c3b0180fbf31bfbfb5f4e5f7800fc134d2c393106efb3320b67a64cd/"},{"source":"arxiv:2601.05053","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2601.05053","title":"Reinforced Efficient Reasoning via Semantically Diverse Exploration","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/74918d32c56962f9c5b2063026f5e2243a84e20c284eb3e37c882bd4681d1a11/"},{"source":"arxiv:2601.07055","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2601.07055","title":"Dr. Zero: Self-Evolving Search Agents without Training Data","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3baa3e7fc5768b75e4ef3e64c3cc5297fa42ef19b03bc918ef5352e437e1c168/"},{"source":"arxiv:2601.14525","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2601.14525","title":"Towards Execution-Grounded Automated AI Research","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/895006e818735c9a565dbe99ae31404366eefd6bdc2278612271c65a8493e4a5/"},{"source":"arxiv:2601.18734","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"},{"source":"arxiv:2601.20802","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"},{"source":"arxiv:2601.20861","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2601.20861","title":"Evolutionary Strategies lead to Catastrophic Forgetting in LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4178758b0cb3ebc95f6a3cbc6423e8ea05c0ae649e11a2f4059a84b3faa6cde0/"},{"source":"arxiv:2601.21343","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2601.21343","title":"Self-Improving Pretraining: using post-trained models to pretrain better models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/43ce1b189d6b21d445c0ce6d4d04b4352d268beb4346da43be19982402befde8/"},{"source":"arxiv:2602.09000","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2602.09000","title":"iGRPO: Self-Feedback-Driven LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd6a66823fe24e8e2d83e817177d3a1e8cda1d7d5b59a4397f8669bee6306ae/"},{"source":"arxiv:2602.18037","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2602.18037","title":"Gradient Regularization Mitigates Reward Hacking in Reinforcement Learning from Human Feedback and Verifiable Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/"},{"source":"arxiv:2604.02288","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2604.02288","title":"Unifying Group-Relative and Self-Distillation Policy Optimization via Sample Routing","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3d1160029d321f2069016f7702d4ebebee0de6cd31a8f296ee220abd6f69c99b/"},{"source":"arxiv:2604.08690","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/"},{"source":"arxiv:2604.13016","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2604.13016","title":"Rethinking On-Policy Distillation of Large Language Models: Phenomenology, Mechanism, and Recipe","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d6d7ff1e2e7b3b9c528f19f7144098920bf573f2cb411fcac5879d5d2cac93e0/"},{"source":"arxiv:2604.19341","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2604.19341","title":"Structured Scaling of AI Discovery Across Diverse Scientific Domains","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/33ad3375247f86197210840a19064548b997e8617475e606d7a7ac2cda64740e/"},{"source":"arxiv:2604.20659","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2604.20659","title":"GRPO-VPS: Enhancing Group Relative Policy Optimization with Verifiable Process Supervision for Effective Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/42caf0f9c992f5df8bac4d0f7e45c3a5bf57879661f41febd01a047694d52628/"},{"source":"arxiv:2605.06241","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2605.06241","title":"Rethinking RL for LLM Reasoning: It's Sparse Policy Selection, Not Capability Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9e4fdd6e0c24f5a635ff70c2bca19da19c4012a05f69ceb20fdc0a86fd8b775/"},{"source":"arxiv:2605.12969","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2605.12969","title":"Revisiting Reinforcement Learning with Verifiable Rewards from a Contrastive Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/30e87f5eb9beda8e09b3945d6ba1707dec761abed83c49d129a21adaed02ce45/"},{"source":"arxiv:2605.22074","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2605.22074","title":"From Reasoning Chains to Verifiable Subproblems: Curriculum Reinforcement Learning Enables Credit Assignment for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e51c03db071e17a4d5403433f1037e79c3cda795e0b20f6a490df35167411dd0/"},{"source":"arxiv:2606.23740","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2606.23740","title":"Weight-Space Geometry of Offline Reasoning Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e4479f0d0930797db9347855247f1c640a12d96f63d43b83ba9635561e3ff352/"},{"source":"arxiv:2607.07847","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2607.07847","title":"When Does Continual Learning Require Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/86e88c6cf21a66879eab5c156913fc85d14eb69c394c78c33c0e81d144e528eb/"},{"source":"arxiv:2608.13040","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2608.13040","title":"Latent On-Policy Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2cfd504d6ca36ba88d75aba018620867d5fafd7433b03f3b2c9f51408422a02/"},{"source":"arxiv:2608.19197","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2608.19197","title":"SPADE: Self-Play in Adaptive Synthetic Executable Environments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c524376d6034b5d79661d6f1a444591404f4a250bcb0790d802c05e02f8249df/"},{"source":"arxiv:2608.27448","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2608.27448","title":"TTPO: Test-Time Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/"},{"source":"arxiv:2609.30652","target":"arxiv:2402.03300","evidence":"title","id":"arxiv:2609.30652","title":"Recursive Self-Improvement via On-Policy Distillation for Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/9e52af299c264dd7185057305368429cdf42deb165dd5e3fc0f9cf6ef1d612bf/"},{"source":"arxiv:2609.37633","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2609.37633","title":"RLTL;DR: Self-improvement by Internalizing Self-generated Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c7a4c1dbb0493059883d9a01c1f89d4db3644209b909e1195c5afa4d3a320999/"},{"source":"url:cameronrwolfe.substack.com/p/grpo-tricks","target":"arxiv:2402.03300","evidence":"id","id":"url:cameronrwolfe.substack.com/p/grpo-tricks","title":"GRPO++: Tricks for Making RL Actually Work (Cameron Wolfe)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d170bc3da5b8e6a055fd511719fcc3764635236bac638947bd0a018412245eb5/"},{"source":"url:thinkingmachines.ai/blog/lora","target":"arxiv:2402.03300","evidence":"id","id":"url:thinkingmachines.ai/blog/lora","title":"LoRA Without Regret (Thinking Machines)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09bfccf7014ee63d12eddd2c954194cd981ce104398b7b64c98873bf2f997b95/"}],"text_parts":[{"start":0,"end":15705,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html"},{"start":15705,"end":30110,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html"},{"start":30110,"end":45630,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html"},{"start":45630,"end":61403,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html"},{"start":61403,"end":69803,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-5.html"}],"contents":[{"title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","offset":19,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"Abstract","offset":105,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"1 Introduction","offset":1177,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"1.1 Contributions","offset":5634,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"1.2 Summary of Evaluations and Metrics","offset":8353,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"2 Math Pre-Training","offset":11023,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"2.1 Data Collection and Decontamination","offset":11047,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"2.2 Validating the Quality of the DeepSeekMath Corpus","offset":14864,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"2.2.1 Training Setting","offset":15676,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-1.html#text"},{"title":"2.2.2 Evaluation Results","offset":17120,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"2.3 Training and Evaluating DeepSeekMath-Base 7B","offset":18646,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"Mathematical Problem Solving with Step-by-Step Reasoning","offset":19795,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"Mathematical Problem Solving with Tool Use","offset":21613,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"Formal Mathematics","offset":22129,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"Natural Language Understanding, Reasoning, and Code","offset":23029,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"3 Supervised Fine-Tuning","offset":23928,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"3.1 SFT Data Curation","offset":23957,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"3.2 Training and Evaluating DeepSeekMath-Instruct 7B","offset":25004,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"4 Reinforcement Learning","offset":29679,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"4.1 Group Relative Policy Optimization","offset":29708,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"4.1.1 From PPO to GRPO","offset":30081,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-2.html#text"},{"title":"4.1.2 Outcome Supervision RL with GRPO","offset":35668,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"4.1.3 Process Supervision RL with GRPO","offset":36595,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"4.1.4 Iterative RL with GRPO","offset":38225,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"4.2 Training and Evaluating DeepSeekMath-RL","offset":38835,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"5 Discussion","offset":40693,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"5.1 Lessons Learnt in Pre-Training","offset":40791,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"5.1.1 Code Training Benefits Mathematical Reasoning","offset":41145,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"Results","offset":43712,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"5.1.2 ArXiv Papers Seem Ineffective in Improving Mathematical Reasoning","offset":45552,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-3.html#text"},{"title":"5.2 Insights of Reinforcement Learning","offset":47718,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"5.2.1 Towards to a Unified Paradigm","offset":47762,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"Observation about Data Source","offset":50166,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"Observation about Gradient Coefficient","offset":51205,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"5.2.2 Why RL Works?","offset":52651,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"5.2.3 How to Achieve More Effective RL?","offset":53654,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"Data Source","offset":54142,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"Algorithms","offset":55018,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"Reward Function","offset":55936,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"6 Conclusion, Limitation, and Future Work","offset":56830,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"Appendix A Appendix","offset":58711,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"A.1 Analysis of Reinforcement Learning","offset":58735,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"A.1.1 Supervised Fine-tuning","offset":58967,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"A.1.2 Rejection Sampling Fine-tuning","offset":59840,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"A.1.3 Online Rejection Sampling Fine-tuning","offset":61353,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-4.html#text"},{"title":"A.1.4 Direct Preference Optimization (DPO)","offset":62033,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-5.html#text"},{"title":"A.1.5 Proximal Policy Optimization (PPO)","offset":64239,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-5.html#text"},{"title":"A.1.6 Group Relative Policy Optimization (GRPO)","offset":66511,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-5.html#text"},{"title":"Instructions for reporting errors","offset":68839,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/part-5.html#text"}],"text_code_points":69803,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/source.md"}