{"id":"arxiv:2506.08007","title":"Reinforcement Pre-Training","date":"2025-06-09","section":"Data, training, optimization","quality":-0.408418,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/","source_url":"https://arxiv.org/abs/2506.08007","text_available":true,"authors":["Qingxiu Dong","Li Dong","Yao Tang"],"outgoing":[{"source":"arxiv:2506.08007","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2506.08007","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2506.08007","target":"arxiv:2504.20571","evidence":"title","id":"arxiv:2504.20571","title":"Reinforcement Learning for Reasoning in Large Language Models with One Training Example","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b2b21c3df49619588c371411dba9467a41f81b3fd4da5441898b5145bb0de120/"},{"source":"arxiv:2506.08007","target":"arxiv:2505.21493","evidence":"id","id":"arxiv:2505.21493","title":"Reinforcing General Reasoning without Verifiers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/"}],"incoming":[{"source":"arxiv:2601.21343","target":"arxiv:2506.08007","evidence":"id","id":"arxiv:2601.21343","title":"Self-Improving Pretraining: using post-trained models to pretrain better models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/43ce1b189d6b21d445c0ce6d4d04b4352d268beb4346da43be19982402befde8/"},{"source":"arxiv:2605.12466","target":"arxiv:2506.08007","evidence":"title","id":"arxiv:2605.12466","title":"Solve the Loop: Attractor Models for Language and Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/eb46f81cd5718eb6b9256c583cf9dd2e97b793b4d14337e487c16285f96e2b9a/"},{"source":"arxiv:2606.31779","target":"arxiv:2506.08007","evidence":"title","id":"arxiv:2606.31779","title":"Bridging the Gap Between Latent and Explicit Reasoning with Looped Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bf07d5c19e507d9ec91a48f9067e6c29fbe8965d001923e2de0e96da143c72e1/"}],"text_parts":[{"start":0,"end":15755,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html"},{"start":15755,"end":28926,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html"}],"contents":[{"title":"Reinforcement Pre-Training","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"Abstract","offset":30,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"1 Introduction","offset":992,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"2 Preliminary","offset":5559,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"Next-Token Prediction (NTP)","offset":5577,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"Reinforcement Learning with Verifiable Rewards (RLVR)","offset":6097,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"3 Reinforcement Pre-Training","offset":6883,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"3.1 Pre-Training Task: Next-Token Reasoning","offset":6916,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"3.2 Pre-Training with Reinforcement Learning","offset":8054,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"3.3 Pre-Training Setup","offset":10206,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"3.4 Evaluation of Pretrained Models","offset":12054,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"Language Modeling","offset":12377,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"Reinforcement Fine-Tuning on Downstream Tasks","offset":12623,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"4 Experiments","offset":13000,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"4.1 Language Modeling","offset":13018,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"4.2 Scaling Properties of Reinforcement Pre-Training","offset":14415,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-1.html#text"},{"title":"4.3 Reinforcement Fine-Tuning with RPT","offset":16002,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"4.4 Zero-Shot Performance on End Tasks","offset":17139,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"4.5 Next-Token Reasoning Pattern Analysis","offset":18406,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"5 Related Work","offset":20736,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Scaling Paradigms of Large Language Models","offset":20755,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Reinforcement Learning for Large Language Models","offset":21395,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"6 Conclusion and Future Work","offset":22227,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Acknowledgement","offset":24133,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Appendix A Design Choices of Reward","offset":24402,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Appendix B Hyperparameters Used for Reinforcement Pre-Training","offset":26327,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Appendix C Detailed Results on End Tasks","offset":26603,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Appendix D Impact of Prompt Templates","offset":27044,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Appendix E Keywords for Reasoning Pattern Analysis","offset":27650,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Appendix F Case Studies","offset":27794,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"},{"title":"Instructions for reporting errors","offset":27962,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/part-2.html#text"}],"text_code_points":28926,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/source.md"}