{"id":"arxiv:2501.00656","title":"2 OLMo 2 Furious","date":"2024-12-31","section":"LLMs: architectures, context, training","quality":0.235048,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/","source_url":"https://arxiv.org/abs/2501.00656","text_available":true,"authors":["Team OLMo","Pete Walsh","Luca Soldaini","Dirk Groeneveld"],"outgoing":[{"source":"arxiv:2501.00656","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2501.00656","target":"arxiv:2002.05202","evidence":"title","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2501.00656","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2501.00656","target":"arxiv:2406.11813","evidence":"id","id":"arxiv:2406.11813","title":"How Do Large Language Models Acquire Factual Knowledge During Pretraining?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a7b36ce24eb2182e7f0bba96c0cddcfe6efe610b830d5e8ad8aac2b7d14696c0/"},{"source":"arxiv:2501.00656","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[{"source":"arxiv:2502.05171","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"},{"source":"arxiv:2505.19590","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2505.19590","title":"Learning to Reason without External Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/"},{"source":"arxiv:2508.02513","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2508.02513","title":"Modular Arithmetic: Language Models Solve Math Digit by Digit","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/"},{"source":"arxiv:2512.13961","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"},{"source":"arxiv:2601.11061","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2601.11061","title":"Spurious Rewards Paradox: Mechanistically Understanding How RLVR Activates Memorization Shortcuts in LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/96002027cf4c274573f7320398481a0cd73dbf72f18d7f37c783605f5158ea7b/"}],"text_parts":[{"start":0,"end":15735,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html"},{"start":15735,"end":31320,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html"},{"start":31320,"end":46977,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html"},{"start":46977,"end":62408,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html"},{"start":62408,"end":78173,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html"},{"start":78173,"end":94125,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html"},{"start":94125,"end":106433,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html"}],"contents":[{"title":"2 OLMo 2 Furious","offset":2200,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"Abstract","offset":2220,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"1 Introduction","offset":3665,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"2 OLMo 2 Family","offset":8758,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"2.1 Model Architecture","offset":9520,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"2.2 Tokenizer","offset":12449,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"2.3 Base Model Training Recipe","offset":13802,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"Stage 1: Pretraining","offset":14566,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"Stage 2: Mid-training","offset":15318,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-1.html#text"},{"title":"Model Merging or “Souping”","offset":16157,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"Overall","offset":16867,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"2.4 Base Model Data","offset":17174,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"2.4.1 Pretraining data: OLMo 2 Mix 1124","offset":17294,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"2.4.2 Mid-training data: Dolmino Mix 1124","offset":18850,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"2.5 Evaluation and Results","offset":19617,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"Base Model Evaluation:","offset":19874,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"Post-Training Recipe and Evaluation","offset":21918,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"3 Deep Dive: Pretraining Stability","offset":23020,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"3.1 Repeated n-Grams","offset":25266,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"3.2 Model Initialization","offset":26916,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"Gradient and activation growth","offset":28141,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"Hyperparameter transfer across width","offset":30129,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"Spike score","offset":31301,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-2.html#text"},{"title":"Empirical results","offset":31858,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"3.3 Architecture Improvements","offset":32426,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"3.3.1 Nonparametric layer norm and RMSNorm","offset":32461,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"3.3.2 Reordered norm and QK-norm","offset":32922,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"3.3.3 Z-Loss","offset":34272,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"3.4 Hyperparameter Improvements","offset":35696,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"3.4.1 ϵ \\mathbf{\\epsilon} in AdamW","offset":35733,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"3.4.2 Weight decay on embeddings","offset":36245,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"4 Deep Dive: Mid-training Recipe","offset":37324,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"4.1 Learning rate annealing","offset":38375,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"4.2 Data Curriculum: Dolmino Mix 1124","offset":43679,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"4.3 Dolmino Mix 1124 : High Quality Sources","offset":46194,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"High quality web","offset":46515,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-3.html#text"},{"title":"Instruction data and Q&A pairs","offset":47796,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Code","offset":48981,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Academic, encyclopedic and other reference content","offset":49304,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Math","offset":49660,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"4.4 Dolmino Mix 1124 : Math Mix","offset":51690,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"4.4.1 Math Sources","offset":52423,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"TuluMath","offset":52448,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"DolminoSynthMath","offset":53393,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"TinyGSM-MIND","offset":54432,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"MathCoder2-Synthetic","offset":55141,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"ProofPile OWM-Filtered","offset":55950,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"GSM8K-Train","offset":56136,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"4.4.2 Evaluating Math Data with Microanneals","offset":56229,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Microanneal experiment 1: domain specific data is helpful even in small proportions","offset":58005,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Microanneal experiment 2: some duplication is beneficial","offset":58721,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Microanneal experiment 3: rewriting can help dramatically","offset":59199,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"4.5 Final Midtraining mix and Checkpoint Soups","offset":59975,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Mid-training model merging or “soups”","offset":61344,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"5 Deep Dive: Post-training Pipeline","offset":62368,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-4.html#text"},{"title":"Supervised Finetuning (SFT)","offset":63448,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Preference Finetuning (PreFT) with DPO","offset":65409,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Reinforcement Learning with Verifiable Rewards (RLVR)","offset":66877,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Hyperparameter selection","offset":68469,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Evaluation of OLMo 2-Instruct","offset":71597,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"6 Deep Dive: Infrastructure as a Research Catalyst","offset":73218,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"6.1 Clusters","offset":74235,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"6.1.1 Jupiter","offset":74688,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Compute","offset":74839,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Storage","offset":75070,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Interconnect","offset":75534,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Cooling","offset":75962,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"6.1.2 Augusta Cluster","offset":76511,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Compute","offset":76670,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Storage","offset":76875,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Interconnect","offset":77091,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Cooling","offset":77500,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"6.2 Beaker","offset":77668,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Portability","offset":77829,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"Isolation","offset":78156,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-5.html#text"},{"title":"6.3 Stability and Operations","offset":79310,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"GPU health checks","offset":79772,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Cordoning","offset":80321,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Active monitoring","offset":80604,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"6.4 Maximizing hardware utilization","offset":80869,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Taking advantage of compilation","offset":81779,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Minimizing host-device syncs","offset":82461,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Asynchronous bookkeeping with a separate backend","offset":84103,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Explicit Python garbage collection","offset":86442,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"6.5 Environmental Impact","offset":87723,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Conclusion","offset":91516,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Author Contributions","offset":92435,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Acknowledgments","offset":94026,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-6.html#text"},{"title":"Appendix A OLMo 2 Evaluation Framework","offset":95669,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"A.1 Base Model Eval","offset":95976,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Multiple-choice tasks","offset":96349,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Generative tasks","offset":97363,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Held-out tasks","offset":98470,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"A.2 Instruct Model Eval","offset":99883,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Instruct tasks","offset":99912,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Appendix B OLMo 2 1B","offset":100911,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"B.1 Difficulties with OLMo 2 1B","offset":101481,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Training","offset":101810,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Base Results","offset":102156,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Analysis","offset":102413,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Appendix C Additional Instruct Details","offset":103658,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"C.1 Additional Hyperparameters","offset":103701,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"C.2 Additional RLVR Learning Curves","offset":104093,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"C.3 OLMo 2-Instruct Preview Models","offset":104234,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Appendix D Additional Hyperparameters","offset":105260,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Appendix E Annealing Data Details","offset":105431,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"},{"title":"Instructions for reporting errors","offset":105469,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/part-7.html#text"}],"text_code_points":106433,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/source.md"}