{"id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","date":"2025-02-07","section":"Reasoning and the \"physics\" of language models","quality":0.22566,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/","source_url":"https://arxiv.org/abs/2502.05171","text_available":true,"authors":["Jonas Geiping","Sean McLeish","Neel Jain","John Kirchenbauer"],"outgoing":[{"source":"arxiv:2502.05171","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:1807.03819","title":"Universal Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/"},{"source":"arxiv:2502.05171","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2502.05171","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2502.05171","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/"},{"source":"arxiv:2502.05171","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2203.08913","title":"Memorizing Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/"},{"source":"arxiv:2502.05171","target":"arxiv:2212.14034","evidence":"title","id":"arxiv:2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/"},{"source":"arxiv:2502.05171","target":"arxiv:2301.05217","evidence":"title","id":"arxiv:2301.05217","title":"Progress measures for grokking via mechanistic interpretability","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f71e3b65a8b3f35beb171d17f6c65878e0914227a0042ccba59b88952554baef/"},{"source":"arxiv:2502.05171","target":"arxiv:2309.14316","evidence":"title","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2502.05171","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2502.05171","target":"arxiv:2405.14838","evidence":"id","id":"arxiv:2405.14838","title":"From Explicit CoT to Implicit CoT: Learning to Internalize CoT Step by Step","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a242b6148f6f84466ce4315ac4937278586a242dd3f847146e142f3c7e27c39c/"},{"source":"arxiv:2502.05171","target":"arxiv:2405.18392","evidence":"title","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"},{"source":"arxiv:2502.05171","target":"arxiv:2412.06769","evidence":"id","id":"arxiv:2412.06769","title":"Training Large Language Models to Reason in a Continuous Latent Space","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/68c71dc1a042dedd5e3c4d9c1ad32c9d046fd31b8dc79080bd6dff1c9800abc7/"},{"source":"arxiv:2502.05171","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2501.00656","title":"2 OLMo 2 Furious","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/"}],"incoming":[{"source":"arxiv:2412.06769","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2412.06769","title":"Training Large Language Models to Reason in a Continuous Latent Space","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/68c71dc1a042dedd5e3c4d9c1ad32c9d046fd31b8dc79080bd6dff1c9800abc7/"},{"source":"arxiv:2501.14082","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2501.14082","title":"Communicating Activations Between Language Model Agents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e67d4addcda44093e242074332925d000726f87bd9538f6b318bdfc454a7dc13/"},{"source":"arxiv:2506.21734","target":"arxiv:2502.05171","evidence":"title","id":"arxiv:2506.21734","title":"Hierarchical Reasoning Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3604312101ff09ab9903e7d6a32c7b2b8a9a7ece6a9d145d04ea01c9d13ffa3a/"},{"source":"arxiv:2507.10524","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2507.10524","title":"Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/"},{"source":"arxiv:2509.20317","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2509.20317","title":"SIM-CoT: Supervised Implicit Chain-of-Thought","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/72b64dd6a30c71d99920cafabbb2c909d03febab8991eeaf3c091b16cb7ed348/"},{"source":"arxiv:2511.09149","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2511.09149","title":"Enabling Agents to Communicate Entirely in Latent Space","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02979751e22005b3284e4d7caa19baae335ca63c9ec46e27d833e95d088e9688/"},{"source":"arxiv:2604.09168","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2604.09168","title":"ELT: Elastic Looped Transformers for Visual Generation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/63d4e5f1a267b96ed4474fa233f7d3bb2171aae949c0139db37b58b4e45eca47/"},{"source":"arxiv:2604.11791","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2604.11791","title":"A Mechanistic Analysis of Looped Reasoning Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae8c682d5708c6c47d65c4eb540dd5bccb877c5e332852765507128d6d1b985d/"},{"source":"arxiv:2604.17121","target":"arxiv:2502.05171","evidence":"title","id":"arxiv:2604.17121","title":"The Topological Trouble With Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f9d762bbe192e1208f6d7dc289b7e79c23d4bdde0b1b3999ff5fda55b0dcd1a/"},{"source":"arxiv:2604.21999","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2604.21999","title":"Universal Transformers Need Memory: Depth-State Trade-offs in Adaptive Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/963f6bb396c25bff462a25854c82e7d92b77803fa60c891fadd9963cc56c8fea/"},{"source":"arxiv:2605.12466","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2605.12466","title":"Solve the Loop: Attractor Models for Language and Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/eb46f81cd5718eb6b9256c583cf9dd2e97b793b4d14337e487c16285f96e2b9a/"},{"source":"arxiv:2605.19376","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2605.19376","title":"Generative Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/795bad9bb1b1a063090fbb925ba6e68bf30211de0ecffd75d56f858b28321ae7/"},{"source":"arxiv:2605.21488","target":"arxiv:2502.05171","evidence":"title","id":"arxiv:2605.21488","title":"Equilibrium Reasoners: Learning Attractors Enables Scalable Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48b3beca0f016a3683df4c6c71fc4be9f13a2c12c61d4ad17d1117c6139bb11e/"},{"source":"arxiv:2606.18206","target":"arxiv:2502.05171","evidence":"title","id":"arxiv:2606.18206","title":"Fixed-Point Reasoners: Stable and Adaptive Deep Looped Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/37ea2d83a92ece4ab3ba4434af71f291d1fdc5040676f75110210f40aad3b81a/"},{"source":"arxiv:2606.29150","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2606.29150","title":"Flow Reasoning Models: Turning Flows Into Efficient Recurrent Reasoners","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8be1801f0f562711039ced2b42f324434c6ab829f544b8d736bc85405b64021b/"},{"source":"arxiv:2606.31779","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2606.31779","title":"Bridging the Gap Between Latent and Explicit Reasoning with Looped Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bf07d5c19e507d9ec91a48f9067e6c29fbe8965d001923e2de0e96da143c72e1/"},{"source":"arxiv:2607.27372","target":"arxiv:2502.05171","evidence":"title","id":"arxiv:2607.27372","title":"Explorative Modeling: Unlocking a Third Pretraining Axis and End-to-End Generation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/542a5e85b84624381953f643ec6075a588c08c542c4f945e57d39c9164c9a835/"},{"source":"arxiv:2608.09888","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2608.09888","title":"BDH-CQ: In-Context Learning with Recurrent Latent Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0c77a8a0f50254e9efdf9a511e08fc2a40677bfa029864bdd0a9cbce561c666/"},{"source":"arxiv:2608.17981","target":"arxiv:2502.05171","evidence":"title","id":"arxiv:2608.17981","title":"Recirculation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd91ffc957070136795d3728f662c997738234065b624eb720acd8547d2edaf6/"},{"source":"arxiv:2609.01117","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2609.01117","title":"Latent Recurrent Thoughts: Recurrent Refinement of Proposed Latents for Reasoning with Frozen LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e9d4f61afc3930fa770d5ea53d3f43a5157fa4904fcdfaa514786e617905e20b/"},{"source":"arxiv:2609.33540","target":"arxiv:2502.05171","evidence":"title","id":"arxiv:2609.33540","title":"Reasoning on the Simplex: Geometric Fixed-Point Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8882e1cc48049a4b6b9a911701665da57fcc16f9e52f2b403b1a5b334653d9c/"}],"text_parts":[{"start":0,"end":15729,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html"},{"start":15729,"end":30981,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html"},{"start":30981,"end":46703,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html"},{"start":46703,"end":60609,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html"},{"start":60609,"end":66233,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-5.html"}],"contents":[{"title":"Abstract","offset":237,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"1 Scaling by Thinking in Continuous Space","offset":1176,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"2 Why Train Models with Recurrent Depth?","offset":4001,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"———————— Table of Contents ————————","offset":6552,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"3 A scalable recurrent architecture","offset":7278,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"3.1 Macroscopic Design","offset":7665,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"Motivation for this Design.","offset":9910,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"Remark 3.1 (Is this a Diffusion Model?) .","offset":11639,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"3.2 Microscopic Design","offset":12591,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"3.3 Training Objective","offset":15650,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"Training Recurrent Models through Unrolling.","offset":15678,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-1.html#text"},{"title":"Truncated Backpropagation.","offset":17197,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"4 Training a large-scale recurrent-depth Language Model","offset":18029,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"4.1 Training Setup","offset":18432,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"Pretraining Data.","offset":18667,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"Tokenization and Packing Details.","offset":19503,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"Architecture and Initialization.","offset":20440,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"Locked-Step Sampling.","offset":21934,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"Optimizer and Learning Rate Schedule.","offset":22553,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"4.2 Compute Setup and Hardware","offset":23173,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"Device Speed and Parallelization Strategy.","offset":23586,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"Training Timeline.","offset":25149,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"4.3 Importance of Norms and Initializations at Scale","offset":25762,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"5 Benchmark Results","offset":28526,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"5.1 Standard Benchmarks","offset":29100,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"5.2 Math and Coding Benchmarks","offset":30945,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-2.html#text"},{"title":"5.3 Where does recurrence help most?","offset":31940,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"Recurrence and Context","offset":33401,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"5.4 Improvements through Weight Averaging","offset":34361,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"6 Recurrent Depth simplifies LLMs","offset":35030,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"6.1 Zero-Shot Adaptive Compute at Test-Time","offset":35355,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"Remark 6.1 (What about missing KV-cache entries?) .","offset":36984,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"6.2 Zero-Shot KV-cache Sharing","offset":37875,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"6.3 Zero-Shot Continuous Chain-of-Thought","offset":38819,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"6.4 Zero-Shot Self-Speculative Decoding","offset":40437,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"7 What Mechanisms Emerge at Scale in Recurrent-Depth Models","offset":41462,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"Path Independence.","offset":44878,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"8 Related Work Overview","offset":45365,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-3.html#text"},{"title":"9 Future Work","offset":49081,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"10 Conclusions","offset":51507,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Acknowledgements","offset":52674,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Appendix A Additional Information","offset":54304,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Potential Implications of This Work","offset":54342,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"A.1 Classical Reasoning Problems","offset":55485,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"A.2 Implementation Details","offset":55617,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Device Speed Details","offset":55649,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Parallelization Strategy","offset":57214,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"RCCL Interconnect Handling","offset":57987,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Pretraining Metrics.","offset":59471,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Appendix B Latent Space Visualizations","offset":60165,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Appendix C Pretraining Data","offset":60577,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-4.html#text"},{"title":"Instructions for reporting errors","offset":65269,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/part-5.html#text"}],"text_code_points":66233,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/source.md"}