{"id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","date":"2020-01-23","section":"Data, training, optimization","quality":0.370895,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/","source_url":"https://arxiv.org/abs/2001.08361","text_available":true,"authors":["Jared Kaplan","Sam McCandlish","Tom Henighan","Tom B. Brown"],"outgoing":[{"source":"arxiv:2001.08361","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:1807.03819","title":"Universal Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/"}],"incoming":[{"source":"arxiv:2206.04615","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/"},{"source":"arxiv:2207.02098","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2207.02098","title":"Neural Networks and the Chomsky Hierarchy","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/"},{"source":"arxiv:2210.10760","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2212.14034","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/"},{"source":"arxiv:2301.06627","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2301.06627","title":"Dissociating language and thought in large language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3791d7ed420be517ded1ee7ae5558cf6afa948ea5b841a76ec3da8a9134ec565/"},{"source":"arxiv:2304.15004","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2304.15004","title":"Are Emergent Abilities of Large Language Models a Mirage?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/"},{"source":"arxiv:2305.14342","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/"},{"source":"arxiv:2306.13575","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2306.13575","title":"Scaling MLPs: A Tale of Inductive Bias","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/"},{"source":"arxiv:2307.06440","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2404.05405","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2404.05405","title":"Physics of Language Models: Part 3.3, Knowledge Capacity Scaling Laws","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/"},{"source":"arxiv:2404.09173","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2404.09173","title":"TransformerFAM: Feedback attention is working memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/"},{"source":"arxiv:2405.04517","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2405.04517","title":"xLSTM: Extended Long Short-Term Memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/"},{"source":"arxiv:2405.07987","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2405.07987","title":"The Platonic Representation Hypothesis","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/"},{"source":"arxiv:2405.16684","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2405.16684","title":"gzip Predicts Data-dependent Scaling Laws","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/"},{"source":"arxiv:2405.18392","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"},{"source":"arxiv:2406.11813","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2406.11813","title":"How Do Large Language Models Acquire Factual Knowledge During Pretraining?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a7b36ce24eb2182e7f0bba96c0cddcfe6efe610b830d5e8ad8aac2b7d14696c0/"},{"source":"arxiv:2407.20311","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2407.20311","title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/"},{"source":"arxiv:2407.21783","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2412.14135","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2412.14135","title":"Scaling of Search and Learning: A Roadmap to Reproduce o1 from Reinforcement Learning Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/"},{"source":"arxiv:2501.00656","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2501.00656","title":"2 OLMo 2 Furious","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/"},{"source":"arxiv:2501.19393","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2501.19393","title":"s1: Simple test-time scaling","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2a88818f7605b71cd0160831faebd51b3274c24ffd29626b4d519390b4bffffe/"},{"source":"arxiv:2502.03387","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2502.03387","title":"LIMO: Less is More for Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/714392528bb842fb3a61cee5367942f697536f0131b0fcac85c6eb8d2212125c/"},{"source":"arxiv:2502.05171","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"},{"source":"arxiv:2502.09992","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2502.09992","title":"Large Language Diffusion Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/"},{"source":"arxiv:2504.16084","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2505.15134","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2505.15134","title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/"},{"source":"arxiv:2505.24832","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2505.24832","title":"How much do language models memorize?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d14faeb604373b83ad90ae653998a11cc09492020341f29419e0deed45a300c/"},{"source":"arxiv:2506.08007","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2506.08007","title":"Reinforcement Pre-Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5ae77d4a2dcd6843e856083ecc6c30a72563e6896503d6efdfae117b8b007e6/"},{"source":"arxiv:2507.10524","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2507.10524","title":"Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/"},{"source":"arxiv:2510.04871","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2510.04871","title":"Less is More: Recursive Reasoning with Tiny Networks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4345eebe9dc7e67f82617896772f55f0970971516995c1688b6812f9ebf054f6/"},{"source":"arxiv:2510.05491","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2510.05491","title":"NorMuon: Making Muon more efficient and scalable","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/"},{"source":"arxiv:2510.13786","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2510.13786","title":"The Art of Scaling Reinforcement Learning Compute for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/72580bcba3d9b32b2f3e83d074f626234b26dc18126edd22a5eacef927cf87da/"},{"source":"arxiv:2510.26622","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2510.26622","title":"Encoder-Decoder or Decoder-Only? Revisiting Encoder-Decoder Large Language Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dc73f87f798d016e2d83acd1d362b4b3525af826e68b055bb5196f120d23e080/"},{"source":"arxiv:2510.27688","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2510.27688","title":"Continuous Autoregressive Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/"},{"source":"arxiv:2512.24695","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/"},{"source":"arxiv:2604.08224","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2604.08224","title":"Externalization in LLM Agents: A Unified Review of Memory, Skills, Protocols and Harness Engineering","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/392317a8b3bedec8258ff8288d69ac72bc9e60299c5f7bec8962838bb448c2ce/"},{"source":"arxiv:2604.19341","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2604.19341","title":"Structured Scaling of AI Discovery Across Diverse Scientific Domains","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/33ad3375247f86197210840a19064548b997e8617475e606d7a7ac2cda64740e/"},{"source":"arxiv:2606.25010","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2606.25010","title":"Emergent Capabilities Arise Randomly from Learning Sparse Attention Patterns","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f87db387bad54ae4fbc0589d2a702f78aeb0817b28f20844f6502bc552a89c2f/"},{"source":"doi:10.1038/s42256-023-00754-x","target":"arxiv:2001.08361","evidence":"id","id":"doi:10.1038/s42256-023-00754-x","title":"A social path to human-like artificial intelligence","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/25e744318984a723282df54f9f39bf66e3a17e6f3b09c90a80af90c250e799be/"},{"source":"url:ai.googleblog.com/2022/11/characterizing-emergent-phenomena-in.html","target":"arxiv:2001.08361","evidence":"id","id":"url:ai.googleblog.com/2022/11/characterizing-emergent-phenomena-in.html","title":"Characterizing emergent phenomena in LLMs (Google Research)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/749249d7d689468b1404142d896fe5bd9048709780cc20f3a0f984a02bcc3aca/"},{"source":"url:anthropic.com/news/mapping-mind-language-model","target":"arxiv:2001.08361","evidence":"id","id":"url:anthropic.com/news/mapping-mind-language-model","title":"Mapping the mind of a large language model (Anthropic)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edf322b5841fac48b6247e69ebcb8c3a4598b4af752c1d567c0f9392b778738d/"},{"source":"url:openai.com/index/deep-double-descent","target":"arxiv:2001.08361","evidence":"title","id":"url:openai.com/index/deep-double-descent","title":"Deep Double Descent (OpenAI)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dbfa3f03c433f18313852b6668892ef9cbe993f6949854e9270622c7cd29b701/"}],"text_parts":[{"start":0,"end":15528,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html"},{"start":15528,"end":31205,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html"},{"start":31205,"end":46825,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-3.html"},{"start":46825,"end":61257,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html"},{"start":61257,"end":70566,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html"}],"contents":[{"title":"Scaling Laws for Neural Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Abstract","offset":43,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"1 Introduction","offset":865,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"1.1 Summary","offset":2141,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Performance depends strongly on scale, weakly on model shape:","offset":2228,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Smooth power laws:","offset":2654,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Universality of overfitting:","offset":3043,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Universality of training:","offset":3481,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Transfer improves with test performance:","offset":3773,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Sample efficiency:","offset":4205,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Convergence is inefficient:","offset":4412,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"Optimal batch size:","offset":4960,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"1.2 Summary of Scaling Laws","offset":5529,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"1.3 Notation","offset":11466,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"2 Background and Methods","offset":13093,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"2.1 Parameter and Compute Scaling of Transformers","offset":13749,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-1.html#text"},{"title":"2.2 Training Procedures","offset":16096,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"2.3 Datasets","offset":16760,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"3 Empirical Results and Basic Power Laws","offset":17825,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"3.1 Approximate Transformer Shape and Hyperparameter Independence","offset":18520,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"3.2 Performance with Non-Embedding Parameter Count N N","offset":19514,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"3.2.1 Comparing to LSTMs and Universal Transformers","offset":20784,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"3.2.2 Generalization Among Data Distributions","offset":21653,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"3.3 Performance with Dataset Size and Compute","offset":22316,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"4 Charting the Infinite Data Limit and Overfitting","offset":24102,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"4.1 Proposed L ⁡ ( N , D ) L(N,D) Equation","offset":24632,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"4.2 Results","offset":27783,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"5 Scaling Laws with Model Size and Training Time","offset":30436,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"5.1 Adjustment for Training at B crit ​ ( L ) B_{\\rm crit}(L)","offset":31138,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-2.html#text"},{"title":"5.2 Results for L ⁡ ( N , S min ) L(N,S_{\\rm min}) and Performance with Model Size and Compute","offset":35519,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-3.html#text"},{"title":"5.3 Lower Bound on Early Stopping Step","offset":37187,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-3.html#text"},{"title":"6 Optimal Allocation of the Compute Budget","offset":38630,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-3.html#text"},{"title":"6.1 Optimal Performance and Allocations","offset":39950,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-3.html#text"},{"title":"6.2 Predictions from L ⁡ ( N , S min ) L(N,S_{\\rm min})","offset":41903,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-3.html#text"},{"title":"6.3 Contradictions and a Conjecture","offset":43206,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-3.html#text"},{"title":"7 Related Work","offset":48255,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"8 Discussion","offset":51432,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"Acknowledgements","offset":54805,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"Appendix A Summary of Power Laws","offset":55064,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"Appendix B Empirical Model of Compute-Efficient Frontier","offset":55321,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"B.1 Defining Equations","offset":55596,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"B.2 Efficient Training","offset":58060,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"B.3 Comparison to Inefficient","offset":59851,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"B.4 Suboptimal Model Sizes","offset":61225,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-4.html#text"},{"title":"Appendix C Caveats","offset":63170,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"Appendix D Supplemental Figures","offset":65538,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.1 Early Stopping and Test vs Train","offset":65574,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.2 Universal Transformers","offset":65854,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.3 Batch Size","offset":66202,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.4 Sample Efficiency vs Model Size","offset":66378,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.5 Context Dependence","offset":66668,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.6 Learning Rate Schedules and Error Analysis","offset":68209,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.7 Fit Details and Power Law Quality","offset":69580,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"},{"title":"D.8 Generalization and Architecture","offset":70304,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/part-5.html#text"}],"text_code_points":70566,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/source.md"}