{"id":"arxiv:2404.05405","title":"Physics of Language Models: Part 3.3, Knowledge Capacity Scaling Laws","date":"2024-04-08","section":"Reasoning and the \"physics\" of language models","quality":0.359542,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/","source_url":"https://arxiv.org/abs/2404.05405","text_available":true,"authors":["Zeyuan Allen-Zhu","Yuanzhi Li"],"outgoing":[{"source":"arxiv:2404.05405","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2404.05405","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2404.05405","target":"arxiv:2305.13673","evidence":"id","id":"arxiv:2305.13673","title":"Physics of Language Models: Part 1, Learning Hierarchical Language Structures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a72219d4bd65d1e84ce31079b6c788d7e6c297e71d4b083af5d4cffccfdcff72/"},{"source":"arxiv:2404.05405","target":"arxiv:2309.14316","evidence":"id","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2404.05405","target":"arxiv:2309.14402","evidence":"id","id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/"}],"incoming":[{"source":"arxiv:2305.13673","target":"arxiv:2404.05405","evidence":"title","id":"arxiv:2305.13673","title":"Physics of Language Models: Part 1, Learning Hierarchical Language Structures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a72219d4bd65d1e84ce31079b6c788d7e6c297e71d4b083af5d4cffccfdcff72/"},{"source":"arxiv:2309.14316","target":"arxiv:2404.05405","evidence":"id","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2309.14402","target":"arxiv:2404.05405","evidence":"id","id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/"},{"source":"arxiv:2407.15017","target":"arxiv:2404.05405","evidence":"title","id":"arxiv:2407.15017","title":"Knowledge Mechanisms in Large Language Models: A Survey and Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8900dc2821aa61e59c5b1219cf9eb36dfaabf303c17d0064deaca653b5ec9a77/"},{"source":"arxiv:2407.20311","target":"arxiv:2404.05405","evidence":"id","id":"arxiv:2407.20311","title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/"},{"source":"arxiv:2503.21676","target":"arxiv:2404.05405","evidence":"id","id":"arxiv:2503.21676","title":"How do language models learn facts? Dynamics, curricula and hallucinations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ef59e7bcfcfc0e4bab638835504ad8a0d2f50665a9797edc09e8628e9b3c0b6/"},{"source":"arxiv:2505.24832","target":"arxiv:2404.05405","evidence":"id","id":"arxiv:2505.24832","title":"How much do language models memorize?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d14faeb604373b83ad90ae653998a11cc09492020341f29419e0deed45a300c/"},{"source":"url:thinkingmachines.ai/blog/lora","target":"arxiv:2404.05405","evidence":"id","id":"url:thinkingmachines.ai/blog/lora","title":"LoRA Without Regret (Thinking Machines)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09bfccf7014ee63d12eddd2c954194cd981ce104398b7b64c98873bf2f997b95/"}],"text_parts":[{"start":0,"end":15536,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html"},{"start":15536,"end":31195,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html"},{"start":31195,"end":47118,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html"},{"start":47118,"end":62941,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html"},{"start":62941,"end":78924,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html"},{"start":78924,"end":94760,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html"},{"start":94760,"end":110104,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-7.html"},{"start":110104,"end":121161,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-8.html"}],"contents":[{"title":"Physics of Language Models: Part 3.3, Knowledge Capacity Scaling Laws Thanks: Submitted for Meta internal review on March 14, 2024. We would like to thank Lin Xiao and Yuchen Zhang for many helpful conversations. We would like to extend special thanks to Ian Clark, Gourab De, Anmol Mann, and Max Pfeifer from W&B , as well as Lucca Bertoncini, Liao Hu, Caleb Ho, Apostolos Kokolis, and Shubho Sengupta from Meta FAIR NextSys; Henry Estela, Wil Johnson, Rizwan Hashmi, and Lucas Noah from Meta Cloud Foundation; without their invaluable support, the extensive experiments in this paper would not have been possible.","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Abstract","offset":619,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"1 Introduction","offset":2120,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Example .","offset":7704,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Remark 1.1 .","offset":8599,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Remark 1.2 .","offset":9231,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Remark 1.3 .","offset":10137,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Remark 1.4 .","offset":10931,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Remark 1.5 .","offset":12112,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Remark 1.6 .","offset":12303,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"2 Preliminaries","offset":15228,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"2.1 Knowledge (Theoretical Setting)","offset":15495,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-1.html#text"},{"title":"Remark 2.1 .","offset":17166,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"Definition 2.2 ( bioD data generation) .","offset":17903,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"Proposition 2.3 (trivial, bit complexity upper bound) .","offset":19086,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"2.2 Knowledge (Empirical Setting)","offset":19804,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"2.3 Models and Training","offset":23503,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"3 Bit Complexity Lower Bound","offset":24826,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"Remark 3.1 .","offset":27437,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"Theorem 3.2 (bit complexity lower bound) .","offset":28566,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"Corollary 3.3 (no-error case) .","offset":29915,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"Remark 3.4 (why “sum of 3”) .","offset":30682,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"Remark 3.5 (why “random data”) .","offset":30883,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-2.html#text"},{"title":"4 Capacity Ratio","offset":32104,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"Definition 4.1 .","offset":32223,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"Remark 4.2 .","offset":33298,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"Definition 4.3 .","offset":34203,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"Remark 4.4 .","offset":35806,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"5 Base Scaling Laws","offset":35935,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"Remark 5.1 .","offset":37653,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"5.1 Data Formats — Diversity and Rewriting","offset":39229,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"5.2 Parameterized Scaling Laws","offset":41875,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"6 Training Time vs Scaling Law","offset":42613,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"7 Model Architecture vs Scaling Law","offset":43833,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"7.1 Insufficient Training Regime and a Closer Comparison","offset":45921,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-3.html#text"},{"title":"8 Quantization vs Scaling Laws","offset":48236,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"Corollary 8.1 .","offset":49123,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"8.1 Where Is the Knowledge Stored?","offset":49506,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"9 Mixture of Experts vs Scaling Laws","offset":50642,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"Remark 9.1 (topk) .","offset":52565,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"Remark 9.2 .","offset":52993,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"10 Junk Data vs Scaling Laws","offset":53246,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"11 Conclusion","offset":57676,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"Appendix A More on GPT2 Scaling Laws","offset":59048,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"A.1 Base Scaling Laws","offset":62521,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"Parameter 1 ( Figure 1(a) ) .","offset":62903,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-4.html#text"},{"title":"Remark A.1 (fp16 vs bf16) .","offset":64114,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Remark A.2 (parameters) .","offset":64266,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Remark A.3 (# GPUs) .","offset":65168,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Parameter 2 ( Figure 1(b) ) .","offset":65709,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"A.2 Knowledge Memorization vs. Extraction","offset":67038,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Parameter 3 ( Figure 10 ) .","offset":69514,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"A.3 Other Biography Datasets","offset":70588,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Parameter 4 ( Figure 11 ) .","offset":72333,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"A.4 More on Parameterized Scaling Laws","offset":72492,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Parameter 5 ( Figure 2 , 12 , 13 ) .","offset":72677,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Remark A.4 (parameters) .","offset":72876,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Appendix B More on Model Architectures","offset":74130,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"B.1 1000-Exposure Setting","offset":76813,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Parameter 6 ( Figure 3 ) .","offset":77110,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"Remark B.1 (bf16 on gated MLP) .","offset":78673,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-5.html#text"},{"title":"B.2 100-Exposure Setting","offset":79357,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Parameter 7 ( Figure 4 ) .","offset":82368,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Parameter 8 ( Figure 5 ) .","offset":85599,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Appendix C More on Quantization","offset":86058,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Appendix D More on Mixture of Experts","offset":86448,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Parameter 9 ( Figure 16 , Figure 17 ) .","offset":88270,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Appendix E More on Junk Data vs. Scaling Laws","offset":90543,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Parameter 10 ( Figure 8 ) .","offset":93125,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-6.html#text"},{"title":"Appendix F Proof of Theorem 3.2","offset":98335,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-7.html#text"},{"title":"Lemma F.1 .","offset":98550,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-7.html#text"},{"title":"Proof of Lemma F.1 .","offset":100060,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-7.html#text"},{"title":"F.1 Warmup Examples","offset":101644,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-7.html#text"},{"title":"F.2 Main Proof","offset":107536,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-7.html#text"},{"title":"Proof of Theorem 3.2 .","offset":107666,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-7.html#text"},{"title":"Appendix G Missing Remark","offset":119103,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-8.html#text"},{"title":"Remark G.1 .","offset":119133,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/part-8.html#text"}],"text_code_points":121161,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/source.md"}