{"id":"arxiv:2407.20311","title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process","date":"2024-07-29","section":"Reasoning and the \"physics\" of language models","quality":0.322996,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/","source_url":"https://arxiv.org/abs/2407.20311","text_available":true,"authors":["Tian Ye","Zicheng Xu","Yuanzhi Li","Zeyuan Allen-Zhu"],"outgoing":[{"source":"arxiv:2407.20311","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2407.20311","target":"arxiv:2305.13673","evidence":"id","id":"arxiv:2305.13673","title":"Physics of Language Models: Part 1, Learning Hierarchical Language Structures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a72219d4bd65d1e84ce31079b6c788d7e6c297e71d4b083af5d4cffccfdcff72/"},{"source":"arxiv:2407.20311","target":"arxiv:2309.14316","evidence":"id","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2407.20311","target":"arxiv:2309.14402","evidence":"id","id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/"},{"source":"arxiv:2407.20311","target":"arxiv:2404.05405","evidence":"id","id":"arxiv:2404.05405","title":"Physics of Language Models: Part 3.3, Knowledge Capacity Scaling Laws","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/"}],"incoming":[{"source":"arxiv:2305.13673","target":"arxiv:2407.20311","evidence":"title","id":"arxiv:2305.13673","title":"Physics of Language Models: Part 1, Learning Hierarchical Language Structures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a72219d4bd65d1e84ce31079b6c788d7e6c297e71d4b083af5d4cffccfdcff72/"},{"source":"arxiv:2309.14316","target":"arxiv:2407.20311","evidence":"title","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2309.14402","target":"arxiv:2407.20311","evidence":"title","id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/"},{"source":"arxiv:2407.15017","target":"arxiv:2407.20311","evidence":"id","id":"arxiv:2407.15017","title":"Knowledge Mechanisms in Large Language Models: A Survey and Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8900dc2821aa61e59c5b1219cf9eb36dfaabf303c17d0064deaca653b5ec9a77/"},{"source":"arxiv:2502.09992","target":"arxiv:2407.20311","evidence":"id","id":"arxiv:2502.09992","title":"Large Language Diffusion Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/"},{"source":"arxiv:2504.01928","target":"arxiv:2407.20311","evidence":"id","id":"arxiv:2504.01928","title":"Is the Reversal Curse a Binding Problem? Uncovering Limitations of Transformers from a Basic Generalization Failure","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/"},{"source":"arxiv:2510.00184","target":"arxiv:2407.20311","evidence":"id","id":"arxiv:2510.00184","title":"Why Can't Transformers Learn Multiplication? Reverse-Engineering Reveals Long-Range Dependency Pitfalls","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7e395b2ae5bfc2a4d8bfc1e8669b4f324739b6a9b11161cba150bfdffc31746c/"},{"source":"arxiv:2512.16902","target":"arxiv:2407.20311","evidence":"id","id":"arxiv:2512.16902","title":"In-Context Algebra","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5aa7087a88619eafd156aad9217e16f6d1ce284c65534322239be0e02f04e553/"}],"text_parts":[{"start":0,"end":15291,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-1.html"},{"start":15291,"end":31282,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html"},{"start":31282,"end":46416,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html"},{"start":46416,"end":61120,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html"},{"start":61120,"end":76962,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html"},{"start":76962,"end":81876,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-6.html"}],"contents":[{"title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process Thanks: Project page + video: https://physics.allen-zhu.com/part-2-grade-school-math/part-2-1 . We would like to thank Lin Xiao, Chunting Zhou for many helpful conversations. We would like to extend special thanks to Lucca Bertoncini, Liao Hu, Caleb Ho, Wil Johnson, Apostolos Kokolis, Parth Malani, Alexander Miller, Junjie Qian and Shubho Sengupta from Meta FAIR; Henry Estela, Rizwan Hashmi, Lucas Noah, and Maxwell Taylor from the Meta Cloud Foundation team; as well as Ian Clark, Gourab De, Anmol Mann, and Max Pfeifer from W&B . Without their invaluable support, the experiments in this paper would not have been possible.","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-1.html#text"},{"title":"Abstract","offset":721,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-1.html#text"},{"title":"1 Introduction","offset":1775,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-1.html#text"},{"title":"2 Result 1: Data Generation","offset":10109,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-1.html#text"},{"title":"2.1 Step 1: Graph Construction and Problem Generation","offset":11546,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-1.html#text"},{"title":"Remark 2.1 .","offset":12952,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-1.html#text"},{"title":"2.2 Step 2: Solution Construction (CoT)","offset":16895,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html#text"},{"title":"2.3 Difficulty Control","offset":18949,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html#text"},{"title":"2.4 Train and Test Datasets","offset":20378,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html#text"},{"title":"Proposition 2.2 .","offset":22550,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html#text"},{"title":"3 Result 2-3: Summarize Model’s Behavior Process","offset":24172,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html#text"},{"title":"4 Result 4-5: Discover Model’s Mental Process","offset":28137,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html#text"},{"title":"4.1 V-Probing: A Nearly-Linear Probing Method","offset":30044,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-2.html#text"},{"title":"4.2 Probing Results and Findings","offset":32103,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html#text"},{"title":"Remark 4.1 .","offset":33376,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html#text"},{"title":"5 Result 6: Explain Model’s Mistakes","offset":35970,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html#text"},{"title":"6 Result 7-8: Depth vs. Reasoning Length","offset":38653,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html#text"},{"title":"7 Conclusion","offset":42523,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html#text"},{"title":"Appendix A Result 1 — An Example in iGSM-hard with op = 21 \\textsf{op}=21","offset":44730,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html#text"},{"title":"Appendix B Results 4-5 — Details on V-probing","offset":44808,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-3.html#text"},{"title":"Appendix C Result 8 — Additional Figure","offset":48009,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"Appendix D Result 1 Details — Math Data Generation","offset":48053,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"D.1 Generate Structure Graph","offset":48867,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"D.1.1 Attach English","offset":50213,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"D.2 Generate Dependency Graph","offset":51006,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"Definition D.1 (operation) .","offset":53288,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"Remark D.2 .","offset":54964,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"D.2.1 Construction of G 𝖽 𝗇𝖾𝖼𝖾𝟣 , G 𝖽 𝗇𝖾𝖼𝖾𝟤 G_{\\mathsf{d}}^{\\mathsf{nece1}},G_{\\mathsf{d}}^{\\mathsf{nece2}}","offset":56148,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"D.2.2 Construction of G 𝖽 𝗇𝖾𝖼𝖾𝟥 G_{\\mathsf{d}}^{\\mathsf{nece3}}","offset":57508,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"Proposition D.3 .","offset":60836,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"Remark D.4 .","offset":61099,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-4.html#text"},{"title":"D.2.3 Construction of G 𝖽 𝗇𝖾𝖼𝖾 G_{\\mathsf{d}}^{\\mathsf{nece}}","offset":62648,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"D.2.4 Construction of G 𝖽 G_{\\mathsf{d}}","offset":63593,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"Remark D.5 .","offset":64841,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"D.3 Generate English: Problem, Question and Solution","offset":65236,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"Remark D.6 .","offset":69354,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"D.4 Putting Altogether","offset":69627,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"Remark D.7 .","offset":71891,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"Remark D.8 .","offset":72306,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"Appendix E Data Details: Probing Data Preparation","offset":72617,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"Appendix F Experiment Details","offset":74017,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"F.1 Pretrain Experiment Details","offset":75389,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-5.html#text"},{"title":"F.2 V-probing","offset":77919,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-6.html#text"},{"title":"Appendix G Failure Examples on GPT-4 / GPT-4o","offset":79722,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/part-6.html#text"}],"text_code_points":81876,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/source.md"}