{"id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","date":"2023-09-25","section":"Reasoning and the \"physics\" of language models","quality":0.28396,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/","source_url":"https://arxiv.org/abs/2309.14402","text_available":true,"authors":["Zeyuan Allen-Zhu","Yuanzhi Li"],"outgoing":[{"source":"arxiv:2309.14402","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2101.02235","title":"Did Aristotle Use a Laptop? A Question Answering Benchmark with Implicit Reasoning Strategies","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/"},{"source":"arxiv:2309.14402","target":"arxiv:2305.13673","evidence":"id","id":"arxiv:2305.13673","title":"Physics of Language Models: Part 1, Learning Hierarchical Language Structures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a72219d4bd65d1e84ce31079b6c788d7e6c297e71d4b083af5d4cffccfdcff72/"},{"source":"arxiv:2309.14402","target":"arxiv:2309.12288","evidence":"id","id":"arxiv:2309.12288","title":"The Reversal Curse: LLMs trained on \"A is B\" fail to learn \"B is A\"","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02782a7d9697d5df8f8ae84424becf682d10e8671ee5e18fd75499de0e0a6694/"},{"source":"arxiv:2309.14402","target":"arxiv:2309.14316","evidence":"id","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2309.14402","target":"arxiv:2404.05405","evidence":"id","id":"arxiv:2404.05405","title":"Physics of Language Models: Part 3.3, Knowledge Capacity Scaling Laws","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/"},{"source":"arxiv:2309.14402","target":"arxiv:2407.20311","evidence":"title","id":"arxiv:2407.20311","title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/"}],"incoming":[{"source":"arxiv:2305.13673","target":"arxiv:2309.14402","evidence":"title","id":"arxiv:2305.13673","title":"Physics of Language Models: Part 1, Learning Hierarchical Language Structures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a72219d4bd65d1e84ce31079b6c788d7e6c297e71d4b083af5d4cffccfdcff72/"},{"source":"arxiv:2309.12288","target":"arxiv:2309.14402","evidence":"title","id":"arxiv:2309.12288","title":"The Reversal Curse: LLMs trained on \"A is B\" fail to learn \"B is A\"","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02782a7d9697d5df8f8ae84424becf682d10e8671ee5e18fd75499de0e0a6694/"},{"source":"arxiv:2309.14316","target":"arxiv:2309.14402","evidence":"id","id":"arxiv:2309.14316","title":"Physics of Language Models: Part 3.1, Knowledge Storage and Extraction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b87376d90e99d2bb590e1a02ed50a55c90881591ed7d3e00c3d5be2aaacbb2af/"},{"source":"arxiv:2404.05405","target":"arxiv:2309.14402","evidence":"id","id":"arxiv:2404.05405","title":"Physics of Language Models: Part 3.3, Knowledge Capacity Scaling Laws","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/"},{"source":"arxiv:2405.15071","target":"arxiv:2309.14402","evidence":"title","id":"arxiv:2405.15071","title":"Grokked Transformers are Implicit Reasoners: A Mechanistic Journey to the Edge of Generalization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd7fa83eca0b480c3bf9e2490c043b70e937777252a40b196ae7801a0620141/"},{"source":"arxiv:2406.11813","target":"arxiv:2309.14402","evidence":"title","id":"arxiv:2406.11813","title":"How Do Large Language Models Acquire Factual Knowledge During Pretraining?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a7b36ce24eb2182e7f0bba96c0cddcfe6efe610b830d5e8ad8aac2b7d14696c0/"},{"source":"arxiv:2407.15017","target":"arxiv:2309.14402","evidence":"title","id":"arxiv:2407.15017","title":"Knowledge Mechanisms in Large Language Models: A Survey and Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8900dc2821aa61e59c5b1219cf9eb36dfaabf303c17d0064deaca653b5ec9a77/"},{"source":"arxiv:2407.20311","target":"arxiv:2309.14402","evidence":"id","id":"arxiv:2407.20311","title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/"},{"source":"arxiv:2502.09992","target":"arxiv:2309.14402","evidence":"id","id":"arxiv:2502.09992","title":"Large Language Diffusion Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/"},{"source":"arxiv:2504.01928","target":"arxiv:2309.14402","evidence":"title","id":"arxiv:2504.01928","title":"Is the Reversal Curse a Binding Problem? Uncovering Limitations of Transformers from a Basic Generalization Failure","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/"}],"text_parts":[{"start":0,"end":15384,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html"},{"start":15384,"end":31299,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-2.html"},{"start":31299,"end":47075,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-3.html"},{"start":47075,"end":62989,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html"},{"start":62989,"end":68856,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-5.html"}],"contents":[{"title":"Physics of Language Models: Part 3.2, Knowledge Manipulation Thanks: Project page: https://physics.allen-zhu.com/part-3-knowledge/part-3-2 . An extended video of this paper is available at https://youtu.be/YSHzKmEianc . V1 was circulated internally at Meta on Sep 18, 2023, and appeared on arXiv on Sep 25, 2023. V2 polishes writing and includes additional Llama/Mistral experiments and larger data; but the conclusions remain unchanged. We would like to thank Lin Xiao, Chunting Zhou, Xiaodong Liu, Zhijie Zhou for many helpful conversations. We would like to extend special thanks to Nabib Ahmed, Giri Anantharaman, Lucca Bertoncini, Henry Estela, Liao Hu, Caleb Ho, Wil Johnson, Apostolos Kokolis, and Shubho Sengupta from Meta FAIR, as well as Ian Clark, Gourab De, Anmol Mann, and Max Pfeifer from W&B ; without their invaluable support, the experiments in this paper would not have been possible.","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html#text"},{"title":"Abstract","offset":906,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html#text"},{"title":"1 Introduction","offset":2185,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html#text"},{"title":"1.1 Our Results","offset":6463,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html#text"},{"title":"Remark 1.1 .","offset":11654,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html#text"},{"title":"1.2 Our Contributions","offset":12114,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html#text"},{"title":"2 Preliminaries","offset":15223,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-1.html#text"},{"title":"3 Results 1-2: Knowledge Dual and Partial Retrievals","offset":19379,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-2.html#text"},{"title":"4 Results 3-6: Knowledge Classification and Comparison","offset":22825,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-2.html#text"},{"title":"5 Results 7-9: Knowledge Inverse Search","offset":33140,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-3.html#text"},{"title":"6 Conclusion","offset":40045,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-3.html#text"},{"title":"Appendix A More Details on Data Preparation","offset":41644,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-3.html#text"},{"title":"A.1 BIO dataset bioS","offset":42043,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-3.html#text"},{"title":"Remark A.1 .","offset":46178,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-3.html#text"},{"title":"A.1.1 Adding Reverse Knowledge","offset":46475,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-3.html#text"},{"title":"A.2 BIO dataset bioR","offset":47623,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html#text"},{"title":"Appendix B More Details on Knowledge Retrieval","offset":49458,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html#text"},{"title":"Remark B.1 .","offset":53800,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html#text"},{"title":"Appendix C More Details on Knowledge Classification and Comparison","offset":54199,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html#text"},{"title":"Appendix D More Details on Knowledge Inverse Search","offset":58962,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html#text"},{"title":"Appendix E More Details on ChatGPT Experiments","offset":61620,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html#text"},{"title":"E.1 Inverse Knowledge Search","offset":61827,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-4.html#text"},{"title":"E.2 Knowledge Classification and Comparison","offset":67079,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/part-5.html#text"}],"text_code_points":68856,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/source.md"}