{"id":"arxiv:2502.00873","title":"Language Models Use Trigonometry to Do Addition","date":"2025-02-02","section":"Reasoning and the \"physics\" of language models","quality":0.39366,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/","source_url":"https://arxiv.org/abs/2502.00873","text_available":true,"authors":["Subhash Kantamneni","Max Tegmark"],"outgoing":[{"source":"arxiv:2502.00873","target":"arxiv:2301.05217","evidence":"title","id":"arxiv:2301.05217","title":"Progress measures for grokking via mechanistic interpretability","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f71e3b65a8b3f35beb171d17f6c65878e0914227a0042ccba59b88952554baef/"},{"source":"arxiv:2502.00873","target":"arxiv:2309.08600","evidence":"title","id":"arxiv:2309.08600","title":"Sparse Autoencoders Find Highly Interpretable Features in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/311cc8e986e42947a9cd36e64cb7dad93d7352e1f5ad495664fce8d0f6fca5da/"},{"source":"arxiv:2502.00873","target":"arxiv:2406.03445","evidence":"title","id":"arxiv:2406.03445","title":"Pre-trained Large Language Models Use Fourier Features to Compute Addition","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/169a3e64467f1e0862508b86b2198187fc8eeb3ab114a47d2bd4caaea4a9ee2b/"},{"source":"arxiv:2502.00873","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2502.00873","target":"arxiv:2410.21272","evidence":"title","id":"arxiv:2410.21272","title":"Arithmetic Without Algorithms: Language Models Solve Math With a Bag of Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/397b826ba289bd4f14f457562942cb092c38550a4c49451e95ffcc7b376c8cb0/"}],"incoming":[{"source":"arxiv:2502.19981","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2502.19981","title":"The Lookahead Limitation: Why Multi-Operand Addition is Hard for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/"},{"source":"arxiv:2508.02513","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2508.02513","title":"Modular Arithmetic: Language Models Solve Math Digit by Digit","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/"},{"source":"arxiv:2510.00184","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2510.00184","title":"Why Can't Transformers Learn Multiplication? Reverse-Engineering Reveals Long-Range Dependency Pitfalls","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7e395b2ae5bfc2a4d8bfc1e8669b4f324739b6a9b11161cba150bfdffc31746c/"},{"source":"arxiv:2512.16902","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2512.16902","title":"In-Context Algebra","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5aa7087a88619eafd156aad9217e16f6d1ce284c65534322239be0e02f04e553/"},{"source":"arxiv:2602.10416","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2602.10416","title":"AI-rithmetic","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a644c16d12f96071c95ea707eb5c5b3cc44be3dda2f9fe49a5f38735695fdad9/"},{"source":"arxiv:2605.05115","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2605.05115","title":"Manifold Steering Reveals the Shared Geometry of Neural Network Representation and Behavior","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4d9659f959b746810c945d693de7b0e4d1a73aeaa7ec183aaa7b8f68549e3d50/"},{"source":"arxiv:2606.03982","target":"arxiv:2502.00873","evidence":"title","id":"arxiv:2606.03982","title":"Language Models Compare Quantities Using Number-specific and Unit-specific Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48f07d03c9308ca8fb43ecd2ac22978f6dcbd4bc081056f022ca511711942dbf/"}],"text_parts":[{"start":0,"end":15679,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html"},{"start":15679,"end":31564,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html"},{"start":31564,"end":47174,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html"},{"start":47174,"end":51000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-4.html"}],"contents":[{"title":"Language Models Use Trigonometry to Do Addition","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"Abstract","offset":51,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"Keywords:","offset":1214,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"1 Introduction","offset":1269,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"2 Related Work","offset":3624,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"3 Problem Setup","offset":6497,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"4 LLMs Represent Numbers as a Helix","offset":9051,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"4.1 Investigating Numerical Structure","offset":9811,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"4.2 Parameterizing the Structure as a Helix","offset":11519,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"4.3 Fitting a Helix","offset":13030,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"4.4 Evaluating the Quality of the Helical Fit","offset":14421,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-1.html#text"},{"title":"4.5 Is the Helix the Full Picture?","offset":17891,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5 LLMs Use the Clock Algorithm to Compute Addition","offset":19350,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5.1 Introducing the Clock Algorithm","offset":19405,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5.2 Investigating Attention Heads","offset":21379,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5.3 Looking at MLPs","offset":24914,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5.4 Zooming in on Neurons","offset":27022,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5.4.1 Modeling Neuron Preactivations","offset":27523,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5.4.2 Understanding MLP Inputs","offset":29165,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"5.5 Limitations of Our Understanding","offset":30473,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-2.html#text"},{"title":"6 Conclusion","offset":32449,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Acknowledgments","offset":33632,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Appendix A Performance of all models on a + b = a+b=","offset":33967,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Appendix B Additional Results on the Structure of Numbers","offset":34829,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Appendix C Additional Helix Fitting Results","offset":36749,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"C.1 Helix Properties","offset":36797,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"C.2 Additional Causal Experiments for Helix Fits","offset":37971,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Appendix D Additional Clock algorithm evidence","offset":39734,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"D.1 Attention Heads","offset":40730,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"D.2 MLPs and Neurons","offset":42954,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Appendix E Why Use the Clock Algorithm at All?","offset":45154,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Appendix F Investigating Model Errors","offset":46598,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-3.html#text"},{"title":"Appendix G Tooling and Compute","offset":49710,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-4.html#text"},{"title":"Instructions for reporting errors","offset":50036,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/part-4.html#text"}],"text_code_points":51000,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/source.md"}