{"id":"arxiv:2508.02513","title":"Modular Arithmetic: Language Models Solve Math Digit by Digit","date":"2025-08-04","section":"Reasoning and the \"physics\" of language models","quality":0.091161,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/","source_url":"https://arxiv.org/abs/2508.02513","text_available":true,"authors":["Tanja Baeumel","Daniil Gurgurov","Yusser al Ghussin","Josef van Genabith"],"outgoing":[{"source":"arxiv:2508.02513","target":"arxiv:2301.05217","evidence":"id","id":"arxiv:2301.05217","title":"Progress measures for grokking via mechanistic interpretability","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f71e3b65a8b3f35beb171d17f6c65878e0914227a0042ccba59b88952554baef/"},{"source":"arxiv:2508.02513","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2508.02513","target":"arxiv:2410.21272","evidence":"id","id":"arxiv:2410.21272","title":"Arithmetic Without Algorithms: Language Models Solve Math With a Bag of Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/397b826ba289bd4f14f457562942cb092c38550a4c49451e95ffcc7b376c8cb0/"},{"source":"arxiv:2508.02513","target":"arxiv:2501.00656","evidence":"id","id":"arxiv:2501.00656","title":"2 OLMo 2 Furious","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/"},{"source":"arxiv:2508.02513","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2502.00873","title":"Language Models Use Trigonometry to Do Addition","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/"},{"source":"arxiv:2508.02513","target":"arxiv:2502.19981","evidence":"title","id":"arxiv:2502.19981","title":"The Lookahead Limitation: Why Multi-Operand Addition is Hard for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/"},{"source":"arxiv:2508.02513","target":"url:transformer-circuits.pub/2025/attribution-graphs/biology.html","evidence":"url","id":"url:transformer-circuits.pub/2025/attribution-graphs/biology.html","title":"Attribution graphs, the biology of the model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8b3e14c537e89eed9a4402e9c14094d69913ed24d45fdab1a82ba4b8280660e2/"}],"incoming":[{"source":"arxiv:2602.10416","target":"arxiv:2508.02513","evidence":"id","id":"arxiv:2602.10416","title":"AI-rithmetic","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a644c16d12f96071c95ea707eb5c5b3cc44be3dda2f9fe49a5f38735695fdad9/"}],"text_parts":[{"start":0,"end":15964,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html"},{"start":15964,"end":31810,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html"},{"start":31810,"end":44622,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html"}],"contents":[{"title":"Modular Arithmetic: Language Models Solve Math Digit by Digit","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"Abstract","offset":65,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"1 Introduction","offset":1183,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"2 Method","offset":4523,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"2.1 Overview of our Approach","offset":4536,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"2.2 Data for Neuron Identification Experiments","offset":5985,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"2.3 Data for Intervention Experiments","offset":6849,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"2.4 Models","offset":8140,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"3 Circuit Localization","offset":9348,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"3.1 Step 1: Identifying Digit-Specific Neurons","offset":9616,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"Intuition.","offset":9668,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"Neuron Selection for Digit-Position Circuits.","offset":11967,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"Statistics on Selected Circuits.","offset":13034,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"3.2 Step 2: Causal Verification via Interventions","offset":14009,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"Circuit Depth.","offset":14354,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"Experiment.","offset":15281,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-1.html#text"},{"title":"Results.","offset":16409,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"4 Supplementary Experiments","offset":18932,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"4.1 Addition and Subtraction Circuits are Largely Distinct","offset":19087,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"4.2 Additions with Carry Bits Employ the Same Mechanisms","offset":19658,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"(1) Carry from Units to Tens.","offset":20416,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"(2) Carry from Tens to Hundreds.","offset":20999,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"Results.","offset":21601,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"4.3 MLP Similarity Reflects Task Overlap","offset":22191,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"5 Structured Arithmetic Processing or a Bag of Heuristics?","offset":23085,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"6 Related Work","offset":25902,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"7 Conclusion","offset":28474,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"Limitations","offset":29184,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"Acknowledgements","offset":30262,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"Appendix A Implementation Details","offset":30478,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"Appendix B Results on Single Digit Tokenization Models","offset":30822,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"Intuition: Single- vs Multi-digit Numeric Tokenization.","offset":31038,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-2.html#text"},{"title":"Localizing Digit-Specific Neurons.","offset":32448,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Intervention.","offset":33652,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix C Layers Involved in Arithmetic","offset":34053,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix D Fisher Score for Digit-Sensitivity of Neurons.","offset":35784,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix E Statistics on Arithmetic Circuits","offset":38450,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix F Effect of chosen Threshold on Digit-Circuit Intervention","offset":40755,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix G Intervention Results on D a ​ d ​ d , o ​ p ​ 1 D_{add,op1} and D s ​ u ​ b , o ​ p ​ 1 D_{sub,op1}","offset":40827,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix H Ablation: Deeper Circuits","offset":42411,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix I Similarity of Addition and Subtraction Circuits","offset":42452,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Appendix J Examples of Neuron Heuristics","offset":43613,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"},{"title":"Instructions for reporting errors","offset":43658,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/part-3.html#text"}],"text_code_points":44622,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/source.md"}