{"id":"arxiv:2502.19981","title":"The Lookahead Limitation: Why Multi-Operand Addition is Hard for LLMs","date":"2025-02-27","section":"Reasoning and the \"physics\" of language models","quality":-0.150939,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/","source_url":"https://arxiv.org/abs/2502.19981","text_available":true,"authors":["Tanja Baeumel","Josef van Genabith","Simon Ostermann"],"outgoing":[{"source":"arxiv:2502.19981","target":"arxiv:2301.05217","evidence":"id","id":"arxiv:2301.05217","title":"Progress measures for grokking via mechanistic interpretability","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f71e3b65a8b3f35beb171d17f6c65878e0914227a0042ccba59b88952554baef/"},{"source":"arxiv:2502.19981","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2502.19981","target":"arxiv:2410.21272","evidence":"id","id":"arxiv:2410.21272","title":"Arithmetic Without Algorithms: Language Models Solve Math With a Bag of Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/397b826ba289bd4f14f457562942cb092c38550a4c49451e95ffcc7b376c8cb0/"},{"source":"arxiv:2502.19981","target":"arxiv:2502.00873","evidence":"id","id":"arxiv:2502.00873","title":"Language Models Use Trigonometry to Do Addition","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7766b2640c6b5ab53a295e8d60920cb2419bfbbd812810d9b35d4b7c323922ed/"}],"incoming":[{"source":"arxiv:2508.02513","target":"arxiv:2502.19981","evidence":"title","id":"arxiv:2508.02513","title":"Modular Arithmetic: Language Models Solve Math Digit by Digit","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/83aede757707b76dc76fc15431a436e5dec05563a9f2f28ebcc650775c878bcc/"},{"source":"arxiv:2602.10416","target":"arxiv:2502.19981","evidence":"title","id":"arxiv:2602.10416","title":"AI-rithmetic","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a644c16d12f96071c95ea707eb5c5b3cc44be3dda2f9fe49a5f38735695fdad9/"}],"text_parts":[{"start":0,"end":15820,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html"},{"start":15820,"end":31626,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html"},{"start":31626,"end":38555,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html"}],"contents":[{"title":"The Lookahead Limitation: Why Multi-Operand Addition is Hard for LLMs","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"Abstract","offset":73,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"1 Introduction","offset":1049,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"2 LLMs Struggle with Multi-Operand Addition","offset":5794,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"2.1 Models and Data","offset":6001,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"Models.","offset":6026,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"Data.","offset":6629,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"2.2 LLM Accuracy on Addition Tasks","offset":7721,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"3 Probing LLMs on Digits in Two-Operand Addition Tasks","offset":8173,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"3.1 Methodology and Experiments","offset":9318,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"Data.","offset":9355,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"Probing Setup.","offset":9699,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"3.2 Results","offset":10586,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"4 The Carry Heuristic of LLMs","offset":12570,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"4.1 Formalization of Left-to-Right Addition in Base 10","offset":12958,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"4.2 A Naive Heuristic for Solving Addition Left-to-Right","offset":14306,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"Examples.","offset":15525,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"147 + 293.","offset":15803,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-1.html#text"},{"title":"147 + 255.","offset":16403,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"5 H1 Predicts Difficulties of LLMs in Two-Operand Addition","offset":16916,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"5.1 Predicted Accuracy","offset":17108,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"5.2 Finegrained Analysis","offset":18104,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"Data.","offset":18246,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"Results.","offset":20150,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"6 H1 Predicts the Deterioration of Accuracy in Multi-Operand Addition","offset":21396,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"6.1 Multi-Operand Performance Predicted by H1","offset":22064,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"6.2 Empricial Evidence on Multi-Operand Addition","offset":24416,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"7 Multi-Digit Tokenization Models Employ the Same Heuristic","offset":25471,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"Experimental Setup.","offset":26126,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"Carry Scenarios.","offset":26857,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"Expected Outcomes.","offset":27628,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"Results.","offset":28104,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"8 Related Work","offset":28419,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"9 Conclusion","offset":30929,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-2.html#text"},{"title":"Limitations","offset":31846,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Acknowledgements","offset":33318,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Appendix A Example Addition According to Formalization","offset":33542,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Appendix B Generation Accuracies for 2-Operand, 3-Digit Addition","offset":34463,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Appendix C Example: H1 Failure on 4-Operand Addition","offset":34997,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Appendix D Zero-shot Generation Accuracy","offset":35760,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Appendix E Accuracy Prediction of Heuristic","offset":36073,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Appendix F Generation Accuracy on All Datasets","offset":36551,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Appendix G Probing Accuracy on Carry Scenarios","offset":36617,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Results.","offset":36801,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"},{"title":"Instructions for reporting errors","offset":37591,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/part-3.html#text"}],"text_code_points":38555,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc024e63f0774183f41cf84f17bb5a4238ff43a8ec18a6f8afb9a1abcd113488/source.md"}