{"id":"arxiv:2504.01928","title":"Is the Reversal Curse a Binding Problem? Uncovering Limitations of Transformers from a Basic Generalization Failure","date":"2025-04-02","section":"Reasoning and the \"physics\" of language models","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/","source_url":"https://arxiv.org/abs/2504.01928","text_available":true,"authors":["Boshi Wang","Huan Sun"],"outgoing":[{"source":"arxiv:2504.01928","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2301.08243","title":"Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/"},{"source":"arxiv:2504.01928","target":"arxiv:2309.12288","evidence":"title","id":"arxiv:2309.12288","title":"The Reversal Curse: LLMs trained on \"A is B\" fail to learn \"B is A\"","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02782a7d9697d5df8f8ae84424becf682d10e8671ee5e18fd75499de0e0a6694/"},{"source":"arxiv:2504.01928","target":"arxiv:2309.14402","evidence":"title","id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/"},{"source":"arxiv:2504.01928","target":"arxiv:2407.20311","evidence":"id","id":"arxiv:2407.20311","title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/"},{"source":"arxiv:2504.01928","target":"openreview:BZ5a1r-kVsf","evidence":"url","id":"openreview:BZ5a1r-kVsf","title":"A Path Towards Autonomous Machine Intelligence (LeCun, 2022)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b7a7789ddc6c70bf2f3e56e6df28529a3eefac2506f6104eb6b2460f5571d657/"},{"source":"arxiv:2504.01928","target":"url:ai.meta.com/research/publications/large-concept-models-language-modeling-in-a-sentence-representation-space","evidence":"id","id":"url:ai.meta.com/research/publications/large-concept-models-language-modeling-in-a-sentence-representation-space","title":"Large Concept Models: Language Modeling in a Sentence Representation Space","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5cd322d52f1966361afb1d5225560e2d3ce4d22a117bd14cfa27ec669e09673/"}],"incoming":[],"text_parts":[{"start":0,"end":15520,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-1.html"},{"start":15520,"end":30793,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-2.html"},{"start":30793,"end":46617,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html"},{"start":46617,"end":49193,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-4.html"}],"contents":[{"title":"Is the Reversal Curse a Binding Problem? Uncovering Limitations of Transformers from a Basic Generalization Failure","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-1.html#text"},{"title":"Abstract","offset":119,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-1.html#text"},{"title":"1 Introduction","offset":1472,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-1.html#text"},{"title":"2 Learning Reversal at the Concept Level","offset":7517,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-1.html#text"},{"title":"3 The Binding Problem Underlying Surface-level Predictions","offset":11129,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-1.html#text"},{"title":"3.1 Inconsistency of Concept Representations","offset":14001,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-1.html#text"},{"title":"3.2 Entanglements of Concept Representations","offset":16462,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-2.html#text"},{"title":"3.3 Experiments","offset":20566,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-2.html#text"},{"title":"4 Arithmetic Reasoning via Parametric Forward-Chaining","offset":28134,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-2.html#text"},{"title":"5 Related Work","offset":32459,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"6 Discussion & Conclusion","offset":34761,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Acknowledgement","offset":36374,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Ethics Statement","offset":36997,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Reproducibility Statement","offset":37467,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Appendix A Hyperparameters and Training Details","offset":38286,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Appendix B Additional Discussion & Results on JEPA","offset":39153,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"JEPA Framework.","offset":39208,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Experimental Stability.","offset":40809,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Appendix C Arithmetic Reasoning","offset":41321,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"C.1 Learning Basic Arithmetic Deductions","offset":41357,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"C.2 Scaling the Reasoning Challenge","offset":43916,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-3.html#text"},{"title":"Appendix D Use of Large Language Models","offset":47788,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-4.html#text"},{"title":"Instructions for reporting errors","offset":48229,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/part-4.html#text"}],"text_code_points":49193,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/source.md"}