{"id":"arxiv:1807.03819","title":"Universal Transformers","date":"2018-07-10","section":"LLMs: architectures, context, training","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/","source_url":"https://arxiv.org/abs/1807.03819","text_available":true,"authors":["Mostafa Dehghani","Stephan Gouws","Oriol Vinyals","Jakob Uszkoreit"],"outgoing":[],"incoming":[{"source":"arxiv:2001.08361","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2006.16236","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/"},{"source":"arxiv:2405.15071","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2405.15071","title":"Grokked Transformers are Implicit Reasoners: A Mechanistic Journey to the Edge of Generalization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1cd7fa83eca0b480c3bf9e2490c043b70e937777252a40b196ae7801a0620141/"},{"source":"arxiv:2502.05171","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"},{"source":"arxiv:2506.21734","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2506.21734","title":"Hierarchical Reasoning Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3604312101ff09ab9903e7d6a32c7b2b8a9a7ece6a9d145d04ea01c9d13ffa3a/"},{"source":"arxiv:2507.10524","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2507.10524","title":"Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/"},{"source":"arxiv:2509.25239","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2509.25239","title":"A Formal Comparison Between Chain of Thought and Latent Thought","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8399816be05fff6a7002f5c3439ec43ffe7b468424759b4cc70f5efc7739ea9e/"},{"source":"arxiv:2512.14693","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2512.14693","title":"Universal Reasoning Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcb1a389a132d64ca93fac5551d8f8b653c5e093908cb515e2a2d901524941a7/"},{"source":"arxiv:2604.09168","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2604.09168","title":"ELT: Elastic Looped Transformers for Visual Generation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/63d4e5f1a267b96ed4474fa233f7d3bb2171aae949c0139db37b58b4e45eca47/"},{"source":"arxiv:2604.11791","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2604.11791","title":"A Mechanistic Analysis of Looped Reasoning Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae8c682d5708c6c47d65c4eb540dd5bccb877c5e332852765507128d6d1b985d/"},{"source":"arxiv:2604.17121","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2604.17121","title":"The Topological Trouble With Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f9d762bbe192e1208f6d7dc289b7e79c23d4bdde0b1b3999ff5fda55b0dcd1a/"},{"source":"arxiv:2604.21999","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2604.21999","title":"Universal Transformers Need Memory: Depth-State Trade-offs in Adaptive Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/963f6bb396c25bff462a25854c82e7d92b77803fa60c891fadd9963cc56c8fea/"},{"source":"arxiv:2605.12466","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2605.12466","title":"Solve the Loop: Attractor Models for Language and Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/eb46f81cd5718eb6b9256c583cf9dd2e97b793b4d14337e487c16285f96e2b9a/"},{"source":"arxiv:2605.19376","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2605.19376","title":"Generative Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/795bad9bb1b1a063090fbb925ba6e68bf30211de0ecffd75d56f858b28321ae7/"},{"source":"arxiv:2605.21488","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2605.21488","title":"Equilibrium Reasoners: Learning Attractors Enables Scalable Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48b3beca0f016a3683df4c6c71fc4be9f13a2c12c61d4ad17d1117c6139bb11e/"},{"source":"arxiv:2606.06574","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2606.06574","title":"Skip a Layer or Loop It? Learning Program-of-Layers in LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5abe08b6c82ce9a4010a09d82dbee40f8b4b0f88539bf8911ae36048a97bb917/"},{"source":"arxiv:2606.18206","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2606.18206","title":"Fixed-Point Reasoners: Stable and Adaptive Deep Looped Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/37ea2d83a92ece4ab3ba4434af71f291d1fdc5040676f75110210f40aad3b81a/"},{"source":"arxiv:2606.31779","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2606.31779","title":"Bridging the Gap Between Latent and Explicit Reasoning with Looped Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bf07d5c19e507d9ec91a48f9067e6c29fbe8965d001923e2de0e96da143c72e1/"},{"source":"arxiv:2608.17981","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2608.17981","title":"Recirculation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd91ffc957070136795d3728f662c997738234065b624eb720acd8547d2edaf6/"},{"source":"arxiv:2609.01117","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:2609.01117","title":"Latent Recurrent Thoughts: Recurrent Refinement of Proposed Latents for Reasoning with Frozen LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e9d4f61afc3930fa770d5ea53d3f43a5157fa4904fcdfaa514786e617905e20b/"},{"source":"arxiv:2609.33540","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:2609.33540","title":"Reasoning on the Simplex: Geometric Fixed-Point Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8882e1cc48049a4b6b9a911701665da57fcc16f9e52f2b403b1a5b334653d9c/"}],"text_parts":[{"start":0,"end":15711,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html"},{"start":15711,"end":31586,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html"},{"start":31586,"end":40513,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html"}],"contents":[{"title":"Universal Transformers","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"Abstract","offset":26,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"1 Introduction","offset":1806,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"2 Model Description","offset":5843,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"2.1 The Universal Transformer","offset":5867,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"2.2 Dynamic Halting","offset":13371,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"3 Experiments and Analysis","offset":14436,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"3.1 bAbI Question-Answering","offset":14670,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-1.html#text"},{"title":"3.2 Subject-Verb Agreement","offset":18445,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"3.3 LAMBADA Language Modeling","offset":20110,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"3.4 Algorithmic Tasks","offset":23170,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"3.5 Learning to Execute (LTE)","offset":23948,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"3.6 Machine Translation","offset":24890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"4 Discussion","offset":25428,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"5 Conclusion","offset":29748,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"Acknowledgements","offset":31330,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"Appendix A Detailed Schema of the Universal Transformer","offset":31463,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"Appendix B On the Computational Power of UT vs Transformer","offset":31523,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-2.html#text"},{"title":"Appendix C UT with Dynamic Halting","offset":33012,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"Appendix D Description of some of the Tasks/Datasets","offset":33792,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"D.1 bAbI Question-Answering","offset":33992,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"D.2 Subject-Verb Agreement","offset":35057,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"D.3 LAMBADA Language Modeling","offset":36534,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"D.4 Learning to Execute (LTE)","offset":38113,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"Appendix E bAbI Detailed Results","offset":39118,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"Appendix F bAbI Attention Visualization","offset":39155,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"},{"title":"Instructions for reporting errors","offset":39549,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/part-3.html#text"}],"text_code_points":40513,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/source.md"}