{"id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","date":"2020-06-29","section":"LLMs: architectures, context, training","quality":0.112975,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/","source_url":"https://arxiv.org/abs/2006.16236","text_available":true,"authors":["Angelos Katharopoulos","Apoorv Vyas","Nikolaos Pappas","François Fleuret"],"outgoing":[{"source":"arxiv:2006.16236","target":"arxiv:1807.03819","evidence":"id","id":"arxiv:1807.03819","title":"Universal Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/"}],"incoming":[{"source":"arxiv:2305.01625","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2305.01625","title":"Unlimiformer: Long-Range Transformers with Unlimited Length Input","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/"},{"source":"arxiv:2312.00276","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2312.00276","title":"Metalearning Continual Learning Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d7361f404c0455d9f759fc6a8dd7bd1da75146e72717b7a2fbcdb8bac80412a/"},{"source":"arxiv:2312.04927","target":"arxiv:2006.16236","evidence":"id","id":"arxiv:2312.04927","title":"Zoology: Measuring and Improving Recall in Efficient Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/baa792b06bdd30f91619f1e852aa5ae4dbf688fe3b1f2ee0722b339b9993ac5c/"},{"source":"arxiv:2404.07143","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2404.07143","title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/"},{"source":"arxiv:2404.09173","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2404.09173","title":"TransformerFAM: Feedback attention is working memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/"},{"source":"arxiv:2405.04517","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2405.04517","title":"xLSTM: Extended Long Short-Term Memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/"},{"source":"arxiv:2502.05171","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"},{"source":"arxiv:2506.13585","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2506.13585","title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5beb2ec6c7e6362f9d957219d7a205dd1adcc53f739acc0c7fd5f1f2bded4d83/"},{"source":"arxiv:2507.18074","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2507.18074","title":"AlphaGo Moment for Model Architecture Discovery","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1c22da54910a26f1f81b276f5cae3a81e22f7db024f7d1a5fa89ccd4f291ffa8/"},{"source":"arxiv:2512.24695","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/"},{"source":"arxiv:2604.17121","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2604.17121","title":"The Topological Trouble With Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f9d762bbe192e1208f6d7dc289b7e79c23d4bdde0b1b3999ff5fda55b0dcd1a/"},{"source":"arxiv:2607.02303","target":"arxiv:2006.16236","evidence":"id","id":"arxiv:2607.02303","title":"A Hippocampus for Linear Attention: An Exact Memory for What the Recurrent State Forgets","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/499d4b7d5f795ec273c099c3ee7ba3ba4b6efb60063810c5afc4d10082a3168d/"},{"source":"arxiv:2608.09888","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2608.09888","title":"BDH-CQ: In-Context Learning with Recurrent Latent Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0c77a8a0f50254e9efdf9a511e08fc2a40677bfa029864bdd0a9cbce561c666/"},{"source":"arxiv:2608.17981","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2608.17981","title":"Recirculation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd91ffc957070136795d3728f662c997738234065b624eb720acd8547d2edaf6/"},{"source":"url:people.idsia.ch/~juergen/most-cited-neural-nets.html","target":"arxiv:2006.16236","evidence":"title","id":"url:people.idsia.ch/~juergen/most-cited-neural-nets.html","title":"The most cited neural nets of the 20th century (Schmidhuber)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a07ae08185ac4ab008e5384d2dab1a6ce15cf327b9414fb3ccc3afa8e5ac914a/"}],"text_parts":[{"start":0,"end":15260,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html"},{"start":15260,"end":31189,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html"},{"start":31189,"end":45521,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html"}],"contents":[{"title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"Abstract","offset":81,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"1 Introduction","offset":948,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"2 Related Work","offset":3729,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"2.1 Efficient Transformers","offset":4147,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"2.2 Understanding Self-Attention","offset":7094,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"2.3 Linearized softmax","offset":8266,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"3 Linear Transformers","offset":9170,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"3.1 Transformers","offset":9827,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"3.2 Linearized Attention","offset":12393,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"3.2.1 Feature Maps and Computational Cost","offset":14716,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-1.html#text"},{"title":"3.3 Causal Masking","offset":16835,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"3.3.1 Gradient Computation","offset":18460,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"3.3.2 Training and Inference","offset":20677,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"3.4 Transformers are RNNs","offset":21932,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4 Experiments","offset":23947,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4.1 Synthetic Tasks","offset":25791,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4.1.1 Convergence Analysis","offset":25816,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4.1.2 Memory and Computational Requirements","offset":26690,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4.2 Image Generation","offset":27946,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4.2.1 MNIST","offset":28902,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4.2.2 CIFAR-10","offset":31168,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-2.html#text"},{"title":"4.3 Automatic Speech Recognition","offset":33082,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"5 Conclusions","offset":34771,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"Acknowledgements","offset":35786,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"Appendix A Gradient Derivation","offset":36165,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"Appendix B Training Evolution","offset":40371,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"Appendix C Image Generation Throughput Discussion","offset":41586,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"C.1 Stateful softmax attention","offset":41640,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"C.2 Equalizing the batch size","offset":42862,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"Appendix D Qualitative Results on Image Generation","offset":44004,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"},{"title":"Instructions for reporting errors","offset":44557,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/part-3.html#text"}],"text_code_points":45521,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/source.md"}