{"id":"arxiv:2404.07143","title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","date":"2024-04-10","section":"LLMs: architectures, context, training","quality":-0.055248,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/","source_url":"https://arxiv.org/abs/2404.07143","text_available":true,"authors":["Tsendsuren Munkhdalai","Manaal Faruqui","Siddharth Gopal"],"outgoing":[{"source":"arxiv:2404.07143","target":"acl:2022.acl-long.360","evidence":"id","id":"acl:2022.acl-long.360","title":"PRIMERA: Pyramid-based Masked Sentence Pre-training for Multi-document Summarization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7ea72ee4d97ca9c2839284c71ef41df5bb2db7ca0ab28e219c275e13139eaace/"},{"source":"arxiv:2404.07143","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/"},{"source":"arxiv:2404.07143","target":"arxiv:2203.08913","evidence":"id","id":"arxiv:2203.08913","title":"Memorizing Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/"},{"source":"arxiv:2404.07143","target":"arxiv:2304.08467","evidence":"title","id":"arxiv:2304.08467","title":"Learning to Compress Prompts with Gist Tokens","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/"},{"source":"arxiv:2404.07143","target":"arxiv:2305.01625","evidence":"title","id":"arxiv:2305.01625","title":"Unlimiformer: Long-Range Transformers with Unlimited Length Input","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/"},{"source":"arxiv:2404.07143","target":"arxiv:2404.09173","evidence":"id","id":"arxiv:2404.09173","title":"TransformerFAM: Feedback attention is working memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/"}],"incoming":[{"source":"arxiv:2404.09173","target":"arxiv:2404.07143","evidence":"title","id":"arxiv:2404.09173","title":"TransformerFAM: Feedback attention is working memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/"},{"source":"arxiv:2512.24601","target":"arxiv:2404.07143","evidence":"id","id":"arxiv:2512.24601","title":"Recursive Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ac41983c934a3432ce7b6531e367b9e639bddbed93d4c6513781e76407d9a8bd/"},{"source":"arxiv:2604.21999","target":"arxiv:2404.07143","evidence":"id","id":"arxiv:2604.21999","title":"Universal Transformers Need Memory: Depth-State Trade-offs in Adaptive Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/963f6bb396c25bff462a25854c82e7d92b77803fa60c891fadd9963cc56c8fea/"}],"text_parts":[{"start":0,"end":15277,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html"},{"start":15277,"end":31220,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html"},{"start":31220,"end":31904,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-3.html"}],"contents":[{"title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"Abstract","offset":90,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"1 Introduction","offset":883,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"2 Background","offset":4931,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"3 Method","offset":7333,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"3.1 Infini-attention","offset":8332,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"3.1.1 Scaled Dot-product Attention","offset":8976,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"3.1.2 Compressive Memory","offset":10630,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"3.2 Memory and Effective Context Window","offset":15232,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-1.html#text"},{"title":"4 Experiments","offset":18141,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"4.1 Implementation details","offset":18626,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"4.2 Long-context Language Modeling","offset":19690,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"4.3 LLM Continual Pre-training","offset":22066,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"5 Related Work","offset":24746,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"6 Conclusion","offset":28706,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"Acknowledgments","offset":29587,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"Appendix A Additional Training Details","offset":29856,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"Appendix B Passkey Retrieval Task","offset":30388,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"},{"title":"Instructions for reporting errors","offset":30940,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/part-2.html#text"}],"text_code_points":31904,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/source.md"}