{"id":"arxiv:2305.01625","title":"Unlimiformer: Long-Range Transformers with Unlimited Length Input","date":"2023-05-02","section":"LLMs: architectures, context, training","quality":0.178118,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/","source_url":"https://arxiv.org/abs/2305.01625","text_available":true,"authors":["Amanda Bertsch","Uri Alon","Graham Neubig","Matthew R. Gormley"],"outgoing":[{"source":"arxiv:2305.01625","target":"acl:2022.acl-long.360","evidence":"title","id":"acl:2022.acl-long.360","title":"PRIMERA: Pyramid-based Masked Sentence Pre-training for Multi-document Summarization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7ea72ee4d97ca9c2839284c71ef41df5bb2db7ca0ab28e219c275e13139eaace/"},{"source":"arxiv:2305.01625","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/"},{"source":"arxiv:2305.01625","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2203.08913","title":"Memorizing Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/"}],"incoming":[{"source":"arxiv:2404.07143","target":"arxiv:2305.01625","evidence":"title","id":"arxiv:2404.07143","title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/"}],"text_parts":[{"start":0,"end":15866,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html"},{"start":15866,"end":31839,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html"},{"start":31839,"end":47839,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html"},{"start":47839,"end":60487,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-4.html"}],"contents":[{"title":"Unlimiformer: Long-Range Transformers with Unlimited Length Input","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"Abstract","offset":69,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"1 Introduction","offset":1406,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"2 Unlimiformer","offset":4797,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"2.1 Encoding","offset":5213,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"2.2 Retrieval-augmented Cross-Attention","offset":5734,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"2.3 Attention reformulation","offset":6935,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"3 Training Unlimiformer","offset":11317,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"3.1 Low (additional-) Cost Training Methods: Applying Unlimiformer at validation or test-time only","offset":11720,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"3.2 Long-range Training Methods: Applying Unlimiformer at training time","offset":13111,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"4 Experimental Settings","offset":14133,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"4.1 Datasets","offset":14161,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"Metrics","offset":15043,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"4.2 Baselines","offset":15431,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-1.html#text"},{"title":"5 Results","offset":17542,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"5.1 Long Document Summarization","offset":17556,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"Low-cost training","offset":17593,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"Long-range training","offset":18587,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"5.2 Book Summarization","offset":19502,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"6 Analysis","offset":21035,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"Is the long input really needed?","offset":21050,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"7 Related Work","offset":25328,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"8 Conclusions","offset":28246,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"9 Limitations","offset":29898,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"Acknowledgments","offset":31063,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"Appendix A Training details","offset":31807,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-2.html#text"},{"title":"Appendix B WikiSum scraping","offset":32364,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html#text"},{"title":"Appendix C Evaluation details","offset":33151,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html#text"},{"title":"Appendix D Computational Cost","offset":34126,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html#text"},{"title":"GPU-time","offset":34408,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html#text"},{"title":"Memory usage","offset":35352,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html#text"},{"title":"Appendix E Validation Results","offset":36243,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html#text"},{"title":"Appendix F Sample Outputs","offset":36345,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-3.html#text"},{"title":"Appendix G Additional Related Work","offset":56831,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-4.html#text"},{"title":"Instructions for reporting errors","offset":59523,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/part-4.html#text"}],"text_code_points":60487,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/source.md"}