{"id":"arxiv:2203.08913","title":"Memorizing Transformers","date":"2022-03-16","section":"LLMs: architectures, context, training","quality":0.238047,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/","source_url":"https://arxiv.org/abs/2203.08913","text_available":true,"authors":["Yuhuai Wu","Markus N. Rabe","DeLesley Hutchins","Christian Szegedy"],"outgoing":[],"incoming":[{"source":"arxiv:2304.08467","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2304.08467","title":"Learning to Compress Prompts with Gist Tokens","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/"},{"source":"arxiv:2305.01625","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2305.01625","title":"Unlimiformer: Long-Range Transformers with Unlimited Length Input","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/618496c3ed89cfd0171d97f92a102cd2a37c2cdb1c00a197683ee69421fe7279/"},{"source":"arxiv:2404.07143","target":"arxiv:2203.08913","evidence":"id","id":"arxiv:2404.07143","title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/"},{"source":"arxiv:2404.09173","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2404.09173","title":"TransformerFAM: Feedback attention is working memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/"},{"source":"arxiv:2502.05171","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"},{"source":"arxiv:2607.02303","target":"arxiv:2203.08913","evidence":"id","id":"arxiv:2607.02303","title":"A Hippocampus for Linear Attention: An Exact Memory for What the Recurrent State Forgets","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/499d4b7d5f795ec273c099c3ee7ba3ba4b6efb60063810c5afc4d10082a3168d/"}],"text_parts":[{"start":0,"end":15980,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html"},{"start":15980,"end":31869,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html"},{"start":31869,"end":41080,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html"}],"contents":[{"title":"Memorizing Transformers","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"Abstract","offset":27,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"1 Introduction","offset":982,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"2 Related Work","offset":4859,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"3 Method","offset":9652,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"3.1 k k NN-augmented Attention Layer","offset":10909,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"3.2 Distributional Shift","offset":14203,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"3.3 Approximate k k NN","offset":15426,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"4 Experiments","offset":15962,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-1.html#text"},{"title":"4.1 Datasets","offset":16627,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"arXiv Math","offset":16645,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Github","offset":17136,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Formal Math – Isabelle","offset":18204,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"C4(4K + + )","offset":19071,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"PG-19","offset":19354,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"4.2 Experimental Method","offset":19695,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"4.3 Effect of external memory","offset":21698,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Adding external memory results in substantial gains across datasets and architectures,","offset":21733,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Increasing the size of the memory increases the benefit of the memory.","offset":22218,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"4.4 Scaling to larger models","offset":23889,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"4.5 Finetuning on Larger Memories","offset":24759,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Finetuning on a larger memory.","offset":24798,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Finetuning a non-memory model to use memory","offset":25492,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"4.6 Information Retrieval Patterns","offset":26191,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Which tokens show a benefit from memory?","offset":26999,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"What information is being looked up?","offset":28421,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"5 Conclusion","offset":30442,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Acknowledgments","offset":31847,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-2.html#text"},{"title":"Ethics","offset":32379,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Reproducibility","offset":32911,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Appendix A Length of inputs","offset":33354,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"A.1 Ablation studies","offset":33386,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Multiple k k NN layers.","offset":33679,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"k k NN layer index","offset":33868,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Number of neighbors","offset":34281,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Random seeds","offset":34593,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Appendix B What does the model retrieve from memory?","offset":35154,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Retrieving citation names","offset":35211,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Retrieving function names from the codebase","offset":36149,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"B.1 More Retrieving examples in formal theorem proving corpus","offset":36698,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Example 1","offset":36765,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Example 2","offset":37213,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Example 3","offset":37682,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Example 4","offset":38136,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Example 5","offset":38596,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Example 6","offset":39072,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"},{"title":"Instructions for reporting errors","offset":40116,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/part-3.html#text"}],"text_code_points":41080,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/source.md"}