{"id":"arxiv:2404.09173","title":"TransformerFAM: Feedback attention is working memory","date":"2024-04-14","section":"LLMs: architectures, context, training","quality":-0.082563,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/","source_url":"https://arxiv.org/abs/2404.09173","text_available":true,"authors":["Dongseong Hwang","Weiran Wang","Zhuoyuan Huo","Khe Chai Sim"],"outgoing":[{"source":"arxiv:2404.09173","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2404.09173","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/"},{"source":"arxiv:2404.09173","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2203.08913","title":"Memorizing Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/"},{"source":"arxiv:2404.09173","target":"arxiv:2304.01433","evidence":"title","id":"arxiv:2304.01433","title":"TPU v4: An Optically Reconfigurable Supercomputer for Machine Learning with Hardware Support for Embeddings","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/eb01c6279da01afe0e0bb6c4035117303abd37996d3aa65326f44cdccaa5a6fc/"},{"source":"arxiv:2404.09173","target":"arxiv:2304.08467","evidence":"title","id":"arxiv:2304.08467","title":"Learning to Compress Prompts with Gist Tokens","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/"},{"source":"arxiv:2404.09173","target":"arxiv:2404.07143","evidence":"title","id":"arxiv:2404.07143","title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/"},{"source":"arxiv:2404.09173","target":"doi:10.1073/pnas.1611835114","evidence":"title","id":"doi:10.1073/pnas.1611835114","title":"Overcoming catastrophic forgetting in neural networks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/849082368a44d06b93ef3ba95bcfddc6672b7a5e8064ad2b2d19601639266e8b/"}],"incoming":[{"source":"arxiv:2404.07143","target":"arxiv:2404.09173","evidence":"id","id":"arxiv:2404.07143","title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/"}],"text_parts":[{"start":0,"end":15888,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html"},{"start":15888,"end":31659,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html"},{"start":31659,"end":47584,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html"},{"start":47584,"end":58342,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html"}],"contents":[{"title":"TransformerFAM: Feedback attention is working memory","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html#text"},{"title":"Abstract","offset":56,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html#text"},{"title":"1 Introduction","offset":902,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html#text"},{"title":"Assumption 1 .","offset":5102,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html#text"},{"title":"2 TransformerFAM","offset":7257,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html#text"},{"title":"2.1 Block Sliding Window Attention (BSWA)","offset":7278,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html#text"},{"title":"2.2 Feedback Attention Memory","offset":14085,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-1.html#text"},{"title":"3 Experiments","offset":17891,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"3.1 Training","offset":17909,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"3.1.1 Data","offset":20903,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"3.2 PassKey Retrieval","offset":23182,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"3.3 Long Context Tasks","offset":25244,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"4 Related Work","offset":28095,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"5 Conclusion","offset":29287,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"Appendix A Architecture details","offset":30953,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"A.1 Flan-PaLM architecture","offset":30989,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"A.2 FAM hyperparameters","offset":31252,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"Appendix B Additional details of TransformerFAM","offset":31369,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"B.1 FAM initialization","offset":31494,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-2.html#text"},{"title":"B.2 Input Length Extrapolation","offset":33095,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.2.1 FAM Position Encoding","offset":33131,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.2.2 Random Position Offset","offset":33969,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.2.3 Random State Passing","offset":35979,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.3 FAM illustrated","offset":37176,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4 Ablation Studies","offset":37693,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.1 FAM length","offset":37771,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.2 The number of previous FAM blocks","offset":38286,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.3 FAM Position Encoding","offset":38852,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.4 Random Position Offset","offset":39456,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.5 Random State Passing","offset":41375,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.6 Prompt vs. Prefix","offset":42016,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.7 TransformerXL vs. TransformerBSWA","offset":42302,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"B.4.8 Comparison with other methods","offset":42760,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"Appendix C Don’t","offset":44334,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"C.1 Beyond Block-Recurrent Transformers","offset":44608,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"C.2 Feedback Memory Segment","offset":46822,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-3.html#text"},{"title":"C.3 Static Summary Tokens","offset":48518,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"C.4 Diversity Loss","offset":49916,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"C.5 Reconstruction Loss","offset":51619,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"Appendix D Experiments details","offset":52298,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"D.1 PassKey Retrieval","offset":52434,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"D.2 Long Context Tasks","offset":52892,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"D.3 GPT-3 Tasks","offset":53281,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"D.4 Complexity","offset":53408,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"Appendix E Related Work","offset":53720,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"Appendix F Attention Visualization","offset":55338,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"Appendix G Limitations","offset":55790,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"Appendix H Broader Impacts","offset":56433,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"},{"title":"Instructions for reporting errors","offset":57378,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/part-4.html#text"}],"text_code_points":58342,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/source.md"}