{"id":"arxiv:2304.08467","title":"Learning to Compress Prompts with Gist Tokens","date":"2023-04-17","section":"Retrieval, embeddings, benchmarks","quality":-0.070542,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/","source_url":"https://arxiv.org/abs/2304.08467","text_available":true,"authors":["Jesse Mu","Xiang Lisa Li","Noah Goodman"],"outgoing":[{"source":"arxiv:2304.08467","target":"acl:2022.emnlp-main.340","evidence":"title","id":"acl:2022.emnlp-main.340","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e43a06571b7e8d89abd5f20f8f520c8bc3d3eb32ffb156cb7fc9e0469fc14b0a/"},{"source":"arxiv:2304.08467","target":"arxiv:2203.08913","evidence":"title","id":"arxiv:2203.08913","title":"Memorizing Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/76deb2bc758a8a71e7c5c5ad6f4b45a1527845ec73784adc6f75f77462a596aa/"}],"incoming":[{"source":"arxiv:2404.07143","target":"arxiv:2304.08467","evidence":"title","id":"arxiv:2404.07143","title":"Leave No Context Behind: Efficient Infinite Context Transformers with Infini-attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcdbb3ceff27039e0eeeeac187ebf2570e2dc8b6f58ed5ee278f4259e88cfab1/"},{"source":"arxiv:2404.09173","target":"arxiv:2304.08467","evidence":"title","id":"arxiv:2404.09173","title":"TransformerFAM: Feedback attention is working memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cbd04248dee4a37ffe8957d53e0b260196e25a477e8aa86db09efcd9fee74326/"},{"source":"arxiv:2510.27688","target":"arxiv:2304.08467","evidence":"title","id":"arxiv:2510.27688","title":"Continuous Autoregressive Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/"}],"text_parts":[{"start":0,"end":15759,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html"},{"start":15759,"end":31625,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html"},{"start":31625,"end":40309,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html"}],"contents":[{"title":"Learning to Compress Prompts with Gist Tokens","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"Abstract","offset":49,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"1 Introduction","offset":1022,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"2 Gisting","offset":3592,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"2.1 A Context Distillation Perspective","offset":6656,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"3 Learning Gisting by Masking","offset":9243,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"4 Experiments","offset":11659,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"4.1 Data","offset":11677,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"4.2 Models","offset":13360,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"Positive Control.","offset":13962,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"Negative Control.","offset":14172,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"Discrete Compression with TF-IDF.","offset":14444,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"4.3 Evaluation","offset":15335,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"ROUGE-L.","offset":15450,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"ChatGPT.","offset":15744,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-1.html#text"},{"title":"Human eval.","offset":17036,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"5 Results","offset":17962,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"5.1 Failure cases","offset":21541,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"6 Compute, Memory, and Storage Efficiency","offset":22270,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"7 Additional Related Work","offset":25774,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Adapting LMs without Backprop.","offset":26003,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Compression and memory in transformers.","offset":26809,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Sparse attention mechanisms.","offset":27790,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"8 Discussion and Limitations","offset":28437,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Acknowledgments and Disclosure of Funding","offset":30753,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Appendix A Example PyTorch Implementation of Gist Masking","offset":31198,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Appendix B Data, Training, Evaluation, and Compute Details","offset":31462,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Data.","offset":31613,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-2.html#text"},{"title":"Training.","offset":32323,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Evaluation.","offset":33367,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Compute.","offset":33540,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Appendix C Details of ChatGPT Evaluation","offset":33923,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Appendix D Additional Human Evaluation Details and Results","offset":35324,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"D.1 Experimental Details","offset":35387,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"D.2 Additional Results","offset":36065,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Appendix E Exact Match Results","offset":36358,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Appendix F Additional FLOPs details","offset":36519,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Embeddings","offset":36960,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Attention (Single Layer)","offset":37036,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Dense Block","offset":37591,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Final Logits","offset":37696,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Total Forward Pass FLOPs","offset":37774,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Appendix G Additional Examples of Model Outputs","offset":38975,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"},{"title":"Instructions for reporting errors","offset":39345,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/part-3.html#text"}],"text_code_points":40309,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/source.md"}