{"id":"arxiv:2405.12250","title":"Your Transformer is Secretly Linear","date":"2024-05-19","section":"LLMs: architectures, context, training","quality":-0.404708,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/","source_url":"https://arxiv.org/abs/2405.12250","text_available":true,"authors":["Anton Razzhigaev","Matvey Mikhalchuk","Elizaveta Goncharova","Nikolai Gerasimenko"],"outgoing":[{"source":"arxiv:2405.12250","target":"arxiv:2402.02342","evidence":"title","id":"arxiv:2402.02342","title":"MetaOptimize: A Framework for Optimizing Step Sizes and Other Meta-parameters","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/"},{"source":"arxiv:2405.12250","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[],"text_parts":[{"start":0,"end":15824,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html"},{"start":15824,"end":20961,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-2.html"}],"contents":[{"title":"Your Transformer is Secretly Linear","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"Abstract","offset":39,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"1 Introduction","offset":1169,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"2 Related Work","offset":3445,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"Structure-based pruning","offset":6679,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"3 Analysis of Pretrained Architectures","offset":8529,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"3.1 Linearity Score","offset":8768,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"3.2 Linearity Dynamics at Pretraining and Fine-tuning","offset":11856,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"4 Improving Linearity with Regularized Pretraining","offset":13737,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-1.html#text"},{"title":"5 Exploiting Linearity for Pruning","offset":16853,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-2.html#text"},{"title":"6 Conclusion","offset":17829,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-2.html#text"},{"title":"7 Limitations","offset":18802,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-2.html#text"},{"title":"8 Ethics Statement","offset":19559,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-2.html#text"},{"title":"Appendix A Error Distribution by Layers","offset":19821,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-2.html#text"},{"title":"Instructions for reporting errors","offset":19997,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/part-2.html#text"}],"text_code_points":20961,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/source.md"}