{"id":"arxiv:2405.04517","title":"xLSTM: Extended Long Short-Term Memory","date":"2024-05-07","section":"LLMs: architectures, context, training","quality":-0.312713,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/","source_url":"https://arxiv.org/abs/2405.04517","text_available":true,"authors":["Maximilian Beck","Korbinian Pöppel","Markus Spanring","Andreas Auer"],"outgoing":[{"source":"arxiv:2405.04517","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2405.04517","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/"},{"source":"arxiv:2405.04517","target":"arxiv:2207.02098","evidence":"title","id":"arxiv:2207.02098","title":"Neural Networks and the Chomsky Hierarchy","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/"},{"source":"arxiv:2405.04517","target":"arxiv:2302.10866","evidence":"title","id":"arxiv:2302.10866","title":"Hyena Hierarchy: Towards Larger Convolutional Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/617c7279bfc8f8250cb7fb2d0bb4a2f7be1d1d692516fd45f80c7229632b16c7/"},{"source":"arxiv:2405.04517","target":"arxiv:2312.04927","evidence":"title","id":"arxiv:2312.04927","title":"Zoology: Measuring and Improving Recall in Efficient Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/baa792b06bdd30f91619f1e852aa5ae4dbf688fe3b1f2ee0722b339b9993ac5c/"}],"incoming":[{"source":"arxiv:2512.24695","target":"arxiv:2405.04517","evidence":"id","id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/"},{"source":"arxiv:2606.25010","target":"arxiv:2405.04517","evidence":"id","id":"arxiv:2606.25010","title":"Emergent Capabilities Arise Randomly from Learning Sparse Attention Patterns","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f87db387bad54ae4fbc0589d2a702f78aeb0817b28f20844f6502bc552a89c2f/"}],"text_parts":[{"start":0,"end":16000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-1.html"},{"start":16000,"end":32000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-2.html"},{"start":32000,"end":48000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-3.html"},{"start":48000,"end":64000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-4.html"},{"start":64000,"end":80000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-5.html"},{"start":80000,"end":96000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-6.html"},{"start":96000,"end":112000,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-7.html"},{"start":112000,"end":121152,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-8.html"},{"start":121152,"end":137152,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-9.html"},{"start":137152,"end":153152,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-10.html"},{"start":153152,"end":169152,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-11.html"},{"start":169152,"end":177404,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-12.html"},{"start":177404,"end":193404,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-13.html"},{"start":193404,"end":209404,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-14.html"},{"start":209404,"end":225404,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-15.html"},{"start":225404,"end":241404,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-16.html"},{"start":241404,"end":256449,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html"},{"start":256449,"end":272023,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html"},{"start":272023,"end":288023,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-19.html"},{"start":288023,"end":301912,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-20.html"},{"start":301912,"end":317912,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-21.html"},{"start":317912,"end":333912,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-22.html"},{"start":333912,"end":349912,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-23.html"},{"start":349912,"end":360474,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-24.html"},{"start":360474,"end":376474,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-25.html"},{"start":376474,"end":392474,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-26.html"},{"start":392474,"end":407558,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-27.html"},{"start":407558,"end":423526,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-28.html"},{"start":423526,"end":439417,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html"},{"start":439417,"end":455379,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html"}],"contents":[{"title":"xLSTM: Extended Long Short-Term Memory","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-1.html#text"},{"title":"Abstract","offset":42,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-1.html#text"},{"title":"1 Introduction","offset":1444,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-1.html#text"},{"title":"2 Extended Long Short-Term Memory","offset":35867,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-3.html#text"},{"title":"2.1 Review of the Long Short-Term Memory","offset":37263,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-3.html#text"},{"title":"2.2 sLSTM","offset":66749,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-5.html#text"},{"title":"New Memory Mixing.","offset":174763,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-12.html#text"},{"title":"2.3 mLSTM","offset":175447,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-12.html#text"},{"title":"2.4 xLSTM Architecture","offset":245566,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"xLSTM Blocks.","offset":245594,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"xLSTM Architecture.","offset":247066,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"2.5 Memory and Speed Considerations","offset":247383,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"3 Related Work","offset":248398,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Linear Attention.","offset":248417,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"State Space Models.","offset":249219,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Recurrent Neural Networks.","offset":249783,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Gating.","offset":250348,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Covariance Update Rule.","offset":250841,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Most Related.","offset":251271,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Residually Stacking Architectures.","offset":251879,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"4 Experiments","offset":252895,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"4.1 Synthetic Tasks and Long Range Arena","offset":254596,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Test of xLSTM’s Exponential Gating with Memory Mixing.","offset":255026,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Test of xLSTM’s Memory Capacities on Associative Recall Tasks.","offset":256379,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-17.html#text"},{"title":"Test of xLSTM’s Long Context Capabilities on Long Range Arena.","offset":257774,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"4.2 Method Comparison and Ablation Study","offset":258217,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"Comparing xLSTM to Other Methods.","offset":258622,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"Ablation Studies.","offset":260325,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"4.3 xLSTM as Large Language Model","offset":262306,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"Sequence Length Extrapolation.","offset":263329,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"Validation Perplexity and Downstream Tasks.","offset":263819,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"Performance on PALOMA Language Tasks.","offset":268136,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"Scaling Laws.","offset":268949,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"5 Limitations","offset":270547,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"6 Conclusion","offset":272006,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-18.html#text"},{"title":"Acknowledgements","offset":272805,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-19.html#text"},{"title":"Appendix A Extended Long Short-Term Memory","offset":273135,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-19.html#text"},{"title":"A.1 Vanilla Long Short-Term Memory Formulation: Vector Notation","offset":273182,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-19.html#text"},{"title":"A.2 sLSTM","offset":301897,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-20.html#text"},{"title":"Proof of Equivalence for sLSTM Stabilized Version.","offset":360416,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-24.html#text"},{"title":"A.3 mLSTM","offset":407005,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-27.html#text"},{"title":"Recurrent mLSTM Backward Pass.","offset":407232,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-27.html#text"},{"title":"Parallel mLSTM Forward Pass.","offset":411052,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-28.html#text"},{"title":"Parallel mLSTM Backward Pass.","offset":415512,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-28.html#text"},{"title":"A.4 Detailed Block Structure","offset":423526,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Appendix B Experiments","offset":423560,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Training Setup.","offset":423587,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Nearest Neighbor Search Task.","offset":424059,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Wikitext-103 Rare Token Prediction.","offset":425132,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"B.1 Synthetic Tasks and Long Range Arena","offset":425854,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"B.1.1 Test of xLSTM’s Exponential Gating with Memory Mixing.","offset":425900,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Experiment Setup.","offset":426592,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Additional Formal Language Results.","offset":427988,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Individual Task Description.","offset":428309,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"B.1.2 Test of xLSTM’s Memory Capacities on Associative Recall Tasks.","offset":432432,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Why Multi-Query Associative Recall for Memory Tests of LLM Architectures.","offset":432667,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Experiment Setup.","offset":433734,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Extended Results.","offset":436424,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"B.1.3 Test of xLSTM’s Long Range Capabilities on the Long Range Arena.","offset":438037,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-29.html#text"},{"title":"Experiment Setup.","offset":439593,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Results.","offset":440609,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"B.2 Method Comparison and Ablation Study on SlimPajama (15B)","offset":441320,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"General Training Procedure.","offset":441386,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Details on Comparison to Other Methods.","offset":444177,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Details on Training Precision for Baselines.","offset":446270,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"General Details on Ablation Studies.","offset":446794,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Additional Ablation Study on Matrix Memory.","offset":447022,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Details on new xLSTM Components Ablation Study.","offset":448011,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Details on Gating Technique Ablation Study.","offset":449048,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Details on Scaling Experiments.","offset":449470,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"B.3 xLSTM Large Language Models – SlimPajama300B","offset":449803,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"General Training Procedure.","offset":449857,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Details on Downstream Evaluation.","offset":450348,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Details on PALOMA.","offset":451188,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Appendix C Detailed Results on PALOMA Language Model Evaluation","offset":454149,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"},{"title":"Instructions for reporting errors","offset":454415,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/part-30.html#text"}],"text_code_points":455379,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/source.md"}