{"id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","date":"2025-12-31","section":"Data, training, optimization","quality":-0.629827,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/","source_url":"https://arxiv.org/abs/2512.24695","text_available":true,"authors":["Ali Behrouz","Meisam Razaviyayn","Peilin Zhong","Vahab Mirrokni"],"outgoing":[{"source":"arxiv:2512.24695","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2512.24695","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2512.24695","target":"arxiv:2006.16236","evidence":"title","id":"arxiv:2006.16236","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d119844a2dce8a3b156558743b90add84774d00894634b6478d03abb4f5216f9/"},{"source":"arxiv:2512.24695","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"},{"source":"arxiv:2512.24695","target":"arxiv:2302.10866","evidence":"title","id":"arxiv:2302.10866","title":"Hyena Hierarchy: Towards Larger Convolutional Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/617c7279bfc8f8250cb7fb2d0bb4a2f7be1d1d692516fd45f80c7229632b16c7/"},{"source":"arxiv:2512.24695","target":"arxiv:2304.15004","evidence":"title","id":"arxiv:2304.15004","title":"Are Emergent Abilities of Large Language Models a Mirage?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/"},{"source":"arxiv:2512.24695","target":"arxiv:2312.00276","evidence":"title","id":"arxiv:2312.00276","title":"Metalearning Continual Learning Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5d7361f404c0455d9f759fc6a8dd7bd1da75146e72717b7a2fbcdb8bac80412a/"},{"source":"arxiv:2512.24695","target":"arxiv:2405.04517","evidence":"id","id":"arxiv:2405.04517","title":"xLSTM: Extended Long Short-Term Memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/"},{"source":"arxiv:2512.24695","target":"arxiv:2407.21783","evidence":"title","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2512.24695","target":"arxiv:2409.03137","evidence":"title","id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/"},{"source":"arxiv:2512.24695","target":"doi:10.1073/pnas.1611835114","evidence":"title","id":"doi:10.1073/pnas.1611835114","title":"Overcoming catastrophic forgetting in neural networks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/849082368a44d06b93ef3ba95bcfddc6672b7a5e8064ad2b2d19601639266e8b/"}],"incoming":[],"text_parts":[{"start":0,"end":15720,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-1.html"},{"start":15720,"end":31637,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-2.html"},{"start":31637,"end":47395,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-3.html"},{"start":47395,"end":62819,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-4.html"},{"start":62819,"end":77003,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-5.html"},{"start":77003,"end":91982,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-6.html"},{"start":91982,"end":107836,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-7.html"},{"start":107836,"end":123336,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-8.html"},{"start":123336,"end":139098,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-9.html"},{"start":139098,"end":152346,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-10.html"},{"start":152346,"end":167877,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-11.html"},{"start":167877,"end":183293,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-12.html"},{"start":183293,"end":196798,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html"},{"start":196798,"end":211657,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-14.html"}],"contents":[{"title":"Nested Learning: The Illusion of Deep Learning Architecture","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-1.html#text"},{"title":"Abstract","offset":63,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-1.html#text"},{"title":"1 Introduction","offset":2413,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-1.html#text"},{"title":"1.1 Human Brain Perspective and Neurophysiological Motivation","offset":8472,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-1.html#text"},{"title":"1.2 Contributions and Roadmap","offset":15685,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-1.html#text"},{"title":"2 Preliminaries","offset":22996,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-2.html#text"},{"title":"3 Nested Learning","offset":30973,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-2.html#text"},{"title":"3.1 Associative Memory","offset":31609,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-2.html#text"},{"title":"Definition 1 (Associative Memory) .","offset":32539,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-3.html#text"},{"title":"3.2 Nested Optimization Processes","offset":47022,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-3.html#text"},{"title":"Definition 2 (Update Frequency) .","offset":47995,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-4.html#text"},{"title":"Definition 3 (Nested System) .","offset":49290,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-4.html#text"},{"title":"Definition 4 (Nested System of Associative Memories) .","offset":51443,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-4.html#text"},{"title":"3.3 Knowledge Transfer Between Levels","offset":62776,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-4.html#text"},{"title":"4 Optimizers as Learning Modules","offset":72747,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-5.html#text"},{"title":"4.1 Backpropagation as an Associative Memory","offset":73195,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-5.html#text"},{"title":"4.2 Momentum-based Optimizers as Associative Memories","offset":76944,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-5.html#text"},{"title":"4.3 Long Context in Optimizers: An Example of Continual Learning with Orthogonal Tasks","offset":87760,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-6.html#text"},{"title":"4.4 More Expressive Designs for Momentum as an Associative Memory","offset":91392,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-6.html#text"},{"title":"4.5 Going Beyond Simple Gradient Descent and Momentum","offset":100188,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-7.html#text"},{"title":"Definition 5 (Generalized Gradient Descent (GGD) Learning Rule) .","offset":104367,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-7.html#text"},{"title":"5 Existing Architectures as Neural Learning Modules","offset":106753,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-7.html#text"},{"title":"5.1 Revisiting the Human Brain Perspective of Nested Learning","offset":120018,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-8.html#text"},{"title":"6 Takeaways and Revisiting Common Terms","offset":122957,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-8.html#text"},{"title":"7 Continuum Multi-Timescale Memory System","offset":133753,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-9.html#text"},{"title":"7.1 Continuum Memory System (CMS)","offset":134621,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-9.html#text"},{"title":"7.2 Continuum Memory System In Optimizers","offset":143431,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-10.html#text"},{"title":"7.3 Ad-hoc Level Stacking: Initializing CMS with Pre-Trained Models","offset":146431,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-10.html#text"},{"title":"8 Hope : A Self-Referential Learning Module with Continuum Memory","offset":148730,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-10.html#text"},{"title":"8.1 Deep Self-Referential Titans","offset":152308,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-10.html#text"},{"title":"8.2 Fast and Parallelizable Training","offset":161981,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-11.html#text"},{"title":"8.3 Hope Neural Learning Module","offset":167840,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-11.html#text"},{"title":"9 Experiments","offset":170771,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-12.html#text"},{"title":"9.1 Hope : Continual Learning and Long Context Understanding","offset":171618,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-12.html#text"},{"title":"9.2 Hope : Long Context Understanding","offset":180301,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-12.html#text"},{"title":"9.3 Hope : Language Modeling and Common-Sense Reasoning","offset":184398,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"9.4 Hope : In-context Recall Tasks and MAD Synthetic Benchmark","offset":186543,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"9.5 Language Recognition Tasks","offset":187627,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"9.6 Hope : Ablation Studies and Scaling","offset":188667,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"9.7 Expressive Optimizers","offset":190016,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"10 Conclusion","offset":191538,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"Appendix A Generalized Formulation for Nested Learning and Nested Systems","offset":193825,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"Definition 6 ((Generalized) Nested System) .","offset":194013,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"Definition 7 ((Generalized) Nested System of Associative Memories) .","offset":195212,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"Appendix B Adam, AdaGrad, and Other Similar Optimizers as Associative Memory Modules","offset":196709,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-13.html#text"},{"title":"Appendix C Delta Gradient Descent with Normalization","offset":206703,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-14.html#text"},{"title":"Instructions for reporting errors","offset":210693,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/part-14.html#text"}],"text_code_points":211657,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/source.md"}