{"id":"arxiv:2507.10524","title":"Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation","date":"2025-07-14","section":"LLMs: architectures, context, training","quality":0.17613,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/","source_url":"https://arxiv.org/abs/2507.10524","text_available":true,"authors":["Sangmin Bae","Yujin Kim","Reza Bayat","Sungnyun Kim"],"outgoing":[{"source":"arxiv:2507.10524","target":"arxiv:1807.03819","evidence":"title","id":"arxiv:1807.03819","title":"Universal Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1d843516d980c3404cb622a2a490edb535a3afe27a85a980335a54b5fcd2220a/"},{"source":"arxiv:2507.10524","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2507.10524","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2507.10524","target":"arxiv:2412.06769","evidence":"id","id":"arxiv:2412.06769","title":"Training Large Language Models to Reason in a Continuous Latent Space","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/68c71dc1a042dedd5e3c4d9c1ad32c9d046fd31b8dc79080bd6dff1c9800abc7/"},{"source":"arxiv:2507.10524","target":"arxiv:2502.05171","evidence":"id","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"}],"incoming":[{"source":"arxiv:2509.25239","target":"arxiv:2507.10524","evidence":"id","id":"arxiv:2509.25239","title":"A Formal Comparison Between Chain of Thought and Latent Thought","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8399816be05fff6a7002f5c3439ec43ffe7b468424759b4cc70f5efc7739ea9e/"},{"source":"arxiv:2604.09168","target":"arxiv:2507.10524","evidence":"id","id":"arxiv:2604.09168","title":"ELT: Elastic Looped Transformers for Visual Generation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/63d4e5f1a267b96ed4474fa233f7d3bb2171aae949c0139db37b58b4e45eca47/"},{"source":"arxiv:2604.11791","target":"arxiv:2507.10524","evidence":"id","id":"arxiv:2604.11791","title":"A Mechanistic Analysis of Looped Reasoning Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae8c682d5708c6c47d65c4eb540dd5bccb877c5e332852765507128d6d1b985d/"},{"source":"arxiv:2604.17121","target":"arxiv:2507.10524","evidence":"title","id":"arxiv:2604.17121","title":"The Topological Trouble With Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f9d762bbe192e1208f6d7dc289b7e79c23d4bdde0b1b3999ff5fda55b0dcd1a/"},{"source":"arxiv:2604.21999","target":"arxiv:2507.10524","evidence":"id","id":"arxiv:2604.21999","title":"Universal Transformers Need Memory: Depth-State Trade-offs in Adaptive Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/963f6bb396c25bff462a25854c82e7d92b77803fa60c891fadd9963cc56c8fea/"},{"source":"arxiv:2605.12466","target":"arxiv:2507.10524","evidence":"id","id":"arxiv:2605.12466","title":"Solve the Loop: Attractor Models for Language and Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/eb46f81cd5718eb6b9256c583cf9dd2e97b793b4d14337e487c16285f96e2b9a/"},{"source":"arxiv:2605.19376","target":"arxiv:2507.10524","evidence":"id","id":"arxiv:2605.19376","title":"Generative Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/795bad9bb1b1a063090fbb925ba6e68bf30211de0ecffd75d56f858b28321ae7/"},{"source":"arxiv:2605.21488","target":"arxiv:2507.10524","evidence":"title","id":"arxiv:2605.21488","title":"Equilibrium Reasoners: Learning Attractors Enables Scalable Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48b3beca0f016a3683df4c6c71fc4be9f13a2c12c61d4ad17d1117c6139bb11e/"},{"source":"arxiv:2606.06574","target":"arxiv:2507.10524","evidence":"id","id":"arxiv:2606.06574","title":"Skip a Layer or Loop It? Learning Program-of-Layers in LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5abe08b6c82ce9a4010a09d82dbee40f8b4b0f88539bf8911ae36048a97bb917/"},{"source":"arxiv:2606.18206","target":"arxiv:2507.10524","evidence":"title","id":"arxiv:2606.18206","title":"Fixed-Point Reasoners: Stable and Adaptive Deep Looped Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/37ea2d83a92ece4ab3ba4434af71f291d1fdc5040676f75110210f40aad3b81a/"},{"source":"arxiv:2606.31779","target":"arxiv:2507.10524","evidence":"title","id":"arxiv:2606.31779","title":"Bridging the Gap Between Latent and Explicit Reasoning with Looped Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bf07d5c19e507d9ec91a48f9067e6c29fbe8965d001923e2de0e96da143c72e1/"},{"source":"arxiv:2608.17981","target":"arxiv:2507.10524","evidence":"title","id":"arxiv:2608.17981","title":"Recirculation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd91ffc957070136795d3728f662c997738234065b624eb720acd8547d2edaf6/"}],"text_parts":[{"start":0,"end":15101,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html"},{"start":15101,"end":30708,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html"},{"start":30708,"end":46201,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html"},{"start":46201,"end":61677,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html"},{"start":61677,"end":77596,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html"},{"start":77596,"end":81142,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-6.html"}],"contents":[{"title":"Mixture-of-Recursions: Learning Dynamic Recursive Depths for Adaptive Token-Level Computation","offset":15,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Abstract","offset":112,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"1 Introduction","offset":1637,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Contributions.","offset":6094,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"2 Method","offset":7421,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"2.1 Preliminary","offset":7434,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Recursive Transformers.","offset":7455,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Parameter-sharing strategies.","offset":8422,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Enhanced training and inference efficiency in recursive models.","offset":9536,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Limitations in prior works.","offset":10560,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"2.2 Mixture-of-Recursions","offset":11721,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"2.2.1 Routing Strategies: Expert-choice vs. Token-choice","offset":12190,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Expert-choice routing. ( 2(a) )","offset":12253,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Token-choice routing. ( 2(b) )","offset":13961,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-1.html#text"},{"title":"Strengths and limitations. ( Table 2 – Left )","offset":16024,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"2.2.2 KV Caching Strategies: Recursion-wise Caching vs. Recursive Sharing","offset":16827,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"Recursion-wise KV caching. ( 2(c) – Top )","offset":17461,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"Recursive KV sharing. ( 2(c) – Bottom )","offset":18002,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"Strengths and limitations. ( Table 2 – Right )","offset":18642,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"3 Experiments","offset":19552,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"3.1 Main Results","offset":20206,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"MoR outperforms baselines with fewer parameters under equal train compute.","offset":20228,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"MoR outperforms baselines with less compute at equal data.","offset":21106,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"MoR performance varies with routing and caching strategies.","offset":21855,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"3.2 IsoFLOP Analysis","offset":22509,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"Experimental Setup.","offset":22937,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"MoR is a scalable and parameter-efficient architecture.","offset":23259,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"3.3 Inference Throughput Evaluation","offset":24045,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"Experimental Setup.","offset":24493,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"MoR boosts inference throughput with continuous depth-wise batching.","offset":25127,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"4 Ablation Studies","offset":25904,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"4.1 Parameter Sharing Strategies","offset":25927,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"Middle-Cycle is the most effective parameter sharing strategy.","offset":25965,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"4.2 Routing Strategies","offset":26688,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"In expert-choice routing, auxiliary loss and linear router yield the best performance.","offset":26920,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"In token-choice routing, balancing loss yields stable and accurate routing.","offset":27920,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"4.3 KV Caching Strategies","offset":28805,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"KV sharing robustly works even in parameter-shared architectures.","offset":28836,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"KV sharing degrades expert-choice but benefits token-choice routing in MoR.","offset":29691,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"5 Analysis","offset":30577,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"5.1 Compute-optimal Scaling Analysis","offset":30592,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"MoR scaling favors model size over training length under isoFLOPs.","offset":30634,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-2.html#text"},{"title":"5.2 Routing Analysis","offset":31311,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"The allocation of recursion depth reflects contextual predictability of the subsequent token.","offset":31337,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Expert-choice router with auxiliary loss perfectly separates selected from unselected tokens.","offset":31985,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"5.3 Test-time Scaling Analysis","offset":32710,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"MoR enables test-time scaling via deeper recursion.","offset":32746,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"6 Related Work","offset":33468,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Recursive Transformers.","offset":33487,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Adaptive computation.","offset":34995,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Routing mechanism.","offset":36740,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Key-value caching.","offset":38026,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Latent reasoning.","offset":39583,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"7 Conclusion","offset":40911,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"7.1 Limitations and Future Works","offset":41735,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Reasoning MoR models.","offset":41773,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Further scaling model family.","offset":42579,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Adaptive capacity control.","offset":43657,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Compatibility with sparse algorithms.","offset":44357,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Expansion to multimodal and non - text domains.","offset":45157,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"7.2 Acknowledgements","offset":45785,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-3.html#text"},{"title":"Appendix A Details of Design Choices for Mixture-of-Recursions","offset":46789,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"A.1 Parameter-sharing Strategy","offset":47019,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"A.2 Routing Strategy","offset":48927,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Expert-choice routing.","offset":49255,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Token-choice routing.","offset":50638,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"A.3 KV Caching Strategy","offset":53256,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Recursion-wise caching.","offset":53479,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Recursive KV sharing.","offset":53799,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Appendix B Experimental Setup","offset":54545,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Training settings.","offset":54579,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Evaluation settings.","offset":55463,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Model architecture details.","offset":56059,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Appendix C Expanded Results of Main Experiments","offset":56505,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"C.1 Performance Validation at 1.7B Scale","offset":56557,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"C.2 Increasing Recursion Depth under Fixed Parameters","offset":57410,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Appendix D Expanded Results of IsoFLOP Analysis","offset":57959,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"FLOPs approximated calculation of Transformers.","offset":58630,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Trapezoid learning - rate schedule with checkpoint reuse.","offset":60518,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Results at a glance.","offset":61649,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-4.html#text"},{"title":"Appendix E Details of Experimental Settings for Throughput Measurement","offset":62324,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Appendix F Expanded Results of Parameter Sharing Strategy","offset":65828,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Middle - Cycle is consistently the safest choice.","offset":66440,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Behavior under continued pre - training (up - training).","offset":68068,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Appendix G Expanded Results of Design Choices for Router","offset":68631,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"G.1 Details of Design Configurations","offset":68692,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"G.2 Router Performance Evaluation Metrics","offset":70119,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Expert-choice routing.","offset":70166,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Token-choice routing.","offset":70955,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"G.3 Extended Evaluation Results of Router Designs","offset":72275,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Appendix H Expanded Results of KV Cache Sharing Mechanism","offset":73630,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"H.1 Key Value Representation Trends in Recursive Transformers","offset":73692,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"H.2 Performance Comparison of KV Sharing Strategy","offset":75489,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Experimental results of key-value cache sharing.","offset":75544,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Relaxation for key-value sharing constraints.","offset":77008,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-5.html#text"},{"title":"Appendix I Expanded Qualitative Results","offset":78220,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-6.html#text"},{"title":"I.1 Analysis on Adaptive Computation Paths","offset":78264,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-6.html#text"},{"title":"I.2 Analysis on Router Weights","offset":78799,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-6.html#text"},{"title":"Instructions for reporting errors","offset":80178,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/part-6.html#text"}],"text_code_points":81142,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7b25702c35f07446cea97d6a98afc489c8fdda54560c05fbeb1b3b139a9da622/source.md"}