{"id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","date":"2024-09-05","section":"Data, training, optimization","quality":0.319523,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/","source_url":"https://arxiv.org/abs/2409.03137","text_available":true,"authors":["Matteo Pagliardini","Pierre Ablin","David Grangier"],"outgoing":[{"source":"arxiv:2409.03137","target":"arxiv:1904.00962","evidence":"title","id":"arxiv:1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/"},{"source":"arxiv:2409.03137","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"},{"source":"arxiv:2409.03137","target":"arxiv:2305.14342","evidence":"title","id":"arxiv:2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/"},{"source":"arxiv:2409.03137","target":"arxiv:2307.06440","evidence":"title","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2409.03137","target":"arxiv:2405.18392","evidence":"title","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"},{"source":"arxiv:2409.03137","target":"arxiv:2405.20233","evidence":"title","id":"arxiv:2405.20233","title":"Grokfast: Accelerated Grokking by Amplifying Slow Gradients","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/"},{"source":"arxiv:2409.03137","target":"arxiv:2407.21783","evidence":"title","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[{"source":"arxiv:2512.24695","target":"arxiv:2409.03137","evidence":"title","id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/"}],"text_parts":[{"start":0,"end":14717,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-1.html"},{"start":14717,"end":30077,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-2.html"},{"start":30077,"end":45607,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html"},{"start":45607,"end":60460,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html"},{"start":60460,"end":67549,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-5.html"}],"contents":[{"title":"The AdEMAMix Optimizer: Better, Faster, Older","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-1.html#text"},{"title":"Abstract","offset":49,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-1.html#text"},{"title":"1 Introduction","offset":1419,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-1.html#text"},{"title":"2 Related Work","offset":6525,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-1.html#text"},{"title":"3 Our method: AdEMAMix","offset":12686,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-1.html#text"},{"title":"4 Results","offset":23769,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-2.html#text"},{"title":"4.1 Transformer LLM Training","offset":23939,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-2.html#text"},{"title":"4.2 Mamba LM Training","offset":31341,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"4.3 ViT Training","offset":31776,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"5 Conclusion","offset":34845,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"6 Acknowledgements","offset":35429,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"Appendix A Implementation details","offset":35736,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"A.1 Deriving the β 3 \\beta_{3} scheduler","offset":35774,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"A.2 Pytorch implementation","offset":37573,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"A.3 Optax implementation","offset":37828,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"Appendix B Experimental details","offset":38115,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"B.1 Transformer LLM experiments","offset":38151,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"B.2 Mamba LM experiments","offset":40953,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"B.3 ViT experiments","offset":41970,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"Appendix C Additional results","offset":43377,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"C.1 LLM experiments","offset":43411,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"C.1.1 In-Context Learning (ICL) results","offset":43436,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"C.1.2 More results on forgetting","offset":44961,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-3.html#text"},{"title":"C.1.3 Removing the second EMA mid-training","offset":46623,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.1.4 Hyperparameter sensitivity","offset":47281,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.1.5 Impact of training for fewer iterations","offset":48473,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.1.6 Limitations of a single EMA","offset":49271,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.1.7 Removing m 1 m_{1} by setting β 1 = 0 \\beta_{1}=0","offset":51845,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.1.8 Comparing m 1 + α ​ m 2 m_{1}+\\alpha m_{2} versus ( 1 − α ) ​ m 1 + α ​ m 2 (1-\\alpha)m_{1}+\\alpha m_{2}","offset":52954,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.1.9 Miscellaneous results","offset":56550,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.2 ViT experiments","offset":58630,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.3 Comparison with other methods","offset":59129,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.3.1 Comparison with AdMeta and DEMA","offset":59168,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-4.html#text"},{"title":"C.3.2 Comparison with Lion","offset":61876,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-5.html#text"},{"title":"C.3.3 Adding a third momentum term ( ∼ \\sim AggMo)","offset":63968,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-5.html#text"},{"title":"Instructions for reporting errors","offset":66585,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/part-5.html#text"}],"text_code_points":67549,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/source.md"}