{"id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","date":"2023-07-12","section":"Data, training, optimization","quality":0.017146,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/","source_url":"https://arxiv.org/abs/2307.06440","text_available":true,"authors":["Jean Kaddour","Oscar Key","Piotr Nawrot","Pasquale Minervini"],"outgoing":[{"source":"arxiv:2307.06440","target":"acl:2022.emnlp-main.340","evidence":"title","id":"acl:2022.emnlp-main.340","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e43a06571b7e8d89abd5f20f8f520c8bc3d3eb32ffb156cb7fc9e0469fc14b0a/"},{"source":"arxiv:2307.06440","target":"arxiv:1708.07120","evidence":"title","id":"arxiv:1708.07120","title":"Super-Convergence: Very Fast Training of Neural Networks Using Large Learning Rates","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/"},{"source":"arxiv:2307.06440","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2307.06440","target":"arxiv:2002.05202","evidence":"title","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2307.06440","target":"arxiv:2010.01412","evidence":"id","id":"arxiv:2010.01412","title":"Sharpness-Aware Minimization for Efficiently Improving Generalization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/"},{"source":"arxiv:2307.06440","target":"arxiv:2212.14034","evidence":"id","id":"arxiv:2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/"},{"source":"arxiv:2307.06440","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"},{"source":"arxiv:2307.06440","target":"arxiv:2305.14342","evidence":"id","id":"arxiv:2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/"}],"incoming":[{"source":"arxiv:2405.18392","target":"arxiv:2307.06440","evidence":"title","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"},{"source":"arxiv:2409.03137","target":"arxiv:2307.06440","evidence":"title","id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/"}],"text_parts":[{"start":0,"end":15550,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html"},{"start":15550,"end":31387,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html"},{"start":31387,"end":43456,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html"}],"contents":[{"title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Abstract","offset":100,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"1 Introduction","offset":1187,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"2 Comparing Efficient Training Algorithms","offset":4043,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"The Pitfalls of Iterations","offset":4440,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"The Pitfalls of FLOPs","offset":4604,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"The Pitfalls of Wall-Clock Time","offset":4994,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Our Proposal: Reference System Time (RST)","offset":5423,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"2.1 The Pitfall of Comparing Intermediate Performances","offset":6264,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"3 Experimental Setup","offset":8429,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"BERT: Encoder-only.","offset":8950,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"T5: Encoder-Decoder.","offset":10106,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Dataset.","offset":10921,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Learning-rate schedule.","offset":11167,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Hyper-parameter search.","offset":11355,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Reference System.","offset":11615,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"4 Case Study 1: Dynamic Architectures","offset":11836,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"4.1 Layer stacking","offset":11878,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"4.2 Layer dropping","offset":12457,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"4.3 Results","offset":13984,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Training losses.","offset":14001,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"Downstream performance.","offset":14709,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-1.html#text"},{"title":"5 Case Study 2: Batch Selection","offset":16071,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"5.1 Selective Backprop","offset":16725,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"5.2 RHO Loss","offset":17238,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"5.3 Results","offset":18822,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Validation loss.","offset":19614,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Downstream performance.","offset":19999,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"5.4 Ablation: Batch Selection For Free","offset":20385,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"6 Case Study 3: Efficient Optimizers","offset":21649,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Mixed Precision Training with BF16 vs. FP16","offset":22658,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"RMS-Scaling for T5 Pre-training","offset":23401,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"6.1 Results","offset":24060,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"7 Conclusion","offset":24640,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Acknowledgments","offset":25244,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Appendix A Limitations and Future Work","offset":26263,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Appendix B Related Work","offset":27759,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"B.1 Efficient Training Algorithms","offset":27787,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Architecture-centric strategies.","offset":28043,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Data-centric strategies.","offset":29277,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Optimization-centric strategies.","offset":30693,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"Others.","offset":31373,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-2.html#text"},{"title":"B.2 Efficient Training Meta-Studies","offset":31588,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"Budget-centric recipes.","offset":31629,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"Empirical Meta-Studies of Training Transformer-based models.","offset":32047,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"Appendix C Hyper-Parameter Search","offset":33370,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"C.1 Layer stacking : When To Stack","offset":33408,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"C.2 Layer dropping : How to Drop","offset":34026,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"C.3 Selective backprop : Selectivity Scale","offset":34754,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"C.4 RHO loss : Mega-batch Size","offset":35467,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"C.5 Lion : Learning Rate (LR) And Weight Decay (WD)","offset":36914,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"C.6 Sophia : Learning Rate (LR), Weight Decay (WD) and ρ \\rho","offset":37817,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"Appendix D Additional Results","offset":39463,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"D.1 Training BERT, Evaluating on GLUE","offset":39497,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"D.2 Layer dropping on small datasets","offset":40957,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"},{"title":"Instructions for reporting errors","offset":42492,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/part-3.html#text"}],"text_code_points":43456,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/source.md"}