{"id":"arxiv:1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","date":"2019-04-01","section":"Data, training, optimization","quality":0.469974,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/","source_url":"https://arxiv.org/abs/1904.00962","text_available":true,"authors":["Yang You","Jing Li","Sashank Reddi","Jonathan Hseu"],"outgoing":[],"incoming":[{"source":"arxiv:2212.14034","target":"arxiv:1904.00962","evidence":"title","id":"arxiv:2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/"},{"source":"arxiv:2305.14342","target":"arxiv:1904.00962","evidence":"id","id":"arxiv:2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/"},{"source":"arxiv:2409.03137","target":"arxiv:1904.00962","evidence":"title","id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/"}],"text_parts":[{"start":0,"end":15559,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html"},{"start":15559,"end":31254,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html"},{"start":31254,"end":41220,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-3.html"},{"start":41220,"end":57092,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html"},{"start":57092,"end":59584,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-5.html"}],"contents":[{"title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"Abstract","offset":75,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"1 Introduction","offset":1564,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"1.1 Related Work","offset":6416,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"2 Preliminaries","offset":9261,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"Notation.","offset":9281,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"Theorem 1 ( ( Ghadimi & Lan, 2013b ) ) .","offset":13365,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"3 Algorithms","offset":14254,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-1.html#text"},{"title":"3.1 Lars Algorithm","offset":17221,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"Theorem 2 .","offset":18068,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"3.2 Lamb Algorithm","offset":18969,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"Theorem 3 .","offset":20001,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"4 Experiments","offset":22710,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"4.1 Bert Training","offset":24357,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"Comparison with AdamW and Lars .","offset":28686,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"4.2 ImageNet Training with ResNet-50.","offset":29530,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"4.3 Hyperparameters for scaling the batch size","offset":30910,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"5 Conclusion","offset":31237,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-2.html#text"},{"title":"6 Acknowledgement","offset":32087,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-3.html#text"},{"title":"Appendix","offset":32283,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-3.html#text"},{"title":"Appendix A Proof of Theorem 2","offset":32296,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-3.html#text"},{"title":"Proof.","offset":32330,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-3.html#text"},{"title":"Appendix B Proof of Theorem 3","offset":40808,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-3.html#text"},{"title":"Proof.","offset":40842,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-3.html#text"},{"title":"Appendix C Comparison of Convergence Rates of Lars and Sgd","offset":48451,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"Appendix D N-LAMB: Nesterov Momentum for LAMB","offset":49826,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"Appendix E LAMB with learning rate correction","offset":51405,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"Appendix F LAMB with different norms","offset":52194,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"Appendix G Regular Batch Sizes for Small Datasets: MNIST and CIFAR-10.","offset":52638,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"Appendix H Implementation Details and Additional Results","offset":53704,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"H.0.1 BERT","offset":55007,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"H.0.2 ImageNet","offset":55999,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"H.1 Baseline tuning details for ImageNet training with ResNet-50","offset":56298,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-4.html#text"},{"title":"Instructions for reporting errors","offset":58620,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/part-5.html#text"}],"text_code_points":59584,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/source.md"}