{"id":"arxiv:2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","date":"2023-05-23","section":"Data, training, optimization","quality":0.046984,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/","source_url":"https://arxiv.org/abs/2305.14342","text_available":true,"authors":["Hong Liu","Zhiyuan Li","David Hall","Percy Liang"],"outgoing":[{"source":"arxiv:2305.14342","target":"arxiv:1904.00962","evidence":"id","id":"arxiv:1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/"},{"source":"arxiv:2305.14342","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2305.14342","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"}],"incoming":[{"source":"arxiv:2307.06440","target":"arxiv:2305.14342","evidence":"id","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2407.15017","target":"arxiv:2305.14342","evidence":"title","id":"arxiv:2407.15017","title":"Knowledge Mechanisms in Large Language Models: A Survey and Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8900dc2821aa61e59c5b1219cf9eb36dfaabf303c17d0064deaca653b5ec9a77/"},{"source":"arxiv:2409.03137","target":"arxiv:2305.14342","evidence":"title","id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/"}],"text_parts":[{"start":0,"end":15963,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-1.html"},{"start":15963,"end":30020,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-2.html"},{"start":30020,"end":45778,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html"},{"start":45778,"end":61530,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html"},{"start":61530,"end":77484,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html"},{"start":77484,"end":91611,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html"},{"start":91611,"end":103346,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html"}],"contents":[{"title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-1.html#text"},{"title":"Abstract","offset":88,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-1.html#text"},{"title":"1 Introduction","offset":1599,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-1.html#text"},{"title":"2 Method","offset":7957,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-1.html#text"},{"title":"2.1 Motivations","offset":8323,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-1.html#text"},{"title":"2.2 Sophia: S econd- o rder Cli p ped Stoc h astic Opt i miz a tion","offset":15890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-1.html#text"},{"title":"2.3 Diagonal Hessian Estimators","offset":22203,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-2.html#text"},{"title":"3 Experiments","offset":34966,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html#text"},{"title":"3.1 Experimental Setup","offset":35515,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html#text"},{"title":"Hyperparamter tuning strategy.","offset":38631,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html#text"},{"title":"3.2 Evaluation","offset":39946,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html#text"},{"title":"Methodology for comparing the optimizers for LLMs.","offset":39966,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html#text"},{"title":"Technical details.","offset":43730,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html#text"},{"title":"3.3 Results","offset":44212,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-3.html#text"},{"title":"3.4 Analysis","offset":46676,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"3.5 Ablation Study","offset":49420,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"4 Theoretical Analysis","offset":51776,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"Assumption 4.1 .","offset":52270,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"Assumption 4.2 .","offset":52834,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"Theorem 4.3 .","offset":54349,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"5 Related work","offset":55528,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"6 Conclusion","offset":60134,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"Acknowledgements","offset":60505,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"Appendix A Additional Experiment Results","offset":60719,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-4.html#text"},{"title":"Appendix B Additional Experiment Details","offset":61819,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"B.1 Hyperparamter Tuning","offset":61864,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"B.2 Model and Implementation Details","offset":65580,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"B.3 Downtream Evaluation","offset":67480,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Appendix C Limitations","offset":67884,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Appendix D Theoretical Analyses: Details of Section 4","offset":69017,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Lemma D.1 .","offset":69450,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Proof of Lemma D.1 .","offset":69587,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Lemma D.2 .","offset":71012,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Proof of Lemma D.2 .","offset":71304,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Lemma D.3 .","offset":73364,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Proof of Lemma D.3 .","offset":73674,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Lemma D.4 .","offset":74857,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Proof of Lemma D.4 .","offset":75403,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Lemma D.5 .","offset":76984,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Proof of Lemma D.5 .","offset":77455,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-5.html#text"},{"title":"Lemma D.6 .","offset":79275,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Proof of Lemma D.6 .","offset":79665,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Lemma D.7 .","offset":79778,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Proof of Lemma D.7 .","offset":80133,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Lemma D.8 .","offset":81064,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Proof of Lemma D.8 .","offset":81495,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Lemma D.9 .","offset":84093,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Proof of Lemma D.9 .","offset":85062,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-6.html#text"},{"title":"Lemma D.10 (Descent Lemma) .","offset":94148,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"},{"title":"Proof of Lemma D.10 .","offset":95062,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"},{"title":"Lemma D.11 .","offset":98109,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"},{"title":"Proof of Lemma D.11 .","offset":98624,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"},{"title":"D.1 Lower bound for SignGD on 2-dimensional quadratic loss","offset":99747,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"},{"title":"Theorem D.12 .","offset":100213,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"},{"title":"Proof of Theorem D.12 .","offset":100812,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"},{"title":"Instructions for reporting errors","offset":102382,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/part-7.html#text"}],"text_code_points":103346,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/source.md"}