{"id":"arxiv:2405.15682","title":"The Road Less Scheduled","date":"2024-05-24","section":"Data, training, optimization","quality":0.34565,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/","source_url":"https://arxiv.org/abs/2405.15682","text_available":true,"authors":["Aaron Defazio","Xingyu Alice Yang","Harsh Mehta","Konstantin Mishchenko"],"outgoing":[],"incoming":[{"source":"arxiv:2405.18392","target":"arxiv:2405.15682","evidence":"id","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"}],"text_parts":[{"start":0,"end":15819,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html"},{"start":15819,"end":31637,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-2.html"},{"start":31637,"end":46508,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html"},{"start":46508,"end":59899,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-4.html"},{"start":59899,"end":74670,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html"},{"start":74670,"end":90605,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html"}],"contents":[{"title":"The Road Less Scheduled","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"Abstract","offset":27,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"1 Introduction","offset":987,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"1.1 Summary of Results","offset":3684,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"1.2 Notation","offset":5304,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"2 Method","offset":5823,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"Theorem 1 .","offset":9380,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"2.1 General Theory","offset":10233,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"Theorem 2 .","offset":10525,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"2.2 On Large Learning Rates","offset":15786,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-1.html#text"},{"title":"Theorem 3 .","offset":16866,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-2.html#text"},{"title":"2.3 How Schedule-Free Replaces a Schedule","offset":19170,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-2.html#text"},{"title":"3 Related Work","offset":20556,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-2.html#text"},{"title":"4 Experiments","offset":25691,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-2.html#text"},{"title":"4.1 Deep Learning Problems","offset":26092,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-2.html#text"},{"title":"4.2 MLCommons Algorithmic Efficiency benchmark","offset":29717,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-2.html#text"},{"title":"4.3 Convex Problems","offset":32163,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"4.4 Implementation Concerns","offset":32932,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"5 Parameter Sensitivity","offset":35830,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"6 Conclusion","offset":37124,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"7 Contributions","offset":37721,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"Appendix A Proof of Theorem 2","offset":38543,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"Proof.","offset":38584,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"Appendix B Recovering Prior Conversions, and Connections to Momentum","offset":42571,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"Theorem 4 .","offset":42775,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"Proof.","offset":43966,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"B.1 Recovering Linear Decay","offset":46475,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-3.html#text"},{"title":"Appendix C Generalizing Theorem 2 via Bregman Divergences","offset":48761,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-4.html#text"},{"title":"Theorem 5 .","offset":49012,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-4.html#text"},{"title":"Proof.","offset":51859,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-4.html#text"},{"title":"Appendix D Acceleration","offset":58387,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-4.html#text"},{"title":"Corollary 1 .","offset":59102,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-4.html#text"},{"title":"Proof.","offset":59884,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-4.html#text"},{"title":"Corollary 2 .","offset":63142,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html#text"},{"title":"Proof.","offset":64538,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html#text"},{"title":"D.1 An Optimistic Regret Bound","offset":66398,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html#text"},{"title":"Lemma 1 .","offset":66811,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html#text"},{"title":"Proof.","offset":68167,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html#text"},{"title":"Appendix E Strongly Convex Losses","offset":74410,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html#text"},{"title":"Theorem 6 .","offset":74650,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-5.html#text"},{"title":"Proof.","offset":76053,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"Appendix F Large Step size convergence","offset":79321,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"Proof.","offset":79371,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"Appendix G Experimental Setup","offset":82769,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.1 Convex experiments","offset":82803,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.2 CIFAR-10","offset":83119,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.3 CIFAR-100","offset":83740,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.4 SVHN","offset":84285,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.5 ImageNet","offset":84676,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.6 IWSLT14","offset":85323,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.7 NanoGPT","offset":86005,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.8 MAE","offset":86775,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.9 DLRM","offset":87283,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.10 MRI","offset":87904,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"G.11 Algoperf","offset":88517,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"Appendix H Polyak and Primal Averaging Runs","offset":89002,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"Appendix I Additional LR Sensitivity Plots","offset":89594,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"},{"title":"Instructions for reporting errors","offset":89641,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/part-6.html#text"}],"text_code_points":90605,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/source.md"}