{"id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","date":"2024-05-28","section":"Data, training, optimization","quality":0.24444,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/","source_url":"https://arxiv.org/abs/2405.18392","text_available":true,"authors":["Alexander Hägele","Elie Bakouch","Atli Kosson","Loubna Ben Allal"],"outgoing":[{"source":"arxiv:2405.18392","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2405.18392","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2405.18392","target":"arxiv:2307.06440","evidence":"title","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2405.18392","target":"arxiv:2405.15682","evidence":"id","id":"arxiv:2405.15682","title":"The Road Less Scheduled","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/205e2d6291f9c17e2c0e0de4392fe953a5e711d00b6758c8752a76f69af587ff/"},{"source":"arxiv:2405.18392","target":"arxiv:2405.16684","evidence":"id","id":"arxiv:2405.16684","title":"gzip Predicts Data-dependent Scaling Laws","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/"},{"source":"arxiv:2405.18392","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[{"source":"arxiv:2409.03137","target":"arxiv:2405.18392","evidence":"title","id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/"},{"source":"arxiv:2412.13663","target":"arxiv:2405.18392","evidence":"id","id":"arxiv:2412.13663","title":"Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/"},{"source":"arxiv:2502.05171","target":"arxiv:2405.18392","evidence":"title","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"}],"text_parts":[{"start":0,"end":15840,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html"},{"start":15840,"end":31820,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html"},{"start":31820,"end":42136,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html"}],"contents":[{"title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"Abstract","offset":77,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"1 Introduction","offset":1144,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"2 Background: Cosine Learning Rate Schedule for LLMs","offset":4318,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"3 A Different Route: Constant Learning Rate with Cooldown","offset":7398,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"3.1 The Advantages","offset":9117,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"3.2 Experimental Comparison","offset":10855,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"3.3 Scaling Up: 1B and 8B Models","offset":14802,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-1.html#text"},{"title":"4 Do We Even Need to Cooldown?","offset":16711,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html#text"},{"title":"4.1 Stochastic Weight Averaging (SWA)","offset":17147,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html#text"},{"title":"4.2 Schedule-Free Optimizer (SFO)","offset":19293,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html#text"},{"title":"5 The Implications for Scaling Law Research","offset":20763,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html#text"},{"title":"6 Limitations","offset":26426,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html#text"},{"title":"7 Related work","offset":26886,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html#text"},{"title":"8 Conclusion","offset":30374,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-2.html#text"},{"title":"Acknowledgements","offset":31820,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"Appendix A Experimental Details","offset":32127,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"A.1 Overview","offset":32163,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"A.2 Setup for the 1B and 8B Runs","offset":34878,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"A.3 FLOPs computation","offset":36947,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"Appendix B Additional Results","offset":37260,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"B.1 More Results on Cooldown","offset":37294,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"B.2 Additional Results and Compute Savings","offset":39556,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"B.3 Learning Curves for All Models of Scaling Experiments","offset":39732,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"B.4 Additional Experiments on OpenWebText2","offset":39967,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"B.5 Full Results of Large Model Runs","offset":40378,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"},{"title":"Instructions for reporting errors","offset":41172,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/part-3.html#text"}],"text_code_points":42136,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/source.md"}