{"id":"arxiv:2405.20233","title":"Grokfast: Accelerated Grokking by Amplifying Slow Gradients","date":"2024-05-30","section":"Data, training, optimization","quality":0.183974,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/","source_url":"https://arxiv.org/abs/2405.20233","text_available":true,"authors":["Jaerin Lee","Bong Gyun Kang","Kihoon Kim","Kyoung Mu Lee"],"outgoing":[{"source":"arxiv:2405.20233","target":"arxiv:2301.05217","evidence":"id","id":"arxiv:2301.05217","title":"Progress measures for grokking via mechanistic interpretability","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f71e3b65a8b3f35beb171d17f6c65878e0914227a0042ccba59b88952554baef/"}],"incoming":[{"source":"arxiv:2409.03137","target":"arxiv:2405.20233","evidence":"title","id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/"}],"text_parts":[{"start":0,"end":15998,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html"},{"start":15998,"end":31500,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html"},{"start":31500,"end":47384,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html"},{"start":47384,"end":51875,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-4.html"}],"contents":[{"title":"Grokfast : Accelerated Grokking by Amplifying Slow Gradients","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"Abstract","offset":64,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"1 Introduction","offset":1154,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"2 Amplifying the Low-Frequencies of the Stochastic Gradients","offset":6376,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"2.1 Filter Design","offset":6441,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"2.2 Experiment","offset":8756,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"2.3 Discussion","offset":10007,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"Q1. Are both slow and fast gradients necessary?","offset":10404,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"Q2. Exploiting state transition in the training of a model under grokking.","offset":11595,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"Q3. Synergistic effect with weight decay.","offset":12900,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"2.4 Limitations","offset":14103,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"3 Grokfast with Exponential Moving Average Gradient Filter","offset":14870,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"4 Experiment","offset":15981,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-1.html#text"},{"title":"4.1 Algorithmic Data","offset":16372,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"4.2 MNIST","offset":18627,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"4.3 QM9","offset":19532,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"4.4 IMDb","offset":20168,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"5 More Discussion","offset":20807,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"5.1 Difference between Algorithm 2 and the Momentum in Typical Optimizers","offset":20829,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"5.2 Visualizing Trajectories","offset":21994,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"6 Related Work","offset":25685,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"Grokking.","offset":25704,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"Optimization techniques.","offset":27844,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"7 Conclusion","offset":29361,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"Appendix A Frequency Responses of the Parameter Updates under Grokfast","offset":29974,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-2.html#text"},{"title":"Theorem A.1 .","offset":32336,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Proof of Theorem A.1 .","offset":33415,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Proposition A.2 (SGD with momentum) .","offset":39631,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Proof of Proposition A.2 .","offset":40991,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Proposition A.3 (SGD with Nesterov’s momentum) .","offset":41150,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Proof of Proposition A.3 .","offset":42551,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Appendix B Task Details","offset":42723,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"B.1 Binary Operation (Algorithmic Data)","offset":42949,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"B.2 MNIST","offset":44013,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"B.3 QM9","offset":44862,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"B.4 IMDb","offset":45979,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Appendix C Autograd Implementation","offset":46951,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-3.html#text"},{"title":"Appendix D Time and Memory Requirements","offset":47666,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-4.html#text"},{"title":"Appendix E More Visualization","offset":48173,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-4.html#text"},{"title":"Instructions for reporting errors","offset":50911,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/part-4.html#text"}],"text_code_points":51875,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b62d50c44fbd3554702d96f717a5ee25029a28ceed9346ab27ff64d1d08ba07e/source.md"}