{"id":"arxiv:2009.11243","title":"Tasks, stability, architecture, and compute: Training more effective learned optimizers, and using them to train themselves","date":"2020-09-23","section":"Data, training, optimization","quality":0.494615,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/","source_url":"https://arxiv.org/abs/2009.11243","text_available":true,"authors":["Luke Metz","Niru Maheswaranathan","C. Daniel Freeman","Ben Poole"],"outgoing":[],"incoming":[{"source":"arxiv:2402.02342","target":"arxiv:2009.11243","evidence":"id","id":"arxiv:2402.02342","title":"MetaOptimize: A Framework for Optimizing Step Sizes and Other Meta-parameters","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/"}],"text_parts":[{"start":0,"end":15917,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html"},{"start":15917,"end":31601,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html"},{"start":31601,"end":47463,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html"},{"start":47463,"end":63307,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html"},{"start":63307,"end":72056,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html"}],"contents":[{"title":"Tasks, stability, architecture, and compute: Training more effective learned optimizers, and using them to train themselves","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"Abstract","offset":127,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"1 Introduction","offset":1336,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"2 Preliminaries","offset":5198,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"3 Methods: Addressing the three barriers to learned optimizers","offset":6629,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"3.1 Outer-Optimization","offset":6696,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"3.2 Task distributions","offset":8934,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"3.3 Optimizer architecture","offset":11008,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"4 Results","offset":12947,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"4.1 Comparing learned optimizer architectures and training task set sizes","offset":12961,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"4.2 Comparisons with hand-designed optimizers","offset":14222,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-1.html#text"},{"title":"4.3 Understanding optimizer behavior","offset":17411,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"4.4 Generalization along different task axes","offset":18811,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"4.5 Generalization to large-scale problems","offset":20379,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"4.6 Learned optimizers training themselves","offset":21399,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"5 Discussion","offset":22228,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"Acknowledgements","offset":24429,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"Broader Impact","offset":24817,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"Appendix A Extended Related Work","offset":26789,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"A.1 Parameterizations","offset":27028,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"A.1.1 Controller based parameterization","offset":27208,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"A.1.2 Symbolic parameterizations","offset":28513,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"A.1.3 Continuous parameterizations","offset":29255,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-2.html#text"},{"title":"A.2 Data distribution of tasks","offset":32184,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"A.3 Outer-Optimization methods","offset":33477,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"A.3.1 Hyper parameter optimization","offset":33688,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"A.3.2 Reinforcement learning","offset":34512,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"A.3.3 Neural Architecture Search Style","offset":35104,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"A.3.4 Backpropogation / Gradient based","offset":35480,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"A.3.5 Evolutionary Strategies","offset":37746,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"Appendix B Outer Optimization Details","offset":38426,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"Appendix C Learned Optimizer Architecture Details","offset":41975,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"C.1 High level structure of the optimizer","offset":42535,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"C.2 Utilities / components","offset":44046,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"C.2.1 Rolling Features","offset":44205,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"C.2.2 LossAccum","offset":44971,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"C.2.3 RollingClipState","offset":45971,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"C.3 Learned optimizer specific / putting it all together","offset":46890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-3.html#text"},{"title":"C.4 “ThetaMod”: The learned optimizer outer-parameters","offset":48796,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"Appendix D Adam8p","offset":55670,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"Appendix E Performance Table","offset":58014,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"Appendix F Experimental details for figures","offset":59017,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"F.1 Outer training of different learned architectures","offset":59065,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"F.2 Outer training with different sized models","offset":60289,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"F.3 Large scale transfer with Resnet models","offset":60601,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"CIFAR-10 Resnet","offset":61424,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"Imagenet Resnet","offset":61552,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"F.4 Self optimization details","offset":61730,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"Appendix G Self Optimization Extended","offset":62663,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"Appendix H Infrastructure Used","offset":62705,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-4.html#text"},{"title":"H.1 Outer-training cluster","offset":63645,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Workers","offset":63881,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Gradient Storage","offset":64636,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Learner","offset":64957,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Summary aggregators","offset":66019,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Evaluations:","offset":67320,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Evaluation Chief:","offset":68987,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Evaluation workers:","offset":69339,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"H.2 Computational expense","offset":69742,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"},{"title":"Instructions for reporting errors","offset":71092,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/part-5.html#text"}],"text_code_points":72056,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/source.md"}