{"id":"arxiv:2402.02342","title":"MetaOptimize: A Framework for Optimizing Step Sizes and Other Meta-parameters","date":"2024-02-04","section":"Data, training, optimization","quality":-0.208417,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/","source_url":"https://arxiv.org/abs/2402.02342","text_available":true,"authors":["Arsalan Sharifnassab","Saber Salehkaleybar","Richard Sutton"],"outgoing":[{"source":"arxiv:2402.02342","target":"arxiv:2009.11243","evidence":"id","id":"arxiv:2009.11243","title":"Tasks, stability, architecture, and compute: Training more effective learned optimizers, and using them to train themselves","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3813d451f50fe60647b2397ec811ddabe99664b1897350808aac27a456914ed2/"},{"source":"arxiv:2402.02342","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"},{"source":"arxiv:2402.02342","target":"arxiv:2401.17401","evidence":"id","id":"arxiv:2401.17401","title":"Step-size Optimization for Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c114ace9d4e8c21e4f5e7bf6fdaf29c0df7b9527d2935c401329991ca92e94fd/"},{"source":"arxiv:2402.02342","target":"url:paperswithcode.com/paper/optimizing-millions-of-hyperparameters-by","evidence":"title","id":"url:paperswithcode.com/paper/optimizing-millions-of-hyperparameters-by","title":"Optimizing Millions of Hyperparameters by Implicit Differentiation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c5ceae5fd961be6d0bb1cd78bdf473d8b86954dc0615b3fb35ad9124be5e2643/"}],"incoming":[{"source":"arxiv:2405.12250","target":"arxiv:2402.02342","evidence":"title","id":"arxiv:2405.12250","title":"Your Transformer is Secretly Linear","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6ae80c5f07c56011562d6b8323b0984ec0d36194708a43867aae691e2ddc5d86/"}],"text_parts":[{"start":0,"end":12223,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html"},{"start":12223,"end":28204,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html"},{"start":28204,"end":41599,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html"},{"start":41599,"end":51651,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-4.html"},{"start":51651,"end":66080,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-5.html"},{"start":66080,"end":80289,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-6.html"},{"start":80289,"end":92232,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-7.html"},{"start":92232,"end":101927,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-8.html"},{"start":101927,"end":116009,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-9.html"},{"start":116009,"end":128659,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-10.html"}],"contents":[{"title":"MetaOptimize: A Framework for Optimizing Step Sizes and Other Meta-parameters","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html#text"},{"title":"Abstract","offset":81,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html#text"},{"title":"Keywords:","offset":998,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html#text"},{"title":"1 Introduction","offset":1016,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html#text"},{"title":"2 Problem Setting","offset":4430,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html#text"},{"title":"3 Forward and Backward Views","offset":7204,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html#text"},{"title":"4 MetaOptimize","offset":10571,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-1.html#text"},{"title":"Remark 4.1 .","offset":18469,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"5 Reducing Complexity","offset":19333,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"2 × \\times 2 approximation.","offset":19503,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"L-approximation.","offset":20014,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"Intuition.","offset":21314,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"Containing some prior methods as special cases.","offset":22377,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"6 Hessian-Free MetaOptimize","offset":22959,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"7 Experiments","offset":25683,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"7.1 CIFAR10 dataset","offset":26410,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"7.2 Non-stationary CIFAR100","offset":27286,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-2.html#text"},{"title":"7.3 ImageNet dataset","offset":28602,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html#text"},{"title":"7.4 Language modeling","offset":29495,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html#text"},{"title":"7.5 Sensitivity analysis","offset":30596,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html#text"},{"title":"8 Related Works","offset":32098,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html#text"},{"title":"9 Limitations and Future Works","offset":37336,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html#text"},{"title":"Impact Statement","offset":41113,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html#text"},{"title":"Appendix A Step-size Optimization for Different Choices of Base and Meta updates","offset":41334,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-3.html#text"},{"title":"A.1 Derivation of G meta G^{\\textrm{meta}} for Different Meta Updates","offset":45757,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-4.html#text"},{"title":"A.1.1 Meta SGD","offset":49019,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-4.html#text"},{"title":"A.1.2 Meta Adam","offset":51629,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-4.html#text"},{"title":"A.1.3 Meta Lion","offset":59959,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-5.html#text"},{"title":"A.2 Derivation of G base G^{\\textrm{base}} for Different Base Updates","offset":65850,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-5.html#text"},{"title":"A.3 Base SGD","offset":74175,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-6.html#text"},{"title":"A.3.1 Base AdamW","offset":86524,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-7.html#text"},{"title":"A.3.2 Base Lion","offset":101905,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-8.html#text"},{"title":"Appendix B Exiting Step-size Optimization Algorithms as Special Cases of MetaOptimize","offset":115565,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-9.html#text"},{"title":"B.1 IDBD and Its Extensions","offset":115976,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-9.html#text"},{"title":"B.2 Hyper-gradient Descent","offset":120340,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-10.html#text"},{"title":"Appendix C Experiment Details","offset":122319,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-10.html#text"},{"title":"Appendix D Further Experimental Results","offset":125961,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-10.html#text"},{"title":"Instructions for reporting errors","offset":127695,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/part-10.html#text"}],"text_code_points":128659,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/source.md"}