{"id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","date":"2023-02-13","section":"Data, training, optimization","quality":0.415216,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/","source_url":"https://arxiv.org/abs/2302.06675","text_available":true,"authors":["Xiangning Chen","Chen Liang","Da Huang","Esteban Real"],"outgoing":[{"source":"arxiv:2302.06675","target":"arxiv:2010.01412","evidence":"title","id":"arxiv:2010.01412","title":"Sharpness-Aware Minimization for Efficiently Improving Generalization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/"}],"incoming":[{"source":"arxiv:2304.07193","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/"},{"source":"arxiv:2305.14342","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2305.14342","title":"Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afd7fbad6232c6801145c611952e4b6b6aacd1ecb692f56874317ace8a328858/"},{"source":"arxiv:2306.13575","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2306.13575","title":"Scaling MLPs: A Tale of Inductive Bias","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/"},{"source":"arxiv:2307.06440","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2402.02342","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2402.02342","title":"MetaOptimize: A Framework for Optimizing Step Sizes and Other Meta-parameters","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6801d4fe20b85924aed2dec1855a35232b2ec1d98e15ec3bc27a0fbe5dfd1af7/"},{"source":"arxiv:2405.20541","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2405.20541","title":"Perplexed by Perplexity: Perplexity-Based Data Pruning With Small Reference Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/"},{"source":"arxiv:2409.03137","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2409.03137","title":"The AdEMAMix Optimizer: Better, Faster, Older","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4c4e435f36a4ffe28fc019c45f447a6825e58858ce5616b9b4bcd9e9cd245233/"},{"source":"arxiv:2510.05491","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2510.05491","title":"NorMuon: Making Muon more efficient and scalable","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/"},{"source":"arxiv:2512.24695","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/"},{"source":"arxiv:2601.14525","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2601.14525","title":"Towards Execution-Grounded Automated AI Research","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/895006e818735c9a565dbe99ae31404366eefd6bdc2278612271c65a8493e4a5/"},{"source":"arxiv:2608.05136","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2608.05136","title":"The Loss Does Not See the Basis, but Adam Does","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/"},{"source":"doi:10.1038/s41586-023-06924-6","target":"arxiv:2302.06675","evidence":"id","id":"doi:10.1038/s41586-023-06924-6","title":"Mathematical discoveries from program search with large language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70a6b427d42fd28c9d538648e148a97db2fb2614c7764f3231e53f4bebdce887/"}],"text_parts":[{"start":0,"end":15886,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-1.html"},{"start":15886,"end":30530,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html"},{"start":30530,"end":45295,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-3.html"},{"start":45295,"end":60333,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html"}],"contents":[{"title":"Symbolic Discovery of Optimization Algorithms","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-1.html#text"},{"title":"Abstract","offset":49,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-1.html#text"},{"title":"1 Introduction","offset":2074,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-1.html#text"},{"title":"2 Symbolic Discovery of Algorithms","offset":8002,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-1.html#text"},{"title":"2.1 Program Search Space","offset":8792,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-1.html#text"},{"title":"2.2 Efficient Search Techniques","offset":13831,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-1.html#text"},{"title":"2.3 Generalization: Program Selection and Simplification","offset":18466,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html#text"},{"title":"3 Derivation and Analysis of Lion","offset":21508,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html#text"},{"title":"3.1 Derivation","offset":21897,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html#text"},{"title":"3.2 Analysis","offset":22848,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html#text"},{"title":"4 Evaluation of Lion","offset":26469,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html#text"},{"title":"4.1 Image Classification","offset":27202,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html#text"},{"title":"4.2 Vision-Language Contrastive Learning","offset":30100,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-2.html#text"},{"title":"4.3 Diffusion Model","offset":32866,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-3.html#text"},{"title":"4.4 Language Modeling and Fine-tuning","offset":35393,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-3.html#text"},{"title":"4.5 Comparison with Other Popular Optimizers","offset":38861,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-3.html#text"},{"title":"4.6 Ablations","offset":40088,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-3.html#text"},{"title":"5 Hyperparameter Tuning","offset":41660,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-3.html#text"},{"title":"6 Limitations","offset":44394,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-3.html#text"},{"title":"7 Related Work","offset":46585,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"8 Conclusion","offset":48089,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Acknowledgements","offset":48509,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix A Pseudocode for AdamW and Lion","offset":49110,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix B Image Classification Tasks","offset":49155,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix C NLP Tasks","offset":49563,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix D Other Discovered Programs","offset":51057,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix E Architecture Details for Language Modeling","offset":51518,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix F Details of Proxy Tasks","offset":51830,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix G Analysis of Loss Landscape","offset":52455,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix H Available Functions","offset":53427,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Appendix I Abstract Execution","offset":55106,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"},{"title":"Instructions for reporting errors","offset":59369,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/part-4.html#text"}],"text_code_points":60333,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/source.md"}