{"id":"arxiv:2010.01412","title":"Sharpness-Aware Minimization for Efficiently Improving Generalization","date":"2020-10-03","section":"Data, training, optimization","quality":0.564643,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/","source_url":"https://arxiv.org/abs/2010.01412","text_available":true,"authors":["Pierre Foret","Ariel Kleiner","Hossein Mobahi","Behnam Neyshabur"],"outgoing":[],"incoming":[{"source":"arxiv:2302.06675","target":"arxiv:2010.01412","evidence":"title","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"},{"source":"arxiv:2307.06440","target":"arxiv:2010.01412","evidence":"id","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2602.18037","target":"arxiv:2010.01412","evidence":"title","id":"arxiv:2602.18037","title":"Gradient Regularization Mitigates Reward Hacking in Reinforcement Learning from Human Feedback and Verifiable Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/"},{"source":"arxiv:2605.21488","target":"arxiv:2010.01412","evidence":"title","id":"arxiv:2605.21488","title":"Equilibrium Reasoners: Learning Attractors Enables Scalable Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48b3beca0f016a3683df4c6c71fc4be9f13a2c12c61d4ad17d1117c6139bb11e/"}],"text_parts":[{"start":0,"end":15877,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-1.html"},{"start":15877,"end":31309,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html"},{"start":31309,"end":47257,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html"},{"start":47257,"end":51022,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-4.html"}],"contents":[{"title":"Sharpness-Aware Minimization for Efficiently Improving Generalization","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-1.html#text"},{"title":"Abstract","offset":73,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-1.html#text"},{"title":"1 Introduction","offset":1280,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-1.html#text"},{"title":"2 Sharpness-Aware Minimization (SAM)","offset":6165,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-1.html#text"},{"title":"Theorem (stated informally) 1 .","offset":8842,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-1.html#text"},{"title":"3 Empirical Evaluation","offset":15850,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-1.html#text"},{"title":"3.1 Image Classification From Scratch","offset":16412,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"3.2 Finetuning","offset":21863,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"3.3 Robustness to Label Noise","offset":23353,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"4 Sharpness and Generalization Through the Lens of SAM","offset":25193,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"4.1 m m -sharpness","offset":25252,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"4.2 Hessian Spectra","offset":27273,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"5 Related Work","offset":28464,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"6 Discussion and Future Work","offset":30406,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"7 Acknowledgments","offset":31287,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-2.html#text"},{"title":"Appendix A Appendix","offset":31984,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"A.1 PAC Bayesian Generalization Bound","offset":32008,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"Theorem 2 .","offset":32111,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"Proof.","offset":33328,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"Appendix B Additional Experimental Results","offset":42168,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"B.1 SVHN and Fashion-MNIST","offset":42215,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"Appendix C Experiment Details","offset":42788,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"C.1 Hyperparameters for Experiments","offset":42822,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"C.2 Finetuning Details","offset":43917,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"C.3 Experimental results with ρ = 0.05 \\rho=0.05","offset":45558,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"C.4 Ablation of the Second Order Terms","offset":45993,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"C.5 Choice of p-norm","offset":47231,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-3.html#text"},{"title":"C.6 Several Iterations in the Inner Maximization","offset":48487,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-4.html#text"},{"title":"Instructions for reporting errors","offset":50058,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/part-4.html#text"}],"text_code_points":51022,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/source.md"}