{"id":"arxiv:1708.07120","title":"Super-Convergence: Very Fast Training of Neural Networks Using Large Learning Rates","date":"2017-08-23","section":"Data, training, optimization","quality":-0.016091,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/","source_url":"https://arxiv.org/abs/1708.07120","text_available":true,"authors":["Leslie N. Smith","Nicholay Topin"],"outgoing":[{"source":"arxiv:1708.07120","target":"arxiv:1506.01186","evidence":"id","id":"arxiv:1506.01186","title":"Cyclical Learning Rates for Training Neural Networks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e05cf550191656c287184521d0ff011092f5ed6baa72e50f3a5bfd25d0f5986a/"}],"incoming":[{"source":"arxiv:2212.14034","target":"arxiv:1708.07120","evidence":"id","id":"arxiv:2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/"},{"source":"arxiv:2307.06440","target":"arxiv:1708.07120","evidence":"title","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"}],"text_parts":[{"start":0,"end":15808,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-1.html"},{"start":15808,"end":31057,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html"},{"start":31057,"end":44761,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-3.html"}],"contents":[{"title":"Super-Convergence: Very Fast Training of Neural Networks Using Large Learning Rates","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-1.html#text"},{"title":"Abstract","offset":87,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-1.html#text"},{"title":"1 Introduction","offset":1264,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-1.html#text"},{"title":"2 Background","offset":3609,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-1.html#text"},{"title":"3 Super-convergence","offset":6185,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-1.html#text"},{"title":"4 Estimating optimal learning rates","offset":9979,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-1.html#text"},{"title":"5 Experiments and analysis","offset":16178,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"5.1 Other datasets and architectures","offset":18861,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"5.2 Imagenet","offset":21448,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"6 Conclusion","offset":24361,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"Appendix A Supplemental material","offset":25340,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"A.1 Intuitive explanation for super-convergence","offset":25730,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"A.2 Large learning rate regularization","offset":27183,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"A.3 Relationship of super-convergence to SGD and generalization","offset":28691,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-2.html#text"},{"title":"A.4 Hardware, software, and architectures","offset":36247,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-3.html#text"},{"title":"A.5 Hyper-parameters and adaptive learning rate methods","offset":38181,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-3.html#text"},{"title":"A.6 Experimental details and additional results","offset":41063,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-3.html#text"},{"title":"Instructions for reporting errors","offset":43797,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/part-3.html#text"}],"text_code_points":44761,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/source.md"}