{"id":"arxiv:2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","date":"2022-12-28","section":"Data, training, optimization","quality":-0.087484,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/","source_url":"https://arxiv.org/abs/2212.14034","text_available":true,"authors":["Jonas Geiping","Tom Goldstein"],"outgoing":[{"source":"arxiv:2212.14034","target":"arxiv:1708.07120","evidence":"id","id":"arxiv:1708.07120","title":"Super-Convergence: Very Fast Training of Neural Networks Using Large Learning Rates","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/604e53f71d7e9128110137811c408ab3b0c041d945f88d001f5095edbfb1a875/"},{"source":"arxiv:2212.14034","target":"arxiv:1904.00962","evidence":"title","id":"arxiv:1904.00962","title":"Large Batch Optimization for Deep Learning: Training BERT in 76 minutes","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2cea31116143e748e7497985367ddcf86a3da4aadb24ea47451b55947eedc9f0/"},{"source":"arxiv:2212.14034","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"}],"incoming":[{"source":"arxiv:2307.06440","target":"arxiv:2212.14034","evidence":"id","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2412.13663","target":"arxiv:2212.14034","evidence":"title","id":"arxiv:2412.13663","title":"Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/"},{"source":"arxiv:2502.05171","target":"arxiv:2212.14034","evidence":"title","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"}],"text_parts":[{"start":0,"end":15788,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html"},{"start":15788,"end":31071,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html"},{"start":31071,"end":43663,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html"}],"contents":[{"title":"Cramming: Training a Language Model on a Single GPU in One Day","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"Abstract","offset":66,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"1 Scaling Up and Scaling Down","offset":1284,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"2 Tying our hands behind our back: A setup with limited compute","offset":6077,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"3 Related Work on Efficient Transformers","offset":9398,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"How long does it take to train BERT?","offset":9443,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"Studies of Efficient Transformers","offset":11512,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"Scaling Laws","offset":13562,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"4 Investigations","offset":14424,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"4.1 Implementation Details","offset":14799,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"Initial Data Setup","offset":15762,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-1.html#text"},{"title":"4.2 Modifying the Architecture","offset":17536,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Scaling laws hold in the low-resource regime","offset":18542,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Exploiting the scaling law.","offset":20502,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Attention Block:","offset":21589,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Feedforward Block:","offset":23120,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Embedding:","offset":23901,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Layer Structure:","offset":24355,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Head Block:","offset":25061,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"4.3 Modifying the Training Setup","offset":25359,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Objective:","offset":25634,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Choice of Optimizer:","offset":26214,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Learning Rate Schedule and Peak:","offset":27101,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Batch Size Schedule:","offset":27692,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Dropping Dropout","offset":28944,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"4.4 Optimizing the Dataset","offset":29976,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-2.html#text"},{"title":"Vocabulary Size","offset":32710,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"5 Finetuning Performance on GLUE","offset":33566,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"5.1 Ablation - Which Changes Really Mattered?","offset":36653,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"5.2 What Happens When Training Longer?","offset":37443,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"6 Limitations","offset":38443,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"7 Conclusions","offset":39248,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"Reproducibility Statement","offset":39974,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"Appendix","offset":40098,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"Appendix A Other Modifications","offset":40111,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"Appendix B Additional Information","offset":40680,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"B.1 References for Table 1","offset":41098,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"},{"title":"Instructions for reporting errors","offset":42699,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/part-3.html#text"}],"text_code_points":43663,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/source.md"}