{"id":"arxiv:2412.13663","title":"Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference","date":"2024-12-18","section":"LLMs: architectures, context, training","quality":0.288117,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/","source_url":"https://arxiv.org/abs/2412.13663","text_available":true,"authors":["Benjamin Warner","Antoine Chaffin","Benjamin Clavié","Orion Weller"],"outgoing":[{"source":"arxiv:2412.13663","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2412.13663","target":"arxiv:2212.14034","evidence":"title","id":"arxiv:2212.14034","title":"Cramming: Training a Language Model on a Single GPU in One Day","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/cb243af26cd29bcc3d04b00cc68393b3c4b3dfc535c3dd228b44f12e5df26c46/"},{"source":"arxiv:2412.13663","target":"arxiv:2405.18392","evidence":"id","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"},{"source":"arxiv:2412.13663","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[],"text_parts":[{"start":0,"end":15971,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html"},{"start":15971,"end":31916,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html"},{"start":31916,"end":46866,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html"},{"start":46866,"end":49894,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-4.html"}],"contents":[{"title":"Smarter, Better, Faster, Longer : A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"Abstract","offset":138,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"1 Introduction","offset":1047,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2 Methods","offset":5498,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2.1 Architectural Improvements","offset":5512,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2.1.1 Modern Transformer","offset":5991,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2.1.2 Efficiency Improvements","offset":7685,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2.1.3 Model Design","offset":9919,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2.2 Training","offset":11564,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2.2.1 Data","offset":11582,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"2.2.2 Training Settings","offset":12920,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-1.html#text"},{"title":"3 Downstream Evaluation","offset":16944,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"3.1 Evaluation Setting","offset":17762,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"3.1.1 Natural Language Understanding","offset":17790,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"3.1.2 Text Retrieval","offset":19139,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"3.1.3 Long-Context Text Retrieval","offset":22039,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"3.1.4 Code Retrieval","offset":23220,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"3.2 Downstream Results and Discussion","offset":24882,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"4 Efficiency","offset":28665,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"4.1 Evaluation Setting","offset":28682,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"4.2 Results","offset":29984,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"5 Conclusion","offset":31593,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-2.html#text"},{"title":"6 Limitations","offset":33289,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"7 Acknowledgements","offset":34662,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"8 Contribution Statement","offset":35780,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"Appendix A Training Settings","offset":37835,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"A.1 Batch Size Schedule","offset":38223,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"A.2 Weight Tiling","offset":38972,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"A.3 Weight Decay","offset":39666,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"A.4 Final Checkpoints","offset":39892,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"Appendix B Model Design","offset":40543,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"Appendix C Training Log","offset":41615,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"C.1 Sampling Issue","offset":41643,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"C.2 Large Rollback","offset":42444,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"Appendix D Architecture ablations","offset":43229,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"Appendix E Extended results","offset":45839,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"E.1 Full GLUE results","offset":45871,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"E.2 Full BEIR results","offset":46839,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-3.html#text"},{"title":"Appendix F Efficiency","offset":47921,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-4.html#text"},{"title":"Appendix G Licensing","offset":48791,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-4.html#text"},{"title":"Instructions for reporting errors","offset":48930,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/part-4.html#text"}],"text_code_points":49894,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/source.md"}