{"id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","date":"2020-02-12","section":"LLMs: architectures, context, training","quality":-0.585411,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/","source_url":"https://arxiv.org/abs/2002.05202","text_available":true,"authors":["Noam Shazeer"],"outgoing":[{"source":"arxiv:2002.05202","target":"arxiv:1710.05941","evidence":"id","id":"arxiv:1710.05941","title":"Searching for Activation Functions","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/"}],"incoming":[{"source":"arxiv:2304.07193","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/"},{"source":"arxiv:2307.06440","target":"arxiv:2002.05202","evidence":"title","id":"arxiv:2307.06440","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc9b59e6276d3fc1ebf8d77b30466c5047df876d0a884edfdfcdea124d6b76d6/"},{"source":"arxiv:2312.17742","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2312.17742","title":"Learning Vision from Models Rivals Learning Vision from Data","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/71dc1683019904602d643d16348a76b98c556b924ca964e3949d8d6420dc4402/"},{"source":"arxiv:2404.05405","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2404.05405","title":"Physics of Language Models: Part 3.3, Knowledge Capacity Scaling Laws","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/541683d3678177bb2800d37a49a2f541c8a2c81cf802eaefe8840fed359c540e/"},{"source":"arxiv:2405.18392","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"},{"source":"arxiv:2412.13663","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2412.13663","title":"Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ae6c000a40b751c4486e08fd61f5f7788d8bb57f0838e234e27a5a67f894e923/"},{"source":"arxiv:2501.00656","target":"arxiv:2002.05202","evidence":"title","id":"arxiv:2501.00656","title":"2 OLMo 2 Furious","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1ca88dba5d5c59e8cb23b2a304e74370b8ec69077b714a30d426300f7ef2a0b9/"},{"source":"arxiv:2502.05171","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2502.05171","title":"Scaling up Test-Time Compute with Latent Reasoning: A Recurrent Depth Approach","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/24a6ab865777290627cecebd3f06ea3daef01622f102d32a95f7fb2a5a42880f/"},{"source":"arxiv:2502.09992","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2502.09992","title":"Large Language Diffusion Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/"},{"source":"arxiv:2506.21734","target":"arxiv:2002.05202","evidence":"title","id":"arxiv:2506.21734","title":"Hierarchical Reasoning Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3604312101ff09ab9903e7d6a32c7b2b8a9a7ece6a9d145d04ea01c9d13ffa3a/"},{"source":"arxiv:2510.04871","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2510.04871","title":"Less is More: Recursive Reasoning with Tiny Networks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4345eebe9dc7e67f82617896772f55f0970971516995c1688b6812f9ebf054f6/"},{"source":"arxiv:2510.26622","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2510.26622","title":"Encoder-Decoder or Decoder-Only? Revisiting Encoder-Decoder Large Language Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dc73f87f798d016e2d83acd1d362b4b3525af826e68b055bb5196f120d23e080/"},{"source":"arxiv:2510.27688","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2510.27688","title":"Continuous Autoregressive Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/"},{"source":"arxiv:2512.14693","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2512.14693","title":"Universal Reasoning Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fcb1a389a132d64ca93fac5551d8f8b653c5e093908cb515e2a2d901524941a7/"},{"source":"arxiv:2512.24695","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/"},{"source":"arxiv:2605.19376","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2605.19376","title":"Generative Recursive Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/795bad9bb1b1a063090fbb925ba6e68bf30211de0ecffd75d56f858b28321ae7/"}],"text_parts":[{"start":0,"end":8834,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html"}],"contents":[{"title":"GLU Variants Improve Transformer","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"Abstract","offset":36,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"1 Introduction","offset":555,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"2 Gated Linear Units (GLU) and Variants","offset":2170,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"3 Experiments on Text-to-Text Transfer Transformer (T5)","offset":4683,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"3.1 Model Architecture","offset":5053,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"3.2 Pre-Training and Perplexity Results","offset":5673,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"3.3 Fine-Tuning","offset":6863,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"},{"title":"4 Conclusions","offset":8302,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/part-1.html#text"}],"text_code_points":8834,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/source.md"}