{"id":"arxiv:2304.15004","title":"Are Emergent Abilities of Large Language Models a Mirage?","date":"2023-04-28","section":"Reasoning and the \"physics\" of language models","quality":0.369634,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/","source_url":"https://arxiv.org/abs/2304.15004","text_available":true,"authors":["Rylan Schaeffer","Brando Miranda","Sanmi Koyejo"],"outgoing":[{"source":"arxiv:2304.15004","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2304.15004","target":"arxiv:2206.04615","evidence":"id","id":"arxiv:2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/"}],"incoming":[{"source":"arxiv:2311.16452","target":"arxiv:2304.15004","evidence":"title","id":"arxiv:2311.16452","title":"Can Generalist Foundation Models Outcompete Special-Purpose Tuning? Case Study in Medicine","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8e8303d0328f412b165c8cdfca91debf742ea45ed7865a0d96ee3905e5bba919/"},{"source":"arxiv:2406.11813","target":"arxiv:2304.15004","evidence":"title","id":"arxiv:2406.11813","title":"How Do Large Language Models Acquire Factual Knowledge During Pretraining?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a7b36ce24eb2182e7f0bba96c0cddcfe6efe610b830d5e8ad8aac2b7d14696c0/"},{"source":"arxiv:2503.21676","target":"arxiv:2304.15004","evidence":"title","id":"arxiv:2503.21676","title":"How do language models learn facts? Dynamics, curricula and hallucinations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ef59e7bcfcfc0e4bab638835504ad8a0d2f50665a9797edc09e8628e9b3c0b6/"},{"source":"arxiv:2512.13961","target":"arxiv:2304.15004","evidence":"title","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"},{"source":"arxiv:2512.24695","target":"arxiv:2304.15004","evidence":"title","id":"arxiv:2512.24695","title":"Nested Learning: The Illusion of Deep Learning Architectures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d19bf2367894704a701d0741e7db6d8b148719af1d634b55f5797aab5f617d06/"},{"source":"arxiv:2606.25010","target":"arxiv:2304.15004","evidence":"id","id":"arxiv:2606.25010","title":"Emergent Capabilities Arise Randomly from Learning Sparse Attention Patterns","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f87db387bad54ae4fbc0589d2a702f78aeb0817b28f20844f6502bc552a89c2f/"}],"text_parts":[{"start":0,"end":15285,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html"},{"start":15285,"end":31001,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html"},{"start":31001,"end":32843,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-3.html"}],"contents":[{"title":"Are Emergent Abilities of Large Language Models a Mirage?","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"Abstract","offset":61,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"1 Introduction","offset":1669,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"2 Alternative Explanation for Emergent Abilities","offset":6209,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"3 Analyzing InstructGPT/GPT-3’s Emergent Arithmetic Abilities","offset":10860,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"Prediction: Emergent Abilities Disappear With Different Metrics","offset":12357,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"Prediction: Emergent Abilities Disappear With Better Statistics","offset":13262,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"4 Meta-Analysis of Claimed Emergent Abilities","offset":14221,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"Prediction: Emergent Abilities Should Appear with Metrics, not Task-Model Families","offset":15196,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-1.html#text"},{"title":"Prediction: Changing Metric Removes Emergent Abilities","offset":17269,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"5 Inducing Emergent Abilities in Networks on Vision Tasks","offset":18177,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"Emergent Reconstruction of CIFAR100 Natural Images by Nonlinear Autoencoders","offset":18734,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"Emergent Classification of Omniglot Characters by Autoregressive Transformers","offset":20079,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"6 Related Work","offset":20876,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"7 Discussion","offset":22099,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"Appendix A Approximate Behavior of Metrics on Sequential Data","offset":23918,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"A.1 Per-Token Error Probability is Resolution-Limited","offset":24804,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"A.2 Token Edit Distance","offset":25856,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"A.3 Accuracy","offset":27999,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"A.4 ROUGE-L-Sum","offset":29734,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"Appendix B Inducing Emergent Abilities in Networks on Vision Tasks","offset":30843,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"B.1 Emergent Classification of MNIST Handwritten Digits by Convolutional Networks","offset":30914,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-2.html#text"},{"title":"Instructions for reporting errors","offset":31879,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/part-3.html#text"}],"text_code_points":32843,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/source.md"}