{"id":"arxiv:2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","date":"2022-06-09","section":"Retrieval, embeddings, benchmarks","quality":0.491414,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/","source_url":"https://arxiv.org/abs/2206.04615","text_available":true,"authors":["Aarohi Srivastava","Abhinav Rastogi","Abhishek Rao","Abu Awal Md Shoeb"],"outgoing":[{"source":"arxiv:2206.04615","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2206.04615","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2101.02235","title":"Did Aristotle Use a Laptop? A Question Answering Benchmark with Implicit Reasoning Strategies","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/"}],"incoming":[{"source":"arxiv:2212.09196","target":"arxiv:2206.04615","evidence":"id","id":"arxiv:2212.09196","title":"Emergent Analogical Reasoning in Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f74e529a278203e2c144ab7400a343836d9549cf1ae4843c584408448ad3f5a9/"},{"source":"arxiv:2301.06627","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2301.06627","title":"Dissociating language and thought in large language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3791d7ed420be517ded1ee7ae5558cf6afa948ea5b841a76ec3da8a9134ec565/"},{"source":"arxiv:2304.15004","target":"arxiv:2206.04615","evidence":"id","id":"arxiv:2304.15004","title":"Are Emergent Abilities of Large Language Models a Mirage?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7cd8c081ad54f2fdaa2211a4b49dc21f6ff2db7282e287546d210ef90e84323/"},{"source":"arxiv:2311.02462","target":"arxiv:2206.04615","evidence":"id","id":"arxiv:2311.02462","title":"Levels of AGI for Operationalizing Progress on the Path to AGI","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/288b98572ecf9c0b81bcb6d0c89b523e3b71a5b3623301b9e24536f615a07bca/"},{"source":"arxiv:2405.07987","target":"arxiv:2206.04615","evidence":"id","id":"arxiv:2405.07987","title":"The Platonic Representation Hypothesis","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/"},{"source":"arxiv:2405.14838","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2405.14838","title":"From Explicit CoT to Implicit CoT: Learning to Internalize CoT Step by Step","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a242b6148f6f84466ce4315ac4937278586a242dd3f847146e142f3c7e27c39c/"},{"source":"arxiv:2405.20541","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2405.20541","title":"Perplexed by Perplexity: Perplexity-Based Data Pruning With Small Reference Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/"},{"source":"arxiv:2501.19393","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2501.19393","title":"s1: Simple test-time scaling","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2a88818f7605b71cd0160831faebd51b3274c24ffd29626b4d519390b4bffffe/"},{"source":"arxiv:2503.14858","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2503.14858","title":"1000 Layer Networks for Self-Supervised RL: Scaling Depth Can Enable New Goal-Reaching Capabilities","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/"},{"source":"arxiv:2510.13786","target":"arxiv:2206.04615","evidence":"id","id":"arxiv:2510.13786","title":"The Art of Scaling Reinforcement Learning Compute for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/72580bcba3d9b32b2f3e83d074f626234b26dc18126edd22a5eacef927cf87da/"},{"source":"arxiv:2510.26622","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2510.26622","title":"Encoder-Decoder or Decoder-Only? Revisiting Encoder-Decoder Large Language Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dc73f87f798d016e2d83acd1d362b4b3525af826e68b055bb5196f120d23e080/"},{"source":"arxiv:2606.18543","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2606.18543","title":"CEO-Bench: Can Agents Play the Long Game?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/696d7ac1dfd7539fbe8147ed713574b8eb5a7400a35e96805c7f0c9add99165c/"},{"source":"doi:10.1073/pnas.2520095123","target":"arxiv:2206.04615","evidence":"id","id":"doi:10.1073/pnas.2520095123","title":"Evidence from formal logical reasoning reveals that the language of thought is not natural language","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ba9811c4b4f1ea1fa65751f033bad4c2f6543f454b4d0d9e91a52abc6eb407/"}],"text_parts":[{"start":0,"end":15672,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html"},{"start":15672,"end":31339,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html"},{"start":31339,"end":47235,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html"},{"start":47235,"end":62890,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html"},{"start":62890,"end":78787,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-5.html"},{"start":78787,"end":94540,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-6.html"},{"start":94540,"end":110521,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-7.html"},{"start":110521,"end":126352,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-8.html"},{"start":126352,"end":135608,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html"}],"contents":[{"title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"Abstract","offset":96,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"1 Introduction","offset":1968,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"1.1 Quantity has a quality all its own","offset":3330,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"1.2 Limitations of current benchmarks","offset":6159,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"1.3 Beyond the imitation game","offset":8574,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"2 What is in BIG-bench?","offset":10598,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"2.1 The BIG-bench API","offset":12390,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"2.2 BIG-bench Lite","offset":15648,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-1.html#text"},{"title":"2.3 Evaluation targets","offset":16754,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"2.3.1 Language models evaluated on BIG-bench","offset":16782,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"BIG-G.","offset":18068,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"BIG-G sparse.","offset":18754,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"GPT.","offset":19601,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"2.3.2 Human rater baseline performance","offset":20154,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"2.4 Include a canary string in all documents about BIG-bench","offset":22229,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"3 Behavior of language models and human raters on BIG-bench","offset":22862,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"3.1 Aggregate performance improves with model size but is worse than human performance","offset":22926,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"3.2 Model predictions grow better calibrated with increased scale","offset":24711,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"3.3 Model classes behave similarly, with benefits from sparsity","offset":27229,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"3.4 Linearity and breakthroughness: categorizing scaling behaviors","offset":28959,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"3.4.1 Properties of tasks with high linearity or breakthroughness","offset":29744,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"3.4.2 Breakthrough behavior is sensitive to details of task specification","offset":31259,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-2.html#text"},{"title":"Using smoother metrics.","offset":32344,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"Manual decomposition into subtasks.","offset":33065,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"3.4.3 Even programmatic measures of model capability can be highly subjective","offset":35071,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"3.5 Even large language models are brittle","offset":35815,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"3.6 Social bias in large language models","offset":40217,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"What is meant by social bias?","offset":41044,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"Bias typically increases with scale in settings with broad or ambiguous contexts.","offset":41463,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"Bias can decrease with scale in settings with narrow, unambiguous contexts.","offset":42646,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"Bias can potentially be steered through appropriately chosen prompting.","offset":43541,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"3.7 Performance on non-English languages","offset":44768,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"Performance on non-English tasks is worse than on English tasks.","offset":46058,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"Low-resource language tasks are particularly challenging.","offset":46634,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-3.html#text"},{"title":"4 Behavior on selected tasks","offset":48662,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"4.1 Checkmate-in-one task","offset":48981,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"4.2 Periodic elements task","offset":52267,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"Zero-shot behavior across scales:","offset":53227,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"One-shot behavior across scales:","offset":54333,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"5 Additional related work","offset":56398,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"6 Discussion","offset":58699,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"6.1 Overall Findings","offset":59935,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-4.html#text"},{"title":"6.2 Conclusion","offset":65421,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-5.html#text"},{"title":"Broader Impact Statement","offset":65989,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-5.html#text"},{"title":"7 Author contributions","offset":67609,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-5.html#text"},{"title":"7.1 Core contributors","offset":67636,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-5.html#text"},{"title":"7.2 Task authors, and task-specific citations","offset":73779,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-5.html#text"},{"title":"Acknowledgments","offset":126771,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"Appendix","offset":127238,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"Appendix A Model details","offset":127251,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"Appendix B Definitions of linearity and breakthroughness","offset":128946,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"Appendix C Additional evaluation results","offset":130641,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"C.1 Performance at different temperatures","offset":130686,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"C.2 Number of tasks that pass a score threshold","offset":131101,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"C.3 Social bias results per task","offset":131681,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"C.4 Metric dependent performance and breakthroughness","offset":131870,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"Appendix D Performance by keyword","offset":132635,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"D.1 Checkmate in one successful solutions","offset":133132,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"Appendix E Expert evaluation","offset":133299,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"},{"title":"Appendix F Additional Tasks","offset":134757,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/part-9.html#text"}],"text_code_points":135608,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/source.md"}