{"id":"arxiv:2101.02235","title":"Did Aristotle Use a Laptop? A Question Answering Benchmark with Implicit Reasoning Strategies","date":"2021-01-06","section":"Retrieval, embeddings, benchmarks","quality":0.058238,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/","source_url":"https://arxiv.org/abs/2101.02235","text_available":true,"authors":["Mor Geva","Daniel Khashabi","Elad Segal","Tushar Khot"],"outgoing":[],"incoming":[{"source":"arxiv:2203.11171","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2203.11171","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02a15b098d96a94d463bb470f59a34723abc34e59196f35c539f9796d8f3c3df/"},{"source":"arxiv:2206.04615","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/"},{"source":"arxiv:2212.14024","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2212.14024","title":"Demonstrate-Search-Predict: Composing retrieval and language models for knowledge-intensive NLP","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/"},{"source":"arxiv:2309.14402","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/"},{"source":"arxiv:2309.16797","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2309.16797","title":"Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/daac3bcfcd7f39084d995905e9ef96ae01c5aab7bfe8e565bd7622b4e44088f7/"},{"source":"arxiv:2510.26622","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2510.26622","title":"Encoder-Decoder or Decoder-Only? Revisiting Encoder-Decoder Large Language Model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dc73f87f798d016e2d83acd1d362b4b3525af826e68b055bb5196f120d23e080/"},{"source":"arxiv:2609.01117","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2609.01117","title":"Latent Recurrent Thoughts: Recurrent Refinement of Proposed Latents for Reasoning with Frozen LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e9d4f61afc3930fa770d5ea53d3f43a5157fa4904fcdfaa514786e617905e20b/"}],"text_parts":[{"start":0,"end":15668,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html"},{"start":15668,"end":31216,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html"},{"start":31216,"end":44820,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html"}],"contents":[{"title":"Did Aristotle Use a Laptop? A Question Answering Benchmark with Implicit Reasoning Strategies","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Abstract","offset":97,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"1 Introduction","offset":1354,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"2 Strategy Questions","offset":6011,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"2.1 Desiderata","offset":6036,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Multi-step","offset":6381,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Feasible","offset":6935,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Implicit","offset":7634,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Definite","offset":8592,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"2.2 Decomposing Strategy Questions","offset":9599,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"3 Data Collection Pipeline","offset":12371,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"3.1 Creative Question Writing (CQW)","offset":12969,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Task description","offset":14564,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Priming with Wikipedia terms","offset":14808,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Controlling the answer distribution","offset":15341,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Model-in-the-loop filtering","offset":15634,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-1.html#text"},{"title":"Auxiliary sub-task","offset":16333,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"3.2 Strategy Question Decomposition (SQD)","offset":16682,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Task description","offset":17534,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Auxiliary sub-task","offset":17684,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"3.3 Evidence Matching (EVM)","offset":18263,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Task description","offset":18436,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Controlling the matched context","offset":18693,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"3.4 Data Verification Mechanisms","offset":19096,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Task qualifications","offset":19134,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Real-time automatic checks","offset":19550,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Inter-task feedback","offset":20191,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Evidence verification task","offset":20642,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"4 The StrategyQA Dataset","offset":21279,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"4.1 Dataset Statistics","offset":22399,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"4.2 Data Quality","offset":23578,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Do questions in StrategyQA require multi-step implicit reasoning?","offset":23600,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Do questions in StrategyQA have a definitive answer?","offset":24050,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"What is the quality of the decompositions?","offset":24584,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Would different annotators use the same decomposition strategy?","offset":25399,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Is the evidence for strategy questions in Wikipedia?","offset":25981,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Do matched paragraphs provide evidence?","offset":26935,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Do different annotators match the same evidence paragraphs?","offset":28912,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"4.3 Data Diversity","offset":29991,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Reasoning skills","offset":30154,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"Question topics","offset":31194,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-2.html#text"},{"title":"4.4 Human Performance","offset":32450,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"5 Experimental Evaluation","offset":33624,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"5.1 Baseline Models","offset":33941,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Predicting decompositions","offset":35609,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Baseline models","offset":35860,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Online solvers","offset":37578,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"5.2 Results","offset":37776,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Strategy QA performance","offset":37793,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Retrieval evaluation","offset":39334,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Human retrieval performance","offset":41229,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Predicted decompositions","offset":41916,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"6 Related Work","offset":42451,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"7 Conclusion","offset":43737,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"},{"title":"Acknowledgement","offset":44232,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/part-3.html#text"}],"text_code_points":44820,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/source.md"}