{"id":"arxiv:2212.14024","title":"Demonstrate-Search-Predict: Composing retrieval and language models for knowledge-intensive NLP","date":"2022-12-28","section":"Retrieval, embeddings, benchmarks","quality":0.294453,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/","source_url":"https://arxiv.org/abs/2212.14024","text_available":true,"authors":["Omar Khattab","Keshav Santhanam","Xiang Lisa Li","David Hall"],"outgoing":[{"source":"arxiv:2212.14024","target":"arxiv:2101.02235","evidence":"title","id":"arxiv:2101.02235","title":"Did Aristotle Use a Laptop? A Question Answering Benchmark with Implicit Reasoning Strategies","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5b44455f95aaaeefc838faceeaf20aaf1008889ca2e479d95291e245d5365a36/"},{"source":"arxiv:2212.14024","target":"arxiv:2203.11171","evidence":"id","id":"arxiv:2203.11171","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02a15b098d96a94d463bb470f59a34723abc34e59196f35c539f9796d8f3c3df/"}],"incoming":[{"source":"arxiv:2507.19457","target":"arxiv:2212.14024","evidence":"id","id":"arxiv:2507.19457","title":"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c1ef72d96fa3292590a6a3d112ab509840081b59aaa771cf42b8b0e8ef62ba08/"}],"text_parts":[{"start":0,"end":15564,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html"},{"start":15564,"end":31537,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html"},{"start":31537,"end":47217,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html"},{"start":47217,"end":49475,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-4.html"}],"contents":[{"title":"Demonstrate – Search – Predict : Composing retrieval and language models for knowledge-intensive NLP","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Abstract","offset":104,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Keywords:","offset":1367,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"1 Introduction","offset":1385,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"2 Demonstrate – Search – Predict","offset":6918,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"2.1 Pretrained Modules: LM and RM","offset":7494,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Language Model","offset":7633,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Retrieval Model","offset":8398,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"2.2 Datatypes and Control Flow","offset":9463,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"The Example Datatype","offset":9835,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"A DSP Program","offset":11075,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Transformations","offset":11645,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"2.3 Demonstrate","offset":12522,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Bootstrapping Demonstrations","offset":13037,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Case Study","offset":14371,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Discussion","offset":15547,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-1.html#text"},{"title":"Selecting Demonstrations","offset":17813,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Compositions & Extensions","offset":18766,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"2.4 Search","offset":19653,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Search Strategies","offset":20540,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Case Study","offset":21972,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Comparison with self-ask","offset":23667,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Fusing Retrieval Results","offset":25741,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Compositions & Extensions","offset":26626,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"2.5 Predict","offset":27237,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Generating Candidates","offset":27694,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Selecting Predictions","offset":29059,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"Aggregating Information","offset":31168,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-2.html#text"},{"title":"3 Evaluation","offset":32634,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"3.1 Evaluation Methodology","offset":33359,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"3.2 Pretrained Modules","offset":34847,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"RM","offset":34875,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"LM","offset":35687,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"3.3 Baselines","offset":35981,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"Vanilla LM","offset":36000,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"Retrieve-then-Read","offset":36505,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"Self-ask","offset":37083,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"3.4 Proposed DSP Programs","offset":38463,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"3.5 Development Datasets & Results","offset":40814,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"Open-SQuAD","offset":40854,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"HotPotQA","offset":43671,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"QReCC","offset":45317,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"4 Conclusion","offset":46737,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-3.html#text"},{"title":"Acknowledgements","offset":48552,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/part-4.html#text"}],"text_code_points":49475,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/09c4c9bae4e016c9ce276cf84a46742b5b21168f4181202b35c207baf887f71e/source.md"}