{"id":"arxiv:2402.12483","title":"Artifacts or Abduction: How Do LLMs Answer Multiple-Choice Questions Without the Question?","date":"2024-02-19","section":"Retrieval, embeddings, benchmarks","quality":-0.037184,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/","source_url":"https://arxiv.org/abs/2402.12483","text_available":true,"authors":["Nishant Balepur","Abhilasha Ravichander","Rachel Rudinger"],"outgoing":[],"incoming":[],"text_parts":[{"start":0,"end":15689,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html"},{"start":15689,"end":31503,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html"},{"start":31503,"end":47390,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html"},{"start":47390,"end":49037,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-4.html"}],"contents":[{"title":"Artifacts or Abduction: How Do LLMs Answer Multiple-Choice Questions Without the Question?","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"Abstract","offset":94,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"1 Introduction","offset":1503,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"2 Experimental Setup","offset":7904,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"2.1 MCQA Task Definition","offset":7929,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"2.2 Models","offset":9070,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"2.3 Datasets","offset":9543,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"2.4 Evaluation Protocol","offset":10432,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"3 Performing MCQA With No Question","offset":11319,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"3.1 Prompt Design","offset":12093,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"3.2 Results","offset":12609,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"4 Hypothesis 1: Memorization","offset":14362,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"4.1 Prompt Design","offset":14809,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"4.2 Results","offset":15672,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-1.html#text"},{"title":"5 Hypothesis 2: Choice Dynamics","offset":16057,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"5.1 Prompt Design","offset":17690,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"5.2 Converting Individual Scores","offset":18809,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"5.3 Results","offset":20845,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"6 Hypothesis 3: Question Inference","offset":22365,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"6.1 Prompt Design","offset":22827,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"6.2 Quantitative Results","offset":24038,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"6.3 Qualitative Analysis","offset":25789,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"7 Related Work","offset":29755,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"7.1 Dataset Artifacts","offset":29774,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"7.2 MCQA Decision Making","offset":31331,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-2.html#text"},{"title":"8 Conclusion","offset":32691,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"9 Limitations","offset":33739,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"10 Ethical Considerations","offset":35311,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"11 Acknowledgements","offset":36097,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"Appendix A Experimental Setup","offset":36902,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"A.1 Dataset Details","offset":36936,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"A.2 LLM Inference","offset":37349,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"A.3 Prompt Creation","offset":38131,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"A.4 Prompt Box Example","offset":39188,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"A.5 Output Validity Statistics","offset":39875,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"A.6 Metrics","offset":40691,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"Appendix B Detailed Results","offset":40893,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.1 Scaling Laws","offset":40925,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.2 Decoding Strategies","offset":41277,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.3 Partial-input Models vs Prompting","offset":42138,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.4 Falcon Individual Evaluation","offset":43580,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.5 One-Step Question Inference","offset":44017,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.6 Qualitative Analysis","offset":44478,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.6.1 Annotation Procedure Details","offset":44714,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"B.6.2 Additional Results","offset":46026,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"Appendix C Related Work on MCQA Evaluations","offset":47342,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-3.html#text"},{"title":"Instructions for reporting errors","offset":48073,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/part-4.html#text"}],"text_code_points":49037,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1b9723010e10f9e91a6a9cc14c735ef6b1ff56c29c55c1204eeb18b6a43b7f15/source.md"}