{"id":"arxiv:2302.00923","title":"Multimodal Chain-of-Thought Reasoning in Language Models","date":"2023-02-02","section":"Reasoning and the \"physics\" of language models","quality":0.040672,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/","source_url":"https://arxiv.org/abs/2302.00923","text_available":true,"authors":["Zhuosheng Zhang","Aston Zhang","Mu Li","Hai Zhao"],"outgoing":[{"source":"arxiv:2302.00923","target":"arxiv:2203.11171","evidence":"title","id":"arxiv:2203.11171","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02a15b098d96a94d463bb470f59a34723abc34e59196f35c539f9796d8f3c3df/"}],"incoming":[{"source":"arxiv:2605.07654","target":"arxiv:2302.00923","evidence":"title","id":"arxiv:2605.07654","title":"Reliable Chain-of-Thought via Prefix Consistency","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/715ed2b4b2f355b771111eb0e65e113f100dc33739423a76307e8ece18b57d18/"}],"text_parts":[{"start":0,"end":15995,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html"},{"start":15995,"end":31801,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html"},{"start":31801,"end":47547,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html"}],"contents":[{"title":"Multimodal Chain-of-Thought Reasoning in Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"Abstract","offset":60,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"1 Introduction","offset":1162,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"2 Background","offset":5862,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"2.1 CoT Reasoning with LLMs","offset":5979,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"Optimizing Demonstrations","offset":7435,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"Optimizing Reasoning Chains","offset":8816,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"2.2 Eliciting CoT Reasoning by Fine-Tuning Models","offset":9910,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"3 Challenge of Multimodal-CoT","offset":11278,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"3.1 Towards the Role of CoT","offset":11876,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"3.2 Misleading by Hallucinated Rationales","offset":13779,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"3.3 Multimodality Contributes to Effective Rationales","offset":14953,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-1.html#text"},{"title":"4 Multimodal-CoT","offset":16619,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"4.1 Framework Overview","offset":16989,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"4.2 Model Architecture","offset":18866,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"Encoding","offset":19953,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"Interaction","offset":21766,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"Decoding","offset":23214,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"5 Experiments","offset":23345,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"5.1 Dataset","offset":23537,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"5.2 Implementation","offset":24510,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"Experimental Settings","offset":24633,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"Baseline Models","offset":25405,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"5.3 Main Results","offset":27564,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"6 Analysis","offset":28711,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"6.1 Multimodality Boosts Convergence","offset":29187,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"6.2 When Multimodal-CoT Meets Large Models","offset":29996,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"6.3 Effectiveness Across Backbones","offset":31761,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-2.html#text"},{"title":"6.4 Using Different Vision Features","offset":32046,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"6.5 Alignment Strategies for Multimodal Interaction","offset":32826,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"6.6 Generalization to Other Multimodal Reasoning Benchmarks","offset":33590,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"6.7 Error Analysis","offset":34138,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"7 Conclusion","offset":35604,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"Appendix A Extended Analysis for the Challenge of Multimodal-CoT","offset":36359,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"A.1 Additional Examples of Misleading through Hallucinated Rationales","offset":36428,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"A.2 Two-Stage Training Performance with Different Sizes of LMs","offset":36727,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"A.3 Discussion of the Possible Paradigms to Achieve Multimodal-CoT","offset":37749,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"Appendix B Experimental Details","offset":40461,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"B.1 Details of Vision Features","offset":40497,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"B.2 Datasets","offset":41389,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"B.3 Implementation Details of Multimodal-CoT","offset":42416,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"Appendix C Further Analysis","offset":43591,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"C.1 Examples of Rationale Generation with Large Models","offset":43623,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"C.2 Detailed Results of Multimodal-CoT on Different Backbone Models","offset":44718,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"Appendix D Examples of Case Studies","offset":45052,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"},{"title":"Instructions for reporting errors","offset":46583,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/part-3.html#text"}],"text_code_points":47547,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0fe1320410a5c35c29f2ac16d69a70adbf3c490670902745c2b4faf88460b5c7/source.md"}