{"id":"arxiv:2302.14045","title":"Language Is Not All You Need: Aligning Perception with Language Models","date":"2023-02-27","section":"LLMs: architectures, context, training","quality":0.261634,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/","source_url":"https://arxiv.org/abs/2302.14045","text_available":true,"authors":["Shaohan Huang","Li Dong","Wenhui Wang","Yaru Hao"],"outgoing":[{"source":"arxiv:2302.14045","target":"arxiv:2208.10442","evidence":"title","id":"arxiv:2208.10442","title":"Image as a Foreign Language: BEiT Pretraining for All Vision and Vision-Language Tasks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/"}],"incoming":[],"text_parts":[{"start":0,"end":15833,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html"},{"start":15833,"end":31674,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html"},{"start":31674,"end":41312,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html"}],"contents":[{"title":"Language Is Not All You Need: Aligning Perception with Language Models Thanks: Equal contribution. † \\dagger Corresponding author.","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"Abstract","offset":134,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"1 Introduction: From LLMs to MLLMs","offset":1615,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"From LLMs to MLLMs.","offset":4407,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"Language models as general-purpose interfaces.","offset":5313,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"New capabilities of MLLMs.","offset":5935,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"2 Kosmos-1 : A Multimodal Large Language Model","offset":6482,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"2.1 Input Representation","offset":7307,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"2.2 Multimodal Large Language Models (MLLMs)","offset":8486,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"Magneto","offset":9670,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"xPos","offset":10132,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"2.3 Training Objective","offset":10546,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"3 Model Training","offset":11802,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"3.1 Multimodal Training Data","offset":11823,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"Text Corpora","offset":12024,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"Image-Caption Pairs","offset":12660,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"Interleaved Image-Text Data","offset":13125,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"3.2 Training Setup","offset":13844,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"3.3 Language-Only Instruction Tuning","offset":15235,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-1.html#text"},{"title":"4 Evaluation","offset":16506,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.1 Perception-Language Tasks","offset":17071,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.1.1 Evaluation Setup","offset":17524,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.1.2 Results","offset":18969,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"Image Captioning","offset":18989,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"Visual Question Answering","offset":19848,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.2 IQ Test: Nonverbal Reasoning","offset":20612,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.2.1 Evaluation Setup","offset":21484,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.2.2 Results","offset":22791,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.3 OCR-Free Language Understanding","offset":23526,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.3.1 Evaluation Setup","offset":24039,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.3.2 Results","offset":24612,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.4 Web Page Question Answering","offset":25134,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.4.1 Evaluation Setup","offset":25584,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.4.2 Results","offset":26457,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.5 Multimodal Chain-of-Thought Prompting","offset":26946,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.5.1 Evaluation Setup","offset":27607,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.5.2 Results","offset":28020,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.6 Zero-Shot Image Classification","offset":28417,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.6.1 Evaluation Setup","offset":28791,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.6.2 Results","offset":29352,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.7 Zero-Shot Image Classification with Descriptions","offset":29712,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.7.1 Evaluation Setup","offset":30233,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.7.2 Results","offset":31283,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.8 Language Tasks","offset":31650,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-2.html#text"},{"title":"4.8.1 Evaluation Setup","offset":31892,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"4.8.2 Results","offset":32608,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"4.9 Cross-modal Transfer","offset":33248,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"4.9.1 Transfer from Language to Multimodal: Language-Only Instruction Tuning","offset":33613,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"4.9.2 Transfer from Multimodal to Language: Visual Commonsense Reasoning","offset":34471,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"Evaluation Setup","offset":34995,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"Results","offset":35709,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"5 Conclusion","offset":36419,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"Appendix A Hyperparameters","offset":37136,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"A.1 Training","offset":37167,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"A.2 Language-Only Instruction Tuning","offset":37306,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"Appendix B Datasets","offset":37422,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"B.1 Pretraning","offset":37446,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"B.1.1 Text Corpora","offset":37466,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"B.1.2 Image-Caption Pairs","offset":38174,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"B.1.3 Interleaved Data","offset":38981,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"B.2 Data Format","offset":39703,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"Appendix C Evaluation","offset":39782,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"C.1 Input Format Used for Perception-Language Tasks","offset":39808,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"C.2 Language Tasks","offset":39961,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"C.3 WebSRC Task Examples","offset":40303,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"},{"title":"Instructions for reporting errors","offset":40348,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/part-3.html#text"}],"text_code_points":41312,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/source.md"}