{"id":"arxiv:2208.10442","title":"Image as a Foreign Language: BEiT Pretraining for All Vision and Vision-Language Tasks","date":"2022-08-22","section":"Self-supervised learning and vision","quality":0.235929,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/","source_url":"https://arxiv.org/abs/2208.10442","text_available":true,"authors":["Wenhui Wang","Hangbo Bao","Li Dong","Johan Bjorck"],"outgoing":[],"incoming":[{"source":"arxiv:2302.14045","target":"arxiv:2208.10442","evidence":"title","id":"arxiv:2302.14045","title":"Language Is Not All You Need: Aligning Perception with Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fc11ce0896e97836324fcdf28bbcaf2c69f0e5e408a67265741605e450246035/"},{"source":"arxiv:2304.12210","target":"arxiv:2208.10442","evidence":"id","id":"arxiv:2304.12210","title":"A Cookbook of Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/"}],"text_parts":[{"start":0,"end":15582,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html"},{"start":15582,"end":23578,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html"}],"contents":[{"title":"Image as a Foreign Language: BEiT Pretraining for All Vision and Vision-Language Tasks","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Abstract","offset":90,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"1 Introduction: The Big Convergence","offset":1117,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"2 BEiT-3 : A General-Purpose Multimodal Foundation Model","offset":5822,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"2.1 Backbone Network: Multiway Transformers","offset":6116,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"2.2 Pretraining Task: Masked Data Modeling","offset":7444,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"2.3 Scaling Up: BEiT-3 Pretraining","offset":9088,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Backbone Network","offset":9128,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Pretraining Data","offset":10015,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Pretraining Settings","offset":10617,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"3 Experiments on Vision and Vision-Language Tasks","offset":11927,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"3.1 Vision-Language Downstream Tasks","offset":12222,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Visual Question Answering (VQA)","offset":12516,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Visual Reasoning","offset":13454,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Image Captioning","offset":14218,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"Image-Text Retrieval","offset":15555,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-1.html#text"},{"title":"3.2 Vision Downstream Tasks","offset":17563,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"Object Detection and Instance Segmentation","offset":17944,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"Semantic Segmentation","offset":18800,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"Image Classification","offset":19514,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"4 Conclusion","offset":20525,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"Appendix A Effects of Intermediate Finetuning for Retrieval","offset":21632,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"Appendix B Hyperparameters Used for Pretraining","offset":22511,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"Appendix C Hyperparameters Used for Finetuning","offset":22563,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"},{"title":"Instructions for reporting errors","offset":22614,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/part-2.html#text"}],"text_code_points":23578,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/source.md"}