{"id":"arxiv:2311.06242","title":"Florence-2: Advancing a Unified Representation for a Variety of Vision Tasks","date":"2023-11-10","section":"LLMs: architectures, context, training","quality":0.228032,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/","source_url":"https://arxiv.org/abs/2311.06242","text_available":true,"authors":["Bin Xiao","Haiping Wu","Weijian Xu","Xiyang Dai"],"outgoing":[{"source":"arxiv:2311.06242","target":"arxiv:2006.09882","evidence":"title","id":"arxiv:2006.09882","title":"Unsupervised Learning of Visual Features by Contrasting Cluster Assignments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6601018611a96dff6f5324f2e4f39f62484917cf4d8f8a8d5baa7f4be1deb433/"}],"incoming":[{"source":"arxiv:2407.21783","target":"arxiv:2311.06242","evidence":"title","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"text_parts":[{"start":0,"end":15518,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html"},{"start":15518,"end":31468,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html"},{"start":31468,"end":47034,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html"},{"start":47034,"end":62124,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html"},{"start":62124,"end":68820,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-5.html"}],"contents":[{"title":"Florence-2: Advancing a Unified Representation for a Variety of Vision Tasks","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"Abstract","offset":80,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"1 Introduction","offset":1253,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"2 Rethinking Vision Model Pre-training","offset":7667,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"2.1 Comprehensive Multitask Learning","offset":9122,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"3 Model","offset":11156,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"Task formulation.","offset":11972,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"Vision encoder.","offset":13946,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"Multi-modality encoder decoder.","offset":14365,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"Optimization objective.","offset":15105,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"4 Data Engine","offset":15500,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-1.html#text"},{"title":"4.1 Image Collection","offset":16121,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"4.2 Data Annotation","offset":16702,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Initial annotation with specialist models.","offset":17373,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Data filtering and enhancement.","offset":18846,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Iterative data refinement.","offset":20081,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"4.3 Annotation-specific Variations","offset":21202,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Text.","offset":21414,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Region-text pairs.","offset":22545,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Text-phrase-region triplets.","offset":23730,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"5 Dataset","offset":24459,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"5.1 Overview","offset":24798,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"5.2 Data Analysis","offset":25706,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Annotation statistics.","offset":25729,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Semantic coverage.","offset":27254,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"Spatial coverage.","offset":28951,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"6 Experiments","offset":29964,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"6.1 Setup","offset":30718,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-2.html#text"},{"title":"6.2 Zero-shot Evaluation Across Tasks","offset":31838,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"6.3 Generalist Model with Public Supervised Data","offset":32930,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Simple design for strong performance.","offset":33465,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Competitive performance with fewer parameters.","offset":34131,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Adaptable generalization across task levels.","offset":34558,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"6.4 Downstream Tasks Fine-tuning","offset":35261,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Object detection and segmentation.","offset":35699,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Semantic segmentation.","offset":38328,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"6.5 Ablation Studies","offset":39274,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Multitask transfer.","offset":39300,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Model scaling.","offset":41993,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Data scaling.","offset":42468,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"Training settings.","offset":44819,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"7 Related Works","offset":46971,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"7.1 Vision-Language Foundation Models","offset":46991,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-3.html#text"},{"title":"7.2 Vision Datasets","offset":49169,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Comprehensive annotations.","offset":49194,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Scalable annotations.","offset":50044,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"8 Conclusion","offset":51250,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Acknowledgment.","offset":52168,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Appendix A Supported Tasks and Annotations in Florence-2","offset":53124,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Appendix B Supervised Data Collection for Generalist Model Fine-tuning","offset":53185,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Appendix C Model Configuration","offset":53260,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Appendix D More Examples of Annotations in FLD-5B","offset":53295,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Appendix E Qualitative Evaluation and Visualization Results","offset":53349,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"E.1 Detailed Image Caption","offset":53413,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"E.2 Visual Grounding","offset":55063,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"E.3 Dense Region Caption","offset":57413,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"E.4 Open Vocabulary Detection","offset":57443,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"E.5 OCR","offset":57739,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"E.6 Region to segmentation","offset":58248,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Appendix F Comparision with LMMs on Detailed Caption Task","offset":58901,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-4.html#text"},{"title":"Appendix G Comarison with Kosmos-2 on Detailed Caption and Grounding Tasks","offset":66777,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-5.html#text"},{"title":"Instructions for reporting errors","offset":67856,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/part-5.html#text"}],"text_code_points":68820,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/edc2679fff98a10c70395e79c58b56eaf9ec0001691dbca2b33ba94a401a8398/source.md"}