{"id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","date":"2023-04-14","section":"Self-supervised learning and vision","quality":0.201878,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/","source_url":"https://arxiv.org/abs/2304.07193","text_available":true,"authors":["Maxime Oquab","Timothée Darcet","Théo Moutakanni","Huy Vo"],"outgoing":[{"source":"arxiv:2304.07193","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2304.07193","target":"arxiv:2006.09882","evidence":"title","id":"arxiv:2006.09882","title":"Unsupervised Learning of Visual Features by Contrasting Cluster Assignments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6601018611a96dff6f5324f2e4f39f62484917cf4d8f8a8d5baa7f4be1deb433/"},{"source":"arxiv:2304.07193","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"},{"source":"arxiv:2304.07193","target":"arxiv:2111.07832","evidence":"title","id":"arxiv:2111.07832","title":"iBOT: Image BERT Pre-Training with Online Tokenizer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/"},{"source":"arxiv:2304.07193","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2301.08243","title":"Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/"},{"source":"arxiv:2304.07193","target":"arxiv:2302.06675","evidence":"id","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"}],"incoming":[{"source":"arxiv:2304.12210","target":"arxiv:2304.07193","evidence":"title","id":"arxiv:2304.12210","title":"A Cookbook of Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/"},{"source":"arxiv:2312.17742","target":"arxiv:2304.07193","evidence":"id","id":"arxiv:2312.17742","title":"Learning Vision from Models Rivals Learning Vision from Data","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/71dc1683019904602d643d16348a76b98c556b924ca964e3949d8d6420dc4402/"},{"source":"arxiv:2405.07987","target":"arxiv:2304.07193","evidence":"title","id":"arxiv:2405.07987","title":"The Platonic Representation Hypothesis","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/"},{"source":"arxiv:2511.08544","target":"arxiv:2304.07193","evidence":"id","id":"arxiv:2511.08544","title":"LeJEPA: Provable and Scalable Self-Supervised Learning Without the Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e2c5835a26dabc321e6364380a37c47c10425ee830ec522564b0604733025a94/"},{"source":"arxiv:2604.11962","target":"arxiv:2304.07193","evidence":"title","id":"arxiv:2604.11962","title":"The Linear Centroids Hypothesis: Features as Directions Learned by Local Experts","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8428fd0cce9753855f4d0f801609b0205a0be52a459d53ae44e7860d00348485/"},{"source":"arxiv:2606.02572","target":"arxiv:2304.07193","evidence":"title","id":"arxiv:2606.02572","title":"VISReg: Variance-Invariance-Sketching Regularization for JEPA training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/262c1938984761f99c55d9ae1237e686b2d6ab94b69d7181d11dceab2a7210fb/"},{"source":"arxiv:2607.27372","target":"arxiv:2304.07193","evidence":"title","id":"arxiv:2607.27372","title":"Explorative Modeling: Unlocking a Third Pretraining Axis and End-to-End Generation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/542a5e85b84624381953f643ec6075a588c08c542c4f945e57d39c9164c9a835/"}],"text_parts":[{"start":0,"end":15817,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html"},{"start":15817,"end":31679,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html"},{"start":31679,"end":47527,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html"},{"start":47527,"end":60610,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html"}],"contents":[{"title":"DINOv2: Learning Robust Visual Features without Supervision","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"Abstract","offset":63,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"1 Introduction","offset":1376,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"2 Related Work","offset":6241,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"Intra-image self-supervised training.","offset":6260,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"Discriminative self-supervised learning.","offset":7582,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"Scaling self-supervised pretraining.","offset":8675,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"Automatic data curation.","offset":9495,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"3 Data Processing","offset":10421,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"4 Discriminative Self-supervised Pre-training","offset":13297,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-1.html#text"},{"title":"5 Efficient implementation","offset":17379,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"Fast and memory-efficient attention.","offset":17855,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"Sequence packing.","offset":18761,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"Efficient stochastic depth.","offset":19863,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"Fully-Sharded Data Parallel (FSDP).","offset":20488,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"Model distillation.","offset":21985,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"6 Ablation Studies","offset":23107,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"6.1 Improved Training Recipe","offset":23400,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"6.2 Pretraining Data Source","offset":24054,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"6.3 Model Size and Data","offset":25578,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"6.4 Loss Components","offset":25965,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"6.5 Impact of Knowledge Distillation","offset":26937,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"6.6 Impact of Resolution","offset":27503,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"7 Results","offset":28655,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"7.1 ImageNet Classification","offset":30305,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"How far are we from weakly-supervised models?","offset":31626,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-2.html#text"},{"title":"Can we finetune the encoders?","offset":32574,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"Robustness analysis.","offset":33780,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"7.2 Additional Image and Video classification Benchmarks","offset":34660,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"7.3 Instance Recognition","offset":37440,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"7.4 Dense Recognition Tasks","offset":38419,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"Semantic segmentation.","offset":38709,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"Frozen backbone in a SOTA pipeline.","offset":39926,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"Depth estimation.","offset":40630,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"7.5 Qualitative Results","offset":42594,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"Semantic Segmentation and Depth Estimation.","offset":42730,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"Out-of-distribution generalization.","offset":43849,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"PCA of patch features.","offset":44263,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"Patch matching.","offset":45221,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"8 Fairness and Bias Analysis","offset":46068,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"8.1 Geographical Fairness","offset":46296,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"8.2 Gender, Skintones and Age","offset":47492,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-3.html#text"},{"title":"9 Estimating the Environmental Impact of Training our Models","offset":49168,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Carbon footprint of the whole project.","offset":51604,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"10 Future work and Discussion","offset":52367,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Acknowledgments.","offset":53888,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Appendix A Data Processing","offset":54329,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"A.1 Data selection","offset":54360,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"A.2 Image similarity","offset":54594,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"A.3 Deduplication","offset":55129,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Self-deduplication.","offset":55152,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Relative deduplication","offset":55726,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"A.4 Retrieval","offset":56295,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Appendix B Implementation Details","offset":57405,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"B.1 Unsupervised pre-training","offset":57443,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"KoLeo regularization.","offset":57635,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"EMA update for the teacher.","offset":57840,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"B.2 High-Resolution adaptation","offset":58116,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"B.3 Linear probing evaluation","offset":58516,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Appendix C List of Datasets used","offset":59527,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"},{"title":"Instructions for reporting errors","offset":59646,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/part-4.html#text"}],"text_code_points":60610,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/source.md"}