{"id":"arxiv:2304.12210","title":"A Cookbook of Self-Supervised Learning","date":"2023-04-24","section":"Self-supervised learning and vision","quality":-0.556775,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/","source_url":"https://arxiv.org/abs/2304.12210","text_available":true,"authors":["Randall Balestriero","Mark Ibrahim","Vlad Sobal","Ari Morcos"],"outgoing":[{"source":"arxiv:2304.12210","target":"arxiv:2006.09882","evidence":"title","id":"arxiv:2006.09882","title":"Unsupervised Learning of Visual Features by Contrasting Cluster Assignments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6601018611a96dff6f5324f2e4f39f62484917cf4d8f8a8d5baa7f4be1deb433/"},{"source":"arxiv:2304.12210","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"},{"source":"arxiv:2304.12210","target":"arxiv:2105.04906","evidence":"title","id":"arxiv:2105.04906","title":"VICReg: Variance-Invariance-Covariance Regularization for Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/822b6414fb02bd6c746e0d559fd3dcc60766476142ebae7795b02bc6c45223ce/"},{"source":"arxiv:2304.12210","target":"arxiv:2111.07832","evidence":"title","id":"arxiv:2111.07832","title":"iBOT: Image BERT Pre-Training with Online Tokenizer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/"},{"source":"arxiv:2304.12210","target":"arxiv:2208.10442","evidence":"id","id":"arxiv:2208.10442","title":"Image as a Foreign Language: BEiT Pretraining for All Vision and Vision-Language Tasks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/51f033b7e43981722ced32d8001d10cdd68eafee7e0dad056b9001eeca29f358/"},{"source":"arxiv:2304.12210","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2301.08243","title":"Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/"},{"source":"arxiv:2304.12210","target":"arxiv:2304.07193","evidence":"title","id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/"}],"incoming":[{"source":"arxiv:2511.08544","target":"arxiv:2304.12210","evidence":"id","id":"arxiv:2511.08544","title":"LeJEPA: Provable and Scalable Self-Supervised Learning Without the Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e2c5835a26dabc321e6364380a37c47c10425ee830ec522564b0604733025a94/"}],"text_parts":[{"start":0,"end":15416,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-1.html"},{"start":15416,"end":31124,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-2.html"},{"start":31124,"end":46113,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html"},{"start":46113,"end":61969,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html"},{"start":61969,"end":77936,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html"},{"start":77936,"end":93347,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html"},{"start":93347,"end":109042,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-7.html"},{"start":109042,"end":119971,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-8.html"}],"contents":[{"title":"A Cookbook of Self-Supervised Learning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-1.html#text"},{"title":"1 What is Self-Supervised Learning and Why Bother?","offset":42,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-1.html#text"},{"title":"1.1 Why a Cookbook for Self-Supervised Learning?","offset":3138,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-1.html#text"},{"title":"2 The Families and Origins of SSL","offset":6726,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-1.html#text"},{"title":"2.1 Origins of SSL","offset":6991,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-1.html#text"},{"title":"2.2 The Deep Metric Learning Family: SimCLR/NNCLR/MeanSHIFT/SCL","offset":15347,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-1.html#text"},{"title":"2.3 The Self-Distillation Family: BYOL/SimSIAM/DINO","offset":21241,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-2.html#text"},{"title":"2.4 The Canonical Correlation Analysis Family: VICReg/BarlowTwins/SWAV/W-MSE","offset":29033,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-2.html#text"},{"title":"2.5 Masked Image Modeling","offset":32604,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"2.6 A Theoretical Unification Of Self-Supervised Learning","offset":37253,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"2.6.1 Theoretical Study of SSL","offset":37316,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"Relationship between Contrastive Learning and Other Objectives.","offset":38139,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"Unified contrastive losses.","offset":40193,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"Hard negative sampling.","offset":41722,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"Study of the projector.","offset":42313,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"2.6.2 Dimensional Collapse of Representations","offset":43356,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"2.7 Pretraining Data","offset":44999,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"Curated (standard)","offset":45025,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"Training with data from the wild","offset":45355,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-3.html#text"},{"title":"Weakly-curated training data","offset":47218,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"3 A Cook’s Guide to Successful SSL Training and Deployment","offset":47838,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"3.1 Role of Data-Augmentation","offset":47901,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"3.1.1 Role of multi-crop","offset":52341,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"3.2 Role of the Projector","offset":55001,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"Using a projector to handle noisy image augmentations.","offset":57172,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"Influence of the projector’s output dimension.","offset":58373,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"Influence of the backbone’s output dimension.","offset":59484,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"Properties of the representation induced by the projector.","offset":60437,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"Training a SSL without a projector.","offset":61546,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"3.3 The Uniform Prior in SSL or the Failure of SSL on Unbalanced Data","offset":61894,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-4.html#text"},{"title":"3.4 Teacher-Student Architecture Specific Tricks","offset":63080,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.4.1 Role of the Moving Average Teacher","offset":63134,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.4.2 Role of the Predictor in Self-Labeling SSL","offset":64379,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.5 Role of Standard Hyper-Parameters","offset":65760,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.5.1 Role of Mini-Batch Size","offset":66157,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.5.2 Role of Learning Rate (Schedulers) and Optimizers","offset":67388,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.5.3 Role of Weight-Decay","offset":68543,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.5.4 Vision Transformers Considerations","offset":69386,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.6 Techniques for High Performance Masked Image Modeling","offset":73045,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.7 Evaluating Your SSL Models","offset":75620,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"3.7.1 Evaluation with labels","offset":75656,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"KNN","offset":76613,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-5.html#text"},{"title":"Linear","offset":78092,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"MLP","offset":79421,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"Full Fine-tuning","offset":80410,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"3.7.2 Evaluation without labels","offset":81732,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"3.7.3 Going beyond classification","offset":83825,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"3.7.4 Visual Evaluation","offset":84755,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"3.8 Speeding up Training","offset":86646,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"3.8.1 Distributed Training","offset":86676,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"3.8.2 Even Faster Training with FFCV and Other Speedups","offset":90375,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"3.8.3 Speeding Up Training of Vision Transformers","offset":91046,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"4 Extending Self-Supervised Learning Beyond Images and Classification","offset":92740,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"4.1 Strategies for Other Data Domains","offset":92814,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-6.html#text"},{"title":"4.2 Incorporating Multiple Modalities into SSL Training","offset":107858,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-7.html#text"},{"title":"4.3 Building Feature Extractors with Localization for Dense Prediction Tasks","offset":111187,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-8.html#text"},{"title":"5 Conclusion","offset":119219,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/part-8.html#text"}],"text_code_points":119971,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/source.md"}