{"id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","date":"2021-04-29","section":"Self-supervised learning and vision","quality":0.474747,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/","source_url":"https://arxiv.org/abs/2104.14294","text_available":true,"authors":["Mathilde Caron","Hugo Touvron","Ishan Misra","Hervé Jégou"],"outgoing":[{"source":"arxiv:2104.14294","target":"arxiv:2006.09882","evidence":"title","id":"arxiv:2006.09882","title":"Unsupervised Learning of Visual Features by Contrasting Cluster Assignments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6601018611a96dff6f5324f2e4f39f62484917cf4d8f8a8d5baa7f4be1deb433/"}],"incoming":[{"source":"arxiv:2111.07832","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2111.07832","title":"iBOT: Image BERT Pre-Training with Online Tokenizer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/"},{"source":"arxiv:2301.08243","target":"arxiv:2104.14294","evidence":"id","id":"arxiv:2301.08243","title":"Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/"},{"source":"arxiv:2304.07193","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/"},{"source":"arxiv:2304.09355","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2304.09355","title":"To Compress or Not to Compress- Self-Supervised Learning and Information Theory: A Review","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/"},{"source":"arxiv:2304.12210","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2304.12210","title":"A Cookbook of Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/"},{"source":"arxiv:2306.13575","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2306.13575","title":"Scaling MLPs: A Tale of Inductive Bias","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/"},{"source":"arxiv:2312.17742","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2312.17742","title":"Learning Vision from Models Rivals Learning Vision from Data","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/71dc1683019904602d643d16348a76b98c556b924ca964e3949d8d6420dc4402/"},{"source":"arxiv:2405.07987","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2405.07987","title":"The Platonic Representation Hypothesis","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/"},{"source":"arxiv:2503.14858","target":"arxiv:2104.14294","evidence":"id","id":"arxiv:2503.14858","title":"1000 Layer Networks for Self-Supervised RL: Scaling Depth Can Enable New Goal-Reaching Capabilities","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/"},{"source":"arxiv:2511.08544","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2511.08544","title":"LeJEPA: Provable and Scalable Self-Supervised Learning Without the Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e2c5835a26dabc321e6364380a37c47c10425ee830ec522564b0604733025a94/"},{"source":"arxiv:2605.26379","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2605.26379","title":"When Does LeJEPA Learn a World Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f140b8fcc13ea085ce6ad170e090380c8d373416f791960778c4b216f6e5c8f6/"},{"source":"arxiv:2606.02572","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2606.02572","title":"VISReg: Variance-Invariance-Sketching Regularization for JEPA training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/262c1938984761f99c55d9ae1237e686b2d6ab94b69d7181d11dceab2a7210fb/"},{"source":"arxiv:2606.06021","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2606.06021","title":"OPRD: On-Policy Representation Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a16a9d7ea78c5748acc7f651c9a96663e0200d72e51d6a52c9528e1739a27d8d/"}],"text_parts":[{"start":0,"end":14662,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html"},{"start":14662,"end":29730,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html"},{"start":29730,"end":45498,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html"},{"start":45498,"end":61205,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html"},{"start":61205,"end":76351,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html"},{"start":76351,"end":81260,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html"}],"contents":[{"title":"Emerging Properties in Self-Supervised Vision Transformers","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"Abstract","offset":62,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"1 Introduction","offset":1088,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"2 Related work","offset":5471,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"Self-supervised learning.","offset":5490,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"Self-training and knowledge distillation.","offset":7290,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"3 Approach","offset":8893,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"3.1 SSL with Knowledge Distillation","offset":8908,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"Teacher network.","offset":12048,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"Network architecture.","offset":13652,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"Avoiding collapse.","offset":14636,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-1.html#text"},{"title":"3.2 Implementation and evaluation protocols","offset":16125,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Vision Transformer.","offset":16311,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Implementation details.","offset":17706,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Evaluation protocols.","offset":18727,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"4 Main Results","offset":20035,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"4.1 Comparing with SSL frameworks on ImageNet","offset":20278,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Comparing with the same architecture.","offset":20430,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Comparing across architectures.","offset":21538,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"4.2 Properties of ViT trained with SSL","offset":22383,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"4.2.1 Nearest neighbor retrieval with DINO ViT","offset":22594,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Image Retrieval.","offset":22891,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Copy detection.","offset":23836,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"4.2.2 Discovering the semantic layout of scenes","offset":24774,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Video instance segmentation.","offset":25099,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Probing the self-attention map.","offset":25922,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"4.2.3 Transfer learning on downstream tasks","offset":26918,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"5 Ablation Study of DINO","offset":27563,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"5.1 Importance of the Different Components","offset":27766,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Importance of the patch size.","offset":28737,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"5.2 Impact of the choice of Teacher Network","offset":29459,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Building different teachers from the student.","offset":29677,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-2.html#text"},{"title":"Analyzing the training dynamic.","offset":30715,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"5.3 Avoiding collapse","offset":31753,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"5.4 Compute requirements","offset":32968,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"5.5 Training with small batches","offset":34468,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"6 Conclusion","offset":35400,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Acknowledgement.","offset":36286,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Appendix","offset":36665,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"A Additional Results","offset":36678,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"k k -NN classification.","offset":36704,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Self-supervised ImageNet pretraining of ViT.","offset":37874,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Low-shot learning on ImageNet.","offset":38915,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"B Methodology Comparison","offset":39552,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Relation to MoCo-v2 and BYOL.","offset":40259,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Relation to SwAV.","offset":41780,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Details on the Softmax(batch) variant.","offset":42777,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Validating our implementation.","offset":43357,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"Relation to other works.","offset":43968,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"C Projection Head","offset":44774,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"BN-free system.","offset":45381,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-3.html#text"},{"title":"L2-normalization bottleneck in projection head.","offset":46146,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Output dimension.","offset":47248,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"GELU activations.","offset":47660,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"D Additional Ablations","offset":48021,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Online centering.","offset":48337,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Sharpening.","offset":48633,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Longer training.","offset":49398,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"The teacher outperforms the student.","offset":49906,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Self-attention maps from supervised versus self-supervised learning.","offset":50618,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Impact of the number of heads in ViT-S.","offset":51148,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"E Multi-crop","offset":51566,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Range of scales in multi-crop.","offset":51666,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Multi-crop in different self-supervised frameworks.","offset":52550,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Training BYOL with multi-crop.","offset":53666,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"F Evaluation Protocols","offset":55042,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"F.1 k k -NN classification","offset":55070,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"F.2 Linear classification","offset":56298,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"ViT-S representations for linear eval.","offset":57149,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"ViT-B representations for linear eval.","offset":57440,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"G Self-Attention Visualizations","offset":57843,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"H Class Representation","offset":58162,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Supplementary Material for Self-Supervised Learning of Visual Transformers with a Momentum Teacher","offset":58776,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Appendix","offset":58920,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"A Additional Results","offset":58933,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"k k -NN classification.","offset":58959,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Self-supervised ImageNet pretraining of ViT.","offset":60127,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"Low-shot learning on ImageNet.","offset":61167,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-4.html#text"},{"title":"B Methodology Comparison","offset":61803,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Relation to MoCo-v2 and BYOL.","offset":62509,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Relation to SwAV.","offset":64192,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Details on the Softmax(batch) variant.","offset":65270,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Validating our implementation.","offset":65849,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Relation to other works.","offset":66459,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"C Projection Head","offset":67265,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"BN-free system.","offset":67872,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"L2-normalization bottleneck in projection head.","offset":68637,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Output dimension.","offset":69739,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"GELU activations.","offset":70151,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"D Additional Ablations","offset":70512,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Online centering.","offset":70828,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Sharpening.","offset":71124,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Longer training.","offset":71889,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"The teacher outperforms the student.","offset":72395,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Self-attention maps from supervised versus self-supervised learning.","offset":73105,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Impact of the number of heads in ViT-S.","offset":73635,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"E Multi-crop","offset":74053,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Range of scales in multi-crop.","offset":74153,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Multi-crop in different self-supervised frameworks.","offset":75037,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"Training BYOL with multi-crop.","offset":76153,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-5.html#text"},{"title":"F Evaluation Protocols","offset":77529,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html#text"},{"title":"F.1 k k -NN classification","offset":77557,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html#text"},{"title":"F.2 Linear classification","offset":78785,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html#text"},{"title":"ViT-S representations for linear eval.","offset":79636,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html#text"},{"title":"ViT-B representations for linear eval.","offset":79927,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html#text"},{"title":"G Self-Attention Visualizations","offset":80330,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html#text"},{"title":"H Class Representation","offset":80648,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/part-6.html#text"}],"text_code_points":81260,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/source.md"}