{"id":"arxiv:2111.07832","title":"iBOT: Image BERT Pre-Training with Online Tokenizer","date":"2021-11-15","section":"Self-supervised learning and vision","quality":0.136624,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/","source_url":"https://arxiv.org/abs/2111.07832","text_available":true,"authors":["Jinghao Zhou","Chen Wei","Huiyu Wang","Wei Shen"],"outgoing":[{"source":"arxiv:2111.07832","target":"arxiv:2006.09882","evidence":"title","id":"arxiv:2006.09882","title":"Unsupervised Learning of Visual Features by Contrasting Cluster Assignments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6601018611a96dff6f5324f2e4f39f62484917cf4d8f8a8d5baa7f4be1deb433/"},{"source":"arxiv:2111.07832","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"}],"incoming":[{"source":"arxiv:2301.08243","target":"arxiv:2111.07832","evidence":"title","id":"arxiv:2301.08243","title":"Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/"},{"source":"arxiv:2304.07193","target":"arxiv:2111.07832","evidence":"title","id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/"},{"source":"arxiv:2304.12210","target":"arxiv:2111.07832","evidence":"title","id":"arxiv:2304.12210","title":"A Cookbook of Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/"},{"source":"arxiv:2312.17742","target":"arxiv:2111.07832","evidence":"id","id":"arxiv:2312.17742","title":"Learning Vision from Models Rivals Learning Vision from Data","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/71dc1683019904602d643d16348a76b98c556b924ca964e3949d8d6420dc4402/"},{"source":"arxiv:2606.02572","target":"arxiv:2111.07832","evidence":"title","id":"arxiv:2606.02572","title":"VISReg: Variance-Invariance-Sketching Regularization for JEPA training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/262c1938984761f99c55d9ae1237e686b2d6ab94b69d7181d11dceab2a7210fb/"}],"text_parts":[{"start":0,"end":15494,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html"},{"start":15494,"end":31218,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html"},{"start":31218,"end":47125,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html"},{"start":47125,"end":60722,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html"}],"contents":[{"title":"iBOT : Image BERT Pre-Training with Online Tokenizer","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"Abstract","offset":56,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"1 Introduction","offset":1439,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"2 Preliminaries","offset":6952,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"2.1 Masked Image Modeling as Knowledge Distillation","offset":6972,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"2.2 Self-Distillation","offset":9135,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"3 iBOT","offset":10837,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"3.1 Framework","offset":11384,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"3.2 Implementation","offset":14407,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"Architecture.","offset":14431,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"Pre-Training Setup.","offset":15467,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-1.html#text"},{"title":"4 Experiment","offset":16402,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"4.1 Classification on ImageNet-1K","offset":16738,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"k k -NN and Linear Probing.","offset":16927,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Fine-Tuning.","offset":18193,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Semi-Supervised and Unsupervised Learning.","offset":18960,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"4.2 Downstream Tasks","offset":19816,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Object Detection and Instance Segmentation on COCO.","offset":19842,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Semantic Segmentation on ADE20K.","offset":20802,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Transfer Learning.","offset":21783,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"4.3 Properties of ViT trained with MIM","offset":22653,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"4.3.1 Discovering the Pattern Layout of Image Patches","offset":22901,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"What Patterns Does MIM Learn?","offset":22961,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"How Does MIM Help Image Recognition?","offset":24118,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"4.3.2 Discriminative Parts in Self-Attention Map","offset":25222,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"4.3.3 Robustness","offset":26186,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"4.4 Ablation Study on Tokenizer","offset":27242,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"5 Related Work","offset":28748,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Visual Representation Learning.","offset":28767,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Masked Prediction in Images.","offset":29994,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"6 Conclusion","offset":31201,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-2.html#text"},{"title":"Appendix A Pseudocode","offset":32383,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Appendix B Multi-Crop","offset":32409,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Stability of MIM Pre-trained with Multi-Crop.","offset":32907,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Stabilizing the Training with Non-Masked Global Crops.","offset":34019,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Range of Scales in Multi-Crop.","offset":34921,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"State-of-the-Art Comparison w/o and w/ Multi-Crop.","offset":35434,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Effective Training Epochs.","offset":36689,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Appendix C Additional Implementations","offset":37675,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Fine-Tuning Recipes of Classification on ImageNet-1K.","offset":37717,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Evaluation Protocols of Semi-Supervised Learning on ImageNet-1K.","offset":39004,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Fine-Tuning Recipes of Object Detection and Instance Segmentation on COCO.","offset":40164,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Fine-Tuning Recipes of Semantic Segmentation on ADE20K.","offset":41035,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Part-Wise Linear Probing.","offset":41985,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Appendix D Additional Results","offset":42247,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Object Detection, Instance Segmentation, and Semantic Segmentation.","offset":42611,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"k k -NN and Linear Probing with ImageNet-22K.","offset":43394,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Nearest Neighbor Retrieval.","offset":43988,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Robustness against Background Change.","offset":45089,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Robustness against Occlusion.","offset":46041,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Robustness against Shuffle.","offset":47090,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-3.html#text"},{"title":"Appendix E Additional Ablations","offset":47885,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Architecture of Projection Head.","offset":48184,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Comparing MIM with Dense Self-Distillation.","offset":49137,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Hard Label versus Soft Label","offset":50649,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Centering and Sharpening.","offset":51139,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Loss Ratio.","offset":51692,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Output Dimension.","offset":52093,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Prediction Ratios.","offset":52535,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Training Epochs.","offset":53468,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Time and Memory Requirements.","offset":54066,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Appendix F Alternative Tokenizers","offset":54632,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Appendix G Visualization","offset":56185,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"G.1 Pattern Layout","offset":56482,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Pattern Layout for Patch Tokens.","offset":56506,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Composing Images with Representative Patterns.","offset":57070,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Comparison with Other Methods.","offset":57526,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Pattern Layout for [CLS] Token.","offset":57985,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"G.2 Self-Attention Visualizations","offset":58475,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"G.3 Sparse Correspondence.","offset":58660,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"},{"title":"Instructions for reporting errors","offset":59758,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/part-4.html#text"}],"text_code_points":60722,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/source.md"}