{"id":"arxiv:2301.08243","title":"Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture","date":"2023-01-19","section":"Self-supervised learning and vision","quality":0.189344,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/","source_url":"https://arxiv.org/abs/2301.08243","text_available":true,"authors":["Mahmoud Assran","Quentin Duval","Ishan Misra","Piotr Bojanowski"],"outgoing":[{"source":"arxiv:2301.08243","target":"arxiv:2006.09882","evidence":"id","id":"arxiv:2006.09882","title":"Unsupervised Learning of Visual Features by Contrasting Cluster Assignments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6601018611a96dff6f5324f2e4f39f62484917cf4d8f8a8d5baa7f4be1deb433/"},{"source":"arxiv:2301.08243","target":"arxiv:2104.14294","evidence":"id","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"},{"source":"arxiv:2301.08243","target":"arxiv:2105.04906","evidence":"id","id":"arxiv:2105.04906","title":"VICReg: Variance-Invariance-Covariance Regularization for Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/822b6414fb02bd6c746e0d559fd3dcc60766476142ebae7795b02bc6c45223ce/"},{"source":"arxiv:2301.08243","target":"arxiv:2111.07832","evidence":"title","id":"arxiv:2111.07832","title":"iBOT: Image BERT Pre-Training with Online Tokenizer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2bd0269c0872fc518db65c2ca98dd30b5d38701f79b0a72fec2ed49648ba589f/"}],"incoming":[{"source":"arxiv:2304.07193","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/"},{"source":"arxiv:2304.12210","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2304.12210","title":"A Cookbook of Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/518a026819227af3e1838911bfadf26c1515786a3af305de5c2e77efb052ecfd/"},{"source":"arxiv:2504.01928","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2504.01928","title":"Is the Reversal Curse a Binding Problem? Uncovering Limitations of Transformers from a Basic Generalization Failure","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b18365db218c3406643d59ec371727d81993de9d22541f1bd099f3f5ff15b94c/"},{"source":"arxiv:2511.08544","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2511.08544","title":"LeJEPA: Provable and Scalable Self-Supervised Learning Without the Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e2c5835a26dabc321e6364380a37c47c10425ee830ec522564b0604733025a94/"},{"source":"arxiv:2605.26379","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2605.26379","title":"When Does LeJEPA Learn a World Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f140b8fcc13ea085ce6ad170e090380c8d373416f791960778c4b216f6e5c8f6/"},{"source":"arxiv:2606.02572","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2606.02572","title":"VISReg: Variance-Invariance-Sketching Regularization for JEPA training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/262c1938984761f99c55d9ae1237e686b2d6ab94b69d7181d11dceab2a7210fb/"},{"source":"arxiv:2607.27372","target":"arxiv:2301.08243","evidence":"title","id":"arxiv:2607.27372","title":"Explorative Modeling: Unlocking a Third Pretraining Axis and End-to-End Generation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/542a5e85b84624381953f643ec6075a588c08c542c4f945e57d39c9164c9a835/"}],"text_parts":[{"start":0,"end":14882,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html"},{"start":14882,"end":30562,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html"},{"start":30562,"end":45454,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html"}],"contents":[{"title":"Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Abstract","offset":87,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"1 Introduction","offset":1126,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"2 Background","offset":5577,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Joint-Embedding Architectures.","offset":6108,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Generative Architectures.","offset":7350,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Joint-Embedding Predictive Architectures.","offset":8367,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"3 Method","offset":9476,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Targets.","offset":10229,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Context.","offset":11510,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Prediction.","offset":12386,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"Loss.","offset":13549,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"4 Related Work","offset":14507,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-1.html#text"},{"title":"5 Image Classification","offset":17700,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"ImageNet-1K.","offset":18256,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Low-Shot ImageNet-1K.","offset":19244,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Transfer learning.","offset":20146,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"6 Local Prediction Tasks","offset":20589,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"7 Scalability","offset":21675,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Model Efficiency.","offset":21693,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Scaling data size.","offset":22634,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Scaling model size.","offset":23007,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"8 Predictor Visualizations","offset":23447,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"9 Ablations","offset":24614,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Predicting in representation space.","offset":24630,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Masking strategy.","offset":25203,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"10 Conclusion","offset":26014,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Appendix A Implementation Details","offset":26538,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"A.1 Pretraining","offset":26576,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Architectures.","offset":26597,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Optimization.","offset":27467,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Masking.","offset":28215,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"A.2 Downstream Tasks","offset":29319,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"Linear evaluation.","offset":29345,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-2.html#text"},{"title":"ImageNet evaluations.","offset":31079,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Low-shot evaluation.","offset":31632,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Appendix B Broader Related Work","offset":32442,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Appendix C Additional Ablations","offset":37605,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Multiblock masking strategy.","offset":37822,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Masking at the output of the target-encoder.","offset":38389,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Predictor depth.","offset":39018,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Weight decay.","offset":39426,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Predictor width.","offset":39987,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Appendix D Finetuning on the full ImageNet","offset":40356,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Appendix E RCDM Visualizations","offset":41408,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"E.1 Encoder Visualization","offset":43107,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"},{"title":"Instructions for reporting errors","offset":44490,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/part-3.html#text"}],"text_code_points":45454,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/475bac0a3b29edd42642bbe10d1a41bfb18a3022ac95b9b1a3db9e5d5d0b95fe/source.md"}