{"id":"arxiv:2304.09355","title":"To Compress or Not to Compress- Self-Supervised Learning and Information Theory: A Review","date":"2023-04-19","section":"Self-supervised learning and vision","quality":-0.3926,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/","source_url":"https://arxiv.org/abs/2304.09355","text_available":true,"authors":["Ravid Shwartz-Ziv","Yann LeCun"],"outgoing":[{"source":"arxiv:2304.09355","target":"arxiv:2006.09882","evidence":"title","id":"arxiv:2006.09882","title":"Unsupervised Learning of Visual Features by Contrasting Cluster Assignments","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6601018611a96dff6f5324f2e4f39f62484917cf4d8f8a8d5baa7f4be1deb433/"},{"source":"arxiv:2304.09355","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"},{"source":"arxiv:2304.09355","target":"arxiv:2105.04906","evidence":"id","id":"arxiv:2105.04906","title":"VICReg: Variance-Invariance-Covariance Regularization for Self-Supervised Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/822b6414fb02bd6c746e0d559fd3dcc60766476142ebae7795b02bc6c45223ce/"}],"incoming":[{"source":"arxiv:2407.15017","target":"arxiv:2304.09355","evidence":"title","id":"arxiv:2407.15017","title":"Knowledge Mechanisms in Large Language Models: A Survey and Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8900dc2821aa61e59c5b1219cf9eb36dfaabf303c17d0064deaca653b5ec9a77/"},{"source":"arxiv:2511.08544","target":"arxiv:2304.09355","evidence":"title","id":"arxiv:2511.08544","title":"LeJEPA: Provable and Scalable Self-Supervised Learning Without the Heuristics","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e2c5835a26dabc321e6364380a37c47c10425ee830ec522564b0604733025a94/"}],"text_parts":[{"start":0,"end":15958,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html"},{"start":15958,"end":31210,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html"},{"start":31210,"end":47201,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html"},{"start":47201,"end":63091,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-4.html"},{"start":63091,"end":78742,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html"},{"start":78742,"end":85002,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-6.html"}],"contents":[{"title":"To Compress or Not to Compress - Self-Supervised Learning and Information Theory: A Review","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html#text"},{"title":"Abstract","offset":94,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html#text"},{"title":"keywords","offset":1400,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html#text"},{"title":"1 Introduction","offset":1417,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html#text"},{"title":"2 Background and Fundamental Concepts","offset":7207,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html#text"},{"title":"2.1 Multiview Representation Learning","offset":7249,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html#text"},{"title":"2.2 Self-Supervised Learning","offset":11308,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-1.html#text"},{"title":"2.3 Semi-Supervised Learning","offset":19279,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"2.4 Representation Learning","offset":20165,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"2.4.1 Minimal Sufficient Statistic","offset":21523,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"Definition 1 .","offset":21661,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"Theorem 2 .","offset":22007,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"Definition 3 .","offset":22459,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"2.4.2 The Information Bottleneck","offset":23037,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"2.5 Representation Learning and the Information Bottleneck","offset":26401,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-2.html#text"},{"title":"3 Information-Theoretic Objectives","offset":32484,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"3.1 Setup and Methodology","offset":33658,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"3.2 Optimization with Labels","offset":37408,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"3.2.1 Single-View Supervised Learning","offset":37936,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"Definition 4 .","offset":38517,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"3.2.2 The Information Bottleneck Theory of Deep Learning","offset":40498,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"3.2.3 Multiview IB Learning","offset":42651,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"3.2.4 Semi-Supervised IB Learning: Leveraging Unlabeled Data","offset":45156,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-3.html#text"},{"title":"3.2.5 Unsupervised IB learning","offset":48843,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-4.html#text"},{"title":"4 Self-Supervised Multiview Information Bottleneck Learning","offset":51228,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-4.html#text"},{"title":"Assumption 1 .","offset":54929,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-4.html#text"},{"title":"4.1 Implicit Compression in Self-Supervised Learning Methods","offset":61542,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-4.html#text"},{"title":"4.2 Beyond the Multiview Assumption","offset":64591,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"4.3 To Compress or Not to Compress?","offset":68407,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"5 Optimizing Information in Deep Neural Networks: Challenges and Approaches","offset":70044,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"Information in Deterministic Networks","offset":70526,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"Measuring Information in High-Dimensional Spaces","offset":72348,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"6 Future Research Directions","offset":75735,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"Self-supervised learning with non-shared information.","offset":75954,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"Self-supervised learning for tabular data.","offset":76755,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"Integrating other learning methods into the information-theoretic framework.","offset":77800,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"Expanding the analysis to usable information.","offset":78689,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-5.html#text"},{"title":"Extending self-supervised learning’s information-based perspective to energy-based model optimization.","offset":79556,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-6.html#text"},{"title":"Expanding the multiview framework to accommodate more views and tasks.","offset":81344,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-6.html#text"},{"title":"7 Conclusion","offset":83028,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/part-6.html#text"}],"text_code_points":85002,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6de01284dffdeadbd92020fc614ff702f27bc9c49215a1658ea3ca8a4a0c36ce/source.md"}