{"id":"arxiv:2405.07987","title":"The Platonic Representation Hypothesis","date":"2024-05-13","section":"Representation alignment","quality":-0.102668,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/","source_url":"https://arxiv.org/abs/2405.07987","text_available":true,"authors":["Minyoung Huh","Brian Cheung","Tongzhou Wang","Phillip Isola"],"outgoing":[{"source":"arxiv:2405.07987","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2405.07987","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"},{"source":"arxiv:2405.07987","target":"arxiv:2206.04615","evidence":"id","id":"arxiv:2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/"},{"source":"arxiv:2405.07987","target":"arxiv:2304.07193","evidence":"title","id":"arxiv:2304.07193","title":"DINOv2: Learning Robust Visual Features without Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b15933596df35497024556ae88bfa696f3b0e8372b678194f04f1bfb9f3be553/"},{"source":"arxiv:2405.07987","target":"openreview:pOoKI3ouv1","evidence":"title","id":"openreview:pOoKI3ouv1","title":"Robust agents learn causal world models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fb0182c1f9cccce44855874d3b587206d37085b39cc41e20c08010a379aef258/"}],"incoming":[{"source":"arxiv:2407.15017","target":"arxiv:2405.07987","evidence":"id","id":"arxiv:2407.15017","title":"Knowledge Mechanisms in Large Language Models: A Survey and Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8900dc2821aa61e59c5b1219cf9eb36dfaabf303c17d0064deaca653b5ec9a77/"},{"source":"arxiv:2501.14082","target":"arxiv:2405.07987","evidence":"id","id":"arxiv:2501.14082","title":"Communicating Activations Between Language Model Agents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e67d4addcda44093e242074332925d000726f87bd9538f6b318bdfc454a7dc13/"},{"source":"arxiv:2507.01098","target":"arxiv:2405.07987","evidence":"id","id":"arxiv:2507.01098","title":"Proof of a perfect platonic representation hypothesis","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/39afe7519b574380221abb205b8d481666b312a8b366e8145fe64c41caad5c32/"},{"source":"arxiv:2602.14486","target":"arxiv:2405.07987","evidence":"title","id":"arxiv:2602.14486","title":"Revisiting the Platonic Representation Hypothesis: An Aristotelian View","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f56b2d833094d17524bf9ad90c5d16bc602e9888589afd624ae8efc3f9d9fc0/"},{"source":"arxiv:2604.11962","target":"arxiv:2405.07987","evidence":"title","id":"arxiv:2604.11962","title":"The Linear Centroids Hypothesis: Features as Directions Learned by Local Experts","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8428fd0cce9753855f4d0f801609b0205a0be52a459d53ae44e7860d00348485/"},{"source":"arxiv:2606.06021","target":"arxiv:2405.07987","evidence":"id","id":"arxiv:2606.06021","title":"OPRD: On-Policy Representation Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a16a9d7ea78c5748acc7f651c9a96663e0200d72e51d6a52c9528e1739a27d8d/"},{"source":"arxiv:2608.03893","target":"arxiv:2405.07987","evidence":"title","id":"arxiv:2608.03893","title":"Cross-Model KV Cache Transfer in LLM Families: A Closed-Form Linear Mapping for Prefill Reuse","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c29465e80f45097327908d15926fd9b4a3c4c50431b2d525eb5aa9b92446fb2a/"}],"text_parts":[{"start":0,"end":15916,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html"},{"start":15916,"end":30339,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-2.html"},{"start":30339,"end":45878,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-3.html"},{"start":45878,"end":61502,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html"},{"start":61502,"end":77152,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html"},{"start":77152,"end":87178,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html"}],"contents":[{"title":"The Platonic Representation Hypothesis","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"Abstract","offset":42,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"Keywords:","offset":882,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"1 Introduction","offset":900,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"2 Representations are converging","offset":5533,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"Preliminaries","offset":5570,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"2.1 Different models, with different architectures and objectives, can have aligned representations","offset":8099,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"2.2 Alignment increases with scale and performance","offset":11197,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"2.3 Representations are converging across modalities","offset":13646,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-1.html#text"},{"title":"2.4 Models are increasingly aligning to brains","offset":17532,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-2.html#text"},{"title":"2.5 Does alignment predict downstream performance?","offset":19161,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-2.html#text"},{"title":"3 Why are representations converging?","offset":19601,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-2.html#text"},{"title":"3.1 Convergence via","offset":30057,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-2.html#text"},{"title":"3.2 Convergence via","offset":38133,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-3.html#text"},{"title":"3.3 Convergence via","offset":38795,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-3.html#text"},{"title":"4 What representation are we converging to?","offset":39760,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-3.html#text"},{"title":"4.1 An idealized world","offset":40978,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-3.html#text"},{"title":"4.2 A family of contrastive learners converge to a representation of ℙ ⁡ ( 𝐙 ) \\mathbb{P}(\\mathbf{Z})","offset":42817,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-3.html#text"},{"title":"A study in color","offset":47549,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"5 What are the implications of convergence?","offset":48716,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Scaling is sufficient, but not necessarily efficient","offset":48764,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Training data can be shared across modalities","offset":49457,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Ease of translation and adaptation across modalities","offset":50389,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Scaling may reduce hallucination and bias","offset":52270,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"6 Counterexamples and limitations","offset":53265,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Different modalities may contain different information","offset":53303,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Not all representations are presently converging","offset":55667,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Sociological bias in producing AI models","offset":56248,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Special-purpose intelligences might not converge","offset":56884,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"How do we measure alignment?","offset":57741,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Lots left to explain","offset":58154,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Acknowledgements","offset":58679,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Appendix A Mutual k k -Nearest Neighbor Alignment Metric","offset":59543,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"The choice to use mutual nearest-neighbors","offset":61453,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-4.html#text"},{"title":"Relationship between CKA and Mutual Nearest-Neighbors","offset":62192,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Appendix B Consistency across various metrics","offset":67048,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Vision-vision comparison","offset":67763,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Cross-modal comparison","offset":68002,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Appendix C Experiments on Evaluating Alignment and Convergence","offset":68372,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"C.1 Vision-Vision Alignment and Representation Quality","offset":68603,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"C.2 Cross-Modal Alignment","offset":70234,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Appendix D Color Cooccurrence Experiment","offset":72502,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Perceptual representation from CIELAB color space","offset":72671,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Three representations from cooccurrence in VISION and LANGUAGE","offset":72997,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Appendix E Caption Density Experiments","offset":76777,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-5.html#text"},{"title":"Appendix F Analysis of Contrastive Learners","offset":77799,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html#text"},{"title":"F.1 Contrastive objectives learn pointwise mutual information","offset":77847,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html#text"},{"title":"F.2 Contrastive learners can represent K 𝖯𝖬𝖨 K_{\\mathsf{PMI}} exactly under smoothness conditions","offset":82588,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html#text"},{"title":"Proposition F.1 .","offset":83193,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html#text"},{"title":"Proof.","offset":83765,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html#text"},{"title":"Remark F.2 .","offset":85496,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html#text"},{"title":"Instructions for reporting errors","offset":86214,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/part-6.html#text"}],"text_code_points":87178,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c7695929f8bbcea43ee98da9cd8f2fe806f2ba292f0b4e3e7fea2b64254afb4/source.md"}