{"id":"arxiv:2205.13147","title":"Matryoshka Representation Learning","date":"2022-05-26","section":"Retrieval, embeddings, benchmarks","quality":0.591178,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/","source_url":"https://arxiv.org/abs/2205.13147","text_available":true,"authors":["Aditya Kusupati","Gantavya Bhatt","Aniket Rege","Matthew Wallingford"],"outgoing":[{"source":"arxiv:2205.13147","target":"arxiv:1506.01186","evidence":"title","id":"arxiv:1506.01186","title":"Cyclical Learning Rates for Training Neural Networks","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e05cf550191656c287184521d0ff011092f5ed6baa72e50f3a5bfd25d0f5986a/"}],"incoming":[{"source":"arxiv:2604.09168","target":"arxiv:2205.13147","evidence":"title","id":"arxiv:2604.09168","title":"ELT: Elastic Looped Transformers for Visual Generation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/63d4e5f1a267b96ed4474fa233f7d3bb2171aae949c0139db37b58b4e45eca47/"}],"text_parts":[{"start":0,"end":15437,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html"},{"start":15437,"end":31390,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html"},{"start":31390,"end":46490,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html"},{"start":46490,"end":62453,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html"},{"start":62453,"end":74040,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html"}],"contents":[{"title":"Matryoshka Representation Learning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html#text"},{"title":"Abstract","offset":38,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html#text"},{"title":"1 Introduction","offset":1887,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html#text"},{"title":"2 Related Work","offset":8137,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html#text"},{"title":"Representation Learning.","offset":8156,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html#text"},{"title":"Efficient Classification and Retrieval.","offset":9449,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html#text"},{"title":"3 Matryoshka ​ Representation ​ Learning {\\rm Matryoshka~Representation~Learning}","offset":13350,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-1.html#text"},{"title":"Adaptation to Learning Frameworks.","offset":17903,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html#text"},{"title":"4 Applications","offset":18620,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html#text"},{"title":"4.1 Representation Learning","offset":19117,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html#text"},{"title":"4.2 Classification","offset":21253,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html#text"},{"title":"4.2.1 Adaptive Classification","offset":23669,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html#text"},{"title":"4.3 Retrieval","offset":25449,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html#text"},{"title":"4.3.1 Adaptive Retrieval","offset":28897,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-2.html#text"},{"title":"5 Further Analysis and Ablations","offset":32409,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Robustness.","offset":32446,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Few-shot and Long-tail Learning.","offset":33697,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Disagreement across Dimensions.","offset":34772,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Superclass Accuracy.","offset":35716,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"5.1 Ablations","offset":37059,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"6 Discussion and Conclusions","offset":38111,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Acknowledgments","offset":40184,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Appendix A Code for Matryoshka ​ Representation ​ Learning {\\rm Matryoshka~Representation~Learning} ( MRL {\\rm MRL} )","offset":41017,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Appendix B Datasets","offset":41315,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"ImageNet Robustness Datasets","offset":42125,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Appendix C Matryoshka ​ Representation ​ Learning {\\rm Matryoshka~Representation~Learning} Model Training","offset":42863,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"Appendix D Classification Results","offset":45182,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-3.html#text"},{"title":"D.1 Adaptive Classification ( MRL {\\rm MRL} –AC)","offset":47498,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"D.2 JFT, ALIGN and BERT","offset":50070,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Appendix E Image Retrieval","offset":51940,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Appendix F Adaptive Retrieval","offset":56276,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Funnel Retrieval.","offset":57608,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Appendix G Few-shot and Sample Efficiency","offset":58850,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"ImageNetV2.","offset":59178,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"FLUID.","offset":60130,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Appendix H Robustness Experiments","offset":61158,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Appendix I In Practice Costs","offset":61830,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"MRL {\\rm MRL} models.","offset":62065,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Retrieval","offset":62436,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-4.html#text"},{"title":"Appendix J Analysis of Model Disagreement","offset":63119,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Class Trends","offset":63165,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Discussion of Oracle Accuracy","offset":64426,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Grad-CAM Examples","offset":65577,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Superclass Performance","offset":66386,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Appendix K Ablation Studies","offset":66664,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"K.1 MRL {\\rm MRL} Training Paradigm","offset":66696,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Matryoshka ​ Representations {\\rm Matryoshka~Representations} via Finetuning.","offset":66737,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Relative Importance.","offset":68107,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Matryoshka ​ Representations {\\rm Matryoshka~Representations} at Arbitrary Granularities.","offset":69359,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Lower Dimensionality.","offset":71496,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"K.2 Retrieval","offset":72380,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Adaptive Retrieval.","offset":72399,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"},{"title":"Instructions for reporting errors","offset":73076,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/part-5.html#text"}],"text_code_points":74040,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/803baa0a8ea7ba1bc63bee0ce80006bf4f403ca0db16ada95dd8198da9a4a899/source.md"}