{"id":"arxiv:2505.17117","title":"From Tokens to Thoughts: How LLMs and Humans Trade Compression for Meaning","date":"2025-05-21","section":"NeuroAI","quality":0.424878,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/","source_url":"https://arxiv.org/abs/2505.17117","text_available":true,"authors":["Chen Shani","Liron Soffer","Dan Jurafsky","Yann LeCun"],"outgoing":[{"source":"arxiv:2505.17117","target":"arxiv:2407.21783","evidence":"title","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[],"text_parts":[{"start":0,"end":15925,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html"},{"start":15925,"end":31513,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html"},{"start":31513,"end":47341,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html"},{"start":47341,"end":61829,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html"}],"contents":[{"title":"From Tokens to Thoughts: How LLMs and Humans Trade Compression for Meaning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"Abstract","offset":78,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"1 The Enigma of Meaning in Large Language Models","offset":1668,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"2 Research Questions and Scope","offset":5657,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"3 Benchmarking Against Human Cognition","offset":8795,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"3.1 Human Baselines: Empirical Data from Seminal Cognitive Science","offset":8969,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"3.2 Large Language Models Under Study","offset":11233,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"4 A Framework for Comparing Compression and Meaning","offset":13652,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"4.1 Theoretical Foundations","offset":14092,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"4.2 The ℒ \\mathcal{L} Objective: Quantifying the Trade-off","offset":15712,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-1.html#text"},{"title":"4.2.1 The Complexity Term: Measuring Compression","offset":16355,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"4.2.2 The Distortion Term: Measuring Semantic Coherence","offset":17184,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"4.3 Connecting Framework to Research Questions","offset":18088,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"5 An Empirical Investigation of Representational Strategies","offset":19737,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"5.1 [RQ1] The Big Picture: Alignment of Conceptual Categories","offset":20167,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"5.2 [RQ2] Zooming In: Fidelity to Fine-Grained Semantics","offset":22364,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"5.3 [RQ3] The Efficiency Angle: The Compression-Meaning Trade-off","offset":26490,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"5.4 Emergence During Training: How Divergent Strategies Develop","offset":29583,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-2.html#text"},{"title":"6 Discussion and Conclusion","offset":32045,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"7 Ethics statement","offset":34706,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"8 Reproducibility statement","offset":35690,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"9 Acknowledgments","offset":36567,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Appendix A Cognitive Intuition","offset":36901,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"A.1 A Cognitive Intuition of the compression-meaning tradeoff","offset":36936,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"A.2 Cognitively-Inspired Inductive Biases","offset":38485,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Appendix B Limitations","offset":40439,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Appendix C Dataset Access Details","offset":41191,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Appendix D LLM Details","offset":41450,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Appendix E Contextual Prompts and Pooling Strategies","offset":42767,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Pooling Strategies.","offset":43827,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Prompt Templates.","offset":44085,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Models.","offset":44462,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Appendix F Static vs. Contextual AMI Exploration","offset":44797,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-3.html#text"},{"title":"Appendix G Multilingual Analysis","offset":48201,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix H Training Dynamics","offset":49127,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix I Datasets Polysemy","offset":50426,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix J Tokenizer Analysis","offset":51602,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Model Clustering By Tokenizer Family","offset":53067,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix K Additional Clustering Metrics","offset":53832,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix L Mini-Controlled Experiment (Matched Training Data)","offset":54836,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix M Detailed AMI Scores per Model and Dataset","offset":55537,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix N Correlation between Human Typicality Judgments and LLM Internal Cluster Geometry","offset":55723,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix O Typicality and Cosine Similarity [RQ2]","offset":56813,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix P Theoretical Extreme Case Exploration for ℒ \\mathcal{L}","offset":57334,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix Q Compression Figures","offset":58962,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix R Complexity-Distortion Ratio (on the importance of β \\beta )","offset":59263,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Appendix S ℒ \\mathcal{L} objective vs. downstream task performance","offset":59742,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"},{"title":"Instructions for reporting errors","offset":60865,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/part-4.html#text"}],"text_code_points":61829,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0837eb0f0d0e479142b8ae6d2ef144168e5e16c610ad22479076c9d606be084/source.md"}