{"id":"arxiv:2510.27688","title":"Continuous Autoregressive Language Models","date":"2025-10-31","section":"LLMs: architectures, context, training","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/","source_url":"https://arxiv.org/abs/2510.27688","text_available":true,"authors":["Chenze Shao","Darren Li","Fandong Meng","Jie Zhou"],"outgoing":[{"source":"arxiv:2510.27688","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2510.27688","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2510.27688","target":"arxiv:2304.08467","evidence":"title","id":"arxiv:2304.08467","title":"Learning to Compress Prompts with Gist Tokens","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e77a0eef338c287982221a4a4770c18f85eb99a6d245af931d60a6ec9190ba69/"},{"source":"arxiv:2510.27688","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2510.27688","target":"url:ai.meta.com/research/publications/large-concept-models-language-modeling-in-a-sentence-representation-space","evidence":"id","id":"url:ai.meta.com/research/publications/large-concept-models-language-modeling-in-a-sentence-representation-space","title":"Large Concept Models: Language Modeling in a Sentence Representation Space","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a5cd322d52f1966361afb1d5225560e2d3ce4d22a117bd14cfa27ec669e09673/"}],"incoming":[{"source":"arxiv:2608.05806","target":"arxiv:2510.27688","evidence":"id","id":"arxiv:2608.05806","title":"Hierarchical Latent Prediction for Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/991d6d0a05c5eaa5f6c94943be460ba2a65770ccae4997df9544bb772604d430/"}],"text_parts":[{"start":0,"end":15982,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-1.html"},{"start":15982,"end":31747,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html"},{"start":31747,"end":46620,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html"},{"start":46620,"end":62528,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html"},{"start":62528,"end":78318,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-5.html"},{"start":78318,"end":93139,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html"}],"contents":[{"title":"Continuous Autoregressive Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-1.html#text"},{"title":"Abstract","offset":45,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-1.html#text"},{"title":"1 Introduction","offset":1480,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-1.html#text"},{"title":"2 Autoencoder","offset":8180,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-1.html#text"},{"title":"2.1 High-Fidelity Reconstruction","offset":8198,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-1.html#text"},{"title":"2.2 Robust Vector Representation","offset":11024,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-1.html#text"},{"title":"3 Likelihood-Free Language Modeling","offset":16830,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"3.1 Next-Vector Prediction","offset":16870,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"3.2 Generative Head","offset":19156,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"3.3 Energy Transformer","offset":21084,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"3.3.1 Strictly Proper Scoring Rules","offset":21112,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"3.3.2 Energy Loss","offset":22957,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"3.3.3 Model Architecture","offset":26400,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"4 Likelihood-Free LM Evaluation","offset":29789,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"4.1 Principles of LM Evaluation","offset":29825,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-2.html#text"},{"title":"4.2 BrierLM: Brier for Language Modeling","offset":32535,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"5 Likelihood-Free Temperature Sampling","offset":37688,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"5.1 Exact Temperature Sampling via Rejection","offset":37731,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"Theorem 1 .","offset":40335,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"5.2 Expected Sampling Cost","offset":40687,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"Theorem 2 .","offset":41400,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"Corollary 2.1 .","offset":41921,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"5.3 Batch Approximation","offset":43283,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"Theorem 3 .","offset":45630,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"6 Related Work","offset":46580,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"6.1 Autoencoder","offset":46599,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-3.html#text"},{"title":"6.2 Likelihood-Free Language Modeling","offset":49342,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html#text"},{"title":"6.3 Likelihood-Free LM Evaluation","offset":53198,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html#text"},{"title":"6.4 Likelihood-Free Temperature Sampling","offset":55295,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html#text"},{"title":"7 Experiments","offset":57475,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html#text"},{"title":"7.1 Settings","offset":57493,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html#text"},{"title":"7.2 Main Results","offset":59337,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html#text"},{"title":"7.3 Effect of Autoencoder","offset":62497,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-4.html#text"},{"title":"7.4 Effect of Model Architecture","offset":67203,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-5.html#text"},{"title":"7.5 Temperature Sampling","offset":72615,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-5.html#text"},{"title":"8 Conclusion and Future Work","offset":76393,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-5.html#text"},{"title":"Appendix A Proof","offset":80708,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"A.1 Proof of Theorem 1","offset":80729,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"Proof.","offset":80764,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"A.2 Proof of Theorem 2","offset":83251,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"Proof.","offset":83286,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"Proof.","offset":86468,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"A.3 Proof of Theorem 3","offset":88378,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"Proof.","offset":88413,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"},{"title":"Instructions for reporting errors","offset":92175,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/part-6.html#text"}],"text_code_points":93139,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e954d29144273a3d9e4c1a5ac4fc43a163b8701659926e2bfa94011507f51414/source.md"}