{"id":"arxiv:2502.09992","title":"Large Language Diffusion Models","date":"2025-02-14","section":"LLMs: architectures, context, training","quality":0.498698,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/","source_url":"https://arxiv.org/abs/2502.09992","text_available":true,"authors":["Shen Nie","Fengqi Zhu","Zebin You","Xiaolu Zhang"],"outgoing":[{"source":"arxiv:2502.09992","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2502.09992","target":"arxiv:2002.05202","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2502.09992","target":"arxiv:2309.12288","evidence":"id","id":"arxiv:2309.12288","title":"The Reversal Curse: LLMs trained on \"A is B\" fail to learn \"B is A\"","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02782a7d9697d5df8f8ae84424becf682d10e8671ee5e18fd75499de0e0a6694/"},{"source":"arxiv:2502.09992","target":"arxiv:2309.14402","evidence":"id","id":"arxiv:2309.14402","title":"Physics of Language Models: Part 3.2, Knowledge Manipulation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/79b4d0b92a11dcf3b111e9d2d1b6bb3f516bebce7333326f65e6e583d6ff3cf2/"},{"source":"arxiv:2502.09992","target":"arxiv:2407.20311","evidence":"id","id":"arxiv:2407.20311","title":"Physics of Language Models: Part 2.1, Grade-School Math and the Hidden Reasoning Process","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de5db7bc6b929af401b5b83c845823505a7823c25987b0e455231d2e4fcec3e4/"},{"source":"arxiv:2502.09992","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[{"source":"arxiv:2512.15745","target":"arxiv:2502.09992","evidence":"title","id":"arxiv:2512.15745","title":"LLaDA2.0: Scaling Up Diffusion Language Models to 100B","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/90a25474fdf076c254ea3ddbbc7ddbc1f2f49fb98065168d66b5ad9d8f7dafaa/"},{"source":"arxiv:2603.05454","target":"arxiv:2502.09992","evidence":"id","id":"arxiv:2603.05454","title":"Beyond Scattered Acceptance: Fast and Coherent Inference for DLMs via Longest Stable Prefixes","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/303994b6470f9a9063fca1f6980da95344903a409e61de83c6b35e99146b6329/"},{"source":"arxiv:2604.04943","target":"arxiv:2502.09992","evidence":"id","id":"arxiv:2604.04943","title":"The Illusion of Latent Generalization: Bi-directionality and the Reversal Curse","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/14a8648d9d6f2e9700168f1fd2254dd0307f81e5925488becf1a5a72463af956/"},{"source":"arxiv:2604.08302","target":"arxiv:2502.09992","evidence":"id","id":"arxiv:2604.08302","title":"DMax: Aggressive Parallel Decoding for dLLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/22790263bd218aac28626fe3dab2fcd0ed73ceef94aa8a948bef2cb8bc007f7a/"},{"source":"arxiv:2606.29150","target":"arxiv:2502.09992","evidence":"id","id":"arxiv:2606.29150","title":"Flow Reasoning Models: Turning Flows Into Efficient Recurrent Reasoners","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8be1801f0f562711039ced2b42f324434c6ab829f544b8d736bc85405b64021b/"},{"source":"arxiv:2608.00146","target":"arxiv:2502.09992","evidence":"title","id":"arxiv:2608.00146","title":"DiffusionGemma Technical Report","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/90a7b084496cd65968627c115230fc3a5e78005a66b5ebb34547e874754bc82e/"}],"text_parts":[{"start":0,"end":15520,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html"},{"start":15520,"end":30597,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-2.html"},{"start":30597,"end":46531,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html"},{"start":46531,"end":62139,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-4.html"},{"start":62139,"end":76195,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-5.html"}],"contents":[{"title":"Large Language Diffusion Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"Abstract","offset":35,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"1 Introduction","offset":1293,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"2 Approach","offset":6336,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"2.1 Probabilistic Formulation","offset":6657,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"2.2 Pre-training","offset":9535,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"2.3 Supervised Fine-Tuning","offset":12528,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"2.4 Inference","offset":15329,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-1.html#text"},{"title":"3 Experiments","offset":18457,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-2.html#text"},{"title":"3.1 Scalability of LLaDA on Language Tasks","offset":18675,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-2.html#text"},{"title":"3.2 Benchmark Results","offset":20866,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-2.html#text"},{"title":"3.3 Reversal Reasoning and Analyses","offset":25666,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-2.html#text"},{"title":"3.4 Case Studies","offset":29243,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-2.html#text"},{"title":"4 Related Work","offset":29865,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-2.html#text"},{"title":"5 Conclusion and Discussion","offset":32869,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"Acknowledgements","offset":35528,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"Appendix A Formulation of Masked Diffusion Models","offset":35761,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"A.1 Training","offset":35815,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"A.2 Inference","offset":41717,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"A.3 Algorithms","offset":45122,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"Appendix B Experiments","offset":45606,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"B.1 Data Collection and Preprocessing","offset":45633,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-3.html#text"},{"title":"B.2 Details about Model Training","offset":47995,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-4.html#text"},{"title":"B.3 Ablation on Classifier-free Guidance","offset":50431,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-4.html#text"},{"title":"B.4 Details and Ablation on Sampling Strategies","offset":51210,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-4.html#text"},{"title":"B.5 Ablation on Generated Length","offset":56605,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-4.html#text"},{"title":"B.6 Standard Benchmarks and Evaluation Details","offset":57172,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-4.html#text"},{"title":"B.7 Analysis of Sampling Efficiency","offset":60298,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-4.html#text"},{"title":"B.8 Evaluation on iGSM Dataset","offset":63185,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-5.html#text"},{"title":"B.9 Poem Completion Tasks","offset":66041,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-5.html#text"},{"title":"B.10 More Case Studies","offset":66273,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-5.html#text"},{"title":"Appendix C Impact Statement","offset":66794,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-5.html#text"},{"title":"Instructions for reporting errors","offset":75231,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/part-5.html#text"}],"text_code_points":76195,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a4863ed6f639560bd2feb6c66c13d1e3ac4cf42f8919a65c9818625cc6b6cde9/source.md"}