{"id":"arxiv:2505.15134","title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","date":"2025-05-21","section":"Reasoning and the \"physics\" of language models","quality":0.233035,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/","source_url":"https://arxiv.org/abs/2505.15134","text_available":true,"authors":["Shivam Agarwal","Zimin Zhang","Lifan Yuan"],"outgoing":[{"source":"arxiv:2505.15134","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/"},{"source":"arxiv:2505.15134","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2505.15134","target":"arxiv:2203.11171","evidence":"id","id":"arxiv:2203.11171","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02a15b098d96a94d463bb470f59a34723abc34e59196f35c539f9796d8f3c3df/"},{"source":"arxiv:2505.15134","target":"arxiv:2303.17651","evidence":"title","id":"arxiv:2303.17651","title":"Self-Refine: Iterative Refinement with Self-Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/accef2157d87cf33be93cb3a9af246528b0a3742a9e2e700a194956fac8f5cf4/"},{"source":"arxiv:2505.15134","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2505.15134","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2505.15134","target":"arxiv:2501.19393","evidence":"id","id":"arxiv:2501.19393","title":"s1: Simple test-time scaling","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2a88818f7605b71cd0160831faebd51b3274c24ffd29626b4d519390b4bffffe/"},{"source":"arxiv:2505.15134","target":"arxiv:2504.16084","evidence":"id","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2505.15134","target":"arxiv:2504.20571","evidence":"id","id":"arxiv:2504.20571","title":"Reinforcement Learning for Reasoning in Large Language Models with One Training Example","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b2b21c3df49619588c371411dba9467a41f81b3fd4da5441898b5145bb0de120/"}],"incoming":[{"source":"arxiv:2505.19590","target":"arxiv:2505.15134","evidence":"id","id":"arxiv:2505.19590","title":"Learning to Reason without External Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/"},{"source":"arxiv:2506.10139","target":"arxiv:2505.15134","evidence":"id","id":"arxiv:2506.10139","title":"Unsupervised Elicitation of Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/"},{"source":"arxiv:2509.14234","target":"arxiv:2505.15134","evidence":"title","id":"arxiv:2509.14234","title":"Compute as Teacher: Turning Inference Compute Into Reference-Free Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a6f1443a7e55dd6421fdbe43e65e19c1962a19b135d737fd67f414d3324ce85e/"},{"source":"arxiv:2509.25123","target":"arxiv:2505.15134","evidence":"title","id":"arxiv:2509.25123","title":"From $f(x)$ and $g(x)$ to $f(g(x))$: LLMs Learn New Skills in RL by Composing Old Ones","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fdfe7abe26ee324fe7abad3ce802148163129a7c157d77bdec859c990868e03/"},{"source":"arxiv:2605.06241","target":"arxiv:2505.15134","evidence":"id","id":"arxiv:2605.06241","title":"Rethinking RL for LLM Reasoning: It's Sparse Policy Selection, Not Capability Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9e4fdd6e0c24f5a635ff70c2bca19da19c4012a05f69ceb20fdc0a86fd8b775/"}],"text_parts":[{"start":0,"end":15264,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html"},{"start":15264,"end":31174,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-2.html"},{"start":31174,"end":47129,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html"},{"start":47129,"end":63098,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html"},{"start":63098,"end":64062,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-5.html"}],"contents":[{"title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"Abstract","offset":75,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"1 Introduction","offset":1480,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"2 Background and Related Work","offset":5651,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"2.1 Related Work","offset":5685,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"2.2 Preliminaries","offset":10291,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"3 Entropy Minimization for Post-training","offset":14001,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"3.1 EM-FT : Direct Entropy Minimization through Token-Level Finetuning","offset":14046,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"3.2 EM-RL : Entropy Minimization with Negative Entropy Rewards in RL","offset":14890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-1.html#text"},{"title":"3.3 Training Setup","offset":20169,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-2.html#text"},{"title":"3.4 Results: Entropy Minimization for Post-Training","offset":23439,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-2.html#text"},{"title":"4 EM-INF: Inference-time Logit Optimization","offset":27831,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-2.html#text"},{"title":"Proposition 1 .","offset":31769,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"5 Limitations: Dependence on Base Model Capability and Confidence","offset":36352,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"6 Conclusion","offset":38849,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"Acknowledgments and Disclosure of Funding","offset":39912,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"Appendix A Proof for proposition 1","offset":40975,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"Appendix B Adaptive Temperature Scaling","offset":44412,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"Appendix C Additional Results","offset":44456,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"C.1 Combining EM-INF with Self-consistency and Iterative Self-refinement","offset":44490,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"C.2 Qualitative analysis of EM-INF on SciCode","offset":45344,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-3.html#text"},{"title":"Appendix D Experimental Setup","offset":47711,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.1 Evaluation Benchmarks","offset":47745,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.1.1 Math Reasoning","offset":47776,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"MATH 500","offset":48125,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"AMC","offset":48777,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"AIME","offset":49252,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"Minerva Math","offset":49983,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"OlympiadBench","offset":50719,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.1.2 Coding","offset":51527,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"LeetCode","offset":51546,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"LiveCodeBench","offset":52627,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.1.3 Scientific Reasoning","offset":53951,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"UGPhysics","offset":53984,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"SciCode","offset":55247,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.1.4 Individualistic Value Reasoning","offset":56733,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"IndVal","offset":56777,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.2 Training Data","offset":58477,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"Math Reasoning","offset":58500,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"Coding","offset":58791,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"Individualistic Value Reasoning","offset":59621,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.3 Models","offset":60050,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"Qwen2.5 Family","offset":60066,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"Llama-3.1 Family","offset":61514,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"D.4 FLOPs Computation","offset":61807,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-4.html#text"},{"title":"Instructions for reporting errors","offset":63098,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/part-5.html#text"}],"text_code_points":64062,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/source.md"}