{"id":"arxiv:2506.01939","title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","date":"2025-06-02","section":"Post-training","quality":0.275648,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/","source_url":"https://arxiv.org/abs/2506.01939","text_available":true,"authors":["Shenzhi Wang","Le Yu","Chang Gao"],"outgoing":[{"source":"arxiv:2506.01939","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2506.01939","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2506.01939","target":"arxiv:2501.04519","evidence":"id","id":"arxiv:2501.04519","title":"rStar-Math: Small LLMs Can Master Math Reasoning with Self-Evolved Deep Thinking","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/473ad7b5a354658a1d241813741e15a8ab51e7c2ea34f0c3d22eb06884864a46/"},{"source":"arxiv:2506.01939","target":"arxiv:2503.14476","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2506.01939","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/"},{"source":"arxiv:2506.01939","target":"arxiv:2504.20571","evidence":"id","id":"arxiv:2504.20571","title":"Reinforcement Learning for Reasoning in Large Language Models with One Training Example","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b2b21c3df49619588c371411dba9467a41f81b3fd4da5441898b5145bb0de120/"}],"incoming":[{"source":"arxiv:2504.13837","target":"arxiv:2506.01939","evidence":"title","id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/"},{"source":"arxiv:2506.10947","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2506.10947","title":"Spurious Rewards: Rethinking Training Signals in RLVR","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f48e13a9656d8eff1882eb80988e07a820c4799401298947deac735c004e8592/"},{"source":"arxiv:2506.13585","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2506.13585","title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5beb2ec6c7e6362f9d957219d7a205dd1adcc53f739acc0c7fd5f1f2bded4d83/"},{"source":"arxiv:2508.11408","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2508.11408","title":"On-Policy RL Meets Off-Policy Experts: Harmonizing Supervised Fine-Tuning and Reinforcement Learning via Dynamic Weighting","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/48594da2e3f2cdfeb17195dd033d047235fc4b4ffb65030800f64c6faedc4fbe/"},{"source":"arxiv:2509.03646","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2509.03646","title":"Emergent Hierarchical Reasoning in LLMs through Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/"},{"source":"arxiv:2512.00499","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2512.00499","title":"ESPO: Entropy Importance Sampling Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/342b84df2bf40f64c57ea7839579c1f337374ad9c28b16ce7d477c42e24c67b4/"},{"source":"arxiv:2601.20802","target":"arxiv:2506.01939","evidence":"title","id":"arxiv:2601.20802","title":"Reinforcement Learning via Self-Distillation","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6073ec3d17b9a932a68e79ae078928f4635e40fc79cfcbc31709715d1f1e1ffa/"},{"source":"arxiv:2604.08690","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/"},{"source":"arxiv:2604.20659","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2604.20659","title":"GRPO-VPS: Enhancing Group Relative Policy Optimization with Verifiable Process Supervision for Effective Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/42caf0f9c992f5df8bac4d0f7e45c3a5bf57879661f41febd01a047694d52628/"},{"source":"arxiv:2605.06241","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2605.06241","title":"Rethinking RL for LLM Reasoning: It's Sparse Policy Selection, Not Capability Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9e4fdd6e0c24f5a635ff70c2bca19da19c4012a05f69ceb20fdc0a86fd8b775/"},{"source":"arxiv:2605.22074","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2605.22074","title":"From Reasoning Chains to Verifiable Subproblems: Curriculum Reinforcement Learning Enables Credit Assignment for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e51c03db071e17a4d5403433f1037e79c3cda795e0b20f6a490df35167411dd0/"}],"text_parts":[{"start":0,"end":14943,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html"},{"start":14943,"end":30257,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html"},{"start":30257,"end":46116,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html"},{"start":46116,"end":47260,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-4.html"}],"contents":[{"title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"Abstract","offset":112,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"1 Introduction","offset":2076,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"2 Preliminaries","offset":10030,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"2.1 Token entropy calculation","offset":10050,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"\"Token entropy\" corresponds to the token generation distribution, not a specific token.","offset":11489,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"2.2 RLVR Algorithms","offset":12251,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"Proximal Policy Optimization (PPO)","offset":12276,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"Group Relative Policy Optimization (GRPO)","offset":13704,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"Dynamic sAmpling Policy Optimization (DAPO)","offset":14893,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-1.html#text"},{"title":"3 Analyzing Token Entropy in Chain-of-Thought Reasoning","offset":16406,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"Entropy Pattern 1 in CoTs: Typically, only a minority of tokens are generated with high entropy, while a majority of tokens are outputted with low entropy.","offset":17557,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"Entropy Pattern 2 in CoTs: Tokens with the highest entropy typically serve to bridge the logical connection between two consecutive parts of reasoning, while tokens with the lowest entropy tend to complete the current part of a sentence or finish constructing a word. Other tokens combine these two functions to varying degrees.","offset":18023,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"High-entropy tokens as \"forks\" in chain-of-thoughts","offset":19461,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"4 RLVR Preserves and Reinforces Base Model Entropy Patterns","offset":22009,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"RLVR primarily preserves the existing entropy patterns of the base models","offset":22248,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"RLVR predominantly alters the entropy of high-entropy tokens, whereas the entropy of low-entropy tokens remains comparatively stable with minimal variations.","offset":23245,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"5 High-Entropy Minority Tokens Drive Effective RLVR","offset":24298,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"5.1 Formulation of RLVR Using Only Policy Gradients of the Highest-Entropy Tokens","offset":24980,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"5.2 Experimental Setup","offset":27904,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"Training details","offset":27932,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"Evaluation","offset":29674,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"5.3 Main Results","offset":30161,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"High-entropy tokens drive reinforcement learning for LLM reasoning.","offset":30183,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-2.html#text"},{"title":"Low-entropy tokens contribute minimally to reasoning performance.","offset":31948,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"The effectiveness of high-entropy tokens may lie in their ability to enhance exploration.","offset":32455,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Focusing on forking tokens in the policy gradient loss benefits larger reasoning models.","offset":33686,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"5.4 Analysis","offset":34176,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Generalization ability to other domains","offset":34194,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Unlocking more potential of RLVR with only forking tokens","offset":35356,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Results on models other than Qwen","offset":36027,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"6 Discussions","offset":37347,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Discussion 1: High-entropy minority tokens (i.e., forking tokens) could play a key role in explaining why RL generalizes while SFT memorizes.","offset":37365,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Discussion 2: Unlike traditional RL, LLM reasoning integrates prior knowledge and must produce readable output. Consequently, LLM CoTs contain a mix of low-entropy majority tokens and high-entropy minority tokens, whereas traditional RL can assume uniform action entropy throughout a trajectory.","offset":38430,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Discussion 3: In RLVR, entropy bonus may be suboptimal, as it increases the entropy of low-entropy majority tokens. In contrast, clip-higher effectively promotes entropy in high-entropy minority tokens.","offset":39520,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"7 Related Work","offset":40931,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Reinforcement learning for LLM","offset":40950,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Analysis on reinforcement learning with verifiable rewards","offset":43130,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"8 Conclusion","offset":44903,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Limitations","offset":46098,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-3.html#text"},{"title":"Future Directions","offset":46955,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/part-4.html#text"}],"text_code_points":47260,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/source.md"}