{"id":"arxiv:2509.09675","title":"CDE: Curiosity-Driven Exploration for Efficient Reinforcement Learning in Large Language Models","date":"2025-09-11","section":"Reinforcement learning","quality":0.1136,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/","source_url":"https://arxiv.org/abs/2509.09675","text_available":true,"authors":["Runpeng Dai","Linfeng Song","Haolin Liu","Zhenwen Liang"],"outgoing":[{"source":"arxiv:2509.09675","target":"arxiv:1801.01290","evidence":"title","id":"arxiv:1801.01290","title":"Soft Actor-Critic: Off-Policy Maximum Entropy Deep Reinforcement Learning with a Stochastic Actor","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21e70876ae9603903b2b20dd667a413bdd8b0abf37b9c11e284ea8b5e4124c74/"},{"source":"arxiv:2509.09675","target":"arxiv:2203.11171","evidence":"id","id":"arxiv:2203.11171","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/02a15b098d96a94d463bb470f59a34723abc34e59196f35c539f9796d8f3c3df/"},{"source":"arxiv:2509.09675","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2509.09675","target":"arxiv:2503.14476","evidence":"id","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"}],"incoming":[],"text_parts":[{"start":0,"end":15014,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html"},{"start":15014,"end":30991,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html"},{"start":30991,"end":46513,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html"},{"start":46513,"end":59023,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html"}],"contents":[{"title":"CDE: Curiosity-Driven Exploration for Efficient Reinforcement Learning in Large Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"Abstract","offset":99,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"1 Introduction","offset":1342,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"2 Preliminaries: RLVR, GRPO and PPO","offset":6949,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"2.1 Group Relative Policy Optimization (GRPO, Shao et al. 2024 )","offset":7575,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"2.2 Proximal Policy Optimization (PPO, Schulman et al. 2017 )","offset":9781,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"3 CDE: Curiosity-Driven Exploration","offset":11679,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"3.1 Challenge of Count-based exploration for RLVR","offset":12103,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"3.2 Exploration Guided by Actor Curiosity","offset":14068,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-1.html#text"},{"title":"Intuitions and Theoretical Foundation","offset":17439,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"Theorem 3.1 .","offset":18699,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"3.3 Exploration Guided by Critic Curiosity","offset":19929,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"Theorem 3.2 .","offset":21966,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"4 Experiments","offset":26271,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"4.1 Dataset and Model","offset":26289,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"4.2 Main Results","offset":27056,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"4.3 Understanding the Effect of the PPL Bonus","offset":28849,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-2.html#text"},{"title":"4.4 Further Analysis of the Multi-Head Critic","offset":32278,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"5 Related Work","offset":34358,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"5.1 Reinforcement Learning (RL) for LLM reasoning","offset":34377,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"5.2 Efficient exploration","offset":36068,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"6 Conclusion and Future Work","offset":38154,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"Appendix A Training Details","offset":39341,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"Appendix B Prompt","offset":39531,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"Appendix C Details on Hash-based pseudo count","offset":39595,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"Appendix D Proof for Calibration Theorem","offset":41198,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-3.html#text"},{"title":"Appendix E Proof for Consistency of Multi-head Critic Bonus","offset":47004,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html#text"},{"title":"Assumption E.1 (Linear MDP) .","offset":47096,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html#text"},{"title":"Lemma E.2 (Proposition 2.3 in Jin et al. (2020) ) .","offset":48017,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html#text"},{"title":"Formal Version and Proof of Theorem 3.2","offset":51165,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html#text"},{"title":"Theorem E.3 .","offset":51211,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html#text"},{"title":"Proof.","offset":51582,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html#text"},{"title":"Instructions for reporting errors","offset":58059,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/part-4.html#text"}],"text_code_points":59023,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f5581044978796ff7f35cb68c27963d3abbb391d35d88173ff7801242cbb1f7a/source.md"}