{"id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","date":"2022-10-19","section":"Data, training, optimization","quality":0.215147,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/","source_url":"https://arxiv.org/abs/2210.10760","text_available":true,"authors":["Leo Gao","John Schulman","Jacob Hilton"],"outgoing":[{"source":"arxiv:2210.10760","target":"arxiv:1912.01683","evidence":"title","id":"arxiv:1912.01683","title":"Optimal Policies Tend to Seek Power","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/"},{"source":"arxiv:2210.10760","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2210.10760","target":"arxiv:2206.13353","evidence":"id","id":"arxiv:2206.13353","title":"Is Power-Seeking AI an Existential Risk?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/"},{"source":"arxiv:2210.10760","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[{"source":"arxiv:2412.14135","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2412.14135","title":"Scaling of Search and Learning: A Roadmap to Reproduce o1 from Reinforcement Learning Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5a0b4b0b524698ee5f30eab659b7cd5104f5fe6497c8dec2d0c4b9d332350047/"},{"source":"arxiv:2503.14476","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2505.19590","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2505.19590","title":"Learning to Reason without External Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/"},{"source":"arxiv:2505.21493","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2505.21493","title":"Reinforcing General Reasoning without Verifiers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/06d2ae06e1a3bcda5db4695ecca7ec78df67002bddc755936e262799f3e15d15/"},{"source":"arxiv:2506.03106","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2506.03106","title":"Critique-GRPO: Advancing LLM Reasoning with Natural Language and Numerical Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/"},{"source":"arxiv:2602.18037","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2602.18037","title":"Gradient Regularization Mitigates Reward Hacking in Reinforcement Learning from Human Feedback and Verifiable Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/"},{"source":"arxiv:2604.19341","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2604.19341","title":"Structured Scaling of AI Discovery Across Diverse Scientific Domains","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/33ad3375247f86197210840a19064548b997e8617475e606d7a7ac2cda64740e/"}],"text_parts":[{"start":0,"end":15803,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html"},{"start":15803,"end":31704,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html"},{"start":31704,"end":42270,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html"}],"contents":[{"title":"Scaling Laws for Reward Model Overoptimization","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"Abstract","offset":50,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"1 Introduction","offset":1339,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"2 Methodology","offset":6929,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"2.1 Synthetic Data Setup","offset":8766,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"2.2 Recalibration","offset":9815,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"3 Results","offset":10678,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"3.1 Fitting and validating functional forms","offset":10692,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"3.2 Scaling with RM Parameter Count","offset":11797,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"3.3 Scaling with RM Data Size","offset":12951,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"3.4 Scaling with Policy Size","offset":14649,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"Larger policies see less benefit from optimization against an RM, but don’t overoptimize more.","offset":14916,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"3.5 RL vs BoN","offset":15784,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-1.html#text"},{"title":"RL is far less KL-efficient than BoN.","offset":16293,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"When looking at proxy vs gold RM scores, BoN and RL look more similar.","offset":17031,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"3.6 Effect of KL Penalty","offset":17548,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4 Discussion","offset":18894,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.1 KL as a measure of amount of optimization","offset":18911,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.2 Relation to Goodhart Taxonomy","offset":19655,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.2.1 Regressional Goodhart","offset":19963,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.2.2 Extremal Goodhart","offset":22011,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.2.3 Causal Goodhart","offset":23576,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.2.4 Adversarial Goodhart","offset":24435,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.3 Implications for iterated RLHF","offset":25026,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.4 Policy size independence","offset":26775,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"4.5 Limitations and Future Work","offset":28097,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"5 Related Work","offset":31685,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-2.html#text"},{"title":"Acknowlegements","offset":35388,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html#text"},{"title":"Appendix A Proof of Regressional Goodhart identity","offset":35564,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html#text"},{"title":"Lemma .","offset":35619,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html#text"},{"title":"Proof.","offset":36404,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html#text"},{"title":"Appendix B RL form details","offset":38838,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html#text"},{"title":"Appendix C Hyperparameters","offset":39741,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html#text"},{"title":"Instructions for reporting errors","offset":41306,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/part-3.html#text"}],"text_code_points":42270,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/source.md"}