{"id":"arxiv:2602.18037","title":"Gradient Regularization Mitigates Reward Hacking in Reinforcement Learning from Human Feedback and Verifiable Rewards","date":"2026-02-20","section":"Post-training","quality":0.140712,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/","source_url":"https://arxiv.org/abs/2602.18037","text_available":true,"authors":["Johannes Ackermann","Michael Noukhovitch","Takashi Ishida"],"outgoing":[{"source":"arxiv:2602.18037","target":"arxiv:2010.01412","evidence":"title","id":"arxiv:2010.01412","title":"Sharpness-Aware Minimization for Efficiently Improving Generalization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e8386ef0dc42519cd17f8346b28a923ab9ad12f8bae3942e216a31c995d97f61/"},{"source":"arxiv:2602.18037","target":"arxiv:2205.07802","evidence":"title","id":"arxiv:2205.07802","title":"The Primacy Bias in Deep Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4e2aca370b941bb16bb6fefc359c420f6d4bde5a956bedaf0a84e6e71febc5bc/"},{"source":"arxiv:2602.18037","target":"arxiv:2210.10760","evidence":"title","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2602.18037","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2602.18037","target":"arxiv:2503.20783","evidence":"id","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2602.18037","target":"arxiv:2512.13961","evidence":"id","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"},{"source":"arxiv:2602.18037","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[],"text_parts":[{"start":0,"end":15544,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html"},{"start":15544,"end":30184,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html"},{"start":30184,"end":46164,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html"},{"start":46164,"end":61391,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html"},{"start":61391,"end":76810,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html"},{"start":76810,"end":86040,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html"}],"contents":[{"title":"Gradient Regularization Mitigates Reward Hacking in Reinforcement Learning from Human Feedback and Verifiable Rewards","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"Abstract","offset":121,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"Keywords:","offset":1544,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"1 Introduction","offset":1562,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"2 Background","offset":4787,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"2.1 Reinforcement Learning","offset":4804,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"2.2 Proxy Rewards","offset":7613,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"Reward Models","offset":7798,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"Rule-Based Rewards","offset":9883,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"LLM-as-a-Judge","offset":10604,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"2.3 Gradient Regularization (GR)","offset":11253,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"3 Accurate Proxy Rewards via Gradient Regularization","offset":12951,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"3.1 Problem Formulation and Overview","offset":13008,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"3.2 Gradient Regularization Improves Proxy Reward Accuracy","offset":15480,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-1.html#text"},{"title":"GR Favors Flat Maxima in Parameter Space","offset":16057,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Flat Maxima in Parameter Space imply Robust Maxima in Action Space","offset":17328,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Definition 3.1 ( ℰ − L ^ \\mathcal{E}-\\widehat{L} flat reward maximum) .","offset":17927,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Definition 3.2 ( ( δ , K , ρ ) (\\delta,K,\\rho) -pairwise robust policy) .","offset":18997,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Proposition 3.3 ( ℰ − L ^ \\mathcal{E}-\\widehat{L} flat reward implies ( δ , K , ρ ) (\\delta,K,\\rho) robust policy) .","offset":20060,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Non-Robust Policies Imply Inaccurate Proxy Reward","offset":22560,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Proposition 3.4 .","offset":22876,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Limitations","offset":23980,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"4 Reference Reset as Gradient Regularization","offset":25014,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Setup","offset":26125,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Gradient Norm Tracks Sharpness and RM Accuracy","offset":26767,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Reference Resets Outperform Standard KL","offset":28280,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"Is a Scheduled or Weaker β \\beta a Sufficient Alternative?","offset":29231,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"5 Explicit Gradient Regularization","offset":30145,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-2.html#text"},{"title":"5.1 GR Mitigates Hacking Reward Models in RLHF","offset":31662,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Explicit GR is robust to RM accuracy","offset":32847,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"5.2 GR Prevents Focus on Easy Rule-Based Rewards","offset":33891,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"5.3 GR Mitigates Hacking LLM-as-a-Judge in RLVR","offset":36154,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Limitations","offset":37597,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"6 Conclusion","offset":38522,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Acknowledgements","offset":39520,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Impact Statement","offset":39737,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Appendix A Extended related work","offset":40149,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Reference Resets","offset":40186,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Replacing the KL penalty in Post-training","offset":42527,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Gradient Regularization","offset":43651,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Appendix B Proofs","offset":44700,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"B.1 Flatness in Parameter Space implies ( δ , K , ρ ) (\\delta,K,\\rho) -pairwise robustness","offset":45820,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-3.html#text"},{"title":"Definition B.1 ( D − L ^ D-\\widehat{L} action robust policy, slightly modified from Lee and Yoon (2025) ) .","offset":46164,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Assumption B.2 (Gaussian policy with fixed covariance and full row-rank Jacobian) .","offset":46923,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Assumption B.3 (Action-smooth proxy reward) .","offset":47353,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Assumption B.4 (Lipschitz-continuous true reward) .","offset":47730,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proposition B.5 ( ℰ − L ^ \\mathcal{E}-\\widehat{L} flat return implies D − L ^ D-\\widehat{L} robust policy, slightly modified from Lee and Yoon (2025) ) .","offset":48155,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proof.","offset":48885,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Lemma B.6 (Flatness and β \\beta -smoothness imply bounded gradient) .","offset":51235,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proof.","offset":51864,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Lemma B.7 (Pointwise action-gradient is controlled by Gaussian-smoothing) .","offset":53253,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proof.","offset":53836,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Lemma B.8 (Pairwise robustness is bounded by local smoothness) .","offset":55158,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proof.","offset":55931,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proposition B.9 (From D D - L ^ \\widehat{L} robustness to pairwise sharpness control) .","offset":57487,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proof.","offset":58265,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"B.2 BT-Loss Lower Bound Based on Sharpness","offset":60017,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Proposition B.10 .","offset":60441,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-4.html#text"},{"title":"Lower Bound","offset":63354,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"B.3 Connection of Gradient Regularization and Lipschitz Continuity","offset":67239,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"Appendix C Experiment Details","offset":68295,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"C.1 Gradient regularization implementation","offset":68390,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"C.2 RLHF Details","offset":70377,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"C.3 Gradient Regularization on Alpaca Farm","offset":72310,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"C.4 Reasoning Experiments","offset":72691,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"C.5 LLM Judge Setup","offset":74351,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"Appendix D Additional Experiments","offset":74714,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"D.1 Gradient Regularization γ \\gamma , ε \\varepsilon , and learning rate sweeps","offset":74752,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"D.2 Gradient regularization decreases BT loss","offset":76056,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"D.3 LLM judge ablation","offset":76782,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-5.html#text"},{"title":"D.4 KL penalty schedule","offset":77814,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"},{"title":"D.5 Steps per reset R R","offset":78834,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"},{"title":"D.6 Ablations","offset":79679,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"},{"title":"D.7 Runtime Comparison","offset":81822,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"},{"title":"D.8 Reward hacking example","offset":82391,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"},{"title":"Appendix E Reference Resets in RLHF","offset":82538,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"},{"title":"E.1 Asymptotic analysis","offset":83117,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"},{"title":"Instructions for reporting errors","offset":85076,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/part-6.html#text"}],"text_code_points":86040,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/75161700a0f3e03055116a289c82e9f9fa7d8fee7a6f802e116216eac3a70e31/source.md"}