{"id":"arxiv:2511.20347","title":"Soft Adaptive Policy Optimization","date":"2025-11-25","section":"Post-training","quality":0.121839,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/","source_url":"https://arxiv.org/abs/2511.20347","text_available":true,"authors":["Chang Gao","Chujie Zheng","Xiong-Hui Chen","Kai Dang"],"outgoing":[{"source":"arxiv:2511.20347","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2511.20347","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"}],"incoming":[{"source":"arxiv:2604.08690","target":"arxiv:2511.20347","evidence":"id","id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/"},{"source":"arxiv:2605.12969","target":"arxiv:2511.20347","evidence":"id","id":"arxiv:2605.12969","title":"Revisiting Reinforcement Learning with Verifiable Rewards from a Contrastive Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/30e87f5eb9beda8e09b3945d6ba1707dec761abed83c49d129a21adaed02ce45/"}],"text_parts":[{"start":0,"end":14124,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html"},{"start":14124,"end":29869,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html"},{"start":29869,"end":34749,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-3.html"}],"contents":[{"title":"Soft Adaptive Policy Optimization","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"Abstract","offset":37,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"1 Introduction","offset":1909,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"2 Preliminaries","offset":6101,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"Notation.","offset":6121,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"Group Relative Policy Optimization (GRPO).","offset":6602,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"Group Sequence Policy Optimization (GSPO)","offset":8150,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"3 Soft Adaptive Policy Optimization","offset":9603,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"Why Different Temperatures for Positive and Negative Advantages","offset":14054,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-1.html#text"},{"title":"4 A Gating‑Function Perspective on SAPO’s Connections to GRPO and GSPO","offset":16900,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Unified surrogate","offset":16975,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Algorithm-specific f i , t f_{i,t}","offset":18297,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Gradient form for SAPO/GRPO","offset":19798,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"4.1 SAPO–GSPO Connection: Reduction to a Sequence-Level Soft Gate","offset":20501,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"SAPO’s token-level soft gate","offset":20731,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Assumptions","offset":21379,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Average token gates ⇒ \\Rightarrow sequence gate","offset":22518,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Do the two assumptions (A1) and (A2) hold?","offset":26284,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Advantages over GSPO","offset":27489,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"4.2 SAPO–GRPO Connection: Smooth Token Gates vs. Hard Token Clipping","offset":28070,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"GRPO’s piecewise-hard token gate","offset":28144,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"Advantages over GRPO","offset":29144,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"4.3 Summary","offset":29852,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-2.html#text"},{"title":"5 Experiments","offset":30353,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-3.html#text"},{"title":"5.1 Controlled Experiments","offset":30371,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-3.html#text"},{"title":"5.2 Qwen3‑VL Training","offset":32376,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-3.html#text"},{"title":"6 Conclusion","offset":33661,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/part-3.html#text"}],"text_code_points":34749,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/source.md"}