{"id":"arxiv:2604.08690","title":"Skip-Connected Policy Optimization for Implicit Advantage","date":"2026-04-09","section":"Post-training","quality":0.295907,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/","source_url":"https://arxiv.org/abs/2604.08690","text_available":true,"authors":["Fengwei Teng","Jinyi Bai","Xinhao Yao","Demi Ruohan Wang"],"outgoing":[{"source":"arxiv:2604.08690","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2604.08690","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2604.08690","target":"arxiv:2503.20783","evidence":"title","id":"arxiv:2503.20783","title":"Understanding R1-Zero-Like Training: A Critical Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b49a056f3ce8e269dd65e3b11605927f3c4d0ef074bc68e0b1d7f84e6c9c6c6a/"},{"source":"arxiv:2604.08690","target":"arxiv:2504.16084","evidence":"id","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2604.08690","target":"arxiv:2506.01939","evidence":"id","id":"arxiv:2506.01939","title":"Beyond the 80/20 Rule: High-Entropy Minority Tokens Drive Effective Reinforcement Learning for LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e0f4a55bca612bce5895b6b98f89b0efc14cbd8cf45946a7743347cf6daee1a1/"},{"source":"arxiv:2604.08690","target":"arxiv:2506.03106","evidence":"id","id":"arxiv:2506.03106","title":"Critique-GRPO: Advancing LLM Reasoning with Natural Language and Numerical Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b16b5772846ee30548a2d2c9cce1833ca3af6beb01c4a58f2dd5a69d809d55d4/"},{"source":"arxiv:2604.08690","target":"arxiv:2506.13585","evidence":"id","id":"arxiv:2506.13585","title":"MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/5beb2ec6c7e6362f9d957219d7a205dd1adcc53f739acc0c7fd5f1f2bded4d83/"},{"source":"arxiv:2604.08690","target":"arxiv:2507.18071","evidence":"id","id":"arxiv:2507.18071","title":"Group Sequence Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/69430db904efc7978803673806cc9338004ffd8cf92199f3349dfbfa8a44e440/"},{"source":"arxiv:2604.08690","target":"arxiv:2509.03646","evidence":"id","id":"arxiv:2509.03646","title":"Emergent Hierarchical Reasoning in LLMs through Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/683657af0d699b7d854ce84d95bcd53c12e152aa738c602d0982cc0d11009f08/"},{"source":"arxiv:2604.08690","target":"arxiv:2509.13232","evidence":"id","id":"arxiv:2509.13232","title":"Single-stream Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d699d0e034b21f3b85c3f70ce4fb95064fbf59685ce752935e6d173dac2b71f3/"},{"source":"arxiv:2604.08690","target":"arxiv:2510.00977","evidence":"id","id":"arxiv:2510.00977","title":"It Takes Two: Your GRPO Is Secretly DPO","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/575241376f2282c1929931f107a0ce60fd25f0a5388ecfea6b9abbbed2f03ca6/"},{"source":"arxiv:2604.08690","target":"arxiv:2510.13786","evidence":"id","id":"arxiv:2510.13786","title":"The Art of Scaling Reinforcement Learning Compute for LLMs","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/72580bcba3d9b32b2f3e83d074f626234b26dc18126edd22a5eacef927cf87da/"},{"source":"arxiv:2604.08690","target":"arxiv:2511.20347","evidence":"id","id":"arxiv:2511.20347","title":"Soft Adaptive Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/1e74c1ffcb5ea955bca4e6d610e9ba590965617f13bd8868f7f31e7b42fbfbfd/"},{"source":"arxiv:2604.08690","target":"arxiv:2512.00499","evidence":"id","id":"arxiv:2512.00499","title":"ESPO: Entropy Importance Sampling Policy Optimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/342b84df2bf40f64c57ea7839579c1f337374ad9c28b16ce7d477c42e24c67b4/"}],"incoming":[],"text_parts":[{"start":0,"end":13842,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html"},{"start":13842,"end":29395,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html"},{"start":29395,"end":45197,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html"},{"start":45197,"end":56129,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html"}],"contents":[{"title":"Skip-Connected Policy Optimization for Implicit Advantage","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"Abstract","offset":61,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"Keywords:","offset":1358,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"1 Introduction","offset":1376,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"2 Related Work","offset":6108,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"2.1 Group-Based Policy Optimization","offset":6127,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"2.2 Fine-Grained Reward/Advantage","offset":8333,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"2.3 Multi-Stage Pipeline Designs","offset":10381,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"3 Methodology","offset":12941,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"3.1 Monte Carlo Trajectory Sampling","offset":13745,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"Ideal Token-Level Reward Estimation for Analysis.","offset":13786,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-1.html#text"},{"title":"Upstream-Downstream Instantiation: A Cost-Effective Step Beyond.","offset":16351,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html#text"},{"title":"3.2 Asymmetric Optimization Strategy","offset":21364,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html#text"},{"title":"Downstream Optimization: Group Relative Policy Optimization.","offset":22288,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html#text"},{"title":"Upstream Optimization: Single-stream Policy Optimization.","offset":23839,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html#text"},{"title":"4 Experiments","offset":26516,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html#text"},{"title":"Training Setup.","offset":26534,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html#text"},{"title":"4.1 Main Results","offset":27654,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-2.html#text"},{"title":"Shortcut Validation.","offset":30414,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"4.2 Ablation Studies","offset":32016,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"4.3 Implicit Advantage Analysis","offset":34222,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"Why External Evaluator?","offset":35532,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"5 Conclusion","offset":36745,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"Limitations.","offset":37896,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"Impact Statement","offset":38953,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"Appendix A Why Fine-Grained Rewards Fail with Group-Relative Estimation","offset":39437,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"A.1 Position-Dependent Variance","offset":40437,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"A.2 Simulation Setup and Results","offset":41780,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"A.3 Correctness Distribution Analysis","offset":42651,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"A.4 How SKPO Structurally Partitions the Problem","offset":45143,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-3.html#text"},{"title":"Appendix B Training Dynamics Analysis","offset":46151,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"B.1 Performance Metrics","offset":46620,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"B.2 Optimization Stability","offset":47409,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Entropy Analysis.","offset":47441,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Training Time.","offset":48082,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Appendix C Experimental Setup and Hyperparameters","offset":48784,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Dataset.","offset":49093,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Model Architectures.","offset":49850,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Optimization Hyperparameters.","offset":50364,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"SKPO Configuration.","offset":51173,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Baseline Configurations.","offset":53256,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Appendix D Prompt for Implicit Advantage Analysis","offset":53691,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Why Instruction-Based Continuation?","offset":54406,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"},{"title":"Instructions for reporting errors","offset":55165,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/part-4.html#text"}],"text_code_points":56129,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5ec1244fd1b770994920ee69c200ecc134262e6011043284200eb18eb0d6c87/source.md"}