{"id":"arxiv:2608.27448","title":"TTPO: Test-Time Policy Optimization","date":"2026-08-27","section":"Post-training","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/","source_url":"https://arxiv.org/abs/2608.27448","text_available":true,"authors":["Aozhe Wang","Zhengxi Lu","Jianze Wang","Shangke Lv"],"outgoing":[{"source":"arxiv:2608.27448","target":"arxiv:2402.03300","evidence":"id","id":"arxiv:2402.03300","title":"DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/636fb871904bc06f742cfdea069796f9e9665f95bd58dc63260eaff5154ef985/"},{"source":"arxiv:2608.27448","target":"arxiv:2503.14476","evidence":"title","id":"arxiv:2503.14476","title":"DAPO: An Open-Source LLM Reinforcement Learning System at Scale","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/548e934b912ef84691ace63bf1cfd670e7b0257d9d6986dde926f569e7b2f8b7/"},{"source":"arxiv:2608.27448","target":"arxiv:2504.16084","evidence":"title","id":"arxiv:2504.16084","title":"TTRL: Test-Time Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ad54dd58d24afdfefc8aa5087d07c4aab2143e90ef06f6f7db8d3f3333452c48/"},{"source":"arxiv:2608.27448","target":"arxiv:2601.18734","evidence":"id","id":"arxiv:2601.18734","title":"Self-Distilled Reasoner: On-Policy Self-Distillation for Large Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8a352e37fa79840a4f35f86ebc39661cfc34fbfc4391cd020a69af158c63485a/"},{"source":"arxiv:2608.27448","target":"arxiv:2604.02288","evidence":"id","id":"arxiv:2604.02288","title":"Unifying Group-Relative and Self-Distillation Policy Optimization via Sample Routing","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3d1160029d321f2069016f7702d4ebebee0de6cd31a8f296ee220abd6f69c99b/"}],"incoming":[],"text_parts":[{"start":0,"end":15993,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html"},{"start":15993,"end":31373,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html"},{"start":31373,"end":41878,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html"}],"contents":[{"title":"TTPO: Test-Time Policy Optimization","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"Abstract","offset":39,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"1 Introduction","offset":1449,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"2 Related Work","offset":6854,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"2.1 Test-Time Training for Reasoning","offset":6873,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"2.2 On-Policy Self-Distillation","offset":7614,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"2.3 Token-Level Weighting and Masking","offset":8530,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"3 Method","offset":9363,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"3.1 Preliminaries and Problem Setup","offset":9376,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"Majority-vote pseudo-labeling.","offset":9789,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"Answer-conditioned teacher.","offset":10248,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"3.2 Motivation: Why Asymmetric?","offset":10899,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"3.3 Positive Samples: OPSD Branch","offset":12154,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"Token weighting.","offset":12610,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"3.4 Negative Samples: GRPO Branch","offset":13486,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"Token masking.","offset":14499,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"3.5 Unified Objective","offset":15559,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-1.html#text"},{"title":"4 Experiments","offset":15993,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"4.1 Experimental Setup","offset":16011,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Implementation.","offset":16039,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Baselines.","offset":16623,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Evaluation.","offset":17007,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"TTPO-specific hyperparameters.","offset":17561,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"4.2 Main Results","offset":18069,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Labeled training data.","offset":18091,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Label-free test-time training.","offset":18923,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"4.3 Ablation Studies","offset":20136,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Token-level selection.","offset":20162,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Update strategy.","offset":21113,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"4.4 Analysis","offset":22317,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Privileged information.","offset":22335,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Generalization beyond the target task.","offset":23395,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Upper bound with labeled supervision.","offset":23861,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Sustainable self-evolution.","offset":24872,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"5 Conclusion","offset":25709,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Appendix A Implementation Details","offset":26195,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Appendix B Prompt Templates","offset":27063,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Appendix C Case Study: Token-Level Selection Visualization","offset":27399,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"C.1 Token Weighting","offset":28215,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"C.2 Token Masking","offset":29088,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"C.3 Complementarity","offset":29996,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"Appendix D Additional Experiments","offset":30456,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"D.1 Training Dynamics","offset":30494,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"D.2 Non-Thinking Evaluation","offset":31340,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-2.html#text"},{"title":"D.3 Additional Ablations","offset":32213,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"RL weight λ \\lambda .","offset":32243,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"K train K_{\\text{train}} positive-negative fraction.","offset":33065,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"K train K_{\\text{train}} selection strategy.","offset":34052,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Appendix E Theoretical Analysis","offset":35289,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"E.1 Setup","offset":35325,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"E.2 FKL Signal Analysis","offset":35779,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Positive samples ( y k ∈ 𝒫 y_{k}\\in\\mathcal{P} , a k = a ^ a_{k}=\\hat{a} ).","offset":35808,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Negative samples ( y k ∈ 𝒩 y_{k}\\in\\mathcal{N} , a k ≠ a ^ a_{k}\\neq\\hat{a} ).","offset":36345,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Method comparison.","offset":37163,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"E.3 Majority-Vote vs. Ground-Truth Routing","offset":37674,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"GT routing","offset":38028,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Vote routing","offset":38818,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Appendix F Limitations and Future Work","offset":39722,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Dependence on majority-vote quality.","offset":39765,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Domain scope.","offset":40229,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Dynamic training curriculum.","offset":40564,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"},{"title":"Instructions for reporting errors","offset":40914,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/part-3.html#text"}],"text_code_points":41878,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3473ad1e90105976912e40eabe716441e830da401749e9d85b5162e1c7bbd964/source.md"}