{"id":"arxiv:2606.17056","title":"The Value Axis: Language Models Encode Whether They're on the Right Track","date":"2026-06-15","section":"AI safety and consciousness","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/","source_url":"https://arxiv.org/abs/2606.17056","text_available":true,"authors":["Nick Jiang","Isaac Kauvar","Jack Lindsey"],"outgoing":[],"incoming":[],"text_parts":[{"start":0,"end":15848,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html"},{"start":15848,"end":31056,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html"},{"start":31056,"end":46938,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html"}],"contents":[{"title":"The Value Axis: Language Models Encode Whether They’re on the Right Track","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Abstract","offset":77,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"1 Introduction","offset":1379,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"2 The Value Axis","offset":4433,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"2.1 Construction","offset":4454,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"2.2 Evaluation","offset":8014,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Generalization across held-out scenarios.","offset":8034,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Similarity across layers.","offset":8535,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Logit lens analysis.","offset":8989,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"3 The Value Axis Measures and Modulates Task Confidence","offset":9523,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"3.1 The Value Axis Tracks Task Confidence","offset":9718,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Correlation with verbalized confidence.","offset":9765,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Correlation with backtracking events.","offset":10633,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Correlation with code correctness.","offset":11215,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"3.2 The Value Axis Causally Modulates Task Confidence","offset":12203,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Verbalized confidence in AIME questions.","offset":13063,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Backtracking presence on AIME questions.","offset":13654,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Coding verbosity.","offset":14196,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"4 How Does Preference Learning Affect the Internal Value Function?","offset":14787,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"Training the models to prefer words increases the value of those words.","offset":15770,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-1.html#text"},{"title":"The value increase on preferred words generalizes to natural sentences.","offset":17350,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"Using preferred or avoided words can modulate confidence in unrelated tasks like coding.","offset":18108,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"5 In-the-Wild Case Studies","offset":19494,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"5.1 Chatbot Arena Conversations","offset":19859,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"5.2 Supervised Fine-Tuning on Benchmarks","offset":20719,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"5.3 Eval Awareness","offset":21588,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"6 Related Work","offset":23484,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"7 Limitations","offset":26094,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"8 Discussion","offset":26978,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"Acknowledgments and Disclosure of Funding","offset":28228,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"Appendix A ICRL Conversation Generation","offset":28505,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"Procedure.","offset":28549,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"Game system prompt (shown to the player).","offset":29852,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"Meta-prompt (Claude Opus role-playing the player).","offset":30999,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-2.html#text"},{"title":"Construction criteria.","offset":34096,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Appendix B Supplemental Materials for Evaluating the Value Axis","offset":34218,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"B.1 Steering effects across multiple layers","offset":34478,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Ordinary least squares.","offset":35226,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"B.2 Backtracking detection","offset":36327,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"B.3 Coding steering example","offset":37082,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Appendix C Supplemental Materials for Preference Learning","offset":37655,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"C.1 Training setup","offset":37912,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"C.2 User-prompt analysis","offset":38720,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"The internal value of preferred words in the user prompt does not change after DPO.","offset":38750,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Steering the option within the user prompt.","offset":40030,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"C.3 Per-model internal value changes","offset":40743,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"C.4 Qualitative examples: value priming in coding","offset":41240,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Preferred words.","offset":41540,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Avoided words.","offset":41648,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Appendix D Logit Lens: Top Promoted Tokens","offset":41752,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Appendix E Supplemental Materials for In-the-Wild Case Studies","offset":42120,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Chatbot Arena (Section 5.1 ).","offset":42187,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Supervised fine-tuning (Section 5.2 ).","offset":43541,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Eval awareness (Section 5.3 ).","offset":44400,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"},{"title":"Instructions for reporting errors","offset":45974,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/part-3.html#text"}],"text_code_points":46938,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/969b67b08341367524f91a84c873d4fedbe44a95fa24841e03b2f8e111caa9a6/source.md"}