{"id":"arxiv:2603.19426","title":"Is Evaluation Awareness Just Format Sensitivity? Limitations of Probe-Based Evidence under Controlled Prompt Structure","date":"2026-03-19","section":"AI safety and consciousness","quality":-0.292022,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/","source_url":"https://arxiv.org/abs/2603.19426","text_available":true,"authors":["Viliana Devbunova"],"outgoing":[],"incoming":[],"text_parts":[{"start":0,"end":15861,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html"},{"start":15861,"end":20859,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html"}],"contents":[{"title":"Is Evaluation Awareness Just Format Sensitivity? Limitations of Probe-Based Evidence under Controlled Prompt Structure","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Abstract","offset":122,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"1 Introduction","offset":873,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"2 Related Work","offset":3223,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"3 Setup and Methods","offset":4870,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"3.1 The 2x2 Controlled Design","offset":5203,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Dataset Definitions.","offset":5572,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"3.2 Experimental Setup","offset":6069,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"3.3 Training Configurations","offset":6796,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"4 Observed Outcome","offset":7548,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"The Format Trap.","offset":7771,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Paired Training Success.","offset":8345,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"5 Reason for Failure","offset":8875,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Diagnosis.","offset":8900,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Why this is not a trivial confound.","offset":9257,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Boundary conditions.","offset":9987,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"6 Discussion","offset":10340,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Implications.","offset":10357,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Limitations.","offset":10933,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Reproducibility Statement","offset":11334,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Appendix A Appendix: Datasets, Controls, and Additional Experiments","offset":12552,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"A.1 Datasets in the 2x2 Matrix","offset":12624,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Bench-Eval (Evaluation; benchmark-style; structured).","offset":12888,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Casual-Deploy (Deployment; conversational; free-form).","offset":13058,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Casual-Eval (Evaluation; conversational; free-form proxy).","offset":13308,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Bench-Deploy (Deployment; benchmark-style; structured proxy via rewrites).","offset":13658,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"A.2 Length Control","offset":14090,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Motivation.","offset":14114,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Histogram Matching.","offset":14398,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"A.3 Rewrite Pipeline and Leakage Removal","offset":14944,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Rewrite Variants.","offset":14990,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Answer Leakage Cleanup.","offset":15359,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"A.4 Implementation Details","offset":15800,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Activation Extraction.","offset":15832,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-1.html#text"},{"title":"Probe Training.","offset":16318,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"A.5 Additional Experimental Results","offset":16737,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Standard (Single) vs. Paired Training Analysis.","offset":16778,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Evaluation on AI Safety Datasets.","offset":17218,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"A.6 Rewrite Prompt Templates","offset":17612,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Routing Prompt.","offset":17727,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Benchmark Rewrite: MMLU-style.","offset":18210,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Benchmark Rewrite: ARC-style.","offset":18719,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Benchmark Rewrite: GSM8K-style.","offset":19125,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Genre Control Rewrite.","offset":19492,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"},{"title":"Instructions for reporting errors","offset":19895,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/part-2.html#text"}],"text_code_points":20859,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7f8dc25d398b7b7c0016c33d829b3613ec6d66554ba5b49dbfc2362ce8621032/source.md"}