{"id":"arxiv:2505.13763","title":"Language Models Are Capable of Metacognitive Monitoring and Control of Their Internal Activations","date":"2025-05-19","section":"Reasoning and the \"physics\" of language models","quality":-0.0936,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/","source_url":"https://arxiv.org/abs/2505.13763","text_available":true,"authors":["Li Ji-An","Hua-Dong Xiong","Robert C. Wilson"],"outgoing":[{"source":"arxiv:2505.13763","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2505.13763","target":"url:transformer-circuits.pub/2025/attribution-graphs/biology.html","evidence":"url","id":"url:transformer-circuits.pub/2025/attribution-graphs/biology.html","title":"Attribution graphs, the biology of the model","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8b3e14c537e89eed9a4402e9c14094d69913ed24d45fdab1a82ba4b8280660e2/"},{"source":"arxiv:2505.13763","target":"url:transformer-circuits.pub/2025/attribution-graphs/methods.html","evidence":"url","id":"url:transformer-circuits.pub/2025/attribution-graphs/methods.html","title":"Circuit tracing / attribution graphs (methods)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7d4ba254716fd13d3a1ffb6518788e304eaa7077bd9d7bce201094eb293518b4/"}],"incoming":[],"text_parts":[{"start":0,"end":15011,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html"},{"start":15011,"end":30462,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-2.html"},{"start":30462,"end":46364,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html"},{"start":46364,"end":62349,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html"},{"start":62349,"end":78336,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-5.html"},{"start":78336,"end":82578,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-6.html"}],"contents":[{"title":"Language Models Are Capable of Metacognitive Monitoring and Control of Their Internal Activations","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"Abstract","offset":101,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"1 Introduction","offset":1580,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"2 Neurofeedback paradigm","offset":7640,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"2.1 Neurofeedback in neuroscience","offset":7669,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"2.2 Neurofeedback for LLMs","offset":8488,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"2.3 Defining neurofeedback labels","offset":9767,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"2.4 Models and datasets","offset":11818,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"2.5 Choice of target axes","offset":12774,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"3 LLMs can report their neural activations","offset":14398,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-1.html#text"},{"title":"4 LLMs can control their neural activations","offset":17021,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-2.html#text"},{"title":"4.1 Explicit control","offset":17636,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-2.html#text"},{"title":"4.2 Implicit control","offset":20508,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-2.html#text"},{"title":"4.3 Controlling the LR axis","offset":22091,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-2.html#text"},{"title":"4.4 Accumulation mechanisms of control effects","offset":23962,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-2.html#text"},{"title":"5 Discussion","offset":25034,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-2.html#text"},{"title":"Broader impacts.","offset":33230,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"Acknowledgments and Disclosure of Funding","offset":35800,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"Appendix A Additional methods","offset":36384,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.1 Code and Reproducibility","offset":36418,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.2 Models","offset":36659,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.2.1 LLMs used in the study","offset":36675,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.2.2 LLMs with and without instruction fine-tuning","offset":37066,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.2.3 Transformer and residual stream","offset":37684,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.3 Datasets","offset":39708,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.4 Formal definitions of first-order and second-order processes in LLMs","offset":41437,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-3.html#text"},{"title":"A.5 Metacognitive Tasks","offset":47057,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"A.5.1 Computing quantized neurofeedback labels","offset":47086,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"A.5.2 Task prompts","offset":48305,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"A.5.3 Quantifying reporting accuracy","offset":49194,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"A.5.4 Explicit and implicit control experiments","offset":49928,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"A.5.5 Quantifying control effect","offset":51413,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"Appendix B Additional results","offset":52808,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.1 Target and off-target control effects","offset":52842,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.2 Control Precision","offset":54710,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.3 LLMs’ reporting accuracy varies across layers and models","offset":56575,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.4 Reporting performance of an ideal observer","offset":56641,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.5 Summarized control effects of Qwen2.5 7B","offset":57307,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.6 Summarized metacognitive effects on four datasets","offset":57357,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.7 Control accumulation effects on other three datasets","offset":57416,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.8 Summarized control effects of Llama3.1 8B with fine-grained neurofeedback labels","offset":57478,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.9 Detailed results for control in Llama3.1 8B and Qwen2.5 7B","offset":57568,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"B.10 Defining axes from hidden states aggregated across multiple layers","offset":57636,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"Appendix C Experiment compute resources","offset":58537,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"NeurIPS Paper Checklist","offset":59034,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-4.html#text"},{"title":"Instructions for reporting errors","offset":81614,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/part-6.html#text"}],"text_code_points":82578,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/95c4e7c49af162594458624b663c82546ad42167854a2235442954821ea69c65/source.md"}