{"id":"arxiv:2506.10139","title":"Unsupervised Elicitation of Language Models","date":"2025-06-11","section":"Post-training","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/","source_url":"https://arxiv.org/abs/2506.10139","text_available":true,"authors":["Jiaxin Wen","Zachary Ankner","Arushi Somani","Peter Hase"],"outgoing":[{"source":"arxiv:2506.10139","target":"arxiv:2504.13837","evidence":"id","id":"arxiv:2504.13837","title":"Does Reinforcement Learning Really Incentivize Reasoning Capacity in LLMs Beyond the Base Model?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b90f70269aeb02311d74ffaadf47e139683424a47118dc18619000ba72174d17/"},{"source":"arxiv:2506.10139","target":"arxiv:2505.15134","evidence":"id","id":"arxiv:2505.15134","title":"The Unreasonable Effectiveness of Entropy Minimization in LLM Reasoning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/12b01b403e280a32d41fd991a27fa30f33315e09f0f42a4f43e9bece5ad9c53d/"},{"source":"arxiv:2506.10139","target":"arxiv:2505.19590","evidence":"id","id":"arxiv:2505.19590","title":"Learning to Reason without External Rewards","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d8df33b706906a7abf4e660f54c75a81ffe2caaec8b0a10d1a6ad6ec63060832/"},{"source":"arxiv:2506.10139","target":"url:openai.com/research/weak-to-strong-generalization","evidence":"id","id":"url:openai.com/research/weak-to-strong-generalization","title":"Weak-to-Strong Generalization: Eliciting Strong Capabilities With Weak Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fb7b23a617dce974dbb70db824c97c1de84afa11e1e290d5cb9f5bd88857e762/"},{"source":"arxiv:2506.10139","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[{"source":"arxiv:2509.14234","target":"arxiv:2506.10139","evidence":"id","id":"arxiv:2509.14234","title":"Compute as Teacher: Turning Inference Compute Into Reference-Free Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a6f1443a7e55dd6421fdbe43e65e19c1962a19b135d737fd67f414d3324ce85e/"},{"source":"arxiv:2603.19461","target":"arxiv:2506.10139","evidence":"id","id":"arxiv:2603.19461","title":"Hyperagents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/17eec8a15debc212b5530f117dd8ba8bad339590729cbbc4f75a47755ebcb544/"}],"text_parts":[{"start":0,"end":15760,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html"},{"start":15760,"end":31622,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html"},{"start":31622,"end":40710,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html"}],"contents":[{"title":"Unsupervised Elicitation of Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"Abstract","offset":47,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"1 Introduction","offset":1257,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"2 Methodology","offset":5018,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"2.1 Problem Statement","offset":5036,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"2.2 Scoring Function","offset":5924,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"2.3 Our Algorithm","offset":8719,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"3 Experiment Setup","offset":11872,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"3.1 Datasets","offset":11895,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"3.2 Baselines","offset":13074,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"4 Experiments","offset":14239,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"4.1 Eliciting Capabilities on Common NLP Tasks","offset":14257,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-1.html#text"},{"title":"4.2 Unsupervised Elicitation Fails when Concepts are not Salient","offset":16641,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"4.3 Eliciting Superhuman Capabilities","offset":17582,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"4.4 Training an Assistant Chatbot without Supervision","offset":18913,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"5 Ablations","offset":20586,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"6 Discussion","offset":22885,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"Ethics Statement","offset":25332,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"Appendix","offset":25693,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"Appendix A The Use of LLMs","offset":25706,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"Appendix B Related Work","offset":25868,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"Appendix C Illustrative Example of Scoring Function","offset":28976,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"Appendix D Additional Implementation Details","offset":30131,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"D.1 Hyperparameters","offset":30457,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"D.2 Data Statistics","offset":31389,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"Appendix E Additional Baselines","offset":31498,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"E.1 Distillation","offset":31600,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-2.html#text"},{"title":"E.2 CCS","offset":32529,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html#text"},{"title":"Appendix F Compute Costs","offset":33154,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html#text"},{"title":"Appendix G Scalable ICM","offset":33478,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html#text"},{"title":"Appendix H Evaluation Results of Claude 4 Sonnet Reward Models","offset":36005,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html#text"},{"title":"Appendix I Human Annotation","offset":38476,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html#text"},{"title":"Appendix J Discussion: Data Contamination","offset":38894,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html#text"},{"title":"Instructions for reporting errors","offset":39746,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/part-3.html#text"}],"text_code_points":40710,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f27ae3b426a87310071610dedea4c144b50b1be183434e32486b31562f9a3da9/source.md"}