{"id":"arxiv:2406.11741","title":"Transcendence: Generative Models Can Outperform The Experts That Train Them","date":"2024-06-17","section":"Reasoning and the \"physics\" of language models","quality":0.099613,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/","source_url":"https://arxiv.org/abs/2406.11741","text_available":true,"authors":["Edwin Zhang","Vincent Zhu","Naomi Saphra","Anat Kleiman"],"outgoing":[{"source":"arxiv:2406.11741","target":"arxiv:2005.01643","evidence":"id","id":"arxiv:2005.01643","title":"Offline Reinforcement Learning: Tutorial, Review, and Perspectives on Open Problems","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/2ae228e6f0a418b4cd46f033b57c7452ba84a9c8cb985b198224186333c17d97/"},{"source":"arxiv:2406.11741","target":"url:openai.com/research/weak-to-strong-generalization","evidence":"id","id":"url:openai.com/research/weak-to-strong-generalization","title":"Weak-to-Strong Generalization: Eliciting Strong Capabilities With Weak Supervision","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fb7b23a617dce974dbb70db824c97c1de84afa11e1e290d5cb9f5bd88857e762/"}],"incoming":[{"source":"arxiv:2505.21444","target":"arxiv:2406.11741","evidence":"title","id":"arxiv:2505.21444","title":"Can Large Reasoning Models Self-Train?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/206224ca50098e0bdc230140722feec107cfc90c1cef2ed8d3ed52355ef56bca/"}],"text_parts":[{"start":0,"end":15039,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html"},{"start":15039,"end":30828,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html"},{"start":30828,"end":46435,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html"},{"start":46435,"end":49235,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-4.html"}],"contents":[{"title":"Transcendence: Generative Models Can Outperform The Experts That Train Them","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"Abstract","offset":79,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"1 Introduction","offset":1157,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"2 Definition of Transcendence","offset":5394,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"Definition 1 .","offset":8193,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"Remark 1 .","offset":9070,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"3 Conditions for Transcendence","offset":9506,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"3.1 Low-Temperature Sampling is Necessary for Transcendence","offset":9968,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"Proposition 1 .","offset":10465,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"3.2 Transcendence with Low-Temperature Sampling","offset":10965,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"Proposition 2 .","offset":12440,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"3.3 Denoising a Single Expert","offset":13601,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"Proposition 3 .","offset":14726,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"3.4 Transcendence from Multiple Experts","offset":14994,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-1.html#text"},{"title":"Proposition 4 .","offset":16154,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"4 Experiments","offset":16761,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"4.1 Experimental Setup","offset":17584,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"Training Details.","offset":17612,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"Evaluation.","offset":20128,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"4.2 Experimental Results","offset":21772,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"Main Result: Low-temperature sampling enables transcendence.","offset":21802,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"Lowering temperature increases rewards in expectation on specific states, leading to transcendence over the full game.","offset":22903,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"Dataset diversity is essential for transcendence.","offset":28128,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"4.3 Additional Settings","offset":30799,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-2.html#text"},{"title":"5 Related Work","offset":32304,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Chess and AI.","offset":32323,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Diversity beats Strength.","offset":33232,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Offline Reinforcement Learning.","offset":34712,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"6 Discussion and Future Work","offset":35342,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Limitations.","offset":35996,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Future Work.","offset":36610,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Broader Impact.","offset":37015,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Acknowledgements","offset":37577,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Appendix A Proofs","offset":37951,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Proof.","offset":38158,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Proof.","offset":38869,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Proof.","offset":40018,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Proposition 5 .","offset":40454,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Proof.","offset":41195,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Proof.","offset":43354,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Appendix B Additional Denoising Visualizations","offset":44522,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Appendix C Intuition of low temperature sampling inducing transcendence","offset":44573,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Appendix D Further Related Work","offset":45109,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"D.1 Label Disagreement","offset":45145,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"D.2 Offline Reinforcement Learning","offset":45759,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-3.html#text"},{"title":"Appendix E Training Details","offset":46948,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-4.html#text"},{"title":"Appendix F Compute Resources","offset":47376,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-4.html#text"},{"title":"Appendix G Full t-SNE","offset":47521,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-4.html#text"},{"title":"Instructions for reporting errors","offset":48271,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/part-4.html#text"}],"text_code_points":49235,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/009cee51dbce762b177f0413e99ef4c30535501f77f54ed03aeb95de376319fd/source.md"}