{"id":"arxiv:2206.13353","title":"Is Power-Seeking AI an Existential Risk?","date":"2022-06-16","section":"AI safety and consciousness","quality":0.002809,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/","source_url":"https://arxiv.org/abs/2206.13353","text_available":true,"authors":["Joseph Carlsmith"],"outgoing":[{"source":"arxiv:2206.13353","target":"arxiv:1912.01683","evidence":"title","id":"arxiv:1912.01683","title":"Optimal Policies Tend to Seek Power","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/"}],"incoming":[{"source":"arxiv:2206.13477","target":"arxiv:2206.13353","evidence":"title","id":"arxiv:2206.13477","title":"Parametrically Retargetable Decision-Makers Tend To Seek Power","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/"},{"source":"arxiv:2210.10760","target":"arxiv:2206.13353","evidence":"id","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2304.06528","target":"arxiv:2206.13353","evidence":"id","id":"arxiv:2304.06528","title":"Power-seeking can be probable and predictive for trained agents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/"}],"text_parts":[{"start":0,"end":15536,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html"},{"start":15536,"end":30899,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-2.html"},{"start":30899,"end":46141,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-3.html"},{"start":46141,"end":61885,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-4.html"},{"start":61885,"end":77742,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-5.html"},{"start":77742,"end":92040,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-6.html"},{"start":92040,"end":107553,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-7.html"},{"start":107553,"end":123149,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-8.html"},{"start":123149,"end":138317,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-9.html"},{"start":138317,"end":153109,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-10.html"},{"start":153109,"end":169059,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-11.html"},{"start":169059,"end":183988,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-12.html"},{"start":183988,"end":199946,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-13.html"},{"start":199946,"end":215663,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-14.html"},{"start":215663,"end":231633,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-15.html"},{"start":231633,"end":232429,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-16.html"}],"contents":[{"title":"Is Power-Seeking AI an Existential Risk?","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"Abstract","offset":44,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"1 Introduction","offset":1620,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"1.1 Preliminaries","offset":4722,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"1.2 Backdrop","offset":9513,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"1.2.1 Intelligence","offset":9922,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"1.2.2 Agency","offset":12927,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"1.2.3 Playing with fire","offset":15222,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-1.html#text"},{"title":"1.2.4 Power","offset":19624,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-2.html#text"},{"title":"2 Timelines","offset":23411,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-2.html#text"},{"title":"2.1 Three key properties","offset":23745,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-2.html#text"},{"title":"2.1.1 Advanced capabilities","offset":23899,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-2.html#text"},{"title":"2.1.2 Agentic planning","offset":28288,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-2.html#text"},{"title":"2.1.3 Strategic awareness","offset":38060,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-3.html#text"},{"title":"2.2 Likelihood by 2070","offset":39821,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-3.html#text"},{"title":"3 Incentives","offset":41535,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-3.html#text"},{"title":"3.1 Usefulness","offset":45698,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-3.html#text"},{"title":"3.2 Available techniques","offset":54273,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-4.html#text"},{"title":"3.3 Byproducts of sophistication","offset":55621,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-4.html#text"},{"title":"4 Alignment","offset":57826,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-4.html#text"},{"title":"4.1 Definitions and clarifications","offset":58144,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-4.html#text"},{"title":"4.2 Power-seeking","offset":72925,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-5.html#text"},{"title":"4.3 The challenge of practical PS-alignment","offset":93918,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-7.html#text"},{"title":"4.3.1 Controlling objectives","offset":94940,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-7.html#text"},{"title":"4.3.1.1 Problems with proxies","offset":96695,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-7.html#text"},{"title":"4.3.1.2 Problems with search","offset":103109,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-7.html#text"},{"title":"4.3.1.3 Myopia","offset":109415,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-8.html#text"},{"title":"4.3.2 Controlling capabilities","offset":112711,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-8.html#text"},{"title":"4.3.2.1 Specialization","offset":113559,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-8.html#text"},{"title":"4.3.2.2 Preventing problematic improvements","offset":118232,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-8.html#text"},{"title":"4.3.2.3 Scaling","offset":119815,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-8.html#text"},{"title":"4.3.3 Controlling circumstances","offset":121289,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-8.html#text"},{"title":"4.4 Unusual difficulties","offset":124690,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-9.html#text"},{"title":"4.4.1 Barriers to understanding","offset":126386,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-9.html#text"},{"title":"4.4.2 Adversarial dynamics","offset":129977,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-9.html#text"},{"title":"4.4.3 Stakes of error","offset":133901,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-9.html#text"},{"title":"4.5 Overall difficulty","offset":135882,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-9.html#text"},{"title":"5 Deployment","offset":137632,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-9.html#text"},{"title":"5.1 Timing of problems","offset":141259,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-10.html#text"},{"title":"5.2 Decisions","offset":145783,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-10.html#text"},{"title":"5.3 Key risk factors","offset":149604,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-10.html#text"},{"title":"5.3.1 Externalities and competition","offset":150037,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-10.html#text"},{"title":"5.3.2 Number of relevant actors","offset":155437,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-11.html#text"},{"title":"5.3.3 Bottlenecks on usefulness","offset":158258,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-11.html#text"},{"title":"5.3.4 Deception","offset":163412,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-11.html#text"},{"title":"5.4 Overall risk of problematic deployment","offset":166369,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-11.html#text"},{"title":"6 Correction","offset":168151,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-11.html#text"},{"title":"6.1 Take-off","offset":169041,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-11.html#text"},{"title":"6.2 Warning shots","offset":171760,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-12.html#text"},{"title":"6.3 Competition for power","offset":179279,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-12.html#text"},{"title":"6.3.1 Mechanisms","offset":182895,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-12.html#text"},{"title":"6.4 Corrective feedback loops","offset":193594,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-13.html#text"},{"title":"6.5 Sharing power","offset":199923,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-13.html#text"},{"title":"7 Catastrophe","offset":201588,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-14.html#text"},{"title":"8 Probabilities","offset":210298,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-14.html#text"},{"title":"Acknowledgments","offset":228674,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-15.html#text"},{"title":"9 Appendix","offset":229262,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/part-15.html#text"}],"text_code_points":232429,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/source.md"}