{"id":"arxiv:2304.06528","title":"Power-seeking can be probable and predictive for trained agents","date":"2023-04-13","section":"AI safety and consciousness","quality":-0.33811,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/","source_url":"https://arxiv.org/abs/2304.06528","text_available":true,"authors":["Victoria Krakovna","Janos Kramar"],"outgoing":[{"source":"arxiv:2304.06528","target":"arxiv:1912.01683","evidence":"id","id":"arxiv:1912.01683","title":"Optimal Policies Tend to Seek Power","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/"},{"source":"arxiv:2304.06528","target":"arxiv:2206.13353","evidence":"id","id":"arxiv:2206.13353","title":"Is Power-Seeking AI an Existential Risk?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/"},{"source":"arxiv:2304.06528","target":"arxiv:2206.13477","evidence":"id","id":"arxiv:2206.13477","title":"Parametrically Retargetable Decision-Makers Tend To Seek Power","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/"}],"incoming":[],"text_parts":[{"start":0,"end":15564,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html"},{"start":15564,"end":24255,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-2.html"}],"contents":[{"title":"Power-seeking can be probable and predictive for trained agents","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Abstract","offset":67,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"1 Introduction","offset":1009,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"2 Preliminaries from the power-seeking literature","offset":4465,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Notation and assumptions:","offset":4764,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 1 (Orbit of a reward vector - Def 3.1 in RDSP).","offset":5494,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 2 (Orbit subset where an action set is preferred - from Def 3.5 in RDSP).","offset":5865,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 3 (Preference for an action set A 1 A_{1} - Def 3.2 in RDSP).","offset":6356,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 4 (Multiply retargetable function from A 0 A_{0} to A 1 A_{1} - Def 3.5 in RDSP).","offset":6953,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Theorem 1 (Multiply retargetable functions prefer action set A 1 A_{1} - Thm 3.6 in RDSP).","offset":8458,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"3 Training-compatible goal set","offset":9147,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 5 (Partition of the state space).","offset":9182,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 6 (Training-compatible goal set).","offset":9391,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Example 1 (CoinRun).","offset":10255,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"4 Power-seeking for training-compatible goals","offset":10957,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 7 (Shutdown setting).","offset":11632,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 8 (Revisiting policy).","offset":12137,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Proposition 1 (Reach-and-revisit policy exists).","offset":12524,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Proof.","offset":12892,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Definition 9 (Expected discounted visit count).","offset":14428,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Proposition 2 (Visit count goes to infinity).","offset":14954,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Proof.","offset":15187,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-1.html#text"},{"title":"Proposition 3 (Retargetability to recurrent states).","offset":17366,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-2.html#text"},{"title":"Proof.","offset":18502,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-2.html#text"},{"title":"Theorem 2 (Retargetability from the shutdown action in new situations).","offset":20121,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-2.html#text"},{"title":"Proof.","offset":21072,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-2.html#text"},{"title":"5 Conclusion","offset":23283,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-2.html#text"},{"title":"Acknowledgements.","offset":24127,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/part-2.html#text"}],"text_code_points":24255,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/source.md"}