{"id":"arxiv:2206.13477","title":"Parametrically Retargetable Decision-Makers Tend To Seek Power","date":"2022-06-27","section":"AI safety and consciousness","quality":-0.098112,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/","source_url":"https://arxiv.org/abs/2206.13477","text_available":true,"authors":["Alexander Matt Turner","Prasad Tadepalli"],"outgoing":[{"source":"arxiv:2206.13477","target":"arxiv:1912.01683","evidence":"title","id":"arxiv:1912.01683","title":"Optimal Policies Tend to Seek Power","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/"},{"source":"arxiv:2206.13477","target":"arxiv:2004.12919","evidence":"title","id":"arxiv:2004.12919","title":"First return, then explore","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3cd790c501eaa633193918533947c58e30d1939bc08577606f1d88c56d368285/"},{"source":"arxiv:2206.13477","target":"arxiv:2206.13353","evidence":"title","id":"arxiv:2206.13353","title":"Is Power-Seeking AI an Existential Risk?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/"}],"incoming":[{"source":"arxiv:2304.06528","target":"arxiv:2206.13477","evidence":"id","id":"arxiv:2304.06528","title":"Power-seeking can be probable and predictive for trained agents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/"}],"text_parts":[{"start":0,"end":15226,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html"},{"start":15226,"end":30540,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html"},{"start":30540,"end":45943,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html"},{"start":45943,"end":61814,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html"},{"start":61814,"end":77415,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html"},{"start":77415,"end":92906,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html"},{"start":92906,"end":108618,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html"},{"start":108618,"end":124362,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-8.html"},{"start":124362,"end":139390,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html"},{"start":139390,"end":154428,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html"},{"start":154428,"end":170300,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html"},{"start":170300,"end":183867,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html"},{"start":183867,"end":191049,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-13.html"}],"contents":[{"title":"Parametrically Retargetable Decision-Makers Tend To Seek Power","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html#text"},{"title":"Abstract","offset":66,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html#text"},{"title":"1 Introduction","offset":1254,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html#text"},{"title":"2 Statistical tendencies for a range of decision-making algorithms","offset":3916,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html#text"},{"title":"3 Formal notions of retargetability and decision-making tendencies","offset":12907,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html#text"},{"title":"Definition 3.1 (Orbit of a parameter) .","offset":14077,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html#text"},{"title":"Definition 3.2 (Inequalities which hold for most orbit elements) .","offset":15151,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-1.html#text"},{"title":"Definition 3.3 (Simply-retargetable function) .","offset":17747,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"Proposition 3.4 (Simply-retargetable functions have orbit-level tendencies) .","offset":18951,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"Definition 3.5 (Multiply retargetable function) .","offset":19565,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"Theorem 3.6 (Multiply retargetable functions have orbit-level tendencies) .","offset":20985,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"Proof outline (full proof in Appendix B ).","offset":21256,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"4 Decision-making tendencies in Montezuma’s Revenge","offset":22034,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"Terminology.","offset":22774,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"4.1 Tendencies for initial action selection","offset":24156,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"4.2 Tendencies for maximizing reward over the final observation","offset":29551,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-2.html#text"},{"title":"4.3 Tendencies for rl on featurized reward over the final observation","offset":34112,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"Exploring the first room.","offset":34754,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"Exploring four rooms.","offset":36898,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"Exploring the entire level.","offset":38296,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"5 Retargetability can imply power-seeking tendencies","offset":39289,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"5.1 Generalizing the power-seeking theorems for Markov decision processes","offset":39346,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"5.2 Better rl algorithms tend to be more retargetable","offset":41413,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"6 Discussion","offset":44231,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"6.1 Prior work","offset":44456,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"6.2 Future work and limitations","offset":45227,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-3.html#text"},{"title":"6.3 Conclusion","offset":48058,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Broader impacts","offset":49014,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Acknowledgements","offset":49327,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Appendix A Retargetability over outcome lotteries","offset":49478,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.1 (Outcome lotteries) .","offset":50918,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.2 (Optimality indicator function) .","offset":51891,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.3 (Anti-optimality indicator function) .","offset":54355,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.4 (Boltzmann rationality [ Baker et al., 2007 ] ) .","offset":54953,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.5 (Satisficing) .","offset":55749,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"A.1 A range of decision-making functions are retargetable","offset":57177,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.6 (Similarity of vector sets) .","offset":58486,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.7 (Containment of set copies) .","offset":59111,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.8 (Targeting parameter distribution assumptions) .","offset":61073,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.9 (Pushforward distribution of a permutation [ Turner et al., 2021 ] ) .","offset":61720,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-4.html#text"},{"title":"Definition A.10 (Orbit of a probability distribution [ Turner et al., 2021 ] ) .","offset":62063,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Proposition A.11 (Orbit incentives for different rationalities) .","offset":62824,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Definition A.12 (EU-determined functions) .","offset":66494,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Theorem A.13 (Orbit tendencies occur for EU-determined decision-making functions) .","offset":68253,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Appendix B Theoretical results","offset":70081,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Remark .","offset":70125,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Lemma B.1 (Limited transitivity of ≥ most \\geq_{\\text{most}} ) .","offset":70520,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Proof.","offset":71181,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Lemma B.2 (Order inversion for ≥ most \\geq_{\\text{most}} ) .","offset":73453,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Proof.","offset":73903,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Remark .","offset":75077,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Lemma B.3 (Orbital fraction which agrees on (weak) inequality) .","offset":75152,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Proof.","offset":75781,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"B.1 General results on retargetable functions","offset":76549,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Definition B.4 (Functions which are increasing under joint permutation) .","offset":76600,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Lemma B.5 (Expectations of joint-permutation-increasing functions are also joint-permutation-increasing) .","offset":77300,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-5.html#text"},{"title":"Proof.","offset":78083,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Lemma B.6 (Closure of orbit incentives under increasing functions) .","offset":79854,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Proof.","offset":81007,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Remark .","offset":82376,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Proof.","offset":82538,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Proof.","offset":84412,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"B.2 Helper results on retargetable functions","offset":85067,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Lemma B.7 (Quantitative general orbit lemma) .","offset":85117,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Proof.","offset":86728,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Disjointness.","offset":87884,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Definition B.8 (Superset-of-copy containment) .","offset":90547,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Lemma B.9 (Looser sufficient conditions for orbit-level incentives) .","offset":90952,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Proof.","offset":91827,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-6.html#text"},{"title":"Lemma B.10 (Hiding an argument which is invariant under certain permutations) .","offset":93537,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Proof.","offset":94532,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"B.2.1 EU-determined functions","offset":95353,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Lemma B.11 (EU-determined functions are invariant under joint permutation) .","offset":95613,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Proof.","offset":96165,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Proof.","offset":98124,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"B.3 Particular results on retargetable functions","offset":99216,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Definition B.12 (Quantilization, closed form) .","offset":99270,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Remark .","offset":100700,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Proof.","offset":101457,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-7.html#text"},{"title":"Conjecture B.13 (Orbit tendencies occur for more quantilizer base distributions) .","offset":125246,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Appendix C Detailed analyses of mr scenarios","offset":125808,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"C.1 Action selection","offset":125857,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Lemma C.1 (Lower bound on success probability of the train bandit) .","offset":126560,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Proof.","offset":126933,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Proposition C.2 (The train bandit is 4-retargetable) .","offset":127731,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Proof.","offset":127898,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Corollary C.3 (The train bandit has orbit-level tendencies) .","offset":130798,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Proof.","offset":131030,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"C.2 Observation reward maximization","offset":131090,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Proposition C.4 (Final reward maximization has strong orbit-level incentives in mr ) .","offset":131280,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Proof.","offset":131697,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Lemma C.5 (Room-status inequalities for mr ) .","offset":135938,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Proof.","offset":136378,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-9.html#text"},{"title":"Corollary C.6 (Final reward maximizers tend to leave the first room in mr ) .","offset":141796,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html#text"},{"title":"Proof.","offset":142059,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html#text"},{"title":"C.3 Featurized reward maximization","offset":142681,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html#text"},{"title":"Lemma C.7 ( FracOptimal \\mathrm{FracOptimal} inequalities) .","offset":146716,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html#text"},{"title":"Proof.","offset":147238,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html#text"},{"title":"Subset case.","offset":149733,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html#text"},{"title":"Disjoint case.","offset":154406,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-10.html#text"},{"title":"Conjecture C.8 (Generalizing Lemma C.7 ) .","offset":157382,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Proposition C.9 (Featurized reward maximizers tend to leave the first room in mr ) .","offset":157549,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Proof.","offset":157817,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"C.4 Reasoning for why dqn can’t explore well","offset":166464,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Appendix D Lower bounds on mdp power-seeking incentives for optimal policies","offset":168097,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Definition D.1 (Non-dominated linear functionals) .","offset":169343,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Definition D.2 (Bounded reward function distribution) .","offset":169852,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Remark .","offset":170046,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Lemma D.3 (Quantitative expectation superiority lemma) .","offset":170235,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-11.html#text"},{"title":"Proof.","offset":170972,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Definition D.4 (Linear functional optimality probability [ Turner et al., 2021 ] ) .","offset":173669,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Lemma D.5 (Quantitative optimality probability superiority lemma) .","offset":174238,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Proof.","offset":175099,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Definition D.6 (Rewardless mdp [ Turner et al., 2021 ] ) .","offset":178886,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Definition D.7 (1-cycle states [ Turner et al., 2021 ] ) .","offset":179322,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Definition D.8 (State visit distribution [ Sutton and Barto, 1998 ] ) .","offset":179858,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Definition D.9 (Recurrent state distributions [ Puterman, 2014 ] ) .","offset":180483,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Definition D.10 (Average-optimal policies [ Turner et al., 2021 ] ) .","offset":180973,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Remark .","offset":181883,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Theorem D.11 (Quantitatively, average-optimal policies tend to end up in “larger” sets of rsd s) .","offset":182073,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Proof.","offset":182843,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-12.html#text"},{"title":"Corollary D.12 (Quantitatively, average-optimal policies tend not to end up in any given 1-cycle) .","offset":186564,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-13.html#text"},{"title":"Proof.","offset":187358,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-13.html#text"},{"title":"Conjecture D.13 (Fractional quantitative optimality probability superiority lemma) .","offset":189228,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-13.html#text"},{"title":"Instructions for reporting errors","offset":190085,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/part-13.html#text"}],"text_code_points":191049,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/source.md"}