{"id":"arxiv:1912.01683","title":"Optimal Policies Tend to Seek Power","date":"2019-12-03","section":"AI safety and consciousness","quality":0.047714,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/","source_url":"https://arxiv.org/abs/1912.01683","text_available":true,"authors":["Alexander Matt Turner","Logan Smith","Rohin Shah","Andrew Critch"],"outgoing":[],"incoming":[{"source":"arxiv:2206.13353","target":"arxiv:1912.01683","evidence":"title","id":"arxiv:2206.13353","title":"Is Power-Seeking AI an Existential Risk?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/21192da140d682c8aacb0e8dfccbbf481a855b35ee245ee548f905eb993754d4/"},{"source":"arxiv:2206.13477","target":"arxiv:1912.01683","evidence":"title","id":"arxiv:2206.13477","title":"Parametrically Retargetable Decision-Makers Tend To Seek Power","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e47c3ca813aa251e657f4fd4b126d747e91e7518bcab8f7e9139a9cf5c8d95fb/"},{"source":"arxiv:2210.10760","target":"arxiv:1912.01683","evidence":"title","id":"arxiv:2210.10760","title":"Scaling Laws for Reward Model Overoptimization","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/fd55acb4c34a2366d8f9c1fc0672a20be0ac4da8643118d0cbf627e891346d2d/"},{"source":"arxiv:2304.06528","target":"arxiv:1912.01683","evidence":"id","id":"arxiv:2304.06528","title":"Power-seeking can be probable and predictive for trained agents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d7ff8707221493f94b2bbe97040d72c861160d33c5bfcce90eebf1e0ee3d3af8/"}],"text_parts":[{"start":0,"end":15750,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html"},{"start":15750,"end":31329,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html"},{"start":31329,"end":47054,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html"},{"start":47054,"end":62841,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html"},{"start":62841,"end":78769,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html"},{"start":78769,"end":94206,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html"},{"start":94206,"end":108627,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html"},{"start":108627,"end":124570,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html"},{"start":124570,"end":140463,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html"},{"start":140463,"end":156253,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html"},{"start":156253,"end":171009,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html"},{"start":171009,"end":186895,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-12.html"},{"start":186895,"end":201642,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html"},{"start":201642,"end":217561,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-14.html"},{"start":217561,"end":232856,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html"},{"start":232856,"end":237234,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-16.html"}],"contents":[{"title":"Optimal Policies Tend To Seek Power","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Abstract","offset":39,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"1 Introduction","offset":950,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"2 Related work","offset":3464,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"3 State visit distribution functions quantify the agent’s available options","offset":6030,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 3.1 (Rewardless mdp ) .","offset":6487,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 3.2 (1-cycle states) .","offset":6899,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 3.3 (State visit distribution [ Sutton and Barto, 1998 ] ) .","offset":7749,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 3.4 ( ℱ \\Fop single-state restriction) .","offset":9103,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 3.5 (Value function) .","offset":10196,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 3.6 (Non-domination) .","offset":10845,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"4 Some actions have a greater probability of being optimal","offset":11828,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 4.1 (Optimal policy set function) .","offset":12443,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 4.2 (Reward function distributions) .","offset":13566,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 4.3 (Visit distribution optimality probability) .","offset":15174,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"Definition 4.4 (Action optimality probability) .","offset":15693,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-1.html#text"},{"title":"5 Some states give the agent more control over the future","offset":17921,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 5.1 (Average optimal value) .","offset":18970,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 5.2 ( Power ) .","offset":20242,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Lemma 5.3 (Continuity of Power ) .","offset":20901,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Proposition 5.4 (Maximal Power ) .","offset":21094,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Proposition 5.5 ( Power is smooth across reversible dynamics) .","offset":21455,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 5.6 ( Power -seeking actions) .","offset":22189,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"6 Certain environmental symmetries produce power-seeking tendencies","offset":23319,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 6.1 (Similarity of vector sets) .","offset":24542,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 6.2 (Similarity of vector function sets) .","offset":25489,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 6.3 (Pushforward distribution of a permutation) .","offset":27481,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 6.4 (Orbit of a probability distribution) .","offset":27826,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Definition 6.5 (Inequalities which hold for most probability distributions) .","offset":29503,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"Proposition 6.6 (States with “more options” have more Power ) .","offset":30690,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-2.html#text"},{"title":"6.1 Keeping options open tends to be Power -seeking and tends to be optimal","offset":33281,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Definition 6.7 (Equivalent actions) .","offset":33608,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Definition 6.8 (States reachable after taking an action) .","offset":34024,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Proposition 6.9 (Keeping options open tends to be Power -seeking and tends to be optimal) .","offset":34241,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"6.2 When γ = 1 \\gamma=1 , optimal policies tend to navigate towards “larger” sets of cycles","offset":37698,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Definition 6.10 (Recurrent state distributions [ Puterman, 2014 ] ) .","offset":38379,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Definition 6.11 (Average-optimal policies) .","offset":39707,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Proposition 6.12 (When γ = 1 \\gamma=1 , rsd s control Power ) .","offset":40829,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Theorem 6.13 (Average-optimal policies tend to end up in “larger” sets of rsd s) .","offset":42485,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"Corollary 6.14 (Average-optimal policies tend not to end up in any given 1-cycle) .","offset":43417,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"6.3 How to reason about other environments","offset":45922,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-3.html#text"},{"title":"7 Discussion","offset":49255,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Future work.","offset":49888,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Societal impact.","offset":52450,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Conclusion.","offset":52919,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Acknowledgments","offset":53897,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Appendix A Comparing Power with information-theoretic empowerment","offset":54668,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Appendix B Seeking Power can be a detour","offset":56835,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Remark .","offset":56880,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Proposition B.1 (Greater Power 𝒟 bound \\text{{Power}}_{\\mathcal{D}_{\\text{bound}}} does not imply greater ℙ 𝒟 bound \\Prb_{\\mathcal{D}_{\\text{bound}}} ) .","offset":57067,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Proof.","offset":57576,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Lemma B.2 (Fraction of orbits which agree on weak optimality) .","offset":58861,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Proof.","offset":59672,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Lemma B.3 ( ≥ most \\geq_{\\text{most}} and trivial orbits) .","offset":60558,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Proof.","offset":61140,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Proposition B.4 (Actions which tend to seek Power do not necessarily tend to be optimal) .","offset":61669,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Proof.","offset":62126,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-4.html#text"},{"title":"Appendix C Sub-optimal Power","offset":64305,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Definition C.1 (Suboptimal Power ) .","offset":66242,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Appendix D Lists of results","offset":70223,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"D.1 Contributions of independent interest","offset":70312,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"D.1.1 Optimal value theory","offset":70576,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"D.1.2 Optimal policy theory","offset":71674,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"D.1.3 Visit distribution theory","offset":72400,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Appendix E Theoretical results","offset":73131,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Lemma E.1 (A policy is optimal iff it induces an optimal visit distribution at every state) .","offset":73166,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Proof.","offset":73455,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Definition E.2 (Transition matrix induced by a policy) .","offset":73706,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Proposition E.3 (Properties of visit distribution functions) .","offset":74126,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Proof.","offset":74628,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Lemma E.4 ( 𝐟 ∈ ℱ ⁡ ( s ) \\mathbf{f}\\in\\Fop(s) is multivariate rational on γ \\gamma ) .","offset":76154,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Proof.","offset":76364,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Corollary E.5 (On-policy value is rational on γ \\gamma ) .","offset":77778,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Proof.","offset":77984,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"E.1 Non-dominated visit distribution functions","offset":78331,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Definition E.6 (Continuous reward function distribution) .","offset":78383,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Remark .","offset":78562,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Lemma E.7 (Distinct linear functionals disagree almost everywhere on their domains) .","offset":78675,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-5.html#text"},{"title":"Proof.","offset":79053,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Corollary E.8 (Unique maximization of almost all vectors) .","offset":79614,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Proof.","offset":79984,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"E.1.1 Generalized non-domination results","offset":80720,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Definition E.9 (Non-dominated linear functionals) .","offset":80950,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Lemma E.10 (All vectors are maximized by a non-dominated linear functional) .","offset":81459,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Proof.","offset":81925,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Corollary E.11 (Maximal value is invariant to restriction to non-dominated functionals) .","offset":84364,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Proof.","offset":84786,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Lemma E.12 (How non-domination containment affects optimal value) .","offset":84868,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Proof.","offset":85632,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Definition E.13 (Non-dominated vector functions) .","offset":86585,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Remark .","offset":87233,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Definition E.14 (Affine transformation of visit distribution sets) .","offset":87347,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Lemma E.15 (Invariance of non-domination under positive affine transform) .","offset":87995,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Proof.","offset":88908,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"E.1.2 Inequalities which hold under most reward function distributions","offset":91229,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Lemma E.16 (Helper lemma for demonstrating ≥ most : 𝔇 any \\geq_{\\text{{most}}\\text{: }\\mathfrak{D}_{\\text{any}}} ) .","offset":91315,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Proof.","offset":91977,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Lemma E.17 (A helper result for expectations of functions) .","offset":93184,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Proof.","offset":94191,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-6.html#text"},{"title":"Definition E.18 (Support of 𝒟 any \\mathcal{D}_{\\text{any}} ) .","offset":97428,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Definition E.19 (Linear functional optimality probability) .","offset":97776,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Proposition E.20 (Non-dominated linear functionals and their optimality probability) .","offset":98320,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Proof.","offset":98858,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Lemma E.21 (Expected value of similar linear functional sets) .","offset":100930,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Proof.","offset":102151,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Lemma E.22 (For continuous iid distributions 𝒟 X ​ - iid \\mathcal{D}_{X\\text{-}\\textsc{iid}} , ∃ b < c : ( b , c ) | 𝒮 | ⊆ supp ⁡ ( 𝒟 X ​ - iid ) \\exists b<c\\mathrel{\\mathop{\\ordinarycolon}}(b,c)^{\\left|\\mathcal{S}\\right|}\\subseteq\\optSupp(\\mathcal{D}_{X\\text{-}\\textsc{iid}}) ) .","offset":105900,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Proof.","offset":106189,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Definition E.23 (Bounded, continuous iid reward) .","offset":106628,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Lemma E.24 (Expectation superiority lemma) .","offset":106925,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Proof.","offset":108396,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-7.html#text"},{"title":"Definition E.25 (Indicator function) .","offset":113540,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html#text"},{"title":"Lemma E.26 (Optimality probability inclusion relations) .","offset":113744,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html#text"},{"title":"Proof.","offset":114563,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html#text"},{"title":"Lemma E.27 (Optimality probability of similar linear functional sets) .","offset":118452,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html#text"},{"title":"Proof.","offset":119588,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html#text"},{"title":"Lemma E.28 (Optimality probability superiority lemma) .","offset":122657,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html#text"},{"title":"Proof.","offset":123858,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-8.html#text"},{"title":"Lemma E.29 (Limit probability inequalities which hold for most distributions) .","offset":127004,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proof.","offset":128358,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"E.1.3 ℱ nd \\Fndop results","offset":131311,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proof.","offset":131362,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Definition E.30 (Evaluating sets of visit distribution functions at γ \\gamma ) .","offset":135132,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Lemma E.31 (Non-domination across γ \\gamma values for expectations of visit distributions) .","offset":135683,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proof.","offset":136295,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Lemma E.32 ( ∀ γ ∈ ( 0 , 1 ) : 𝐝 ∈ ℱ nd ⁡ ( s , γ ) \\forall\\gamma\\in(0,1)\\mathrel{\\mathop{\\ordinarycolon}}\\mathbf{d}\\in\\Fndop(s,\\gamma) iff 𝐝 ∈ ND ​ ( ℱ ⁡ ( s , γ ) ) \\mathbf{d}\\in\\text{{ND}}\\left(\\Fop(s,\\gamma)\\right) ) .","offset":137506,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proof.","offset":137737,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proof.","offset":138277,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"E.2 Some actions have greater probability of being optimal","offset":138483,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Lemma E.33 (Optimal policy shift bound) .","offset":138547,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proof.","offset":138844,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proposition E.34 (Optimality probability’s limits exist) .","offset":139274,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Proof.","offset":139756,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-9.html#text"},{"title":"Lemma E.35 (Optimality probability identity) .","offset":141583,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Proof.","offset":142017,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"E.3 Basic properties of Power","offset":143713,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Lemma E.36 ( Power identities) .","offset":143748,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Proof.","offset":144947,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Definition E.37 (Discount-normalized value function) .","offset":147523,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Lemma E.38 (Normalized value functions have uniformly bounded derivative) .","offset":147881,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Proof.","offset":148382,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Proof.","offset":153802,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Proof.","offset":155306,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-10.html#text"},{"title":"Lemma E.39 (Lower bound on current Power based on future Power ) .","offset":156998,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Proof.","offset":157602,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Proof.","offset":160411,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"E.4 Seeking Power is often more probable under optimality","offset":163458,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"E.4.1 Keeping options open tends to be Power -seeking and tends to be optimal","offset":163521,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Definition E.40 (Normalized visit distribution function) .","offset":163605,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Remark .","offset":164291,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Lemma E.41 (Normalized visit distribution functions are continuous) .","offset":164413,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Proof.","offset":164834,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Lemma E.42 (Non-domination of normalized visit distribution functions) .","offset":166057,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Proof.","offset":166639,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Lemma E.43 ( Power limit identity) .","offset":168960,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Proof.","offset":169481,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-11.html#text"},{"title":"Lemma E.44 (Lemma for Power superiority) .","offset":172565,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-12.html#text"},{"title":"Proof.","offset":174498,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-12.html#text"},{"title":"Proof.","offset":184944,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-12.html#text"},{"title":"Lemma E.45 (Non-dominated visit distribution functions never agree with other visit distribution functions at that state) .","offset":186895,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Proof.","offset":187301,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Corollary E.46 (Cardinality of non-dominated visit distributions) .","offset":187858,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Proof.","offset":188174,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Lemma E.47 (Optimality probability and state bottlenecks) .","offset":189398,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Proof.","offset":191120,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Case: γ ∈ ( 0 , 1 ) \\gamma\\in(0,1) .","offset":195309,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Case: γ = 1 \\gamma=1 , γ = 0 \\gamma=0 .","offset":197972,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Lemma E.48 (Action optimality probability is a special case of visit distribution optimality probability) .","offset":199693,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Proof.","offset":200022,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-13.html#text"},{"title":"Proof.","offset":204501,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-14.html#text"},{"title":"E.4.2 When γ = 1 \\gamma=1 , optimal policies tend to navigate towards “larger” sets of cycles","offset":219708,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Lemma E.49 ( Power identity when γ = 1 \\gamma=1 ) .","offset":219808,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Proof.","offset":220362,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Proof.","offset":221914,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Proof.","offset":223711,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Proposition E.50 ( Rsd properties) .","offset":231261,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Proof.","offset":231469,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Lemma E.51 (When reachable with probability 1, 1-cycles induce non-dominated rsd s) .","offset":232558,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Proof.","offset":232841,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-15.html#text"},{"title":"Proof.","offset":233664,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-16.html#text"},{"title":"Instructions for reporting errors","offset":236270,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/part-16.html#text"}],"text_code_points":237234,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/7fe89b6251f3cacab8ed2ad8b728272f92e704949d342d0ab8b0ea97548860cb/source.md"}