{"id":"arxiv:2312.13327","title":"In-Context Reinforcement Learning for Variable Action Spaces","date":"2023-12-20","section":"Reinforcement learning","quality":-0.055241,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/","source_url":"https://arxiv.org/abs/2312.13327","text_available":true,"authors":["Viacheslav Sinii","Alexander Nikulin","Vladislav Kurenkov","Ilya Zisman"],"outgoing":[],"incoming":[],"text_parts":[{"start":0,"end":15861,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html"},{"start":15861,"end":31777,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html"},{"start":31777,"end":47461,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html"},{"start":47461,"end":48911,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-4.html"}],"contents":[{"title":"In-Context Reinforcement Learning for Variable Action Spaces","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"Abstract","offset":64,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"Keywords:","offset":1137,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"1 Introduction","offset":1155,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"2 Algorithm Distillation Struggles with Novel Action Spaces","offset":4606,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"3 Headless-AD","offset":6477,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"4 Experiments","offset":11172,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"4.1 Bernoulli Bandit","offset":11796,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"4.2 Contextual Bandit","offset":14495,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"4.3 Darkroom","offset":15498,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-1.html#text"},{"title":"5 Ablations","offset":20633,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"5.1 Action Set Prompt","offset":20723,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"5.2 Contrastive Loss","offset":22150,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"5.3 Orthonormal Action Embeddings","offset":24010,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"6 Related Work","offset":25335,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"7 Conclusion","offset":27512,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"Impact Statement","offset":30149,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"Appendix A Background","offset":30784,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-2.html#text"},{"title":"Appendix B Related Work","offset":34041,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"B.1 Transformers in Reinforcement Learning","offset":34069,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"B.2 Offline Meta-RL","offset":35672,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"B.3 In-Context Learning in RL","offset":37155,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"B.4 Discarding the Linear Layer","offset":41012,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix C Algorithm Distillation on Permuted Train Sets","offset":43167,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix D Across-Environment Generalization","offset":43228,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix E Visual Darkroom","offset":43960,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix F Darkroom. Alternative Split","offset":45063,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix G MSE-Headless-AD In-Context Curves on Bernoulli Bandit","offset":45906,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix H Sampling of Orthonormal Vectors","offset":45975,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix I Algorithms’ Training Times","offset":46227,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix J Model Hyperparameters","offset":46269,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix K Linear Dependence of Different Types of Action Embeddings","offset":47388,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-3.html#text"},{"title":"Appendix L Code Sample","offset":47920,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-4.html#text"},{"title":"Instructions for reporting errors","offset":47947,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/part-4.html#text"}],"text_code_points":48911,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/03e08bdf5d5c670f49cce235ed39b6ddb047bc31370ae7d24b68d13f81e232f6/source.md"}