{"id":"arxiv:2503.14858","title":"1000 Layer Networks for Self-Supervised RL: Scaling Depth Can Enable New Goal-Reaching Capabilities","date":"2025-03-19","section":"Reinforcement learning","quality":0.017695,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/","source_url":"https://arxiv.org/abs/2503.14858","text_available":true,"authors":["Kevin Wang","Ishaan Javali","Michał Bortkiewicz","Tomasz Trzciński"],"outgoing":[{"source":"arxiv:2503.14858","target":"arxiv:1710.05941","evidence":"title","id":"arxiv:1710.05941","title":"Searching for Activation Functions","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/"},{"source":"arxiv:2503.14858","target":"arxiv:2104.14294","evidence":"id","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"},{"source":"arxiv:2503.14858","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/"},{"source":"arxiv:2503.14858","target":"arxiv:2305.19452","evidence":"title","id":"arxiv:2305.19452","title":"Bigger, Better, Faster: Human-level Atari with human-level efficiency","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/727a7ee43fd005401d34d4591a2f49d85175f9366ebf2c9e7a862b340787614e/"},{"source":"arxiv:2503.14858","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[],"text_parts":[{"start":0,"end":15741,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html"},{"start":15741,"end":31331,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html"},{"start":31331,"end":47323,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html"},{"start":47323,"end":61535,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-4.html"}],"contents":[{"title":"1000 Layer Networks for Self-Supervised RL: Scaling Depth Can Enable New Goal-Reaching Capabilities","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"Abstract","offset":103,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"1 Introduction","offset":1309,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"2 Related Work","offset":5875,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"3 Preliminaries","offset":9022,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"Goal-Conditioned Reinforcement Learning","offset":9277,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"Contrastive Reinforcement Learning.","offset":11704,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"Residual Connections","offset":13883,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"4 Experiments","offset":14982,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"4.1 Experimental Setup","offset":15000,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"Environments.","offset":15028,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"Architectural Components","offset":15710,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-1.html#text"},{"title":"4.2 Scaling Depth in Contrastive RL","offset":16451,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"4.3 Emergent Policies Through Depth","offset":17913,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"4.4 What Matters for CRL Scaling","offset":19534,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Width vs. Depth","offset":19572,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Scaling the Actor vs. Critic Networks","offset":20782,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Deep Networks Unlock Batch Size Scaling","offset":21570,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Training Contrastive RL with 1000+ Layers","offset":23238,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"4.5 Why Scaling Happens","offset":23899,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Depth Enhances Contrastive Representations","offset":23928,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Depth Enhances Exploration and Expressivity in a Synergized Way","offset":25273,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Deep Networks Learn to Allocate Greater Representational Capacity to States Near the Goal","offset":27137,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Deeper Networks Enable Partial Experience Stitching","offset":27992,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"The (CRL) Algorithm is Key","offset":29131,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"4.6 Does Depth Scaling Improve Offline Contrastive RL?","offset":30599,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"5 Conclusion","offset":31314,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-2.html#text"},{"title":"Limitations.","offset":32365,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"Impact Statement","offset":32709,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"Acknowledgments.","offset":32932,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"Appendix A Additional Experiments","offset":33602,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.1 Scaled CRL Outperforms All Other Baselines on 8 out of 10 Environments","offset":33640,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.2 The CRL Algorithm is Key: Depth Scaling is Not Effective on Other Baselines","offset":34157,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.3 Additional Scaling Experiments: Offline GCBC, BC, and QRL","offset":34804,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.4 Can Depth Scaling also be Effective for Quasimetric Architectures?","offset":35469,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.5 Additional Architectural Ablations: Layer Norm and Swish Activation","offset":36242,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.6 Can We Integrate Novel Architectural Innovations from the Emerging RL Scaling Literature?","offset":36784,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.7 Residuals Norms in Deep Networks","offset":37434,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"A.8 Scaling Depth for Offline Goal-conditioned RL","offset":37791,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"Appendix B Experimental Details","offset":37846,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"B.1 Environment Setup and Hyperparameters","offset":37882,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"B.2 Python Environment Differences","offset":38301,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"B.3 Wall-clock Time of Our Approach","offset":38924,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"B.4 Wall-clock Time: Comparison to Baselines","offset":39274,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"NeurIPS Paper Checklist","offset":39700,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-3.html#text"},{"title":"Instructions for reporting errors","offset":60571,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/part-4.html#text"}],"text_code_points":61535,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/source.md"}