{"id":"arxiv:1803.03635","title":"The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks","date":"2018-03-09","section":"Data, training, optimization","quality":0.139358,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/","source_url":"https://arxiv.org/abs/1803.03635","text_available":true,"authors":["Jonathan Frankle","Michael Carbin"],"outgoing":[],"incoming":[{"source":"arxiv:2301.05217","target":"arxiv:1803.03635","evidence":"id","id":"arxiv:2301.05217","title":"Progress measures for grokking via mechanistic interpretability","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f71e3b65a8b3f35beb171d17f6c65878e0914227a0042ccba59b88952554baef/"},{"source":"arxiv:2309.08600","target":"arxiv:1803.03635","evidence":"id","id":"arxiv:2309.08600","title":"Sparse Autoencoders Find Highly Interpretable Features in Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/311cc8e986e42947a9cd36e64cb7dad93d7352e1f5ad495664fce8d0f6fca5da/"},{"source":"arxiv:2312.13558","target":"arxiv:1803.03635","evidence":"title","id":"arxiv:2312.13558","title":"The Truth is in There: Improving Reasoning in Language Models with Layer-Selective Rank Reduction","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e6ddb16378e3684d190823b8f271c8c3273b47f7bada2fc2c692b828d40e3384/"},{"source":"arxiv:2407.15017","target":"arxiv:1803.03635","evidence":"title","id":"arxiv:2407.15017","title":"Knowledge Mechanisms in Large Language Models: A Survey and Perspective","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8900dc2821aa61e59c5b1219cf9eb36dfaabf303c17d0064deaca653b5ec9a77/"},{"source":"arxiv:2606.23740","target":"arxiv:1803.03635","evidence":"title","id":"arxiv:2606.23740","title":"Weight-Space Geometry of Offline Reasoning Training","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e4479f0d0930797db9347855247f1c640a12d96f63d43b83ba9635561e3ff352/"},{"source":"doi:10.1038/s41586-024-07711-7","target":"arxiv:1803.03635","evidence":"title","id":"doi:10.1038/s41586-024-07711-7","title":"Maintaining Plasticity in Deep Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5d46327415fe38f2b9160b874735530939961a71536f7f1ac4c0dc7a3af3224/"}],"text_parts":[{"start":0,"end":15390,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-1.html"},{"start":15390,"end":31378,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-2.html"},{"start":31378,"end":47230,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html"},{"start":47230,"end":63138,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html"},{"start":63138,"end":79094,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html"},{"start":79094,"end":91463,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html"}],"contents":[{"title":"The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-1.html#text"},{"title":"Abstract","offset":76,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-1.html#text"},{"title":"1 Introduction","offset":1525,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-1.html#text"},{"title":"The Lottery Ticket Hypothesis .","offset":3981,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-1.html#text"},{"title":"2 Winning Tickets in Fully-Connected Networks","offset":10192,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-1.html#text"},{"title":"3 Winning Tickets in Convolutional Networks","offset":16393,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-2.html#text"},{"title":"4 VGG and Resnet for CIFAR10","offset":21054,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-2.html#text"},{"title":"5 Discussion","offset":26192,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-2.html#text"},{"title":"6 Limitations and Future Work","offset":31344,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-2.html#text"},{"title":"7 Related Work","offset":33098,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Appendix A Acknowledgments","offset":37276,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Appendix B Iterative Pruning Strategies","offset":37966,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Strategy 1: Iterative pruning with resetting.","offset":38180,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Strategy 2: Iterative pruning with continued training.","offset":38859,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Appendix C Early Stopping Criterion","offset":40119,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Appendix D Training Accuracy for Lottery Ticket Experiments","offset":41756,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Appendix E Comparing Random Reinitialization and Random Sparsity","offset":42865,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"Appendix F Examining Winning Tickets","offset":44100,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"F.1 Winning Ticket Initialization (Adam)","offset":44525,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"F.2 Winning Ticket Initializations (SGD)","offset":46323,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"F.3 Reinitializing from Winning Ticket Initializations","offset":47170,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-3.html#text"},{"title":"F.4 Pruning at Iteration 0","offset":48461,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"F.5 Comparing Initial and Final Weights in Winning Tickets","offset":49231,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"F.6 Winning Ticket Connectivity","offset":51444,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"F.7 Adding Noise to Winning Tickets","offset":53213,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"Appendix G Hyperparameter Exploration for Fully-Connected Networks","offset":54442,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"G.1 Experimental Methodology","offset":54881,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"G.2 Learning Rate","offset":57997,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"G.3 Other Optimization Algorithms","offset":60134,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"G.3.1 SGD","offset":60173,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"G.3.2 Momentum","offset":61066,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"G.4 Iterative Pruning Rate","offset":61739,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-4.html#text"},{"title":"G.5 Initialization Distribution","offset":63440,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"G.6 Network Size","offset":64166,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"Appendix H Hyperparameter Exploration for Convolutional Networks","offset":66708,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.1 Experimental Methodology","offset":67166,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.2 Learning Rate","offset":69892,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.3 Other Optimization Algorithms","offset":73631,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.3.1 SGD","offset":73670,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.3.2 Momentum","offset":75789,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.4 Iterative Pruning Rate","offset":76788,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.5 Learning Rates (Dropout)","offset":78385,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-5.html#text"},{"title":"H.6 Pruning Convolutions vs. Pruning Fully-Connected Layers","offset":82639,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"Appendix I Hyperparameter Exploration for VGG-19 and Resnet-18 on CIFAR10","offset":83422,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"I.1 Global Pruning","offset":83680,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"I.2 VGG-19 Details","offset":85726,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"I.3 Resnet-18 Details","offset":87009,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"I.4 Learning Rate","offset":88545,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"I.5 Warmup Iteration","offset":89090,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"Resnet-18.","offset":89923,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"VGG-19.","offset":90210,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"},{"title":"Instructions for reporting errors","offset":90499,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/part-6.html#text"}],"text_code_points":91463,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a9d7f0235205afcc709e3e74f28cc7707831bcb1d46239c418cc7e4b624d23bd/source.md"}