{"id":"arxiv:2009.11848","title":"How Neural Networks Extrapolate: From Feedforward to Graph Neural Networks","date":"2020-09-24","section":"Data, training, optimization","quality":0.221265,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/","source_url":"https://arxiv.org/abs/2009.11848","text_available":true,"authors":["Keyulu Xu","Mozhi Zhang","Jingling Li","Simon S. Du"],"outgoing":[],"incoming":[],"text_parts":[{"start":0,"end":15904,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html"},{"start":15904,"end":31514,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html"},{"start":31514,"end":46232,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html"},{"start":46232,"end":62139,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-4.html"},{"start":62139,"end":77792,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-5.html"},{"start":77792,"end":91985,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-6.html"},{"start":91985,"end":107629,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-7.html"},{"start":107629,"end":123435,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-8.html"},{"start":123435,"end":139263,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html"},{"start":139263,"end":155121,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html"},{"start":155121,"end":159420,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-11.html"}],"contents":[{"title":"How Neural Networks Extrapolate: From Feedforward to Graph Neural Networks","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"Abstract","offset":78,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"1 Introduction","offset":1513,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"1.1 Related work","offset":6831,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"2 Preliminaries","offset":8320,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"Definition 1 .","offset":9177,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"3 How Feedforward Neural Networks Extrapolate","offset":11495,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"3.1 Linear Extrapolation Behavior of ReLU MLPs","offset":11836,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"Definition 2 .","offset":13019,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"Theorem 1 .","offset":13992,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-1.html#text"},{"title":"3.2 When ReLU MLPs Provably Extrapolate Well","offset":16282,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"Lemma 1 .","offset":16839,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"Theorem 2 .","offset":17788,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"3.3 MLPs with Other Activation Functions","offset":20206,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"4 How Graph Neural Networks Extrapolate","offset":21276,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"4.1 Hypothesis: Linear Algorithmic Alignment Helps Extrapolation","offset":21800,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"Definition 3 .","offset":23785,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"Hypothesis 1 .","offset":24567,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"4.2 Theoretical and Empirical Support","offset":27891,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"Corollary 1 .","offset":28399,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"Theorem 3 .","offset":28699,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-2.html#text"},{"title":"5 Connections to Other Out-of-Distribution Settings","offset":33348,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"6 Conclusion","offset":35767,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"Acknowledgments","offset":36267,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"Appendix A Theoretical Background","offset":37222,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"Lemma 2 .","offset":42031,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"Lemma 3 .","offset":42994,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"Appendix B Proofs","offset":45524,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"B.1 Proof of Theorem 1","offset":45546,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-3.html#text"},{"title":"B.2 Proof of Lemma 1","offset":61460,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-4.html#text"},{"title":"Setup and preparation.","offset":62109,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-4.html#text"},{"title":"Constrained optimization in NTK feature space.","offset":64311,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-5.html#text"},{"title":"Claim 1 .","offset":72449,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-5.html#text"},{"title":"Global optimum via the geometry of training data.","offset":77735,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-5.html#text"},{"title":"B.3 Proof of Theorem 2","offset":87319,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-6.html#text"},{"title":"Asymptotic convergence of data sets.","offset":87952,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-6.html#text"},{"title":"Asymptotic convergence of predictions.","offset":96491,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-7.html#text"},{"title":"B.4 Proof of Corollary 1","offset":99171,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-7.html#text"},{"title":"B.5 Proof of Theorem 3","offset":100495,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-7.html#text"},{"title":"B.6 Proof of Lemma 2","offset":110315,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-8.html#text"},{"title":"B.7 Proof of Lemma 3","offset":113890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-8.html#text"},{"title":"Appendix C Experimental Details","offset":122864,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-8.html#text"},{"title":"Overview.","offset":123068,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-8.html#text"},{"title":"C.1 Learning Simple Non-Linear Functions","offset":127807,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Model and hyperparameter settings.","offset":130538,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Test error and model selection.","offset":131888,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"C.2 R-squared for Out-of-distribution Directions","offset":132700,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Definition.","offset":133070,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Datasets and models.","offset":133335,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Computation.","offset":133600,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Results.","offset":134701,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"C.3 Learning Linear Functions","offset":135185,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Model and hyperparameter settings.","offset":138581,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"Exact computation with neural tangent kernel","offset":138827,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-9.html#text"},{"title":"C.4 MLPs with cosine, quadratic, and tanh Activation","offset":140373,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"Dataset details.","offset":141033,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"Model and hyperparameter settings.","offset":141794,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"C.5 Max Degree","offset":142077,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"Model and hyperparameter settings.","offset":144132,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"Test error and model selection.","offset":145869,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"C.6 Shortest Path","offset":146270,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"Model and hyperparameter settings.","offset":148949,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"C.7 N-Body Problem","offset":150767,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"Dataset details.","offset":153129,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-10.html#text"},{"title":"Model and hyperparameter settings.","offset":155364,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-11.html#text"},{"title":"Appendix D Visualization and Additional Experimental Results","offset":157593,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-11.html#text"},{"title":"D.1 Visualization Results","offset":157658,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-11.html#text"},{"title":"D.2 Extra Experimental Results","offset":158361,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-11.html#text"},{"title":"Instructions for reporting errors","offset":158456,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/part-11.html#text"}],"text_code_points":159420,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/bd1a0149b2db51763f5d8c949f315e618ad47d5f1e2976b08a049f2bed2fbc4c/source.md"}