{"id":"arxiv:2306.13575","title":"Scaling MLPs: A Tale of Inductive Bias","date":"2023-06-23","section":"LLMs: architectures, context, training","quality":0.244662,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/","source_url":"https://arxiv.org/abs/2306.13575","text_available":true,"authors":["Gregor Bachmann","Sotiris Anagnostidis","Thomas Hofmann"],"outgoing":[{"source":"arxiv:2306.13575","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2306.13575","target":"arxiv:2104.14294","evidence":"title","id":"arxiv:2104.14294","title":"Emerging Properties in Self-Supervised Vision Transformers","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4372186d141ca514b08e93bb3b1e26fdbcba9ce62187fb772c14bf223160c06d/"},{"source":"arxiv:2306.13575","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"}],"incoming":[],"text_parts":[{"start":0,"end":15347,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html"},{"start":15347,"end":28082,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html"},{"start":28082,"end":43904,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html"},{"start":43904,"end":46230,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-4.html"}],"contents":[{"title":"Scaling MLPs: A Tale of Inductive Bias","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"Abstract","offset":42,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"1 Introduction","offset":1755,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"2 Background","offset":7595,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"Theoretical Works.","offset":7612,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"MLPs.","offset":9226,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"Convolutions.","offset":11464,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"Vision Transformer.","offset":12874,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"MLP-Mixer.","offset":13684,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"Patchifiying.","offset":14421,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"3 Architecture","offset":15129,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"Standard MLP.","offset":15327,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-1.html#text"},{"title":"Inverted Bottleneck MLP.","offset":16548,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"4 Experiments","offset":17402,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"4.1 Setup","offset":17420,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"4.2 Training from Scratch","offset":18281,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"4.3 Transfer Learning","offset":19765,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"Test-Time Augmentations.","offset":22469,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"ReaL accuary.","offset":23344,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"Large batch-sizes.","offset":24993,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"Role of augmentations.","offset":26199,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"4.4 Scaling Laws","offset":27138,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"Compute.","offset":28067,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-2.html#text"},{"title":"Parameters or examples.","offset":31509,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"4.5 Computational Feasibility","offset":32742,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"5 Related Works","offset":33809,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"6 Discussion","offset":35991,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Appendix A Experimental Details","offset":37046,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"A.1 Resources","offset":37082,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"A.2 Additional Ablations","offset":37457,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Ablations.","offset":37487,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Normalization.","offset":37750,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Label smoothing.","offset":38124,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Architecture.","offset":38600,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Optimization.","offset":39235,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"A.3 Linear Probing","offset":39587,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"A.4 Scaling Laws","offset":40025,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Implementation Details.","offset":40047,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"A.5 Computational Efficiency","offset":40615,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Appendix B Results for Standard MLPs","offset":42093,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"B.1 Transfer Learning","offset":42134,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"B.2 Scaling Laws","offset":42725,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Appendix C Weight visualizations","offset":43199,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-3.html#text"},{"title":"Appendix D Inverted Bottleneck MLP Code","offset":44284,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-4.html#text"},{"title":"Instructions for reporting errors","offset":45266,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/part-4.html#text"}],"text_code_points":46230,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d2a2acb2495d23945ce079a4faaef7a8592b53aeff34afe67e5b9ae884791a22/source.md"}