{"id":"arxiv:1710.05941","title":"Searching for Activation Functions","date":"2017-10-16","section":"LLMs: architectures, context, training","quality":0.112054,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/","source_url":"https://arxiv.org/abs/1710.05941","text_available":true,"authors":["Prajit Ramachandran","Barret Zoph","Quoc V. Le"],"outgoing":[],"incoming":[{"source":"arxiv:2002.05202","target":"arxiv:1710.05941","evidence":"id","id":"arxiv:2002.05202","title":"GLU Variants Improve Transformer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e052a2974593d29355ab45c301766ac6f5545ac1bc81d605e78d3adc71e9e2e4/"},{"source":"arxiv:2108.06325","target":"arxiv:1710.05941","evidence":"id","id":"arxiv:2108.06325","title":"Continual Backprop: Stochastic Gradient Descent with Persistent Randomness","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/73c90c52bffd8b5b2485d93607350545d822f361c29d3c0063b8d3025576e793/"},{"source":"arxiv:2501.05441","target":"arxiv:1710.05941","evidence":"id","id":"arxiv:2501.05441","title":"The GAN is dead; long live the GAN! A Modern GAN Baseline","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a6fcdbec190497e2e52574b7cb2b998385ed1944920c17dc53b710d18f7ba152/"},{"source":"arxiv:2503.14858","target":"arxiv:1710.05941","evidence":"title","id":"arxiv:2503.14858","title":"1000 Layer Networks for Self-Supervised RL: Scaling Depth Can Enable New Goal-Reaching Capabilities","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/dfa04045239215a4a49544cd31c178090abc3a587e98bd9af08e0ea96d658911/"},{"source":"arxiv:2604.11962","target":"arxiv:1710.05941","evidence":"id","id":"arxiv:2604.11962","title":"The Linear Centroids Hypothesis: Features as Directions Learned by Local Experts","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8428fd0cce9753855f4d0f801609b0205a0be52a459d53ae44e7860d00348485/"},{"source":"doi:10.1038/s41586-024-07711-7","target":"arxiv:1710.05941","evidence":"title","id":"doi:10.1038/s41586-024-07711-7","title":"Maintaining Plasticity in Deep Continual Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/b5d46327415fe38f2b9160b874735530939961a71536f7f1ac4c0dc7a3af3224/"}],"text_parts":[{"start":0,"end":14778,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-1.html"},{"start":14778,"end":29962,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html"}],"contents":[{"title":"Searching for Activation Functions","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-1.html#text"},{"title":"Abstract","offset":38,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-1.html#text"},{"title":"1 Introduction","offset":1325,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-1.html#text"},{"title":"2 Methods","offset":4619,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-1.html#text"},{"title":"3 Search Findings","offset":8161,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-1.html#text"},{"title":"4 Swish","offset":13840,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-1.html#text"},{"title":"5 Experiments with Swish","offset":17633,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"},{"title":"5.1 Experimental Set Up","offset":18987,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"},{"title":"5.2 CIFAR","offset":21184,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"},{"title":"5.3 ImageNet","offset":22121,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"},{"title":"5.4 Machine Translation","offset":24573,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"},{"title":"6 Related Work","offset":25776,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"},{"title":"7 Conclusion","offset":28846,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"},{"title":"Acknowledgements","offset":29689,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/part-2.html#text"}],"text_code_points":29962,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8815985e018dbc2066450e5afbb9260b48d5b03a4f41bc23285755ccf3185454/source.md"}