{"id":"arxiv:2207.02098","title":"Neural Networks and the Chomsky Hierarchy","date":"2022-07-05","section":"LLMs: architectures, context, training","quality":0.297018,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/","source_url":"https://arxiv.org/abs/2207.02098","text_available":true,"authors":["Grégoire Delétang","Anian Ruoss","Jordi Grau-Moya","Tim Genewein"],"outgoing":[{"source":"arxiv:2207.02098","target":"arxiv:2001.08361","evidence":"title","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"}],"incoming":[{"source":"arxiv:2305.13673","target":"arxiv:2207.02098","evidence":"title","id":"arxiv:2305.13673","title":"Physics of Language Models: Part 1, Learning Hierarchical Language Structures","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/a72219d4bd65d1e84ce31079b6c788d7e6c297e71d4b083af5d4cffccfdcff72/"},{"source":"arxiv:2405.04517","target":"arxiv:2207.02098","evidence":"title","id":"arxiv:2405.04517","title":"xLSTM: Extended Long Short-Term Memory","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/40fd3e03894caeb940ad2021fb4052c0d2e809706b5022f7454072997b8ba2d5/"}],"text_parts":[{"start":0,"end":14220,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html"},{"start":14220,"end":29740,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html"},{"start":29740,"end":45622,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html"},{"start":45622,"end":61277,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html"},{"start":61277,"end":76804,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html"},{"start":76804,"end":78408,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-6.html"}],"contents":[{"title":"Neural Networks and the Chomsky Hierarchy","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Abstract","offset":45,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"1 Introduction","offset":1184,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"This work","offset":4792,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"2 Related work","offset":6709,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Learning formal languages","offset":6728,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Neural networks and the Chomsky hierarchy","offset":8387,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Memory-augmented networks","offset":10274,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"3 Background","offset":12032,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Formal language","offset":12529,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Generative grammar","offset":13122,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Recognizing a language","offset":14190,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-1.html#text"},{"title":"Transduction vs. recognition","offset":16145,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"4 Methods","offset":18595,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"Problem setup","offset":18609,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"Tasks","offset":20583,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"Architectures","offset":21779,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"Data generation","offset":23712,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"5 Results","offset":24440,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"5.1 Main result","offset":24751,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"5.2 Analysis of learned program structure","offset":26850,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"Regular tasks","offset":27174,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"Deterministic context-free tasks","offset":28497,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"Context-sensitive tasks","offset":29709,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-2.html#text"},{"title":"5.3 LSTMs","offset":30872,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"5.4 Transformers","offset":31804,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"6 Discussion","offset":34268,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"7 Conclusion","offset":36173,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"8 Ethics Statement & Limitations","offset":36989,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Acknowledgments","offset":38024,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Appendix A Experimental details","offset":38197,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"A.1 Language recognition vs. transduction","offset":38233,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"A.2 Models","offset":41386,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"RNN","offset":42243,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Stack-RNN","offset":42323,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"NDStack-RNN","offset":42689,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Tape-RNN","offset":43179,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"LSTM","offset":44247,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Stack-LSTM","offset":44340,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Transformer encoder","offset":44561,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Transformer (autoregressive)","offset":45288,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"Convolutional Neural Network","offset":45586,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-3.html#text"},{"title":"A.3 Tasks","offset":46038,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Even Pairs ( R )","offset":47057,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Modular Arithmetic (Simple) ( R )","offset":47677,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Parity Check ( R )","offset":48294,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Cycle Navigation ( R )","offset":48750,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Modular Arithmetic ( DCF )","offset":49309,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Reverse String ( DCF )","offset":50397,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Solve Equation ( DCF )","offset":50884,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Stack Manipulation ( DCF )","offset":51635,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Binary Addition ( CS )","offset":52327,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Binary Multiplication ( CS )","offset":52734,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Compute Sqrt ( CS )","offset":53155,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Duplicate String ( CS )","offset":53578,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Missing Duplicate ( CS )","offset":53925,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Odds First ( CS )","offset":54498,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Bucket Sort ( CS )","offset":54890,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"A.4 Problem setup","offset":55386,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"A.5 Computational resources","offset":57652,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-4.html#text"},{"title":"Appendix B Additional Experiments","offset":62253,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.1 Main result","offset":62291,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.2 Phase transition under increasing training ranges","offset":63600,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.3 Analysis of state- and memory-dynamics","offset":64468,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.3.1 Stack-RNN program analysis","offset":64516,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.3.2 Transformer program analysis","offset":66236,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"Setup","offset":66277,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"Interpretation of the learned algorithm","offset":67194,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"Failure mode for long sequences","offset":68317,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.4 Convolutional Neural Networks","offset":69100,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.5 Stack-LSTMs","offset":70306,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.6 NDStack-RNN","offset":71649,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.7 Autoregressive Models","offset":72864,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.7.1 Memory-augmented recurrent neural networks","offset":73636,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.7.2 Transformer","offset":74607,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"B.8 Scaling laws and the Chomsky hierarchy","offset":75767,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-5.html#text"},{"title":"Instructions for reporting errors","offset":77444,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/part-6.html#text"}],"text_code_points":78408,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/6c4e7406c7294ace52a03e4eede4ef1a31cd395dcde0958487fdbb546fa2c026/source.md"}