{"id":"arxiv:2405.16684","title":"gzip Predicts Data-dependent Scaling Laws","date":"2024-05-26","section":"Data, training, optimization","quality":-0.037016,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/","source_url":"https://arxiv.org/abs/2405.16684","text_available":true,"authors":["Rohan Pandey"],"outgoing":[{"source":"arxiv:2405.16684","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"}],"incoming":[{"source":"arxiv:2405.18392","target":"arxiv:2405.16684","evidence":"id","id":"arxiv:2405.18392","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f70e803c548e8a0c524f42ec6203808e4b633b9cc2fb45f4a075af70150f41d1/"},{"source":"arxiv:2512.13961","target":"arxiv:2405.16684","evidence":"id","id":"arxiv:2512.13961","title":"Olmo 3","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8daa11a9729da9879a7e95c15d4eef3bab8579d2885a14102ed09db111221802/"}],"text_parts":[{"start":0,"end":15913,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html"},{"start":15913,"end":26027,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-2.html"}],"contents":[{"title":"gzip Predicts Data-dependent Scaling Laws","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"Abstract","offset":45,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"1 Introduction","offset":936,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"2 Related Work","offset":4636,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"2.1 Scaling Laws","offset":4655,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"2.2 Syntax & Information Theory","offset":7147,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"3 Modulating Data Complexity via Syntactic Properties of a PCFG","offset":9377,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"3.1 gzip -compressibility Measures Syntactic Complexity","offset":12250,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"4 Are Scaling Laws Sensitive to Data Complexity?","offset":14383,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"4.1 Computing Data-sensitive Scaling Laws from gzip -compressibility","offset":15709,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-1.html#text"},{"title":"4.2 Eliminating Syntactic Parameters as a Confounder of Compressibility","offset":19644,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-2.html#text"},{"title":"5 Discussion","offset":21184,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-2.html#text"},{"title":"6 Limitations","offset":23822,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-2.html#text"},{"title":"7 Acknowledgements","offset":24495,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-2.html#text"},{"title":"Appendix A Appendix","offset":25039,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-2.html#text"},{"title":"Instructions for reporting errors","offset":25063,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/part-2.html#text"}],"text_code_points":26027,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8d5456f36b61f2ce9a45da0889b5f0318dc816cab90ee57d54bba06246041de3/source.md"}