{"id":"arxiv:2405.20541","title":"Perplexed by Perplexity: Perplexity-Based Data Pruning With Small Reference Models","date":"2024-05-30","section":"Data, training, optimization","quality":0.16386,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/","source_url":"https://arxiv.org/abs/2405.20541","text_available":true,"authors":["Zachary Ankner","Cody Blakeney","Kartik Sreenivasan","Max Marion"],"outgoing":[{"source":"arxiv:2405.20541","target":"arxiv:2206.04615","evidence":"title","id":"arxiv:2206.04615","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/70882486c1f8c46cb7b26cf100847f770eb7b39b69dee56b8a21141141ef814d/"},{"source":"arxiv:2405.20541","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"}],"incoming":[],"text_parts":[{"start":0,"end":15228,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html"},{"start":15228,"end":26571,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html"}],"contents":[{"title":"Perplexed by Perplexity: Perplexity-Based Data Pruning With Small Reference Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"Abstract","offset":86,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"1 Introduction","offset":1156,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"Contributions","offset":5924,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"2 Perplexity-Based Data Pruning","offset":7047,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"3 Experiments","offset":8720,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"3.1 Setup","offset":8738,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"Models.","offset":8753,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"Data.","offset":8995,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"Training and hyperparameters.","offset":9350,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"Evaluation.","offset":10124,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"3.2 Perplexity-Based Data Pruning Improves Downstream Performance","offset":11034,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"3.3 Perplexity-Based Data Pruning Improves Training Efficiency","offset":12291,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"3.4 Perplexity-Based Data Pruning for Over-Trained Models","offset":13490,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"3.5 Perplexity-Based Data Pruning for the Data Constrained Regime","offset":15157,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-1.html#text"},{"title":"3.6 Upstream Perplexity is not a Reliable Evaluation Metric for Data Pruning","offset":17630,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"4 Understanding the Effects of Perplexity-Based Pruning","offset":18823,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"4.1 How Are Reference Perplexities Distributed","offset":19004,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"4.2 How Pruning Affects Domain Composition","offset":19694,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"5 Related Work","offset":20624,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"Classical methods for pruning text data.","offset":20643,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"Neural network based methods for pruning text data.","offset":22080,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"Data pruning on vision tasks.","offset":23540,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"6 Conclusion","offset":24651,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"},{"title":"Acknowledgments","offset":26232,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/part-2.html#text"}],"text_code_points":26571,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0633300a3d5ba43049372a7b551215c2d1a9443d02b8a74adf639b8a5d8476a8/source.md"}