{"id":"arxiv:2410.07095","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","date":"2024-10-09","section":"Retrieval, embeddings, benchmarks","quality":0.31646,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/","source_url":"https://arxiv.org/abs/2410.07095","text_available":true,"authors":["Jun Shern Chan","Neil Chowdhury","Oliver Jaffe","James Aung"],"outgoing":[],"incoming":[{"source":"arxiv:2511.15593","target":"arxiv:2410.07095","evidence":"title","id":"arxiv:2511.15593","title":"What Does It Take to Be a Good AI Research Agent? Studying the Role of Ideation Diversity","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/"},{"source":"arxiv:2601.14525","target":"arxiv:2410.07095","evidence":"title","id":"arxiv:2601.14525","title":"Towards Execution-Grounded Automated AI Research","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/895006e818735c9a565dbe99ae31404366eefd6bdc2278612271c65a8493e4a5/"},{"source":"arxiv:2604.19341","target":"arxiv:2410.07095","evidence":"id","id":"arxiv:2604.19341","title":"Structured Scaling of AI Discovery Across Diverse Scientific Domains","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/33ad3375247f86197210840a19064548b997e8617475e606d7a7ac2cda64740e/"},{"source":"arxiv:2604.25850","target":"arxiv:2410.07095","evidence":"title","id":"arxiv:2604.25850","title":"Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/"},{"source":"arxiv:2606.18543","target":"arxiv:2410.07095","evidence":"title","id":"arxiv:2606.18543","title":"CEO-Bench: Can Agents Play the Long Game?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/696d7ac1dfd7539fbe8147ed713574b8eb5a7400a35e96805c7f0c9add99165c/"},{"source":"arxiv:2609.01437","target":"arxiv:2410.07095","evidence":"id","id":"arxiv:2609.01437","title":"HarnessDev: Can LLMs Create and Evolve Their Own Agent Harness?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0e91a157b5aa6d15aa804a3849a31a55661a73d6c1470ff918d72163f5fe2076/"},{"source":"url:lilianweng.github.io/posts/2026-07-04-harness","target":"arxiv:2410.07095","evidence":"id","id":"url:lilianweng.github.io/posts/2026-07-04-harness","title":"Harnesses for models (Lilian Weng, 2026)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c1ea249334ba128a9fff468e25fa4f8d13d4cccd13661ee130bec020ebb6463b/"},{"source":"url:weco.ai/blog/first-evidence-of-recursive-self-improvement","target":"arxiv:2410.07095","evidence":"id","id":"url:weco.ai/blog/first-evidence-of-recursive-self-improvement","title":"First evidence of recursive self-improvement (AIDE2, Weco)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/00441fa8f2a9743603579837e07f96b97a0a7de239cad14faaccc9aa2e9d770c/"}],"text_parts":[{"start":0,"end":15474,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html"},{"start":15474,"end":31473,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html"},{"start":31473,"end":43161,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html"}],"contents":[{"title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Abstract","offset":81,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"1 Introduction","offset":1108,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"2 MLE-bench","offset":4876,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"2.1 Dataset Curation","offset":5454,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"2.2 Metrics","offset":8638,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Leaderboards","offset":8655,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Medals","offset":9044,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Headline metric","offset":9708,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Raw Scores","offset":10369,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"2.3 Setup","offset":10617,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Validating submissions","offset":11219,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"2.3.1 Rules","offset":11775,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Rule-breaking detection","offset":12236,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Plagiarism detection","offset":12634,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"3 Experiments and Results","offset":13339,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"3.1 Main experiment","offset":14045,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Varying scaffolding","offset":14070,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Varying models","offset":14929,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-1.html#text"},{"title":"Discussion","offset":16134,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"3.2 Increasing number of attempts","offset":17838,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"3.3 Varying amount of compute available","offset":18545,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"3.4 Increasing time available","offset":19471,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"4 Contamination & Plagiarism","offset":20392,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"4.1 Familiarity with top solutions","offset":21399,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"4.2 Obfuscating competition descriptions","offset":22594,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"5 Related Work","offset":23810,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"6 Limitations","offset":27419,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"7 Conclusion","offset":30665,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-2.html#text"},{"title":"Ethics Statement","offset":31473,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"Reproducibility Statement","offset":33115,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"Appendix A Appendix","offset":33768,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.1 Dataset Curation Criteria","offset":33792,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.2 Distribution of Competitions","offset":35247,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.3 Runs analysis","offset":35469,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.4 Plagiarism detection tool","offset":36066,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.5 Setup details","offset":36924,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.6 Scaffold details","offset":37952,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.6.1 AIDE modifications","offset":38739,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.6.2 OpenHands modifications","offset":39841,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.6.3 MLAB modifications","offset":40340,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.7 Dataset","offset":41705,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.8 Obfuscated Descriptions","offset":41866,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"A.9 Performance by complexity, category, and competition date","offset":42130,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"},{"title":"Instructions for reporting errors","offset":42197,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/part-3.html#text"}],"text_code_points":43161,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/source.md"}