{"id":"arxiv:2511.15593","title":"What Does It Take to Be a Good AI Research Agent? Studying the Role of Ideation Diversity","date":"2025-11-19","section":"Agents, open-endedness, AGI","quality":-0.444708,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/","source_url":"https://arxiv.org/abs/2511.15593","text_available":true,"authors":["Alexis Audran-Reiss","Jordi Armengol-Estapé","Karen Hambardzumyan","Amar Budhiraja"],"outgoing":[{"source":"arxiv:2511.15593","target":"arxiv:2410.07095","evidence":"title","id":"arxiv:2410.07095","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/"}],"incoming":[{"source":"arxiv:2603.19461","target":"arxiv:2511.15593","evidence":"id","id":"arxiv:2603.19461","title":"Hyperagents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/17eec8a15debc212b5530f117dd8ba8bad339590729cbbc4f75a47755ebcb544/"}],"text_parts":[{"start":0,"end":15806,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html"},{"start":15806,"end":31783,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html"},{"start":31783,"end":42783,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html"}],"contents":[{"title":"What Does It Take to Be a Good AI Research Agent? Studying the Role of Ideation Diversity","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"Abstract","offset":93,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"1 Introduction","offset":1193,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"1.1 Contributions","offset":4899,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"2 Research Agents and MLE-bench","offset":5771,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"Agents","offset":5807,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"MLE-Bench","offset":6857,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3 Methods","offset":9746,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.1 General Setup","offset":9994,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.1.1 Dataset","offset":10017,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.1.2 Metrics","offset":10229,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.1.3 Agentic Scaffolds","offset":10703,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.1.4 LLM Backbones","offset":12344,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.2 Measuring Ideation Diversity","offset":12906,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.3 Controlling Ideation Diversity","offset":14380,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.3.1 Baseline agents","offset":14635,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"3.3.2 Agents with ablated diversity","offset":15570,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-1.html#text"},{"title":"4 The Ideation Diversity Bottleneck","offset":16001,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.1 Deep-dive on Agentic Trajectories","offset":16350,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"The agent scaffold choice impacts ideation diversity","offset":16393,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"Diversity correlates with performance on MLE-bench","offset":17739,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.2 Impact of Diversity: A Controlled Experiment","offset":19087,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.2.1 Are We Actually Influencing Diversity?","offset":19518,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.2.2 Results","offset":20387,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.3 Evaluating with Alternative Metrics","offset":20989,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.3.1 Alternative Metrics","offset":21388,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.3.2 Data Analysis Results with Additional Metrics","offset":22977,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"4.3.3 Controlled Experiment Results with Additional Metrics","offset":23372,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"5 Discussion","offset":25355,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"What does it take to be a good AI research agent?","offset":25372,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"Importance of the implementation bottleneck.","offset":27228,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"Generalization to other benchmarks.","offset":28392,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"Limitations of MLE-bench evaluations.","offset":28750,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"Limitations of this study.","offset":30243,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"6 Related Work","offset":30702,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"Generation diversity in language models.","offset":30721,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-2.html#text"},{"title":"Diversity in reinforcement learning and population-based reinforcement learning.","offset":31783,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Diversity in multi-agent systems.","offset":32681,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Automated machine learning and AI research agents.","offset":33814,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"7 Conclusions","offset":34592,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Appendix A Appendix","offset":35241,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"A.1 Controlling Ideation Diversity Using Temperature","offset":35265,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"A.2 Alternative Metrics","offset":36931,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"A.2.1 Limitations of MLE-Bench Medal System","offset":36960,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Variable medal thresholds","offset":37363,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Narrow thresholds between medals and top scores","offset":37687,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Different test sets for evaluation","offset":38134,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Competition age","offset":38685,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"A.2.2 Evaluation Principles","offset":39454,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Limited Set of Metrics","offset":39913,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Inclusion of all attempts","offset":40265,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Independence from Human Score Distributions","offset":40535,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Capturing the complexity of hill-climbing","offset":40708,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"A.3 Models Used By Agents For Image Classification Tasks","offset":41111,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"},{"title":"Instructions for reporting errors","offset":41819,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/part-3.html#text"}],"text_code_points":42783,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/be2cc7f3c48da553c3ae6c2de2a5474a30deb993704a409d471ac0570b91dcab/source.md"}