{"id":"arxiv:2604.25850","title":"Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses","date":"2026-04-28","section":"Harness","quality":0.085264,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/","source_url":"https://arxiv.org/abs/2604.25850","text_available":true,"authors":["Jiahang Lin","Shichun Liu","Chengjun Pan","Lizhi Lin"],"outgoing":[{"source":"arxiv:2604.25850","target":"arxiv:2303.17651","evidence":"title","id":"arxiv:2303.17651","title":"Self-Refine: Iterative Refinement with Self-Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/accef2157d87cf33be93cb3a9af246528b0a3742a9e2e700a194956fac8f5cf4/"},{"source":"arxiv:2604.25850","target":"arxiv:2408.08435","evidence":"title","id":"arxiv:2408.08435","title":"Automated Design of Agentic Systems","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9890d5668d1c1b33e29569b74626fd586f456af6a97c657b8c99895c90f59a5/"},{"source":"arxiv:2604.25850","target":"arxiv:2410.07095","evidence":"title","id":"arxiv:2410.07095","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/f0a0c6321ec079098a36ad19b3900d77197a3af1f6bbf6eee8a083e5eb03d42e/"},{"source":"arxiv:2604.25850","target":"arxiv:2507.19457","evidence":"title","id":"arxiv:2507.19457","title":"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c1ef72d96fa3292590a6a3d112ab509840081b59aaa771cf42b8b0e8ef62ba08/"},{"source":"arxiv:2604.25850","target":"arxiv:2603.28052","evidence":"id","id":"arxiv:2603.28052","title":"Meta-Harness: End-to-End Optimization of Model Harnesses","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8bb0b87384dfdd42c419cf26cc7e52f1f0d89f63afa5d5968e246b73a44c99cb/"}],"incoming":[{"source":"arxiv:2609.01437","target":"arxiv:2604.25850","evidence":"id","id":"arxiv:2609.01437","title":"HarnessDev: Can LLMs Create and Evolve Their Own Agent Harness?","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0e91a157b5aa6d15aa804a3849a31a55661a73d6c1470ff918d72163f5fe2076/"},{"source":"url:lilianweng.github.io/posts/2026-07-04-harness","target":"arxiv:2604.25850","evidence":"id","id":"url:lilianweng.github.io/posts/2026-07-04-harness","title":"Harnesses for models (Lilian Weng, 2026)","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c1ea249334ba128a9fff468e25fa4f8d13d4cccd13661ee130bec020ebb6463b/"}],"text_parts":[{"start":0,"end":15307,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html"},{"start":15307,"end":31201,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html"},{"start":31201,"end":47196,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html"},{"start":47196,"end":53256,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-4.html"}],"contents":[{"title":"Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"Abstract","offset":99,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"1 Introduction","offset":2065,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"2 Related Work","offset":7199,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"2.1 Harness Engineering and Evaluation for Coding Agents","offset":7218,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"2.2 Automated Optimization of LLM Agents","offset":8657,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"3 Method","offset":9808,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"3.1 NexAU: an editable, decoupled harness substrate","offset":10890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"3.2 Agent Debugger: layered trajectory evidence","offset":12230,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"3.3 Evolve Agent: evidence-driven, auditable edits","offset":13609,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-1.html#text"},{"title":"4 Experiments","offset":16113,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"4.1 Setup","offset":16367,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Evaluation.","offset":16382,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Models.","offset":17195,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"4.2 RQ1: Main Results","offset":17655,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"AHE outperforms both human-designed and self-evolve baselines.","offset":17991,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Prompt-only self-evolution misses the components that carry AHE’s gain.","offset":18701,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"4.3 RQ2: Transfer to Unseen Tasks and Base Models","offset":19267,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Cross-benchmark transfer.","offset":19655,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Cross-model transfer.","offset":20931,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"4.4 RQ3: Analysis","offset":22500,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"4.4.1 RQ3a: where value accumulates across components","offset":22678,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Each component owns a different failure surface.","offset":23060,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Components interact non-additively, capping the aggregate gain.","offset":23878,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"4.4.2 RQ3b: how reliably the loop’s self-attribution tracks reality","offset":24502,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Evidence-driven targeting.","offset":24935,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Regression blindness.","offset":25320,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"5 Conclusion","offset":25899,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Limitations","offset":26566,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Benchmark scope.","offset":26703,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Evolution operating point.","offset":27043,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Self-modification governance.","offset":27421,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Appendix A Experimental Setup: Full Details","offset":27849,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Seed agent.","offset":28016,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Runtime infrastructure.","offset":28382,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Hyperparameters.","offset":28849,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Terminal-bench difficulty labels.","offset":29015,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"pass@1.","offset":29228,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Token cost and Succ/Mtok.","offset":29928,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"Appendix B Prompts and Configurations","offset":30693,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"B.1 Code Agent Seed System Prompt","offset":31162,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-2.html#text"},{"title":"B.2 Evolve Agent Prompt","offset":31495,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"B.3 Explore Agent Prompts","offset":31809,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"B.3.1 Source-code Exploration Agent","offset":32131,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"B.3.2 Web-research Agent","offset":32173,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"Appendix C Qualitative Case Study","offset":32204,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.1 Trajectories: failing versus passing rollouts","offset":33308,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.1.1 Trajectory 1: db-wal-recovery","offset":33363,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"The task.","offset":33405,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"Trajectory before and after the iteration-2 changes.","offset":33824,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.1.2 Trajectory 2: path-tracing","offset":35555,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"The task.","offset":35884,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"Trajectory before and after the iteration-5 changes.","offset":36164,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.1.3 Trajectory 3: mcmc-sampling-stan","offset":37664,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"The task.","offset":38040,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"Trajectory before and after the iteration-6 changes.","offset":38403,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.1.4 Trajectory 4: configure-git-webserver","offset":40125,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"The task.","offset":40606,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"Trajectory before and after the iteration-8 changes.","offset":40998,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.2 Changes shipped on the four winning rounds","offset":42920,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.2.1 Iteration 2: prompt rules and shell-timeout argument","offset":42972,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.2.2 Iteration 5: publish-state mechanism (prompt rules + shell-tool guard)","offset":43744,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.2.3 Iteration 6: protected entrypoints and execution-risk middleware","offset":45325,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.2.4 Iteration 8: hard blocks and FRAMEWORK reminders","offset":47135,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-3.html#text"},{"title":"C.3 Reading the change-manifest figures","offset":48804,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-4.html#text"},{"title":"Appendix D Per-round Self-attribution Breakdown","offset":50140,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-4.html#text"},{"title":"Appendix E Broader Impact","offset":51202,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-4.html#text"},{"title":"Instructions for reporting errors","offset":52292,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/part-4.html#text"}],"text_code_points":53256,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c2358d2261b7478833ca59529278f0f4872896ca5ae4434e8db5d420532e10bd/source.md"}