{"id":"arxiv:2403.07714","title":"StableToolBench: Towards Stable Large-Scale Benchmarking on Tool Learning of Large Language Models","date":"2024-03-12","section":"Retrieval, embeddings, benchmarks","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/","source_url":"https://arxiv.org/abs/2403.07714","text_available":true,"authors":["Zhicheng Guo","Sijie Cheng","Hao Wang","Shihao Liang"],"outgoing":[],"incoming":[{"source":"arxiv:2602.06130","target":"arxiv:2403.07714","evidence":"id","id":"arxiv:2602.06130","title":"Self-Improving World Modelling with Latent Actions","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ac56340e6b81e232b48b92dcf79fe37efc1c00e676413e55c9d119a809eb644b/"}],"text_parts":[{"start":0,"end":15532,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html"},{"start":15532,"end":31461,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html"}],"contents":[{"title":"StableToolBench: Towards Stable Large-Scale Benchmarking on Tool Learning of Large Language Models","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"Abstract","offset":102,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"1 Introduction","offset":1107,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"2 Stability Analysis on ToolBench","offset":6638,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"2.1 Stability of Performance","offset":6960,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"2.2 Stability of Evaluation","offset":7795,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"2.3 Stability of API Status","offset":9650,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"3 StableToolBench","offset":11221,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"3.1 Virtual API Server","offset":11494,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"Caching System.","offset":11945,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"API Simulator.","offset":13001,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"API Calling Rules.","offset":13729,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"3.2 Stable Evaluation System","offset":14676,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"Judging Solvable Tasks.","offset":14920,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"Metrics.","offset":15517,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-1.html#text"},{"title":"4 Experiment","offset":16207,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"4.1 Performance","offset":16224,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"4.2 Stability of Virtual API Server","offset":16791,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"4.3 Turing Test of API Simulator","offset":17683,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"4.4 Diversity of API Simulator","offset":20059,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"4.5 Effectiveness of Caching System","offset":21385,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"4.6 Human Evaluation of Evaluator","offset":22306,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"5 Related Work","offset":23541,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Tool Learning Benchmarks.","offset":23560,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Tool Inference Methods.","offset":24238,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Evaluation in Tool Learning.","offset":24935,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"6 Conclusion","offset":25985,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Acknowledgement","offset":26692,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Limitations","offset":26939,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix A Comparison of Reported and Reproduced Performance","offset":28237,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix B Statistics of API change information","offset":28384,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix C Stability Test Scores with Virtual API Systems","offset":28517,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix D Call Error Identification and Cache Filtering Rule","offset":28804,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix E Configurations of API Diversity Analysis","offset":29930,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix F Prompts of API simulation","offset":30178,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix G Prompt to Filter Solvable Task","offset":30277,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Appendix H Prompt Used to Make API Calls","offset":30379,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"},{"title":"Instructions for reporting errors","offset":30497,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/part-2.html#text"}],"text_code_points":31461,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/ed03ae4fed851402f7c4c5378410fc34374148218bc20e4e7e92522063d06df9/source.md"}