{"id":"arxiv:2402.16822","title":"Rainbow Teaming: Open-Ended Generation of Diverse Adversarial Prompts","date":"2024-02-26","section":"Retrieval, embeddings, benchmarks","quality":0.383425,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/","source_url":"https://arxiv.org/abs/2402.16822","text_available":true,"authors":["Mikayel Samvelyan","Sharath Chandra Raparthy","Andrei Lupu","Eric Hambro"],"outgoing":[{"source":"arxiv:2402.16822","target":"arxiv:2309.16797","evidence":"title","id":"arxiv:2309.16797","title":"Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/daac3bcfcd7f39084d995905e9ef96ae01c5aab7bfe8e565bd7622b4e44088f7/"},{"source":"arxiv:2402.16822","target":"arxiv:2406.04268","evidence":"title","id":"arxiv:2406.04268","title":"Open-Endedness is Essential for Artificial Superhuman Intelligence","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/13964b2db9a9f793d1fcae07ca83fbe09f65454214269cac56777cb8be423a4e/"},{"source":"arxiv:2402.16822","target":"url:ai.meta.com/research/publications/code-llama-open-foundation-models-for-code","evidence":"title","id":"url:ai.meta.com/research/publications/code-llama-open-foundation-models-for-code","title":"Code Llama: Open Foundation Models for Code","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0dc2d6030e514e45150f64f511c0f50fc010fdc1ddae0bdca7edcd189b084d41/"}],"incoming":[{"source":"arxiv:2406.04268","target":"arxiv:2402.16822","evidence":"title","id":"arxiv:2406.04268","title":"Open-Endedness is Essential for Artificial Superhuman Intelligence","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/13964b2db9a9f793d1fcae07ca83fbe09f65454214269cac56777cb8be423a4e/"},{"source":"arxiv:2407.21783","target":"arxiv:2402.16822","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"},{"source":"arxiv:2501.18837","target":"arxiv:2402.16822","evidence":"id","id":"arxiv:2501.18837","title":"Constitutional Classifiers: Defending against Universal Jailbreaks across Thousands of Hours of Red Teaming","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/"},{"source":"arxiv:2505.22954","target":"arxiv:2402.16822","evidence":"title","id":"arxiv:2505.22954","title":"Darwin Godel Machine: Open-Ended Evolution of Self-Improving Agents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c29f716871f80314868b5647692de387f26e12166b93bd362915fd272b6e081c/"},{"source":"arxiv:2507.19457","target":"arxiv:2402.16822","evidence":"title","id":"arxiv:2507.19457","title":"GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c1ef72d96fa3292590a6a3d112ab509840081b59aaa771cf42b8b0e8ef62ba08/"},{"source":"arxiv:2603.19461","target":"arxiv:2402.16822","evidence":"title","id":"arxiv:2603.19461","title":"Hyperagents","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/17eec8a15debc212b5530f117dd8ba8bad339590729cbbc4f75a47755ebcb544/"},{"source":"arxiv:2608.23875","target":"arxiv:2402.16822","evidence":"title","id":"arxiv:2608.23875","title":"AI Finds A Way","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/afe34dd5ee58c2e9739f9651349c4c9180d95dabff51994a13339f48dfd0bb27/"}],"text_parts":[{"start":0,"end":15461,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html"},{"start":15461,"end":31088,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html"},{"start":31088,"end":47078,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html"},{"start":47078,"end":53896,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html"}],"contents":[{"title":"R a i n b o w T e a m i n g : Open-Ended Generation of Diverse Adversarial Prompts","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"Abstract","offset":86,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"1 Introduction","offset":1500,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"2 Background","offset":7389,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"3 Rainbow Teaming","offset":9396,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"3.1 Prompt Features","offset":12638,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"3.2 Mutation Operator","offset":13366,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"3.3 Preference Model","offset":14815,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-1.html#text"},{"title":"4 Rainbow Teaming for Safety","offset":16275,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html#text"},{"title":"4.1 Results","offset":19634,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html#text"},{"title":"5 Enhancing Robustness with Synthetic Data","offset":24874,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html#text"},{"title":"6 Rainbow Teaming for Other Applications","offset":28180,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html#text"},{"title":"6.1 Question Answering","offset":28225,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html#text"},{"title":"6.2 Cybersecurity","offset":29627,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html#text"},{"title":"7 Related Work","offset":31069,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-2.html#text"},{"title":"8 Conclusion","offset":34239,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"Acknowledgements","offset":35580,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"Appendix A Limitations and Broader Impact","offset":36159,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"Appendix B Algorithm Pseudocode","offset":37755,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"B.1 MAP-Elites","offset":37791,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"B.2 Rainbow Teaming Pseudocode","offset":37919,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"Appendix C Extended Related Work","offset":38567,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"C.1 Token-Level Attacks","offset":38604,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"C.2 Adversarial Training","offset":39334,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"Appendix D Adversarial Prompts as Stepping Stones","offset":41100,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"Appendix E Additional Results","offset":41312,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"E.1 Varying Model Sizes","offset":41346,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"E.2 Role of System Prompts","offset":41705,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"E.3 Human Evaluation","offset":43098,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"E.4 Preference Model Ablation","offset":44062,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"E.5 Full Evaluations","offset":46411,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"E.6 Archive Visualisation","offset":47047,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-3.html#text"},{"title":"E.7 Question Answering Examples","offset":47324,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"Appendix F Additional Details for Preference Models","offset":47460,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"F.1 Question Answering","offset":47516,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"F.2 Cybersecurity","offset":48505,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"Appendix G Inference Cost Analysis","offset":49167,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"Appendix H Feature Descriptors","offset":50723,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"H.1 Safety","offset":50758,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"H.2 Question Answering","offset":51220,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"H.3 Cybersecurity","offset":51644,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"Appendix I Safety Experiment Prompts","offset":51964,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"I.1 Llama Guard Evaluation Prompt","offset":52086,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"I.2 Mutation Prompts","offset":52323,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"I.3 System Prompt of Target LLM","offset":52415,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"Appendix J Question Answering Experiment Prompts","offset":52643,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"Appendix K Hyperparameters","offset":52771,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"},{"title":"Instructions for reporting errors","offset":52932,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/part-4.html#text"}],"text_code_points":53896,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/source.md"}