{"id":"arxiv:2501.18837","title":"Constitutional Classifiers: Defending against Universal Jailbreaks across Thousands of Hours of Red Teaming","date":"2025-01-31","section":"AI safety and consciousness","quality":0.128536,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/","source_url":"https://arxiv.org/abs/2501.18837","text_available":true,"authors":["Mrinank Sharma","Meg Tong","Jesse Mu","Jerry Wei"],"outgoing":[{"source":"arxiv:2501.18837","target":"arxiv:2402.16822","evidence":"id","id":"arxiv:2402.16822","title":"Rainbow Teaming: Open-Ended Generation of Diverse Adversarial Prompts","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/d4803e891a4b9c929eca007e3c3f003c330d2fc741109975c6ddd7d2c32828fc/"},{"source":"arxiv:2501.18837","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[],"text_parts":[{"start":0,"end":15424,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html"},{"start":15424,"end":31048,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-2.html"},{"start":31048,"end":46394,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html"},{"start":46394,"end":61488,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html"},{"start":61488,"end":76810,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html"},{"start":76810,"end":92412,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html"}],"contents":[{"title":"Constitutional Classifiers: Defending against Universal Jailbreaks across Thousands of Hours of Red Teaming","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"Abstract","offset":111,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"1 Introduction","offset":1266,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"2 Problem Formulation: Robust and Practically Viable Defenses Against Universal Jailbreaks","offset":6953,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"3 Constitutional Classifiers","offset":10293,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"3.1 Specifying a constitution","offset":11460,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"3.2 Generating synthetic data","offset":12462,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"3.3 Automated red teaming","offset":14052,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"3.4 Training classifiers","offset":14673,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-1.html#text"},{"title":"4 Constitutional classifiers defend against universal jailbreaks across thousands of hours of human red teaming","offset":21323,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-2.html#text"},{"title":"4.1 Methods","offset":21540,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-2.html#text"},{"title":"4.2 Results","offset":25529,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-2.html#text"},{"title":"4.3 Comparative assessment of defensive components","offset":32104,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html#text"},{"title":"5 Constitutional classifiers are flexible and practically viable defenses","offset":33889,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html#text"},{"title":"5.1 Methodology","offset":34372,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html#text"},{"title":"5.2 Results","offset":38926,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html#text"},{"title":"6 Related Work","offset":41256,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html#text"},{"title":"7 Conclusion","offset":45007,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html#text"},{"title":"Acknowledgements","offset":46372,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-3.html#text"},{"title":"Author Contributions","offset":47105,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"Appendix","offset":49690,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"Appendix A Frequently-asked questions","offset":49703,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"A.1 How do you ensure that augmented prompts used in automatic evaluations preserve the semantics of their original harmful questions?","offset":49746,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"A.2 How is the inference overhead of classifiers calculated?","offset":50627,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"A.3 Why do output classifiers perform better on the out-of-distribution dataset?","offset":53200,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"A.4 In the token-streaming setup, do classifiers block harmful outputs early enough?","offset":54577,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"A.5 What are some missable tips for implementing constitutional classifiers?","offset":56422,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"Appendix B Example constitution","offset":59465,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"Appendix C Simplified models of uplift","offset":60064,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-4.html#text"},{"title":"Appendix D System details","offset":62842,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"D.1 Bug-bounty system","offset":62873,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"D.2 Automated-evaluations system","offset":63813,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"Appendix E Constitutional classifiers","offset":65451,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.1 Classifier types","offset":65494,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.2 False-positive rates on domain-specific datasets","offset":67389,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.3 Scaling few-shot prompted classifiers","offset":68739,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.4 Initializing classifiers from finetuned models","offset":69965,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.5 Robustness drops without a harmless constitution","offset":71390,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.6 Importance of categories in a harmless constitution","offset":72586,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.7 Prompt wrappers","offset":73991,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"E.8 Refusal rates on chemistry-related user queries","offset":74646,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"Appendix F Automated red teaming","offset":76119,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"F.1 Attack generation","offset":76157,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-5.html#text"},{"title":"F.2 Helpful-only overlap grading","offset":78061,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"F.3 Benign catastrophic-risk queries","offset":78892,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"Appendix G Evaluations","offset":79887,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"G.1 Rubric generation","offset":79915,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"G.2 Grading process","offset":81808,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"Appendix H External red teaming","offset":84505,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"H.1 Recruiting red teamers","offset":84542,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"H.2 Experience levels of red teamers","offset":85405,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"H.3 Jailbreak-reporting process","offset":85992,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"H.4 Estimating hours of red-teaming effort","offset":87177,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"H.5 Outputs that score highly on rubrics are less useful than helpful-only outputs","offset":90151,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"},{"title":"Instructions for reporting errors","offset":91448,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/part-6.html#text"}],"text_code_points":92412,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c73837c3edb5465ee3f7cc7c6e16d23e16bdbdef5e1448aeaad2135dfd87de07/source.md"}