{"id":"arxiv:2604.01375","title":"RIFT: A RubrIc Failure Mode Taxonomy and Automated Diagnostics","date":"2026-04-01","section":"Post-training","quality":-0.088875,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/","source_url":"https://arxiv.org/abs/2604.01375","text_available":true,"authors":["Zhengyang Qi","Charles Dickens","Derek Pham","Amanda Dsouza"],"outgoing":[],"incoming":[],"text_parts":[{"start":0,"end":15884,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-1.html"},{"start":15884,"end":31810,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html"},{"start":31810,"end":47606,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html"},{"start":47606,"end":55827,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-4.html"}],"contents":[{"title":"RIFT: A RubrIc Failure Mode Taxonomy and Automated Diagnostics","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-1.html#text"},{"title":"Abstract","offset":66,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-1.html#text"},{"title":"1 Introduction","offset":1414,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-1.html#text"},{"title":"2 Related Work","offset":5491,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-1.html#text"},{"title":"3 The RubrIc Failure mode Taxonomy","offset":8536,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-1.html#text"},{"title":"4 Automated RIFT Evaluators","offset":14959,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-1.html#text"},{"title":"5 Conclusions","offset":19464,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"6 Limitations","offset":21031,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"7 Ethical Statement","offset":22547,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"Appendix A Data Source and Sampling Provenance","offset":23202,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"A.1 Data source distribution.","offset":23411,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"A.2 Sampling.","offset":23594,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"Appendix B Additional Experiment Details","offset":24373,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"B.1 Alignment and Reward Variance Judge Model Ablation","offset":24822,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"B.2 Additional Analysis for non-LLMaJ RIFT Evaluators","offset":25890,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"Appendix C Prompts","offset":27798,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"C.1 LLM-as-a-Judge Annotation Prompt","offset":27971,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"C.2 Taxonomy Refinement Prompt","offset":29815,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-2.html#text"},{"title":"Taxonomy Philosophy CRITICAL: This taxonomy will be used by human annotators. The primary goal is to create a taxonomy that is: • Compact : Aim for 7–10 total failure modes. Fewer distinct categories is ALWAYS better than many granular ones.","offset":31942,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Guidelines","offset":32804,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Task","offset":34036,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Appendix D RIFT: RubrIc Failure Mode Taxonomy","offset":36963,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"D.1 Reliability Failures","offset":37720,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Subjective.","offset":37916,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Non-Atomic.","offset":39443,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Ungrounded.","offset":40529,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"D.2 Content Validity Failures","offset":42514,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Misaligned or Rigid.","offset":42740,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Missing Criteria.","offset":44336,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"D.3 Consequential Validity Failures","offset":45752,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Hackable.","offset":45963,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-3.html#text"},{"title":"Low Signal.","offset":47888,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-4.html#text"},{"title":"Redundant Criteria.","offset":49490,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-4.html#text"},{"title":"Appendix E Qualitative Analysis of Failure of LLMaJ in Detecting Failure Modes","offset":50970,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-4.html#text"},{"title":"Hackable : rewarding presentation instead of task success.","offset":51524,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-4.html#text"},{"title":"Ungrounded : clear requirement, missing verification procedure.","offset":53292,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-4.html#text"},{"title":"Instructions for reporting errors","offset":54863,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/part-4.html#text"}],"text_code_points":55827,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/543e01a0a067771862a0fab32a2a425ee46776e5e22793659e9272fdaf37a01a/source.md"}