{"id":"arxiv:2304.05977","title":"ImageReward: Learning and Evaluating Human Preferences for Text-to-Image Generation","date":"2023-04-12","section":"Self-supervised learning and vision","quality":0.42693,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/","source_url":"https://arxiv.org/abs/2304.05977","text_available":true,"authors":["Jiazheng Xu","Xiao Liu","Yuchen Wu","Yuxuan Tong"],"outgoing":[{"source":"arxiv:2304.05977","target":"url:rlhfbook.com","evidence":"title","id":"url:rlhfbook.com","title":"Reinforcement Learning from Human Feedback","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/20e354559fad09ad4d6338d104c9c3f822a48e7fbf86cd8afa1ddf951b223658/"}],"incoming":[],"text_parts":[{"start":0,"end":15911,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html"},{"start":15911,"end":31543,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html"},{"start":31543,"end":47535,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-3.html"},{"start":47535,"end":63400,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html"},{"start":63400,"end":66707,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-5.html"}],"contents":[{"title":"ImageReward: Learning and Evaluating Human Preferences for Text-to-Image Generation","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"Abstract","offset":87,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"1 Introduction","offset":954,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"2 ImageReward: Learning to Score and Evaluate Human Preferences","offset":5343,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"2.1 Annotation Pipeline Design","offset":5609,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"2.2 RM Training","offset":7495,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"2.3 As Metric: Re-Evaluating Human Preferences on Text-to-Image Models","offset":9429,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"3 ReFL: Reward Feedback Learning Improves Text-to-Image Diffusion","offset":13214,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-1.html#text"},{"title":"4 Experiment","offset":17442,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"4.1 ImageReward: On Human Preference Prediction","offset":17460,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"4.2 ReFL: On Improving Diffusion Models with Human Preference","offset":21378,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"5 Related Work","offset":26015,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"6 Conclusion","offset":28741,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"Acknowledgement","offset":29294,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"Appendix A Details on ImageReward’s Comparison Data Annotation Pipeline","offset":29768,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"A.1 Prompt Selection","offset":29845,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"A.2 Annotation Management","offset":31270,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-2.html#text"},{"title":"A.3 Human Annotation Design","offset":32559,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-3.html#text"},{"title":"A.4 Human Annotation Analysis","offset":36531,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-3.html#text"},{"title":"Appendix B Annotation Document","offset":43363,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-3.html#text"},{"title":"B.1 Annotate the input text","offset":47441,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-3.html#text"},{"title":"B.2 Annotate generated images","offset":48768,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"B.3 Rank generated images","offset":51732,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"B.4 Frequently asked questions","offset":52176,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"Appendix C More Analysis on ImageReward’s Performance and Properties","offset":54002,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"C.1 Average Scores of the Highest/Lowest Ranked Images","offset":54076,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"C.2 Recall/Filter the Best/Worst Image","offset":55241,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"Appendix D Comparison between ImageReward and Other Reward Models","offset":56369,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"D.1 Human Evaluation","offset":56732,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"D.2 Train Set Distribution","offset":57200,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"Appendix E Implementation Details of Related LDM Optimization Methods","offset":57888,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"Appendix F More Results of ReFL","offset":61353,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"F.1 Demonstrations of ReFL","offset":61390,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"F.2 ReFL compares other fine-tuning methods","offset":61491,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"Appendix G Additional Results of ImageReward Compared to Other Typical Image Scorers","offset":61746,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"Appendix H Limitations","offset":61910,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-4.html#text"},{"title":"Appendix I Broader Impact","offset":63804,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-5.html#text"},{"title":"Appendix J Reproducibility","offset":64555,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-5.html#text"},{"title":"Instructions for reporting errors","offset":65743,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/part-5.html#text"}],"text_code_points":66707,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/8244b779cf69cdaae850b1714a9f3d2f3388571e65bc93c56b03cb1f58c0f6dc/source.md"}