{"id":"arxiv:2302.07253","title":"Energy Transformer","date":"2023-02-14","section":"LLMs: architectures, context, training","quality":-0.291157,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/","source_url":"https://arxiv.org/abs/2302.07253","text_available":true,"authors":["Benjamin Hoover","Yuchen Liang","Bao Pham","Rameswar Panda"],"outgoing":[],"incoming":[],"text_parts":[{"start":0,"end":14613,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html"},{"start":14613,"end":29654,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-2.html"},{"start":29654,"end":44781,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html"},{"start":44781,"end":60608,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-4.html"},{"start":60608,"end":72729,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html"}],"contents":[{"title":"Energy Transformer","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html#text"},{"title":"Abstract","offset":22,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html#text"},{"title":"1 Introduction","offset":1149,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html#text"},{"title":"2 Energy Transformer Block","offset":8167,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html#text"},{"title":"Layer Norm","offset":10908,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html#text"},{"title":"Multi-Head Energy Attention","offset":12053,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html#text"},{"title":"Hopfield Network Module","offset":14584,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-1.html#text"},{"title":"Dynamics of Token Updates","offset":16303,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-2.html#text"},{"title":"Relationship to Modern Hopfield Networks and Conventional Attention","offset":18120,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-2.html#text"},{"title":"3 Qualitative Inspection of the ET framework on ImageNet","offset":21255,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-2.html#text"},{"title":"4 Graph Anomaly Detection","offset":24939,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-2.html#text"},{"title":"4.1 Experimental Evaluation","offset":28178,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-2.html#text"},{"title":"5 Graph Classification with ET","offset":32730,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"5.1 Experimental Evaluation","offset":34320,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"6 Discussion and Conclusions","offset":35706,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"Appendix A Notations Used in the Main Text and Appendices","offset":37802,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"Appendix B Details of Training on ImageNet","offset":37917,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"B.1 Exploring the Hopfield Memories","offset":41747,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"B.2 Positional Bias Correlations","offset":42557,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"Appendix C Details of ET training on Anomaly Detection Task","offset":43232,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"C.1 Detailed Model Structure for the Graph Anomaly Detection","offset":44715,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-3.html#text"},{"title":"C.2 Experimental Details","offset":48881,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-4.html#text"},{"title":"Appendix D Graph Classification with ET","offset":50980,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-4.html#text"},{"title":"D.1 Details of Graph Classification ET Model","offset":51481,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-4.html#text"},{"title":"D.2 Experimental Evaluation","offset":57082,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-4.html#text"},{"title":"D.3 Experimental Details","offset":60578,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-4.html#text"},{"title":"Appendix E ET for heterogeneous Graph","offset":65764,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html#text"},{"title":"Appendix F Ablation Study for Attention and Hopfield Network Modules","offset":66705,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html#text"},{"title":"F.1 On Graphs","offset":67455,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html#text"},{"title":"F.2 On Images","offset":68272,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html#text"},{"title":"Appendix G Parameter comparison","offset":70270,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html#text"},{"title":"Appendix H Formal Algorithm for Training and Inference of ET","offset":71700,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html#text"},{"title":"Instructions for reporting errors","offset":71765,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/part-5.html#text"}],"text_code_points":72729,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4f82adc1eb9a92ea3160324c195688d3834a4c68c026ffc40e951352c572746b/source.md"}