{"id":"arxiv:2504.06225","title":"Encoder-Decoder Gemma: Improving the Quality-Efficiency Trade-Off via Adaptation","date":"2025-04-08","section":"LLMs: architectures, context, training","quality":-0.004879,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/","source_url":"https://arxiv.org/abs/2504.06225","text_available":true,"authors":["Biao Zhang","Fedor Moiseev","Joshua Ainslie","Paul Suganthan"],"outgoing":[{"source":"arxiv:2504.06225","target":"arxiv:2407.21783","evidence":"id","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[{"source":"arxiv:2512.14856","target":"arxiv:2504.06225","evidence":"id","id":"arxiv:2512.14856","title":"T5Gemma 2: Seeing, Reading, and Understanding Longer","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/e982edc00d6b6d9e20d11b62004bb8e91131e2d7f717eca944a9914897f273b6/"}],"text_parts":[{"start":0,"end":15743,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html"},{"start":15743,"end":26741,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html"}],"contents":[{"title":"Encoder-Decoder Gemma: Improving the Quality-Efficiency Trade-Off via Adaptation","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"Abstract","offset":84,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"1 Introduction","offset":1608,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"2 Related Work","offset":5696,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"3 Approach: Encoder-Decoder Adaptation","offset":8147,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"3.1 Architecture","offset":8190,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"3.2 Initialization","offset":10136,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"3.3 Pretraining Objective","offset":10953,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"4 Setup","offset":11869,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"Data Setting","offset":11881,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"Model Setting","offset":12563,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"Evaluation","offset":12887,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"5 Results","offset":14970,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"The encoder-decoder adaptation converges rapidly, particularly for balanced architectures.","offset":14984,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-1.html#text"},{"title":"Pretraining objective matters: UL2 and PrefixLM show different characteristics.","offset":16200,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Encoder-decoder LLMs outperform decoder-only LLMs especially after instruction tuning.","offset":17233,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Encoder-decoder LLMs balance quality and inference efficiency more effectively.","offset":19014,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"6 Discussion","offset":20229,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Is the improvement after the adaptation simply due to the extra pretraining compute?","offset":20246,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Does cross-attention warmup matter for unbalanced encoder-decoder?","offset":20755,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Can we switch from grouped-query attention to multi-head self attention for the encoder?","offset":21173,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Does bidirectional self-attention matter for the encoder?","offset":21815,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Would pretraining encoder-decoder LLMs from scratch yield better performance?","offset":22431,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Is IT/SuperGLUE score predicable from PT score?","offset":23014,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Can we get the best of both worlds from PrefixLM and UL2?","offset":23904,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"7 Conclusion and Future Work","offset":24824,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"},{"title":"Acknowledgements","offset":26424,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/part-2.html#text"}],"text_code_points":26741,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/797af53c1331a8303ff731ef2763d562b29b9864407468ebbd7c1a89ab7c0eed/source.md"}