{"id":"arxiv:2405.14862","title":"Bitune: Leveraging Bidirectional Attention to Improve Decoder-Only LLMs","date":"2024-05-23","section":"LLMs: architectures, context, training","quality":null,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/","source_url":"https://arxiv.org/abs/2405.14862","text_available":true,"authors":["Dawid J. Kopiczko","Tijmen Blankevoort","Yuki M. Asano"],"outgoing":[{"source":"arxiv:2405.14862","target":"url:ai.meta.com/research/publications/code-llama-open-foundation-models-for-code","evidence":"id","id":"url:ai.meta.com/research/publications/code-llama-open-foundation-models-for-code","title":"Code Llama: Open Foundation Models for Code","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/0dc2d6030e514e45150f64f511c0f50fc010fdc1ddae0bdca7edcd189b084d41/"}],"incoming":[],"text_parts":[{"start":0,"end":15979,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html"},{"start":15979,"end":31804,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html"},{"start":31804,"end":34630,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-3.html"}],"contents":[{"title":"Bitune: Leveraging Bidirectional Attention to Improve Decoder-Only LLMs","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"Abstract","offset":75,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"1 Introduction","offset":807,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"2 Bidirectional Instruction-Tuning","offset":5035,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"Two Sets of Features.","offset":7044,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"Parameter Efficient Fine-tuning","offset":10350,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"3 Experiments","offset":11193,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"3.1 Instruction-Tuning","offset":11211,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"Results.","offset":15556,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-1.html#text"},{"title":"3.2 Downstream Task Training","offset":16423,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Results.","offset":17319,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"3.3 Chain-of-Thought Reasoning","offset":18147,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Results.","offset":19503,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"3.4 Inference Speed Comparison","offset":19678,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"3.5 Ablations","offset":20235,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Component Removal","offset":20487,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Different PEFT Methods","offset":21965,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Full Finetuning","offset":22737,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Attention Mask of Second Pass","offset":23106,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"4 Related Work","offset":23633,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Bitune vs. Prefix-LM.","offset":25965,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"5 Conclusion","offset":27031,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Limitations","offset":27573,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"Appendix A Appendix","offset":28292,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.1 Pseudocode for Bitune Training Step","offset":28316,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.2 Hyperparameters","offset":28361,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.3 Datasets","offset":28386,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.4 Training Speed & Memory Usage","offset":28516,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.5 Initialization of Mixing Coefficient","offset":29373,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.6 Instruction-tuning on Alpaca Dataset","offset":30282,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.7 Finetuning Larger Model on GSM8K","offset":30904,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.8 Multi-Turn Setting","offset":31383,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-2.html#text"},{"title":"A.9 Chain of Thought Data Generation","offset":32255,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-3.html#text"},{"title":"A.10 Prompt Templates","offset":32635,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-3.html#text"},{"title":"A.11 Results with Standard Deviation","offset":32845,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-3.html#text"},{"title":"A.12 Example Attention Matrices","offset":32969,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-3.html#text"},{"title":"A.13 GSM8K Samples","offset":33462,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-3.html#text"},{"title":"Instructions for reporting errors","offset":33666,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/part-3.html#text"}],"text_code_points":34630,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/314c3b36ea56b4a34d8f177db2dd33f45710b14aa245cb3e779c51ec76e795de/source.md"}