{"id":"arxiv:2510.05491","title":"NorMuon: Making Muon more efficient and scalable","date":"2025-10-07","section":"Data, training, optimization","quality":0.070672,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/","source_url":"https://arxiv.org/abs/2510.05491","text_available":true,"authors":["Zichong Li","Liming Liu","Chen Liang","Weizhu Chen"],"outgoing":[{"source":"arxiv:2510.05491","target":"arxiv:2001.08361","evidence":"id","id":"arxiv:2001.08361","title":"Scaling Laws for Neural Language Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/82302458a6d03bc051b2e4fdb8277893a81d4bd6a029969045df6a8380904eac/"},{"source":"arxiv:2510.05491","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"},{"source":"arxiv:2510.05491","target":"arxiv:2407.21783","evidence":"title","id":"arxiv:2407.21783","title":"The Llama 3 Herd of Models","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/de3cebb4c28bc5d41a843279f219dca14476284991ea5c7c0aeab7068a40e306/"}],"incoming":[],"text_parts":[{"start":0,"end":15963,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html"},{"start":15963,"end":31551,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html"},{"start":31551,"end":33357,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-3.html"}],"contents":[{"title":"NorMuon: Making Muon more efficient and scalable","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"Abstract","offset":52,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"1 Introduction","offset":1901,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"2 Related Works and Background","offset":8661,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"2.1 Related Works","offset":8696,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"2.2 Background: Muon optimizer","offset":12015,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"3 Method","offset":14101,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"3.1 NorMuon","offset":14387,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-1.html#text"},{"title":"3.2 Distributed NorMuon","offset":16694,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"3.3 Overhead Analysis","offset":18764,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"4 Experiments","offset":20234,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"4.1 Experiments on 1.1B and 5.4B Models","offset":20928,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"4.1.1 Setup","offset":20973,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"4.1.2 Main results","offset":23208,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"4.1.3 Ablation studies.","offset":23874,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"4.1.4 Computational and Memory Overhead Analysis","offset":25654,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"4.2 Experiments on Modded-NanoGPT","offset":26838,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"5 Conclusion","offset":27999,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"Acknowledgments","offset":28662,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"Appendix A Additional Experiments","offset":28785,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"A.1 Adam-mini’s results on optimization geometry analysis","offset":28823,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"A.2 Ablation Experiments on Modded-NanoGPT","offset":28886,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"Appendix B Implementation Details","offset":30175,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"Appendix C Experiment Setup of Modded-NanoGPT","offset":30633,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-2.html#text"},{"title":"Instructions for reporting errors","offset":32393,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/part-3.html#text"}],"text_code_points":33357,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/c9ea441f9c2808077269690a88c1b1407189ac1cb03fbc0df89979a4d1d41b98/source.md"}