{"id":"arxiv:2608.05136","title":"The Loss Does Not See the Basis, but Adam Does","date":"2026-08-05","section":"Data, training, optimization","quality":0.347475,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/","source_url":"https://arxiv.org/abs/2608.05136","text_available":true,"authors":["Devender Singh"],"outgoing":[{"source":"arxiv:2608.05136","target":"arxiv:2302.06675","evidence":"title","id":"arxiv:2302.06675","title":"Symbolic Discovery of Optimization Algorithms","url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/3438564652cae2c199823d6d3456a9bf87703e715521d9faaf396c18621f4a5a/"}],"incoming":[],"text_parts":[{"start":0,"end":15859,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html"},{"start":15859,"end":31199,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html"},{"start":31199,"end":46221,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html"},{"start":46221,"end":60903,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html"},{"start":60903,"end":76576,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html"},{"start":76576,"end":91756,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html"},{"start":91756,"end":107230,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html"},{"start":107230,"end":123172,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html"},{"start":123172,"end":138871,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html"},{"start":138871,"end":153649,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html"},{"start":153649,"end":169622,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-11.html"},{"start":169622,"end":171123,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-12.html"}],"contents":[{"title":"The Loss Does Not See the Basis, but Adam Does","offset":0,"level":1,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"Abstract","offset":50,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"1 Introduction","offset":1927,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"Contributions.","offset":5962,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"Scope.","offset":12660,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"2 Related work","offset":13434,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"Implicit bias of gradient methods on factored models.","offset":13453,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"Implicit bias of Adam and adaptive methods.","offset":14717,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-1.html#text"},{"title":"Optimizer design, symmetry, and structured methods.","offset":17002,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Our own prior work.","offset":17376,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"3 Setup: gauge, task, and protocol","offset":18006,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Factored model and gauge transformation.","offset":18045,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Definition 3.1 (Gauge equivariance of an optimizer) .","offset":19218,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Lemma 3.2 (Orthogonal gauge as maximal isometric symmetry) .","offset":21229,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Task: recovery in the interpolation regime.","offset":22202,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Protocol.","offset":24261,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"4 Theory: equivariance as a classifier of update rules","offset":26273,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Lemma 4.1 (Gradient covariance) .","offset":26642,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Proposition 4.2 (Equivariant class) .","offset":27005,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Proposition 4.3 (Nonlinear coordinate-wise maps break the gauge) .","offset":29541,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Proposition 4.4 (Adam has an exact first-step gauge defect) .","offset":31129,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-2.html#text"},{"title":"Theorem 4.5 (Structure: equivariant memoryless rules are Gram-determined left preconditioners) .","offset":33835,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html#text"},{"title":"Theorem 4.6 (Transfer theorem) .","offset":36731,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html#text"},{"title":"Remark 4.7 (Scope of the transfer, and what equivariance does not buy) .","offset":38489,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html#text"},{"title":"5 The optimizer zoo","offset":40080,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html#text"},{"title":"Design.","offset":40104,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html#text"},{"title":"Result.","offset":42671,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html#text"},{"title":"Controls (Appendix D ).","offset":46190,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-3.html#text"},{"title":"6 The same gauge lives in attention heads","offset":47229,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html#text"},{"title":"Attention has the same symmetry.","offset":47275,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html#text"},{"title":"Twin protocol.","offset":49448,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html#text"},{"title":"Results (Table 3 , Figure 2 ).","offset":50919,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html#text"},{"title":"Scale, and real stochastic training.","offset":54244,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html#text"},{"title":"A consequence for model merging.","offset":57270,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html#text"},{"title":"7 Isolating preconditioner anisotropy with a dial","offset":58447,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-4.html#text"},{"title":"8 Inside the equivariant class: the spectral-tail phase diagram","offset":64726,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html#text"},{"title":"Design.","offset":65507,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html#text"},{"title":"The two axes in the sweep.","offset":66537,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html#text"},{"title":"An analytically tractable model.","offset":68052,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html#text"},{"title":"Proposition 8.1 (Greedy vs. equal-rate schedules: a solvable two-timescale scenario) .","offset":68352,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html#text"},{"title":"9 Real data at matched training loss","offset":71202,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html#text"},{"title":"Validation and a second benchmark.","offset":76032,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-5.html#text"},{"title":"10 Repairing our own optimizer using the criterion","offset":78045,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"10.1 Where the bias succeeds and where it does not","offset":82031,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"11 Discussion and limitations","offset":84101,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Scale.","offset":84135,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Two exceptions we left untuned.","offset":85354,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Not a prescription for practice.","offset":87073,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Open theoretical directions.","offset":87792,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Stochasticity.","offset":89216,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Further consequences.","offset":89665,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Reproducibility.","offset":89836,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Acknowledgments","offset":90280,"level":4,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Use of Large Language Models","offset":90552,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Appendix A Inside the equivariant class","offset":90759,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Proposition A.1 (The spectral transfer function: the within-class coordinate) .","offset":91045,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-6.html#text"},{"title":"Remark A.2 (Stateful rules and right preconditioners) .","offset":95669,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Corollary A.3 (Deeper factorizations) .","offset":96900,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Proposition A.4 (Balancedness: a special property of GD-type flows, not a general invariance) .","offset":98248,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Remark A.5 (Comments on discretization and step-size choice) .","offset":99592,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Appendix B Proofs","offset":100527,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"B.1 Lemma 3.2 (maximal isometric gauge)","offset":100549,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Proof.","offset":100594,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"B.2 Lemma 4.1 (gradient covariance)","offset":101320,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Proof.","offset":101361,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"B.3 Proposition 4.2 (equivariant class)","offset":102058,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Proof.","offset":102103,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"B.4 Proposition 4.3 (coordinate-wise rules break the gauge)","offset":107150,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"Proof.","offset":107215,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-7.html#text"},{"title":"B.5 Proposition 4.4 (exact first-step Adam defect)","offset":111906,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Proof.","offset":111962,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"B.6 Theorem 4.5 (structure)","offset":113726,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Proof.","offset":113759,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"B.7 Proposition A.1 (spectral transfer)","offset":115802,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Proof.","offset":115847,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"B.8 Corollary A.3 (deeper factorizations)","offset":117800,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Proof.","offset":117847,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"B.9 Theorem 4.6 (transfer theorem)","offset":118707,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Proof.","offset":118747,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"B.10 Proposition A.4 (balancedness)","offset":120533,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Proof.","offset":120574,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"B.11 Proposition 8.1 (greedy vs. equal-rate phase boundary)","offset":122550,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Proof.","offset":122615,"level":6,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-8.html#text"},{"title":"Appendix C Experimental details","offset":125707,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"Sensing task (§ 5 –§ 7 , § 8 ).","offset":125743,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"Attention task (§ 6 ).","offset":128829,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"Hyperspectral task (§ 9 ).","offset":130516,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"FlowAdam experiments (§ 10 ).","offset":132449,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"Comparing values across tables.","offset":133701,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"Appendix D The control battery","offset":135021,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"D.1 Schedule symmetrization","offset":135540,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"D.2 The learning-rate axis","offset":138087,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-9.html#text"},{"title":"D.3 Learning-rate curves and selection rules","offset":140212,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"D.4 Initialization scale","offset":142261,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"D.5 Attention twins across seeds, initialization draws, and noise levels","offset":142364,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"Scaling the twin diagnostic (GPU, float64 ).","offset":144305,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"D.6 Hyperspectral learning-rate hygiene","offset":146389,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"Two limits of this grid.","offset":147887,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"D.7 Interpolation-bar audits","offset":149991,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"D.8 FlowAdam-p is not an artifact of early stopping","offset":152538,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"D.9 The H100 replication ladder","offset":153612,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-10.html#text"},{"title":"Discussion around the ladder.","offset":156498,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-11.html#text"},{"title":"D.10 Recovery without equivariance: annealed sign descent at long horizons","offset":158130,"level":3,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-11.html#text"},{"title":"Appendix E Extended related work","offset":163868,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-11.html#text"},{"title":"Transformation invariance in optimizer design.","offset":163905,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-11.html#text"},{"title":"Symmetry-driven accounts of implicit bias.","offset":166944,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-11.html#text"},{"title":"Muon and structured optimizers.","offset":168087,"level":5,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-11.html#text"},{"title":"Appendix F Relation to invariant-optimizer design","offset":169622,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-12.html#text"},{"title":"Instructions for reporting errors","offset":170159,"level":2,"url":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/part-12.html#text"}],"text_code_points":171123,"source_markdown":"https://vinnik-dmitry07.github.io/favorite-papers/agents/papers/4b03f7110ecafad60a66ccb0aab5ee9d412451ea522159ec7f58d4b1798b7cfe/source.md"}