[{"data":1,"prerenderedAt":1398},["ShallowReactive",2],{"post-mix_precision\u002Fmain":3},{"id":4,"title":5,"body":6,"cover":1385,"date":1386,"description":12,"draft":1387,"extension":1388,"meta":1389,"navigation":1390,"path":1391,"seo":1392,"stem":1393,"summary":5,"tags":1394,"__hash__":1397},"posts\u002Fposts\u002Fmix_precision\u002Fmain.md","混合精度训练简介",{"type":7,"value":8,"toc":1377},"minimark",[9,13,17,20,313,316,319,327,330,337,348,351,354,361,364,755,758,909,912,916,1062,1065,1173,1188,1191,1207,1211,1214,1217,1361,1364,1367,1374],[10,11,12],"p",{},"传统的模型训练一般使用FP32精度进行训练。近年来，由于模型越来越大，我们希望在有限的显存中训练更大的模型，也希望能够提高训练速度，因此，人们开始研究如何在混合使用低精度的参数进行模型训练时，保持模型的性能。",[14,15,16],"h2",{"id":16},"浮点数据类型介绍",[10,18,19],{},"要理解混合精度训练，最好先把“浮点数”这件事想清楚。计算机里存一个浮点数，并不是直接把十进制小数塞进去，而是把它拆成三块：符号位决定正负，指数位决定这个数大概落在哪个数量级，尾数位决定这个数量级里还能分得多细。粗略写出来就是：",[21,22,25,127],"span",{"className":23},[24],"katex",[21,26,29],{"className":27},[28],"katex-mathml",[30,31,33],"math",{"xmlns":32},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[34,35,36,122],"semantics",{},[37,38,39,44,47,51,72,75,78,81,83,86,88,90,92,94,96],"mrow",{},[40,41,43],"mo",{"stretchy":42},"false","(",[40,45,46],{},"−",[48,49,50],"mn",{},"1",[52,53,54,57],"msup",{},[40,55,56],{"stretchy":42},")",[37,58,59,63,66,69],{},[60,61,62],"mi",{},"s",[60,64,65],{},"i",[60,67,68],{},"g",[60,70,71],{},"n",[40,73,74],{},"×",[60,76,77],{},"m",[60,79,80],{},"a",[60,82,71],{},[60,84,85],{},"t",[60,87,65],{},[60,89,62],{},[60,91,62],{},[60,93,80],{},[40,95,74],{},[52,97,98,101],{},[48,99,100],{},"2",[37,102,103,106,109,111,114,116,118,120],{},[60,104,105],{},"e",[60,107,108],{},"x",[60,110,10],{},[60,112,113],{},"o",[60,115,71],{},[60,117,105],{},[60,119,71],{},[60,121,85],{},[123,124,126],"annotation",{"encoding":125},"application\u002Fx-tex","(-1)^{sign} \\times mantissa \\times 2^{exponent}\n",[21,128,132,220,253],{"className":129,"ariaHidden":131},[130],"katex-html","true",[21,133,136,141,145,149,152,208,213,217],{"className":134},[135],"base",[21,137],{"className":138,"style":140},[139],"strut","height:1.0747em;vertical-align:-0.25em;",[21,142,43],{"className":143},[144],"mopen",[21,146,46],{"className":147},[148],"mord",[21,150,50],{"className":151},[148],[21,153,156,159],{"className":154},[155],"mclose",[21,157,56],{"className":158},[155],[21,160,163],{"className":161},[162],"msupsub",[21,164,167],{"className":165},[166],"vlist-t",[21,168,171],{"className":169},[170],"vlist-r",[21,172,176],{"className":173,"style":175},[174],"vlist","height:0.8247em;",[21,177,179,184],{"style":178},"top:-3.063em;margin-right:0.05em;",[21,180],{"className":181,"style":183},[182],"pstrut","height:2.7em;",[21,185,191],{"className":186},[187,188,189,190],"sizing","reset-size6","size3","mtight",[21,192,194,198,201,205],{"className":193},[148,190],[21,195,62],{"className":196},[148,197,190],"mathnormal",[21,199,65],{"className":200},[148,197,190],[21,202,68],{"className":203,"style":204},[148,197,190],"margin-right:0.0359em;",[21,206,71],{"className":207},[148,197,190],[21,209],{"className":210,"style":212},[211],"mspace","margin-right:0.2222em;",[21,214,74],{"className":215},[216],"mbin",[21,218],{"className":219,"style":212},[211],[21,221,223,227,231,234,237,241,244,247,250],{"className":222},[135],[21,224],{"className":225,"style":226},[139],"height:0.7429em;vertical-align:-0.0833em;",[21,228,230],{"className":229},[148,197],"man",[21,232,85],{"className":233},[148,197],[21,235,65],{"className":236},[148,197],[21,238,240],{"className":239},[148,197],"ss",[21,242,80],{"className":243},[148,197],[21,245],{"className":246,"style":212},[211],[21,248,74],{"className":249},[216],[21,251],{"className":252,"style":212},[211],[21,254,256,260],{"className":255},[135],[21,257],{"className":258,"style":259},[139],"height:0.7936em;",[21,261,263,266],{"className":262},[148],[21,264,100],{"className":265},[148],[21,267,269],{"className":268},[162],[21,270,272],{"className":271},[166],[21,273,275],{"className":274},[170],[21,276,278],{"className":277,"style":259},[174],[21,279,280,283],{"style":178},[21,281],{"className":282,"style":183},[182],[21,284,286],{"className":285},[187,188,189,190],[21,287,289,292,295,298,301,304,307,310],{"className":288},[148,190],[21,290,105],{"className":291},[148,197,190],[21,293,108],{"className":294},[148,197,190],[21,296,10],{"className":297},[148,197,190],[21,299,113],{"className":300},[148,197,190],[21,302,71],{"className":303},[148,197,190],[21,305,105],{"className":306},[148,197,190],[21,308,71],{"className":309},[148,197,190],[21,311,85],{"className":312},[148,197,190],[10,314,315],{},"所以同样是一个 16 bit 或 32 bit 的数字，指数位多一点，能表示的范围就更大；尾数位多一点，相邻两个可表示数字之间就更密，数值就更精细。混合精度训练里的很多选择，本质上都是在这两个方向之间做取舍：我们到底更怕数值溢出，还是更怕舍入误差。",[10,317,318],{},"FP32 是最传统的训练精度，它用 1 bit 存符号、8 bit 存指数、23 bit 存尾数，一共 4 bytes。它的好处是范围够大、精度也够细，所以不管是前向计算、反向梯度，还是优化器里长期累积的小更新，都比较稳。问题也很直接：每个参数 4 bytes，模型一大，权重、梯度、优化器状态很快就把显存吃完；同时 GPU 做矩阵乘时，低精度 Tensor Core 往往能给出更高吞吐，继续全程 FP32 就显得有点浪费。",[10,320,321,322,326],{},"FP16 看起来像是最自然的压缩版本：它只有 2 bytes，显存直接减半，而且 10 bit 尾数比 BF16 更细。但 FP16 把指数位压到了 5 bit，动态范围小很多；按照 IEEE FP16 的编码规则，它的最大有限值是 65504。训练里一旦激活值、loss scale 或梯度中间值稍微大一点，就容易上溢成 ",[323,324,325],"code",{},"inf","；梯度太小的时候又容易下溢成 0。因此早期 FP16 训练通常要配合 loss scaling，把 loss 先放大再反传，尽量让梯度落在 FP16 能表示的区间里。",[10,328,329],{},"BF16 的设计思路刚好不一样。它同样是 2 bytes，但保留了和 FP32 一样的 8 bit 指数，只把尾数缩到了 7 bit。也就是说，BF16 的动态范围几乎和 FP32 一样，训练中不太容易因为量级问题直接爆掉；代价是同一个数量级里的刻度更粗，精度比 FP16 还低。这个取舍非常适合深度学习里的矩阵乘：神经网络本身对一点舍入噪声比较耐受，而大范围可以显著减少上溢、下溢问题。所以现在做大模型训练或微调时，BF16 往往比 FP16 更省心。",[10,331,332,333,336],{},"这也解释了为什么后文会把 BF16 放在前向和反向传播里，却仍然让优化器保存 FP32 主权重。前向\u002F反向主要是大批量矩阵乘和激活函数，低精度误差通常会被模型和 batch 噪声“吃掉”；优化器更新则是在做长期累加，很多时候是把一个很小的 ",[323,334,335],{},"lr * grad"," 加到一个已经存在的权重上。如果权重只保存在 BF16 里，这个微小更新可能连最小刻度都碰不到，直接被舍入掉。于是计算可以低精度，状态累积仍然要高精度，这就是混合精度训练里最核心的分工。",[10,338,339,340,343,344,347],{},"再往下压就是 FP8。PyTorch 里常见的两个 FP8 类型是 ",[323,341,342],{},"torch.float8_e4m3fn"," 和 ",[323,345,346],{},"torch.float8_e5m2","。前者可以理解成 4 bit 指数、3 bit 尾数，精度稍好但范围更小；后者是 5 bit 指数、2 bit 尾数，范围更大但刻度更粗。实际训练里，FP8 通常不会像 BF16 那样直接替换所有计算，而是要配合 scaling，把张量按块或按通道缩放到合适区间，再交给硬件做低精度矩阵乘。直觉上可以把它看成比 BF16 更激进的带宽和算力优化：收益更大，但对数值缩放、算子支持和训练框架的要求也更高。",[14,349,350],{"id":350},"混合精度训练的完整工作流",[10,352,353],{},"下面以 AdamW + BF16 计算为例。先说明一下，这里讨论的是大模型训练里很常见的一种实现方式：前向、反向使用 BF16 权重和激活来降低显存与带宽压力，优化器内部仍然维护 FP32 主权重和 FP32 动量状态。不同框架的细节会有差异，比如 PyTorch AMP 可能让参数本体保持 FP32，只在算子执行时临时 autocast，但背后的数值逻辑是一样的：计算可以低精度，长期累积的训练状态最好保留高精度。",[10,355,356],{},[357,358],"img",{"alt":359,"src":360},"混合精度训练流程（BF16计算，FP32优化器）","\u002Fimages\u002Fposts\u002Fmix_precision\u002Fmix_precision.png",[10,362,363],{},"如果只看模型状态，不考虑激活值，一个参数在 AdamW + BF16 混合精度全量微调中大致会占：",[21,365,367,462],{"className":366},[24],[21,368,370],{"className":369},[28],[30,371,372],{"xmlns":32},[34,373,374,459],{},[37,375,376,378,382,384,387,389,392,394,396,398,401,403,405,408,410,412,415,417,419,421,423,425,432,434,436,438,440,442,449,451,454,457],{},[48,377,100],{},[379,380,381],"mtext",{},"B",[40,383,43],{"stretchy":42},[379,385,386],{},"BF16 权重",[40,388,56],{"stretchy":42},[40,390,391],{},"+",[48,393,100],{},[379,395,381],{},[40,397,43],{"stretchy":42},[379,399,400],{},"BF16 梯度",[40,402,56],{"stretchy":42},[40,404,391],{},[48,406,407],{},"4",[379,409,381],{},[40,411,43],{"stretchy":42},[379,413,414],{},"FP32 主权重",[40,416,56],{"stretchy":42},[40,418,391],{},[48,420,407],{},[379,422,381],{},[40,424,43],{"stretchy":42},[426,427,428,430],"msub",{},[60,429,77],{},[60,431,85],{},[40,433,56],{"stretchy":42},[40,435,391],{},[48,437,407],{},[379,439,381],{},[40,441,43],{"stretchy":42},[426,443,444,447],{},[60,445,446],{},"v",[60,448,85],{},[40,450,56],{"stretchy":42},[40,452,453],{},"=",[48,455,456],{},"16",[379,458,381],{},[123,460,461],{"encoding":125},"2\\text{B}(\\text{BF16 权重}) + 2\\text{B}(\\text{BF16 梯度}) + 4\\text{B}(\\text{FP32 主权重}) + 4\\text{B}(m_t) + 4\\text{B}(v_t) = 16\\text{B}\n",[21,463,465,509,549,590,666,739],{"className":464,"ariaHidden":131},[130],[21,466,468,472,475,482,485,497,500,503,506],{"className":467},[135],[21,469],{"className":470,"style":471},[139],"height:1em;vertical-align:-0.25em;",[21,473,100],{"className":474},[148],[21,476,479],{"className":477},[148,478],"text",[21,480,381],{"className":481},[148],[21,483,43],{"className":484},[144],[21,486,488,492],{"className":487},[148,478],[21,489,491],{"className":490},[148],"BF16 ",[21,493,496],{"className":494},[148,495],"cjk_fallback","权重",[21,498,56],{"className":499},[155],[21,501],{"className":502,"style":212},[211],[21,504,391],{"className":505},[216],[21,507],{"className":508,"style":212},[211],[21,510,512,515,518,524,527,537,540,543,546],{"className":511},[135],[21,513],{"className":514,"style":471},[139],[21,516,100],{"className":517},[148],[21,519,521],{"className":520},[148,478],[21,522,381],{"className":523},[148],[21,525,43],{"className":526},[144],[21,528,530,533],{"className":529},[148,478],[21,531,491],{"className":532},[148],[21,534,536],{"className":535},[148,495],"梯度",[21,538,56],{"className":539},[155],[21,541],{"className":542,"style":212},[211],[21,544,391],{"className":545},[216],[21,547],{"className":548,"style":212},[211],[21,550,552,555,558,564,567,578,581,584,587],{"className":551},[135],[21,553],{"className":554,"style":471},[139],[21,556,407],{"className":557},[148],[21,559,561],{"className":560},[148,478],[21,562,381],{"className":563},[148],[21,565,43],{"className":566},[144],[21,568,570,574],{"className":569},[148,478],[21,571,573],{"className":572},[148],"FP32 ",[21,575,577],{"className":576},[148,495],"主权重",[21,579,56],{"className":580},[155],[21,582],{"className":583,"style":212},[211],[21,585,391],{"className":586},[216],[21,588],{"className":589,"style":212},[211],[21,591,593,596,599,605,608,654,657,660,663],{"className":592},[135],[21,594],{"className":595,"style":471},[139],[21,597,407],{"className":598},[148],[21,600,602],{"className":601},[148,478],[21,603,381],{"className":604},[148],[21,606,43],{"className":607},[144],[21,609,611,614],{"className":610},[148],[21,612,77],{"className":613},[148,197],[21,615,617],{"className":616},[162],[21,618,621,645],{"className":619},[166,620],"vlist-t2",[21,622,624,640],{"className":623},[170],[21,625,628],{"className":626,"style":627},[174],"height:0.2806em;",[21,629,631,634],{"style":630},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[21,632],{"className":633,"style":183},[182],[21,635,637],{"className":636},[187,188,189,190],[21,638,85],{"className":639},[148,197,190],[21,641,644],{"className":642},[643],"vlist-s","​",[21,646,648],{"className":647},[170],[21,649,652],{"className":650,"style":651},[174],"height:0.15em;",[21,653],{},[21,655,56],{"className":656},[155],[21,658],{"className":659,"style":212},[211],[21,661,391],{"className":662},[216],[21,664],{"className":665,"style":212},[211],[21,667,669,672,675,681,684,725,728,732,736],{"className":668},[135],[21,670],{"className":671,"style":471},[139],[21,673,407],{"className":674},[148],[21,676,678],{"className":677},[148,478],[21,679,381],{"className":680},[148],[21,682,43],{"className":683},[144],[21,685,687,690],{"className":686},[148],[21,688,446],{"className":689,"style":204},[148,197],[21,691,693],{"className":692},[162],[21,694,696,717],{"className":695},[166,620],[21,697,699,714],{"className":698},[170],[21,700,702],{"className":701,"style":627},[174],[21,703,705,708],{"style":704},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[21,706],{"className":707,"style":183},[182],[21,709,711],{"className":710},[187,188,189,190],[21,712,85],{"className":713},[148,197,190],[21,715,644],{"className":716},[643],[21,718,720],{"className":719},[170],[21,721,723],{"className":722,"style":651},[174],[21,724],{},[21,726,56],{"className":727},[155],[21,729],{"className":730,"style":731},[211],"margin-right:0.2778em;",[21,733,453],{"className":734},[735],"mrel",[21,737],{"className":738,"style":731},[211],[21,740,742,746,749],{"className":741},[135],[21,743],{"className":744,"style":745},[139],"height:0.6833em;",[21,747,456],{"className":748},[148],[21,750,752],{"className":751},[148,478],[21,753,381],{"className":754},[148],[10,756,757],{},"所以一个常用的粗估是：",[21,759,761,799],{"className":760},[24],[21,762,764],{"className":763},[28],[30,765,766],{"xmlns":32},[34,767,768,796],{},[37,769,770,773,776,779,781,783,786,788,791,793],{},[379,771,772],{},"显存",[40,774,775],{},"≈",[379,777,778],{},"参数量",[40,780,74],{},[48,782,456],{},[379,784,785],{}," bytes",[40,787,391],{},[379,789,790],{},"激活值",[40,792,391],{},[379,794,795],{},"临时 buffer",[123,797,798],{"encoding":125},"\\text{显存} \\approx \\text{参数量} \\times 16\\text{ bytes} + \\text{激活值} + \\text{临时 buffer}\n",[21,800,802,823,845,870,891],{"className":801,"ariaHidden":131},[130],[21,803,805,808,814,817,820],{"className":804},[135],[21,806],{"className":807,"style":745},[139],[21,809,811],{"className":810},[148,478],[21,812,772],{"className":813},[148,495],[21,815],{"className":816,"style":731},[211],[21,818,775],{"className":819},[735],[21,821],{"className":822,"style":731},[211],[21,824,826,830,836,839,842],{"className":825},[135],[21,827],{"className":828,"style":829},[139],"height:0.7667em;vertical-align:-0.0833em;",[21,831,833],{"className":832},[148,478],[21,834,778],{"className":835},[148,495],[21,837],{"className":838,"style":212},[211],[21,840,74],{"className":841},[216],[21,843],{"className":844,"style":212},[211],[21,846,848,852,855,861,864,867],{"className":847},[135],[21,849],{"className":850,"style":851},[139],"height:0.8889em;vertical-align:-0.1944em;",[21,853,456],{"className":854},[148],[21,856,858],{"className":857},[148,478],[21,859,785],{"className":860},[148],[21,862],{"className":863,"style":212},[211],[21,865,391],{"className":866},[216],[21,868],{"className":869,"style":212},[211],[21,871,873,876,882,885,888],{"className":872},[135],[21,874],{"className":875,"style":829},[139],[21,877,879],{"className":878},[148,478],[21,880,790],{"className":881},[148,495],[21,883],{"className":884,"style":212},[211],[21,886,391],{"className":887},[216],[21,889],{"className":890,"style":212},[211],[21,892,894,898],{"className":893},[135],[21,895],{"className":896,"style":897},[139],"height:0.6944em;",[21,899,901,905],{"className":900},[148,478],[21,902,904],{"className":903},[148,495],"临时",[21,906,908],{"className":907},[148]," buffer",[10,910,911],{},"实际估算时我一般会把模型状态按 16 到 18 bytes\u002Fparam 留余量，因为不同实现里还会有参数对齐、通信 bucket、梯度临时副本、fused optimizer workspace 等额外开销。真正容易被低估的往往是激活值，它和 sequence length、micro batch size、是否开启 activation checkpointing 强相关；有时候模型状态算得很准，最后还是爆在激活值上。",[14,913,915],{"id":914},"为什么优化器必须保存-fp32-主权重","为什么优化器必须保存 FP32 主权重？",[10,917,918,919,990,991,1061],{},"最核心的原因是权重更新通常非常小，而 BF16 在某个数值附近能分辨的刻度又比较粗。BF16 只有 7 bit 显式尾数，算上隐含的最高位，大约可以理解成 8 bit 有效精度；在 1.0 附近，相邻两个 BF16 数之间的间隔约为 ",[21,920,922,945],{"className":921},[24],[21,923,925],{"className":924},[28],[30,926,927],{"xmlns":32},[34,928,929,942],{},[37,930,931],{},[52,932,933,935],{},[48,934,100],{},[37,936,937,939],{},[40,938,46],{},[48,940,941],{},"7",[123,943,944],{"encoding":125},"2^{-7}",[21,946,948],{"className":947,"ariaHidden":131},[130],[21,949,951,955],{"className":950},[135],[21,952],{"className":953,"style":954},[139],"height:0.8141em;",[21,956,958,961],{"className":957},[148],[21,959,100],{"className":960},[148],[21,962,964],{"className":963},[162],[21,965,967],{"className":966},[166],[21,968,970],{"className":969},[170],[21,971,973],{"className":972,"style":954},[174],[21,974,975,978],{"style":178},[21,976],{"className":977,"style":183},[182],[21,979,981],{"className":980},[187,188,189,190],[21,982,984,987],{"className":983},[148,190],[21,985,46],{"className":986},[148,190],[21,988,941],{"className":989},[148,190],"，也就是 0.0078125。FP32 的尾数细得多，在 1.0 附近的间隔约为 ",[21,992,994,1017],{"className":993},[24],[21,995,997],{"className":996},[28],[30,998,999],{"xmlns":32},[34,1000,1001,1014],{},[37,1002,1003],{},[52,1004,1005,1007],{},[48,1006,100],{},[37,1008,1009,1011],{},[40,1010,46],{},[48,1012,1013],{},"23",[123,1015,1016],{"encoding":125},"2^{-23}",[21,1018,1020],{"className":1019,"ariaHidden":131},[130],[21,1021,1023,1026],{"className":1022},[135],[21,1024],{"className":1025,"style":954},[139],[21,1027,1029,1032],{"className":1028},[148],[21,1030,100],{"className":1031},[148],[21,1033,1035],{"className":1034},[162],[21,1036,1038],{"className":1037},[166],[21,1039,1041],{"className":1040},[170],[21,1042,1044],{"className":1043,"style":954},[174],[21,1045,1046,1049],{"style":178},[21,1047],{"className":1048,"style":183},[182],[21,1050,1052],{"className":1051},[187,188,189,190],[21,1053,1055,1058],{"className":1054},[148,190],[21,1056,46],{"className":1057},[148,190],[21,1059,1013],{"className":1060},[148,190],"，也就是 1.19e-7。",[10,1063,1064],{},"训练里一次参数更新大概长这样：",[21,1066,1068,1106],{"className":1067},[24],[21,1069,1071],{"className":1070},[28],[30,1072,1073],{"xmlns":32},[34,1074,1075,1103],{},[37,1076,1077,1081,1084,1086,1089,1092,1094,1096,1098,1100],{},[60,1078,1080],{"mathvariant":1079},"normal","Δ",[60,1082,1083],{},"w",[40,1085,453],{},[60,1087,1088],{},"l",[60,1090,1091],{},"r",[40,1093,74],{},[60,1095,68],{},[60,1097,1091],{},[60,1099,80],{},[60,1101,1102],{},"d",[123,1104,1105],{"encoding":125},"\\Delta w = lr \\times grad\n",[21,1107,1109,1131,1155],{"className":1108,"ariaHidden":131},[130],[21,1110,1112,1115,1118,1122,1125,1128],{"className":1111},[135],[21,1113],{"className":1114,"style":745},[139],[21,1116,1080],{"className":1117},[148],[21,1119,1083],{"className":1120,"style":1121},[148,197],"margin-right:0.0269em;",[21,1123],{"className":1124,"style":731},[211],[21,1126,453],{"className":1127},[735],[21,1129],{"className":1130,"style":731},[211],[21,1132,1134,1138,1142,1146,1149,1152],{"className":1133},[135],[21,1135],{"className":1136,"style":1137},[139],"height:0.7778em;vertical-align:-0.0833em;",[21,1139,1088],{"className":1140,"style":1141},[148,197],"margin-right:0.0197em;",[21,1143,1091],{"className":1144,"style":1145},[148,197],"margin-right:0.0278em;",[21,1147],{"className":1148,"style":212},[211],[21,1150,74],{"className":1151},[216],[21,1153],{"className":1154,"style":212},[211],[21,1156,1158,1161,1164,1167,1170],{"className":1157},[135],[21,1159],{"className":1160,"style":851},[139],[21,1162,68],{"className":1163,"style":204},[148,197],[21,1165,1091],{"className":1166,"style":1145},[148,197],[21,1168,80],{"className":1169},[148,197],[21,1171,1102],{"className":1172},[148,197],[10,1174,1175,1176,1179,1180,1183,1184,1187],{},"如果学习率是 1e-5，梯度量级又差不多是 1，那么这一步更新就是 1e-5。对于 FP32 来说，这个变化明显大于 1.0 附近的最小刻度，所以权重会真实发生变化；虽然十进制的 ",[323,1177,1178],{},"1.00001"," 本身不一定能被二进制浮点数精确表示，但这个更新不会被整个抹掉。对于 BF16 来说就不同了，1e-5 远小于 0.0078125，",[323,1181,1182],{},"1.0 + 0.00001"," 四舍五入之后仍然很可能回到 ",[323,1185,1186],{},"1.0","。",[10,1189,1190],{},"这就是所谓的 swamping problem：大数加小数时，小数被低精度格式的刻度吞掉。它麻烦的地方不在于某一步误差稍微大一点，而在于训练本来就是几千步、几万步的小更新累积。如果每一步都在 BF16 权重上直接更新，大量微小变化会反复消失，模型看起来在反传，实际上参数没有按预期移动。",[10,1192,1193,1194,343,1197,1200,1201,1203,1204,1206],{},"AdamW 里的 ",[323,1195,1196],{},"m_t",[323,1198,1199],{},"v_t"," 也有类似问题。它们是梯度的一阶、二阶指数滑动平均，本质上也是长期状态。如果这些状态本身就用低精度保存，早期的细小变化和后期的微调信号都会更容易被量化误差污染。所以常见做法是：计算权重可以是 BF16，优化器读到梯度后，仍然在 FP32 主权重、FP32 ",[323,1202,1196],{}," 和 FP32 ",[323,1205,1199],{}," 上完成更新，再把新的权重同步回低精度计算副本。",[14,1208,1210],{"id":1209},"为什么前向反向传播可以用-bf16","为什么前向\u002F反向传播可以用 BF16？",[10,1212,1213],{},"看起来这好像有点矛盾：既然优化器更新这么怕 BF16，为什么前向和反向又能用 BF16？关键在于，这两类计算对误差的敏感方式不一样。",[10,1215,1216],{},"前向传播主要是一层层矩阵乘、归一化、激活函数和残差连接：",[21,1218,1220,1272],{"className":1219},[24],[21,1221,1223],{"className":1222},[28],[30,1224,1225],{"xmlns":32},[34,1226,1227,1269],{},[37,1228,1229,1232,1234,1236,1239,1241,1243,1245,1247,1249,1251,1253,1255,1257,1260,1262,1264,1267],{},[60,1230,1231],{},"y",[40,1233,453],{},[60,1235,80],{},[60,1237,1238],{},"c",[60,1240,85],{},[60,1242,65],{},[60,1244,446],{},[60,1246,80],{},[60,1248,85],{},[60,1250,65],{},[60,1252,113],{},[60,1254,71],{},[40,1256,43],{"stretchy":42},[60,1258,1259],{},"W",[60,1261,108],{},[40,1263,391],{},[60,1265,1266],{},"b",[40,1268,56],{"stretchy":42},[123,1270,1271],{"encoding":125},"y = activation(Wx + b)\n",[21,1273,1275,1294,1349],{"className":1274,"ariaHidden":131},[130],[21,1276,1278,1282,1285,1288,1291],{"className":1277},[135],[21,1279],{"className":1280,"style":1281},[139],"height:0.625em;vertical-align:-0.1944em;",[21,1283,1231],{"className":1284,"style":204},[148,197],[21,1286],{"className":1287,"style":731},[211],[21,1289,453],{"className":1290},[735],[21,1292],{"className":1293,"style":731},[211],[21,1295,1297,1300,1303,1306,1309,1312,1315,1318,1321,1324,1327,1330,1333,1337,1340,1343,1346],{"className":1296},[135],[21,1298],{"className":1299,"style":471},[139],[21,1301,80],{"className":1302},[148,197],[21,1304,1238],{"className":1305},[148,197],[21,1307,85],{"className":1308},[148,197],[21,1310,65],{"className":1311},[148,197],[21,1313,446],{"className":1314,"style":204},[148,197],[21,1316,80],{"className":1317},[148,197],[21,1319,85],{"className":1320},[148,197],[21,1322,65],{"className":1323},[148,197],[21,1325,113],{"className":1326},[148,197],[21,1328,71],{"className":1329},[148,197],[21,1331,43],{"className":1332},[144],[21,1334,1259],{"className":1335,"style":1336},[148,197],"margin-right:0.1389em;",[21,1338,108],{"className":1339},[148,197],[21,1341],{"className":1342,"style":212},[211],[21,1344,391],{"className":1345},[216],[21,1347],{"className":1348,"style":212},[211],[21,1350,1352,1355,1358],{"className":1351},[135],[21,1353],{"className":1354,"style":471},[139],[21,1356,1266],{"className":1357},[148,197],[21,1359,56],{"className":1360},[155],[10,1362,1363],{},"BF16 和 FP32 一样有 8 bit exponent，动态范围接近 FP32，因此相比 FP16 更不容易因为量级问题上溢或下溢。这里不能说“不会溢出”，极端输入、异常 loss、错误的初始化照样会把数值打爆；更准确的说法是，BF16 把 FP16 最头疼的动态范围问题缓和了很多。尾数少带来的舍入误差依然存在，但神经网络训练本身就带有 mini-batch 采样噪声、dropout、数据增强等随机性，很多局部舍入误差不会像优化器状态那样长期原封不动地累积。",[10,1365,1366],{},"反向传播也类似。梯度当然不是越粗越好，但它本身就是从一个 mini-batch 估计出来的随机量，我们通常更关心整体方向和统计趋势，而不是每个元素最后几位小数完全准确。更重要的是，现代 GPU 上很多低精度矩阵乘并不是“从头到尾都 BF16”：常见路径是 BF16 输入，内部用更高精度做累加，再输出 BF16 或 FP32 结果。框架也经常会让 softmax、LayerNorm、loss 计算这类敏感算子保留更高精度。",[10,1368,1369,1370,1373],{},"所以这里不能简单概括成“乘法可以低精度，累加必须高精度”。矩阵乘本身就包含大量累加，只是这些累加通常发生在受控的算子内部，并且硬件和框架会用更稳的 accumulate 精度处理。优化器更新的问题更特殊：它是在同一个持久化权重上反复加一个很小的增量，",[323,1371,1372],{},"w = w + lr * grad"," 这种“大数吃小数”的场景正好是低尾数精度最容易翻车的地方。",[10,1375,1376],{},"因此混合精度训练的直觉可以总结成一句话：把吞吐量最大的前向\u002F反向计算交给 BF16，把最怕长期量化误差的优化器状态留在 FP32。这样既吃到低精度硬件的速度和显存收益，又尽量不破坏训练过程中真正需要细粒度累积的部分。",{"title":1378,"searchDepth":1379,"depth":1379,"links":1380},"",2,[1381,1382,1383,1384],{"id":16,"depth":1379,"text":16},{"id":350,"depth":1379,"text":350},{"id":914,"depth":1379,"text":915},{"id":1209,"depth":1379,"text":1210},"\u002Fimages\u002Fposts\u002Fmix_precision\u002Fcover.png","2026-04-16",false,"md",{},true,"\u002Fposts\u002Fmix_precision\u002Fmain",{"title":5,"description":12},"posts\u002Fmix_precision\u002Fmain",[1395,1396],"notes","后训练","S-4aoM7SUoMsqd7d8Gp3rk4W3X4J9vb1xRMS7Nnztfw",1782672216594]