[{"data":1,"prerenderedAt":355},["ShallowReactive",2],{"post-verl_0\u002Fmain":3},{"id":4,"title":5,"body":6,"cover":341,"date":342,"description":326,"draft":343,"extension":344,"meta":345,"navigation":346,"path":347,"seo":348,"stem":349,"summary":350,"tags":351,"__hash__":354},"posts\u002Fposts\u002Fverl_0\u002Fmain.md","verl调参指南",{"type":7,"value":8,"toc":325},"minimark",[9,14,18,31,42,48,51,57,66,79,88,97,100,104,114,119,126,135,150,154,161,171,188,194,196,200,207,211,216,232,241,245,251,272,279,283,293],[10,11,13],"h1",{"id":12},"grpo-训练与调参梳理","GRPO 训练与调参梳理",[15,16,17],"h2",{"id":17},"前言",[19,20,21,22,26,27,30],"p",{},"不久前尝试复现了一个 GRPO 训练 Qwen2.5-VL-7B 的工作，当时面对 ",[23,24,25],"code",{},"verl"," 繁多的参数，看个文档也是一知半解，简单粗暴地改小了 ",[23,28,29],{},"train_batch_size"," 试图增加 logging 频率，结果没练几步就爆了。",[19,32,33,38],{},[34,35],"img",{"alt":36,"src":37},"github issue 0","\u002Fimages\u002Fposts\u002Fverl_0\u002Fgrpo_0.png",[34,39],{"alt":40,"src":41},"github issue 1","\u002Fimages\u002Fposts\u002Fverl_0\u002Fgrpo_1.png",[19,43,44,45,47],{},"这次重回 ",[23,46,25],{}," 进行训练，决定先把这些参数背后的逻辑彻底理顺。",[15,49,50],{"id":50},"概念梳理",[19,52,53,54,56],{},"在聊流程之前，我们得先搞清楚 ",[23,55,25],{}," 里这几个容易打架的 Batch Size 概念。这几个参数如果不区分清楚，后面的逻辑很难盘得通：",[19,58,59,60,65],{},"首先是 ",[61,62,63],"strong",{},[23,64,29],{},"，这是宏观层面的一次“采样大部队”。它决定了在一次采样阶段（Rollout），模型总共要处理多少个不同的 Prompt。",[19,67,68,69,74,75,78],{},"其次是 ",[61,70,71],{},[23,72,73],{},"rollout.n","，也就是 Group Size。GRPO 的特点就是对同一个 Prompt 生成多个回复（Group），然后组内比较优势。所以，实际上模型在采样阶段产生的数据总量是 ",[23,76,77],{},"train_batch_size * rollout.n","。",[19,80,81,82,87],{},"有了数据就要训练，这就轮到 ",[61,83,84],{},[23,85,86],{},"ppo_mini_batch_size"," 了。这才是优化器（Optimizer）真正关心的参数——每次更新权重时“吃”多少数据。",[19,89,90,91,96],{},"至于 ",[61,92,93],{},[23,94,95],{},"micro_batch_size","，它纯粹是为了照顾显存的。如果显卡塞不下 mini batch，就切得更碎一点用梯度累加（Gradient Accumulation）来凑，它不影响训练逻辑。",[98,99],"hr",{},[15,101,103],{"id":102},"grpo-的两阶段循环","GRPO 的两阶段循环",[19,105,106,107,110,111,78],{},"GRPO 的训练本质上就是两个阶段的无限循环：",[61,108,109],{},"造数据（Rollout）"," 和 ",[61,112,113],{},"吃数据（Update）",[115,116,118],"h3",{"id":117},"第一阶段造数据-rollout","第一阶段：造数据 (Rollout)",[19,120,121,122,125],{},"这个时候模型是冻结的（",[23,123,124],{},"eval"," 模式），它的任务就是根据当前的策略去“见世面”。",[19,127,128,129,131,132,134],{},"系统会从数据集中捞出 ",[23,130,29],{}," 个 Prompt，然后让模型对着每个 Prompt 生成 ",[23,133,73],{}," 个回答。",[19,136,137,138,141,142,145,146,149],{},"生成完之后，最关键的一步来了：模型会立刻计算这些回答的 ",[23,139,140],{},"old_log_prob","。你可以把这个值看作是一个 ",[61,143,144],{},"“锚点”","。在接下来的训练中，无论模型参数怎么变，这个锚点数值是",[61,147,148],{},"绝对不会变","的。它就像是一个参照系，用来时刻提醒模型：“你现在的策略和刚开始采样时的策略差了多远”。",[115,151,153],{"id":152},"第二阶段吃数据-ppo-update","第二阶段：吃数据 (PPO Update)",[19,155,156,157,160],{},"数据造好了，Reward Model 也打完分了，现在模型切换到训练模式（",[23,158,159],{},"train","），开始参数更新。",[19,162,163,164,167,168,170],{},"通常我们会把这批造好的数据重复利用几次（",[23,165,166],{},"ppo_epochs","），在每一个 Epoch 里，数据被切分成小块（",[23,169,86],{},"）喂给优化器。",[19,172,173,176,177,180,181,184,185,187],{},[61,174,175],{},"这里有个非常重要的细节：","\n我们在 PPO Update 中会进行多次 ",[23,178,179],{},"optimizer.step()","。虽然每一次更新后，模型的参数都变了，新计算出的概率（",[23,182,183],{},"log_prob","）也变了，但我们在第一阶段算好的那个“锚点”（",[23,186,140],{},"）是定死的。",[19,189,190,191,78],{},"这意味着，",[61,192,193],{},"随着更新步数越来越多，现在的模型策略会逐渐偏离采样时的策略",[98,195],{},[15,197,199],{"id":198},"为什么会练崩调参背后的逻辑","为什么会练崩？调参背后的逻辑",[19,201,202,203,206],{},"理解了上面的流程，之前的“崩盘”原因就呼之欲出了。调参的核心，其实就是在平衡**“外循环的广度”",[61,204,205],{},"和","“内循环的深度”**。",[115,208,210],{"id":209},"_1-外循环为什么-train_batch_size-越大越稳","1. 外循环：为什么 train_batch_size 越大越稳？",[19,212,213,215],{},[23,214,29],{}," 决定了模型在改变策略之前，到底看了多少“案例”。",[217,218,219,226],"ul",{},[220,221,222,225],"li",{},[61,223,224],{},"如果 Batch 很小（比如 256）","：\n这就好比一个学生，做了一道题就急着对答案、改思路。这道题可能比较偏，学生改完思路后，再做下一道题发现又不对，于是又改。模型就会陷入这种“做一题改一次”的震荡中，非常容易过拟合当前的局部数据，导致策略剧烈抖动。",[220,227,228,231],{},[61,229,230],{},"如果 Batch 很大（比如 10240）","：\n这就像学生做完了一整套模拟卷（100题），综合了所有题目的得失，才总结出一套改进方案。这种基于大样本量的梯度方向是非常稳健的，因为它看到了数据的“全貌”。",[19,233,234,235],{},"所以结论很简单：",[61,236,237,238,240],{},"只要显存和时间允许，",[23,239,29],{}," 越大越好。",[115,242,244],{"id":243},"_2-内循环ppo_mini_batch_size-的黄金比例","2. 内循环：ppo_mini_batch_size 的黄金比例",[19,246,247,248,250],{},"确定了要采多少数据（Train Batch），接下来就是决定“怎么吃掉这批数据”。这时候 ",[23,249,86],{}," 就很关键了。",[217,252,253,263],{},[220,254,255,258,259,262],{},[61,256,257],{},"吃得太慢（Mini Batch 太小）","：\n这意味着你需要更新很多次参数才能吃完这批数据。就像上面说的，更新次数越多，模型策略偏离“锚点”就越远。\n这就导致到了后面几步，模型现在的想法和采样时的想法已经天差地别了。此时计算出的 Ratio (",[23,260,261],{},"log_prob \u002F old_log_prob",") 会变得极不稳定，PPO 的裁剪机制（Clip）会强制把梯度切零。结果就是：你后面算的这些步数，基本都是无效计算，甚至是有害的。",[220,264,265,268,269,271],{},[61,266,267],{},"吃得太快（Mini Batch 太大）","：\n假如你直接一口吞（Mini Batch = Train Batch），那这一轮 Rollout 辛辛苦苦生成的几万条数据，只换来了一次 ",[23,270,179],{},"。这就太奢侈了，数据利用率极低，训练效率慢得令人发指。",[19,273,274,275,278],{},"所以，这里需要一个**“黄金比例”**。经验法则通常是：",[61,276,277],{},"调节 Mini Batch 的大小，让每一轮 Rollout 的总更新步数保持在 4-8 步左右。"," 既保证了数据被重复利用，又不会让策略偏离太远。",[115,280,282],{"id":281},"_3-实战避坑指南","3. 实战避坑指南",[19,284,285,286,110,289,292],{},"最后总结一下，我们在看 Log 的时候，主要盯着 ",[23,287,288],{},"Clip Ratio",[23,290,291],{},"KL Divergence"," 这两个指标看就行：",[217,294,295,311],{},[220,296,297,303,304,307,308,310],{},[61,298,299,300,302],{},"如果你发现 ",[23,301,288],{}," 特别低 (\u003C 1%)","，而且 Loss 下降得很慢。那说明你太保守了，数据还没被榨干。\n👉 ",[61,305,306],{},"尝试","：减小 ",[23,309,86],{},"（多更几步），或者干脆多跑一个 Epoch。",[220,312,313,318,319,321,322,324],{},[61,314,299,315,317],{},[23,316,288],{}," 飙升 (> 15%)","，或者 KL 散度突然爆炸。那说明你太激进，模型已经“学歪了”。\n👉 ",[61,320,306],{},"：增大 ",[23,323,86],{},"（少更几步），让模型“少吃多餐”变为“多吃少餐”，稳住心态。",{"title":326,"searchDepth":327,"depth":327,"links":328},"",2,[329,330,331,336],{"id":17,"depth":327,"text":17},{"id":50,"depth":327,"text":50},{"id":102,"depth":327,"text":103,"children":332},[333,335],{"id":117,"depth":334,"text":118},3,{"id":152,"depth":334,"text":153},{"id":198,"depth":327,"text":199,"children":337},[338,339,340],{"id":209,"depth":334,"text":210},{"id":243,"depth":334,"text":244},{"id":281,"depth":334,"text":282},"\u002Fimages\u002Fposts\u002Fverl_0\u002Fcover.png","2026-01-08",false,"md",{},true,"\u002Fposts\u002Fverl_0\u002Fmain",{"title":5,"description":326},"posts\u002Fverl_0\u002Fmain","梳理GRPO流程以及参数",[352,353],"notes","后训练","_oKO3pjhfCDakSPQBUJpkFye5oOiGAhgShYOmxmJquI",1782672216594]