| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.9734513274336285, | |
| "eval_steps": 500, | |
| "global_step": 169, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 186.12500381469727, | |
| "epoch": 0.017699115044247787, | |
| "grad_norm": 2.5752858647583037, | |
| "kl": 0.0, | |
| "learning_rate": 8.333333333333333e-08, | |
| "loss": 0.0, | |
| "reward": 0.48444899916648865, | |
| "reward_std": 0.46400773525238037, | |
| "rewards/correct_code_reward_func": 0.3125, | |
| "rewards/len_reward_func": 0.17194896191358566, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 164.375, | |
| "epoch": 0.035398230088495575, | |
| "grad_norm": 10.223894549317365, | |
| "kl": 0.0, | |
| "learning_rate": 1.6666666666666665e-07, | |
| "loss": 0.0, | |
| "reward": 0.47450903058052063, | |
| "reward_std": 0.40167468786239624, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.22450900077819824, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 164.4791717529297, | |
| "epoch": 0.05309734513274336, | |
| "grad_norm": 2.2899995314753827, | |
| "kl": 4.023313522338867e-05, | |
| "learning_rate": 2.5e-07, | |
| "loss": 0.0, | |
| "reward": 0.37769100069999695, | |
| "reward_std": 0.5044284015893936, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.16935766488313675, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 190.12500762939453, | |
| "epoch": 0.07079646017699115, | |
| "grad_norm": 6.009103649538728, | |
| "kl": 8.815526962280273e-05, | |
| "learning_rate": 3.333333333333333e-07, | |
| "loss": 0.0, | |
| "reward": 0.37105801701545715, | |
| "reward_std": 0.47604119777679443, | |
| "rewards/correct_code_reward_func": 0.2083333395421505, | |
| "rewards/len_reward_func": 0.16272468864917755, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 126.10416793823242, | |
| "epoch": 0.08849557522123894, | |
| "grad_norm": 2.3931977443517964, | |
| "kl": 3.49879264831543e-05, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "loss": 0.0, | |
| "reward": 0.5486156344413757, | |
| "reward_std": 0.40741610527038574, | |
| "rewards/correct_code_reward_func": 0.3958333358168602, | |
| "rewards/len_reward_func": 0.15278229862451553, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 166.45833587646484, | |
| "epoch": 0.10619469026548672, | |
| "grad_norm": 1.5486991749328145, | |
| "kl": 5.221366882324219e-05, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0, | |
| "reward": 0.5515251755714417, | |
| "reward_std": 0.47483181953430176, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.25985850393772125, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 161.58333587646484, | |
| "epoch": 0.12389380530973451, | |
| "grad_norm": 5.714527026195051, | |
| "kl": 5.5789947509765625e-05, | |
| "learning_rate": 4.999535676028337e-07, | |
| "loss": 0.0, | |
| "reward": 0.46977874636650085, | |
| "reward_std": 0.461082324385643, | |
| "rewards/correct_code_reward_func": 0.229166679084301, | |
| "rewards/len_reward_func": 0.24061208218336105, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 161.89583587646484, | |
| "epoch": 0.1415929203539823, | |
| "grad_norm": 3.6484747523940286, | |
| "kl": 0.00010442733764648438, | |
| "learning_rate": 4.998142876590749e-07, | |
| "loss": 0.0, | |
| "reward": 0.2943377196788788, | |
| "reward_std": 0.3771718442440033, | |
| "rewards/correct_code_reward_func": 0.1666666679084301, | |
| "rewards/len_reward_func": 0.12767105549573898, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 147.7916717529297, | |
| "epoch": 0.1592920353982301, | |
| "grad_norm": 14.356223088928283, | |
| "kl": 0.00017464160919189453, | |
| "learning_rate": 4.99582211905537e-07, | |
| "loss": 0.0, | |
| "reward": 0.6274224072694778, | |
| "reward_std": 0.43979279696941376, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.21075571328401566, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 188.2916717529297, | |
| "epoch": 0.17699115044247787, | |
| "grad_norm": 3.0299854001506494, | |
| "kl": 8.487701416015625e-05, | |
| "learning_rate": 4.992574265488882e-07, | |
| "loss": 0.0, | |
| "reward": 0.502624899148941, | |
| "reward_std": 0.35237132012844086, | |
| "rewards/correct_code_reward_func": 0.3125000149011612, | |
| "rewards/len_reward_func": 0.19012490659952164, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 109.1875, | |
| "epoch": 0.19469026548672566, | |
| "grad_norm": 1.8891984084011832, | |
| "kl": 0.00060272216796875, | |
| "learning_rate": 4.988400522336303e-07, | |
| "loss": 0.0, | |
| "reward": 0.6938449144363403, | |
| "reward_std": 0.5790197551250458, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.1730116307735443, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 171.2916717529297, | |
| "epoch": 0.21238938053097345, | |
| "grad_norm": 2.7849518039157664, | |
| "kl": 0.0006923675537109375, | |
| "learning_rate": 4.983302439972828e-07, | |
| "loss": 0.0, | |
| "reward": 0.4239259362220764, | |
| "reward_std": 0.48799367249011993, | |
| "rewards/correct_code_reward_func": 0.20833333395421505, | |
| "rewards/len_reward_func": 0.21559256315231323, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 196.2291717529297, | |
| "epoch": 0.23008849557522124, | |
| "grad_norm": 1.392005725111761, | |
| "kl": 0.000652313232421875, | |
| "learning_rate": 4.977281912127938e-07, | |
| "loss": 0.0, | |
| "reward": 0.5957696735858917, | |
| "reward_std": 0.5921532511711121, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.17910300940275192, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 158.64583587646484, | |
| "epoch": 0.24778761061946902, | |
| "grad_norm": 1.5701172166666861, | |
| "kl": 0.000614166259765625, | |
| "learning_rate": 4.970341175181955e-07, | |
| "loss": 0.0, | |
| "reward": 0.46809761226177216, | |
| "reward_std": 0.4979168623685837, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.19726425409317017, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 147.1041717529297, | |
| "epoch": 0.26548672566371684, | |
| "grad_norm": 2.032633618646926, | |
| "kl": 0.0015106201171875, | |
| "learning_rate": 4.962482807335314e-07, | |
| "loss": 0.0, | |
| "reward": 0.5865300893783569, | |
| "reward_std": 0.5591276288032532, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.16986342146992683, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 177.9791717529297, | |
| "epoch": 0.2831858407079646, | |
| "grad_norm": 53.72073405864756, | |
| "kl": 0.018054962158203125, | |
| "learning_rate": 4.95370972765087e-07, | |
| "loss": 0.0, | |
| "reward": 0.8554503321647644, | |
| "reward_std": 0.48426124453544617, | |
| "rewards/correct_code_reward_func": 0.5625000298023224, | |
| "rewards/len_reward_func": 0.2929503321647644, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 187.31250762939453, | |
| "epoch": 0.3008849557522124, | |
| "grad_norm": 2.4065750408913034, | |
| "kl": 0.001964569091796875, | |
| "learning_rate": 4.944025194969586e-07, | |
| "loss": 0.0, | |
| "reward": 0.36077703535556793, | |
| "reward_std": 0.35280461609363556, | |
| "rewards/correct_code_reward_func": 0.1041666716337204, | |
| "rewards/len_reward_func": 0.25661035627126694, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 186.14583587646484, | |
| "epoch": 0.3185840707964602, | |
| "grad_norm": 20.459541697505053, | |
| "kl": 0.021259307861328125, | |
| "learning_rate": 4.933432806700003e-07, | |
| "loss": 0.0, | |
| "reward": 0.5004578679800034, | |
| "reward_std": 0.42624031007289886, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.22962453216314316, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 184.2916717529297, | |
| "epoch": 0.336283185840708, | |
| "grad_norm": 3.4167543405430436, | |
| "kl": 0.008358001708984375, | |
| "learning_rate": 4.921936497481956e-07, | |
| "loss": 0.0, | |
| "reward": 0.3476429730653763, | |
| "reward_std": 0.43119728565216064, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.2643096297979355, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 179.87500762939453, | |
| "epoch": 0.35398230088495575, | |
| "grad_norm": 2.8250290981037574, | |
| "kl": 0.0052490234375, | |
| "learning_rate": 4.909540537725006e-07, | |
| "loss": 0.0, | |
| "reward": 0.476805180311203, | |
| "reward_std": 0.36166805028915405, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.2059718370437622, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 148.9375, | |
| "epoch": 0.37168141592920356, | |
| "grad_norm": 1.9459373869466972, | |
| "kl": 0.002933502197265625, | |
| "learning_rate": 4.896249532022171e-07, | |
| "loss": 0.0, | |
| "reward": 0.5851579010486603, | |
| "reward_std": 0.3626042455434799, | |
| "rewards/correct_code_reward_func": 0.3333333358168602, | |
| "rewards/len_reward_func": 0.2518245801329613, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 213.8541717529297, | |
| "epoch": 0.3893805309734513, | |
| "grad_norm": 0.5828214607615373, | |
| "kl": 0.0029754638671875, | |
| "learning_rate": 4.882068417439492e-07, | |
| "loss": 0.0, | |
| "reward": 0.5553887784481049, | |
| "reward_std": 0.5428541898727417, | |
| "rewards/correct_code_reward_func": 0.291666679084301, | |
| "rewards/len_reward_func": 0.26372211426496506, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 113.43750381469727, | |
| "epoch": 0.40707964601769914, | |
| "grad_norm": 1.0373945570716487, | |
| "kl": 0.0034637451171875, | |
| "learning_rate": 4.867002461682128e-07, | |
| "loss": 0.0, | |
| "reward": 0.49457328021526337, | |
| "reward_std": 0.38711363077163696, | |
| "rewards/correct_code_reward_func": 0.2291666716337204, | |
| "rewards/len_reward_func": 0.26540662348270416, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 145.5416717529297, | |
| "epoch": 0.4247787610619469, | |
| "grad_norm": 0.8868802158083684, | |
| "kl": 0.00489044189453125, | |
| "learning_rate": 4.851057261137608e-07, | |
| "loss": 0.0, | |
| "reward": 0.7876249551773071, | |
| "reward_std": 0.4593852013349533, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.26679155230522156, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 223.50000762939453, | |
| "epoch": 0.4424778761061947, | |
| "grad_norm": 2.7373544353808152, | |
| "kl": 0.005214691162109375, | |
| "learning_rate": 4.83423873879701e-07, | |
| "loss": 0.0, | |
| "reward": 0.45352864265441895, | |
| "reward_std": 0.5192392915487289, | |
| "rewards/correct_code_reward_func": 0.2291666679084301, | |
| "rewards/len_reward_func": 0.22436197102069855, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 172.56250762939453, | |
| "epoch": 0.46017699115044247, | |
| "grad_norm": 1.5744378292792975, | |
| "kl": 0.0035400390625, | |
| "learning_rate": 4.816553142054805e-07, | |
| "loss": 0.0, | |
| "reward": 0.4348207265138626, | |
| "reward_std": 0.40823744237422943, | |
| "rewards/correct_code_reward_func": 0.14583333395421505, | |
| "rewards/len_reward_func": 0.2889874130487442, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 163.5416717529297, | |
| "epoch": 0.4778761061946903, | |
| "grad_norm": 37.42131429307232, | |
| "kl": 0.0279998779296875, | |
| "learning_rate": 4.798007040388211e-07, | |
| "loss": 0.0, | |
| "reward": 0.44616006314754486, | |
| "reward_std": 0.41116151213645935, | |
| "rewards/correct_code_reward_func": 0.18750000558793545, | |
| "rewards/len_reward_func": 0.25866006314754486, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 147.1458396911621, | |
| "epoch": 0.49557522123893805, | |
| "grad_norm": 3.226533563373661, | |
| "kl": 0.013214111328125, | |
| "learning_rate": 4.778607322916895e-07, | |
| "loss": 0.0, | |
| "reward": 0.5665057748556137, | |
| "reward_std": 0.3880573436617851, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.29567238688468933, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 127.56250381469727, | |
| "epoch": 0.5132743362831859, | |
| "grad_norm": 4.362166865135135, | |
| "kl": 0.01708984375, | |
| "learning_rate": 4.758361195843951e-07, | |
| "loss": 0.0, | |
| "reward": 0.8003825545310974, | |
| "reward_std": 0.510110080242157, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.34204918146133423, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 173.37500381469727, | |
| "epoch": 0.5309734513274337, | |
| "grad_norm": 0.8500372624160994, | |
| "kl": 0.00797271728515625, | |
| "learning_rate": 4.7372761797790825e-07, | |
| "loss": 0.0, | |
| "reward": 0.37487032264471054, | |
| "reward_std": 0.32572413980960846, | |
| "rewards/correct_code_reward_func": 0.1458333432674408, | |
| "rewards/len_reward_func": 0.22903696447610855, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 119.14583587646484, | |
| "epoch": 0.5486725663716814, | |
| "grad_norm": 2.2352274664870624, | |
| "kl": 0.014129638671875, | |
| "learning_rate": 4.7153601069450146e-07, | |
| "loss": 0.0, | |
| "reward": 0.5202069282531738, | |
| "reward_std": 0.5186183750629425, | |
| "rewards/correct_code_reward_func": 0.291666679084301, | |
| "rewards/len_reward_func": 0.22854027152061462, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 111.31250381469727, | |
| "epoch": 0.5663716814159292, | |
| "grad_norm": 1.1283808844332623, | |
| "kl": 0.015625, | |
| "learning_rate": 4.6926211182681294e-07, | |
| "loss": 0.0, | |
| "reward": 0.4815867692232132, | |
| "reward_std": 0.294177770614624, | |
| "rewards/correct_code_reward_func": 0.1458333432674408, | |
| "rewards/len_reward_func": 0.3357534259557724, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 111.87500381469727, | |
| "epoch": 0.584070796460177, | |
| "grad_norm": 15.382447167413178, | |
| "kl": 0.0740966796875, | |
| "learning_rate": 4.6690676603544553e-07, | |
| "loss": 0.0001, | |
| "reward": 0.30563782900571823, | |
| "reward_std": 0.5501309335231781, | |
| "rewards/correct_code_reward_func": 0.1458333395421505, | |
| "rewards/len_reward_func": 0.15980449318885803, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 74.31250190734863, | |
| "epoch": 0.6017699115044248, | |
| "grad_norm": 6.16835237121908, | |
| "kl": 0.038818359375, | |
| "learning_rate": 4.644708482352092e-07, | |
| "loss": 0.0, | |
| "reward": 0.7065277397632599, | |
| "reward_std": 0.4479583501815796, | |
| "rewards/correct_code_reward_func": 0.3958333358168602, | |
| "rewards/len_reward_func": 0.3106943815946579, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 96.45833587646484, | |
| "epoch": 0.6194690265486725, | |
| "grad_norm": 6.369953012669968, | |
| "kl": 0.035186767578125, | |
| "learning_rate": 4.6195526327012623e-07, | |
| "loss": 0.0, | |
| "reward": 0.3565668910741806, | |
| "reward_std": 0.39397968351840973, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.273233562707901, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 112.14583587646484, | |
| "epoch": 0.6371681415929203, | |
| "grad_norm": 3.3487214552142164, | |
| "kl": 0.0289306640625, | |
| "learning_rate": 4.5936094557731806e-07, | |
| "loss": 0.0, | |
| "reward": 0.7201341390609741, | |
| "reward_std": 0.3427349328994751, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.3659674823284149, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 101.64583587646484, | |
| "epoch": 0.6548672566371682, | |
| "grad_norm": 5.120880381398817, | |
| "kl": 0.05126953125, | |
| "learning_rate": 4.566888588399006e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5204495191574097, | |
| "reward_std": 0.3199263662099838, | |
| "rewards/correct_code_reward_func": 0.1250000037252903, | |
| "rewards/len_reward_func": 0.39544953405857086, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 120.5625, | |
| "epoch": 0.672566371681416, | |
| "grad_norm": 10.359850576348157, | |
| "kl": 0.09923553466796875, | |
| "learning_rate": 4.5393999562901517e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6964526474475861, | |
| "reward_std": 0.5385148227214813, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.27978597581386566, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 110.85417175292969, | |
| "epoch": 0.6902654867256637, | |
| "grad_norm": 12.273703285519217, | |
| "kl": 0.0859375, | |
| "learning_rate": 4.511153770351288e-07, | |
| "loss": 0.0001, | |
| "reward": 0.551011860370636, | |
| "reward_std": 0.36342084407806396, | |
| "rewards/correct_code_reward_func": 0.1875000111758709, | |
| "rewards/len_reward_func": 0.3635118901729584, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 68.77083587646484, | |
| "epoch": 0.7079646017699115, | |
| "grad_norm": 2.2127761337332514, | |
| "kl": 0.03765869140625, | |
| "learning_rate": 4.482160522887403e-07, | |
| "loss": 0.0, | |
| "reward": 0.7378278374671936, | |
| "reward_std": 0.39128121733665466, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.3003278076648712, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 135.33333587646484, | |
| "epoch": 0.7256637168141593, | |
| "grad_norm": 7.600755671861986, | |
| "kl": 0.1025390625, | |
| "learning_rate": 4.4524309837063504e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6550733149051666, | |
| "reward_std": 0.5409570336341858, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.32173997163772583, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 87.35416793823242, | |
| "epoch": 0.7433628318584071, | |
| "grad_norm": 1.6503719126782672, | |
| "kl": 0.0264892578125, | |
| "learning_rate": 4.4219761961182965e-07, | |
| "loss": 0.0, | |
| "reward": 0.6292450726032257, | |
| "reward_std": 0.4537566006183624, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.3584117591381073, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 65.29166793823242, | |
| "epoch": 0.7610619469026548, | |
| "grad_norm": 4.327974078868625, | |
| "kl": 0.1019287109375, | |
| "learning_rate": 4.390807472833584e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5142460465431213, | |
| "reward_std": 0.41541849076747894, | |
| "rewards/correct_code_reward_func": 0.1875000074505806, | |
| "rewards/len_reward_func": 0.32674603164196014, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 88.62500381469727, | |
| "epoch": 0.7787610619469026, | |
| "grad_norm": 2.8856078275542814, | |
| "kl": 0.068115234375, | |
| "learning_rate": 4.3589363917605234e-07, | |
| "loss": 0.0001, | |
| "reward": 0.48976023495197296, | |
| "reward_std": 0.39692020416259766, | |
| "rewards/correct_code_reward_func": 0.1875, | |
| "rewards/len_reward_func": 0.30226024240255356, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 78.75000381469727, | |
| "epoch": 0.7964601769911505, | |
| "grad_norm": 1.6857493365819893, | |
| "kl": 0.04052734375, | |
| "learning_rate": 4.3263747917046697e-07, | |
| "loss": 0.0, | |
| "reward": 0.5636806637048721, | |
| "reward_std": 0.5081076323986053, | |
| "rewards/correct_code_reward_func": 0.2500000111758709, | |
| "rewards/len_reward_func": 0.3136806935071945, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 78.50000190734863, | |
| "epoch": 0.8141592920353983, | |
| "grad_norm": 1.2755984783224172, | |
| "kl": 0.0218505859375, | |
| "learning_rate": 4.2931347679711924e-07, | |
| "loss": 0.0, | |
| "reward": 0.6265330016613007, | |
| "reward_std": 0.38107292354106903, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.33486635982990265, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 80.29166793823242, | |
| "epoch": 0.831858407079646, | |
| "grad_norm": 1.1240225012044924, | |
| "kl": 0.0335693359375, | |
| "learning_rate": 4.259228667871962e-07, | |
| "loss": 0.0, | |
| "reward": 0.5936128199100494, | |
| "reward_std": 0.4675884544849396, | |
| "rewards/correct_code_reward_func": 0.25, | |
| "rewards/len_reward_func": 0.34361281991004944, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 50.0, | |
| "epoch": 0.8495575221238938, | |
| "grad_norm": 8.693611957424073, | |
| "kl": 0.2470703125, | |
| "learning_rate": 4.224669086139029e-07, | |
| "loss": 0.0002, | |
| "reward": 0.49966277182102203, | |
| "reward_std": 0.3804493248462677, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.41632944345474243, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 92.20833587646484, | |
| "epoch": 0.8672566371681416, | |
| "grad_norm": 1.2744927278879081, | |
| "kl": 0.02978515625, | |
| "learning_rate": 4.1894688602461913e-07, | |
| "loss": 0.0, | |
| "reward": 0.7709261626005173, | |
| "reward_std": 0.3614697605371475, | |
| "rewards/correct_code_reward_func": 0.3958333544433117, | |
| "rewards/len_reward_func": 0.3750927746295929, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 127.62500381469727, | |
| "epoch": 0.8849557522123894, | |
| "grad_norm": 0.566625300559752, | |
| "kl": 0.016326904296875, | |
| "learning_rate": 4.153641065640402e-07, | |
| "loss": 0.0, | |
| "reward": 0.5525736510753632, | |
| "reward_std": 0.42078813910484314, | |
| "rewards/correct_code_reward_func": 0.20833333395421505, | |
| "rewards/len_reward_func": 0.34424029290676117, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 65.375, | |
| "epoch": 0.9026548672566371, | |
| "grad_norm": 12.046039464547933, | |
| "kl": 0.081298828125, | |
| "learning_rate": 4.11719901088477e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6643719673156738, | |
| "reward_std": 0.4581465870141983, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.3310386538505554, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 103.58333587646484, | |
| "epoch": 0.9203539823008849, | |
| "grad_norm": 10.540579232965062, | |
| "kl": 0.0799560546875, | |
| "learning_rate": 4.080156232714976e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6433776319026947, | |
| "reward_std": 0.3912515640258789, | |
| "rewards/correct_code_reward_func": 0.29166667722165585, | |
| "rewards/len_reward_func": 0.3517109453678131, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 62.375003814697266, | |
| "epoch": 0.9380530973451328, | |
| "grad_norm": 2.819698340947531, | |
| "kl": 0.115966796875, | |
| "learning_rate": 4.0425264910109233e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7070481181144714, | |
| "reward_std": 0.44986239075660706, | |
| "rewards/correct_code_reward_func": 0.3125, | |
| "rewards/len_reward_func": 0.39454813301563263, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 59.0625, | |
| "epoch": 0.9557522123893806, | |
| "grad_norm": 3.22095354432732, | |
| "kl": 0.1131591796875, | |
| "learning_rate": 4.004323763685511e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5839295089244843, | |
| "reward_std": 0.26084955781698227, | |
| "rewards/correct_code_reward_func": 0.14583333395421505, | |
| "rewards/len_reward_func": 0.43809613585472107, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 125.58333969116211, | |
| "epoch": 0.9734513274336283, | |
| "grad_norm": 8.551645640447115, | |
| "kl": 0.15509033203125, | |
| "learning_rate": 3.9655622414924007e-07, | |
| "loss": 0.0002, | |
| "reward": 0.528711274266243, | |
| "reward_std": 0.3454602509737015, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.3620445877313614, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 60.000003814697266, | |
| "epoch": 0.9911504424778761, | |
| "grad_norm": 38.727062970707685, | |
| "kl": 0.20166015625, | |
| "learning_rate": 3.92625632275474e-07, | |
| "loss": 0.0002, | |
| "reward": 0.891117125749588, | |
| "reward_std": 0.36153580248355865, | |
| "rewards/correct_code_reward_func": 0.5000000149011612, | |
| "rewards/len_reward_func": 0.39111708104610443, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 91.54167175292969, | |
| "epoch": 1.0, | |
| "grad_norm": 38.727062970707685, | |
| "kl": 0.1865234375, | |
| "learning_rate": 3.886420608016766e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5094150304794312, | |
| "reward_std": 0.4194638133049011, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.3427482843399048, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 96.41666793823242, | |
| "epoch": 1.0176991150442478, | |
| "grad_norm": 13.425409986987928, | |
| "kl": 0.04998779296875, | |
| "learning_rate": 3.846069894620305e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5195242762565613, | |
| "reward_std": 0.3198210597038269, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.3945242464542389, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 79.3125, | |
| "epoch": 1.0353982300884956, | |
| "grad_norm": 1.7429704138905477, | |
| "kl": 0.0509033203125, | |
| "learning_rate": 3.8052191712081595e-07, | |
| "loss": 0.0001, | |
| "reward": 0.655539482831955, | |
| "reward_std": 0.2987876646220684, | |
| "rewards/correct_code_reward_func": 0.1875, | |
| "rewards/len_reward_func": 0.46803948283195496, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 61.187503814697266, | |
| "epoch": 1.0530973451327434, | |
| "grad_norm": 7.538322601947877, | |
| "kl": 0.1343994140625, | |
| "learning_rate": 3.763883612156441e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7049243450164795, | |
| "reward_std": 0.5336888134479523, | |
| "rewards/correct_code_reward_func": 0.3125000149011612, | |
| "rewards/len_reward_func": 0.3924243599176407, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 71.45833587646484, | |
| "epoch": 1.0707964601769913, | |
| "grad_norm": 1.628031440232102, | |
| "kl": 0.053955078125, | |
| "learning_rate": 3.7220785719379095e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5856566280126572, | |
| "reward_std": 0.31656239926815033, | |
| "rewards/correct_code_reward_func": 0.2083333395421505, | |
| "rewards/len_reward_func": 0.37732329964637756, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 80.64583587646484, | |
| "epoch": 1.0884955752212389, | |
| "grad_norm": 0.8807120675882707, | |
| "kl": 0.05035400390625, | |
| "learning_rate": 3.6798195794184134e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5311767309904099, | |
| "reward_std": 0.2926383763551712, | |
| "rewards/correct_code_reward_func": 0.1041666716337204, | |
| "rewards/len_reward_func": 0.42701007425785065, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 112.29167175292969, | |
| "epoch": 1.1061946902654867, | |
| "grad_norm": 1.0053639016562932, | |
| "kl": 0.07916259765625, | |
| "learning_rate": 3.6371223320885484e-07, | |
| "loss": 0.0001, | |
| "reward": 0.4587331861257553, | |
| "reward_std": 0.33583496510982513, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.3753998577594757, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 83.20833587646484, | |
| "epoch": 1.1238938053097345, | |
| "grad_norm": 1.7713543449970612, | |
| "kl": 0.08245849609375, | |
| "learning_rate": 3.594002690232682e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6741357147693634, | |
| "reward_std": 0.43141913414001465, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.4033023715019226, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 84.10416984558105, | |
| "epoch": 1.1415929203539823, | |
| "grad_norm": 25.050556888869668, | |
| "kl": 0.12774658203125, | |
| "learning_rate": 3.5504766710375047e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6237881779670715, | |
| "reward_std": 0.28890369832515717, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.41545483469963074, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 82.27083587646484, | |
| "epoch": 1.1592920353982301, | |
| "grad_norm": 1.3229069926746502, | |
| "kl": 0.0810546875, | |
| "learning_rate": 3.506560442642299e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7232638597488403, | |
| "reward_std": 0.37123511731624603, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.36909720301628113, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 58.00000190734863, | |
| "epoch": 1.176991150442478, | |
| "grad_norm": 14.439064098343355, | |
| "kl": 0.1383056640625, | |
| "learning_rate": 3.462270318133136e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9435772597789764, | |
| "reward_std": 0.2863286882638931, | |
| "rewards/correct_code_reward_func": 0.520833358168602, | |
| "rewards/len_reward_func": 0.42274393141269684, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 71.10416984558105, | |
| "epoch": 1.1946902654867257, | |
| "grad_norm": 7.987124238678158, | |
| "kl": 0.122314453125, | |
| "learning_rate": 3.41762274948323e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7681821584701538, | |
| "reward_std": 0.3728492707014084, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.41401544213294983, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 60.66666793823242, | |
| "epoch": 1.2123893805309733, | |
| "grad_norm": 2.9983488159761817, | |
| "kl": 0.0537109375, | |
| "learning_rate": 3.372634321441702e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7418330609798431, | |
| "reward_std": 0.38279734551906586, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.38766637444496155, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 80.62500381469727, | |
| "epoch": 1.2300884955752212, | |
| "grad_norm": 3.023771746536167, | |
| "kl": 0.08154296875, | |
| "learning_rate": 3.3273217453730205e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6804832518100739, | |
| "reward_std": 0.4174923449754715, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.40964992344379425, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 69.5, | |
| "epoch": 1.247787610619469, | |
| "grad_norm": 1.4050656113052118, | |
| "kl": 0.0853271484375, | |
| "learning_rate": 3.281701853049416e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7291666865348816, | |
| "reward_std": 0.32618680596351624, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 74.47916793823242, | |
| "epoch": 1.2654867256637168, | |
| "grad_norm": 2.8179580865899894, | |
| "kl": 0.0390625, | |
| "learning_rate": 3.23579159039856e-07, | |
| "loss": 0.0, | |
| "reward": 0.75091952085495, | |
| "reward_std": 0.4079275578260422, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.45925286412239075, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 65.52083587646484, | |
| "epoch": 1.2831858407079646, | |
| "grad_norm": 1.1299435463555585, | |
| "kl": 0.0399169921875, | |
| "learning_rate": 3.1896080112088477e-07, | |
| "loss": 0.0, | |
| "reward": 0.7291666865348816, | |
| "reward_std": 0.40629878640174866, | |
| "rewards/correct_code_reward_func": 0.2708333395421505, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 57.000003814697266, | |
| "epoch": 1.3008849557522124, | |
| "grad_norm": 24.705642459397488, | |
| "kl": 0.17333984375, | |
| "learning_rate": 3.1431682707946117e-07, | |
| "loss": 0.0002, | |
| "reward": 0.5623736083507538, | |
| "reward_std": 0.24504420161247253, | |
| "rewards/correct_code_reward_func": 0.12500000558793545, | |
| "rewards/len_reward_func": 0.4373735785484314, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 78.33333587646484, | |
| "epoch": 1.3185840707964602, | |
| "grad_norm": 0.7830442669221305, | |
| "kl": 0.0224609375, | |
| "learning_rate": 3.096489619623621e-07, | |
| "loss": 0.0, | |
| "reward": 0.7488380074501038, | |
| "reward_std": 0.30931201577186584, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.49883799254894257, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 87.60417175292969, | |
| "epoch": 1.336283185840708, | |
| "grad_norm": 2.1746791001290044, | |
| "kl": 0.04168701171875, | |
| "learning_rate": 3.049589396909239e-07, | |
| "loss": 0.0, | |
| "reward": 0.5558712333440781, | |
| "reward_std": 0.20058585330843925, | |
| "rewards/correct_code_reward_func": 0.1041666716337204, | |
| "rewards/len_reward_func": 0.45170454680919647, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 72.14583587646484, | |
| "epoch": 1.3539823008849559, | |
| "grad_norm": 0.41651138876768207, | |
| "kl": 0.0919189453125, | |
| "learning_rate": 3.0024850241696126e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6021915674209595, | |
| "reward_std": 0.22243696451187134, | |
| "rewards/correct_code_reward_func": 0.1458333358168602, | |
| "rewards/len_reward_func": 0.4563582092523575, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 54.916666984558105, | |
| "epoch": 1.3716814159292037, | |
| "grad_norm": 8.88612982008444, | |
| "kl": 0.106689453125, | |
| "learning_rate": 2.9551939987562865e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6803071200847626, | |
| "reward_std": 0.2964301370084286, | |
| "rewards/correct_code_reward_func": 0.2291666679084301, | |
| "rewards/len_reward_func": 0.4511404484510422, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 93.91666793823242, | |
| "epoch": 1.3893805309734513, | |
| "grad_norm": 3.7969039322604865, | |
| "kl": 0.086761474609375, | |
| "learning_rate": 2.9077338873546564e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.34503278136253357, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 56.08333396911621, | |
| "epoch": 1.407079646017699, | |
| "grad_norm": 3.0759876398616743, | |
| "kl": 0.076171875, | |
| "learning_rate": 2.860122319458661e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8750000596046448, | |
| "reward_std": 0.36751919984817505, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 77.58333587646484, | |
| "epoch": 1.424778761061947, | |
| "grad_norm": 5.267849973823214, | |
| "kl": 0.0604248046875, | |
| "learning_rate": 2.8123769808221403e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.4720035046339035, | |
| "rewards/correct_code_reward_func": 0.4375000298023224, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 86.18750190734863, | |
| "epoch": 1.4424778761061947, | |
| "grad_norm": 16.256704067320886, | |
| "kl": 0.05712890625, | |
| "learning_rate": 2.764515606889307e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8077563941478729, | |
| "reward_std": 0.3568233400583267, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.41192303597927094, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 106.89583587646484, | |
| "epoch": 1.4601769911504425, | |
| "grad_norm": 2.2341312424056214, | |
| "kl": 0.021087646484375, | |
| "learning_rate": 2.716555976206747e-07, | |
| "loss": 0.0, | |
| "reward": 0.7582251131534576, | |
| "reward_std": 0.3638576865196228, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.46655844151973724, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 48.41666793823242, | |
| "epoch": 1.4778761061946903, | |
| "grad_norm": 1.3490761289725859, | |
| "kl": 0.05279541015625, | |
| "learning_rate": 2.66851590381942e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7612736523151398, | |
| "reward_std": 0.23646536469459534, | |
| "rewards/correct_code_reward_func": 0.2708333358168602, | |
| "rewards/len_reward_func": 0.490440309047699, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 60.187503814697266, | |
| "epoch": 1.495575221238938, | |
| "grad_norm": 4.538439276716702, | |
| "kl": 0.0347900390625, | |
| "learning_rate": 2.620413234653093e-07, | |
| "loss": 0.0, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.438393235206604, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 138.50000762939453, | |
| "epoch": 1.5132743362831858, | |
| "grad_norm": 0.9112971760232713, | |
| "kl": 0.021392822265625, | |
| "learning_rate": 2.5722658368856814e-07, | |
| "loss": 0.0, | |
| "reward": 0.6596881151199341, | |
| "reward_std": 0.4098925292491913, | |
| "rewards/correct_code_reward_func": 0.1875000074505806, | |
| "rewards/len_reward_func": 0.4721881300210953, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 67.89583587646484, | |
| "epoch": 1.5309734513274336, | |
| "grad_norm": 1.5750222049346805, | |
| "kl": 0.033447265625, | |
| "learning_rate": 2.5240915953099515e-07, | |
| "loss": 0.0, | |
| "reward": 0.9335185885429382, | |
| "reward_std": 0.32536589354276657, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.49601852893829346, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 56.395835876464844, | |
| "epoch": 1.5486725663716814, | |
| "grad_norm": 0.8224337332016152, | |
| "kl": 0.0401611328125, | |
| "learning_rate": 2.4759084046900483e-07, | |
| "loss": 0.0, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.2342708334326744, | |
| "rewards/correct_code_reward_func": 0.3125000149011612, | |
| "rewards/len_reward_func": 0.4375000149011612, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 55.250003814697266, | |
| "epoch": 1.5663716814159292, | |
| "grad_norm": 1.7919585934332407, | |
| "kl": 0.046142578125, | |
| "learning_rate": 2.427734163114319e-07, | |
| "loss": 0.0, | |
| "reward": 0.8125000298023224, | |
| "reward_std": 0.3177132308483124, | |
| "rewards/correct_code_reward_func": 0.3125000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 78.27083587646484, | |
| "epoch": 1.584070796460177, | |
| "grad_norm": 1.6894819978109317, | |
| "kl": 0.0758056640625, | |
| "learning_rate": 2.3795867653469067e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8326875865459442, | |
| "reward_std": 0.27240147814154625, | |
| "rewards/correct_code_reward_func": 0.35416667722165585, | |
| "rewards/len_reward_func": 0.47852087020874023, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 102.83333969116211, | |
| "epoch": 1.6017699115044248, | |
| "grad_norm": 22.765338134366964, | |
| "kl": 0.07025146484375, | |
| "learning_rate": 2.3314840961805802e-07, | |
| "loss": 0.0001, | |
| "reward": 0.686832070350647, | |
| "reward_std": 0.2984095811843872, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.47849874198436737, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 54.31250190734863, | |
| "epoch": 1.6194690265486726, | |
| "grad_norm": 1.6877966926685302, | |
| "kl": 0.0550537109375, | |
| "learning_rate": 2.2834440237932533e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.38857419788837433, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 120.70833969116211, | |
| "epoch": 1.6371681415929205, | |
| "grad_norm": 10.817524146996126, | |
| "kl": 0.0389404296875, | |
| "learning_rate": 2.2354843931106932e-07, | |
| "loss": 0.0, | |
| "reward": 0.7042862474918365, | |
| "reward_std": 0.3615289777517319, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.49595288932323456, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 44.250000953674316, | |
| "epoch": 1.6548672566371683, | |
| "grad_norm": 28.86285310594792, | |
| "kl": 0.064697265625, | |
| "learning_rate": 2.1876230191778597e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.3857583850622177, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 53.62500190734863, | |
| "epoch": 1.672566371681416, | |
| "grad_norm": 3.9097872367150397, | |
| "kl": 0.0594482421875, | |
| "learning_rate": 2.1398776805413398e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.3233579397201538, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 65.00000190734863, | |
| "epoch": 1.6902654867256637, | |
| "grad_norm": 1.5650440517517894, | |
| "kl": 0.0361328125, | |
| "learning_rate": 2.092266112645343e-07, | |
| "loss": 0.0, | |
| "reward": 0.8589357733726501, | |
| "reward_std": 0.3421449810266495, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.48393575847148895, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 68.06250381469727, | |
| "epoch": 1.7079646017699115, | |
| "grad_norm": 2.91895596774148, | |
| "kl": 0.0269775390625, | |
| "learning_rate": 2.044806001243714e-07, | |
| "loss": 0.0, | |
| "reward": 0.9035088419914246, | |
| "reward_std": 0.48539453744888306, | |
| "rewards/correct_code_reward_func": 0.4375000298023224, | |
| "rewards/len_reward_func": 0.466008797287941, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 73.70833396911621, | |
| "epoch": 1.7256637168141593, | |
| "grad_norm": 24.67219025250006, | |
| "kl": 0.0465087890625, | |
| "learning_rate": 1.997514975830388e-07, | |
| "loss": 0.0, | |
| "reward": 0.8952526450157166, | |
| "reward_std": 0.29202311858534813, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.49941930174827576, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 97.75000381469727, | |
| "epoch": 1.7433628318584071, | |
| "grad_norm": 1.6430763507769786, | |
| "kl": 0.0406494140625, | |
| "learning_rate": 1.9504106030907603e-07, | |
| "loss": 0.0, | |
| "reward": 0.7537775337696075, | |
| "reward_std": 0.3211555927991867, | |
| "rewards/correct_code_reward_func": 0.2916666865348816, | |
| "rewards/len_reward_func": 0.46211084723472595, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 52.62500190734863, | |
| "epoch": 1.7610619469026547, | |
| "grad_norm": 3.9570608772498286, | |
| "kl": 0.0634765625, | |
| "learning_rate": 1.9035103803763792e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8125, | |
| "reward_std": 0.3857453167438507, | |
| "rewards/correct_code_reward_func": 0.3125, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 81.35416793823242, | |
| "epoch": 1.7787610619469025, | |
| "grad_norm": 2.731852842110385, | |
| "kl": 0.033447265625, | |
| "learning_rate": 1.8568317292053891e-07, | |
| "loss": 0.0, | |
| "reward": 0.9219505190849304, | |
| "reward_std": 0.2703188881278038, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.48445048928260803, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 83.31250381469727, | |
| "epoch": 1.7964601769911503, | |
| "grad_norm": 2.4613714386546595, | |
| "kl": 0.022613525390625, | |
| "learning_rate": 1.8103919887911523e-07, | |
| "loss": 0.0, | |
| "reward": 0.7291666865348816, | |
| "reward_std": 0.30859364569187164, | |
| "rewards/correct_code_reward_func": 0.2291666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 79.33333587646484, | |
| "epoch": 1.8141592920353982, | |
| "grad_norm": 4.215262416117561, | |
| "kl": 0.040771484375, | |
| "learning_rate": 1.7642084096014402e-07, | |
| "loss": 0.0, | |
| "reward": 0.7859848439693451, | |
| "reward_std": 0.4127810299396515, | |
| "rewards/correct_code_reward_func": 0.291666679084301, | |
| "rewards/len_reward_func": 0.4943181872367859, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 58.52083396911621, | |
| "epoch": 1.831858407079646, | |
| "grad_norm": 4.745442984631991, | |
| "kl": 0.08740234375, | |
| "learning_rate": 1.7182981469505847e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8333333730697632, | |
| "reward_std": 0.45660629868507385, | |
| "rewards/correct_code_reward_func": 0.354166679084301, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 112.64583587646484, | |
| "epoch": 1.8495575221238938, | |
| "grad_norm": 1.7418907068386247, | |
| "kl": 0.0325927734375, | |
| "learning_rate": 1.672678254626979e-07, | |
| "loss": 0.0, | |
| "reward": 0.6364757716655731, | |
| "reward_std": 0.3013785034418106, | |
| "rewards/correct_code_reward_func": 0.18750000558793545, | |
| "rewards/len_reward_func": 0.4489757716655731, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 57.5, | |
| "epoch": 1.8672566371681416, | |
| "grad_norm": 3.579303393703891, | |
| "kl": 0.0543212890625, | |
| "learning_rate": 1.6273656785582984e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8536822199821472, | |
| "reward_std": 0.45756329596042633, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.45784883201122284, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 83.52083587646484, | |
| "epoch": 1.8849557522123894, | |
| "grad_norm": 2.1418835681881925, | |
| "kl": 0.0758056640625, | |
| "learning_rate": 1.5823772505167698e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.49633626639842987, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 94.20833587646484, | |
| "epoch": 1.9026548672566372, | |
| "grad_norm": 32.45107688241668, | |
| "kl": 0.025146484375, | |
| "learning_rate": 1.5377296818668635e-07, | |
| "loss": 0.0, | |
| "reward": 0.854166716337204, | |
| "reward_std": 0.37177951633930206, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 72.89583396911621, | |
| "epoch": 1.920353982300885, | |
| "grad_norm": 1.7762554154885124, | |
| "kl": 0.04364013671875, | |
| "learning_rate": 1.4934395573577013e-07, | |
| "loss": 0.0, | |
| "reward": 0.7892157137393951, | |
| "reward_std": 0.3116895519196987, | |
| "rewards/correct_code_reward_func": 0.3125000149011612, | |
| "rewards/len_reward_func": 0.47671568393707275, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 66.39583587646484, | |
| "epoch": 1.9380530973451329, | |
| "grad_norm": 1.3218901012489044, | |
| "kl": 0.02496337890625, | |
| "learning_rate": 1.4495233289624958e-07, | |
| "loss": 0.0, | |
| "reward": 0.7881554365158081, | |
| "reward_std": 0.3200981765985489, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.4964887797832489, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 58.291669845581055, | |
| "epoch": 1.9557522123893807, | |
| "grad_norm": 0.9195423141452592, | |
| "kl": 0.066162109375, | |
| "learning_rate": 1.4059973097673185e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.3382536321878433, | |
| "rewards/correct_code_reward_func": 0.2500000111758709, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 94.41667175292969, | |
| "epoch": 1.9734513274336283, | |
| "grad_norm": 89.52879883380525, | |
| "kl": 0.0601806640625, | |
| "learning_rate": 1.3628776679114514e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.31285394728183746, | |
| "rewards/correct_code_reward_func": 0.3125, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 100.27083587646484, | |
| "epoch": 1.991150442477876, | |
| "grad_norm": 2.390953296031483, | |
| "kl": 0.056396484375, | |
| "learning_rate": 1.320180420581587e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6097994744777679, | |
| "reward_std": 0.18817735463380814, | |
| "rewards/correct_code_reward_func": 0.12500000558793545, | |
| "rewards/len_reward_func": 0.48479947447776794, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 63.875, | |
| "epoch": 2.0, | |
| "grad_norm": 3.601264581967262, | |
| "kl": 0.02001953125, | |
| "learning_rate": 1.2779214280620908e-07, | |
| "loss": 0.0, | |
| "reward": 0.6666666865348816, | |
| "reward_std": 0.17817416787147522, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 48.64583396911621, | |
| "epoch": 2.017699115044248, | |
| "grad_norm": 3.9192662388145028, | |
| "kl": 0.05908203125, | |
| "learning_rate": 1.2361163878435594e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6176470816135406, | |
| "reward_std": 0.2277354598045349, | |
| "rewards/correct_code_reward_func": 0.12500000558793545, | |
| "rewards/len_reward_func": 0.49264705181121826, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 51.41666793823242, | |
| "epoch": 2.0353982300884956, | |
| "grad_norm": 2.2242727716589967, | |
| "kl": 0.04486083984375, | |
| "learning_rate": 1.1947808287918403e-07, | |
| "loss": 0.0, | |
| "reward": 0.7083333730697632, | |
| "reward_std": 0.2994871214032173, | |
| "rewards/correct_code_reward_func": 0.2083333395421505, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 69.04166793823242, | |
| "epoch": 2.0530973451327434, | |
| "grad_norm": 5.407021499840827, | |
| "kl": 0.0469970703125, | |
| "learning_rate": 1.1539301053796949e-07, | |
| "loss": 0.0, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.2946278229355812, | |
| "rewards/correct_code_reward_func": 0.4791666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 47.83333396911621, | |
| "epoch": 2.0707964601769913, | |
| "grad_norm": 29.2301422488625, | |
| "kl": 0.08349609375, | |
| "learning_rate": 1.1135793919832334e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9467871785163879, | |
| "reward_std": 0.43978360295295715, | |
| "rewards/correct_code_reward_func": 0.4791666716337204, | |
| "rewards/len_reward_func": 0.46762049198150635, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 90.41666793823242, | |
| "epoch": 2.088495575221239, | |
| "grad_norm": 1.804477992356325, | |
| "kl": 0.02587890625, | |
| "learning_rate": 1.0737436772452602e-07, | |
| "loss": 0.0, | |
| "reward": 0.7708333432674408, | |
| "reward_std": 0.3584126979112625, | |
| "rewards/correct_code_reward_func": 0.2708333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 150.75000762939453, | |
| "epoch": 2.106194690265487, | |
| "grad_norm": 24.510631619520034, | |
| "kl": 0.0252685546875, | |
| "learning_rate": 1.0344377585075997e-07, | |
| "loss": 0.0, | |
| "reward": 0.684719979763031, | |
| "reward_std": 0.30992700159549713, | |
| "rewards/correct_code_reward_func": 0.1875000074505806, | |
| "rewards/len_reward_func": 0.4972199648618698, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 79.85416793823242, | |
| "epoch": 2.1238938053097347, | |
| "grad_norm": 2.667553033080654, | |
| "kl": 0.0318603515625, | |
| "learning_rate": 9.95676236314489e-08, | |
| "loss": 0.0, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.31142252683639526, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 91.56250381469727, | |
| "epoch": 2.1415929203539825, | |
| "grad_norm": 100.44717449922435, | |
| "kl": 0.2076416015625, | |
| "learning_rate": 9.574735089890764e-08, | |
| "loss": 0.0002, | |
| "reward": 0.7708333432674408, | |
| "reward_std": 0.30231600999832153, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 87.70833396911621, | |
| "epoch": 2.15929203539823, | |
| "grad_norm": 77.23064615733736, | |
| "kl": 0.22283935546875, | |
| "learning_rate": 9.198437672850248e-08, | |
| "loss": 0.0002, | |
| "reward": 0.770833358168602, | |
| "reward_std": 0.3205290399491787, | |
| "rewards/correct_code_reward_func": 0.2916666865348816, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 62.312503814697266, | |
| "epoch": 2.1769911504424777, | |
| "grad_norm": 17.734240204261667, | |
| "kl": 0.0535888671875, | |
| "learning_rate": 8.8280098911523e-08, | |
| "loss": 0.0001, | |
| "reward": 0.875, | |
| "reward_std": 0.2994871288537979, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 65.41666793823242, | |
| "epoch": 2.1946902654867255, | |
| "grad_norm": 28.039058681720235, | |
| "kl": 0.0321044921875, | |
| "learning_rate": 8.463589343595976e-08, | |
| "loss": 0.0, | |
| "reward": 1.0833333730697632, | |
| "reward_std": 0.5049939155578613, | |
| "rewards/correct_code_reward_func": 0.5833333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 98.64583396911621, | |
| "epoch": 2.2123893805309733, | |
| "grad_norm": 6.872794457305926, | |
| "kl": 0.0435791015625, | |
| "learning_rate": 8.105311397538084e-08, | |
| "loss": 0.0, | |
| "reward": 0.7077258378267288, | |
| "reward_std": 0.37921955436468124, | |
| "rewards/correct_code_reward_func": 0.25000000558793545, | |
| "rewards/len_reward_func": 0.4577258378267288, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 71.22916984558105, | |
| "epoch": 2.230088495575221, | |
| "grad_norm": 2.9985031046208857, | |
| "kl": 0.0357666015625, | |
| "learning_rate": 7.753309138609704e-08, | |
| "loss": 0.0, | |
| "reward": 0.7608647048473358, | |
| "reward_std": 0.35784420371055603, | |
| "rewards/correct_code_reward_func": 0.2708333358168602, | |
| "rewards/len_reward_func": 0.490031361579895, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 72.87500190734863, | |
| "epoch": 2.247787610619469, | |
| "grad_norm": 1.1881439057403123, | |
| "kl": 0.03509521484375, | |
| "learning_rate": 7.407713321280375e-08, | |
| "loss": 0.0, | |
| "reward": 0.854166716337204, | |
| "reward_std": 0.30231600999832153, | |
| "rewards/correct_code_reward_func": 0.354166679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 95.39583587646484, | |
| "epoch": 2.265486725663717, | |
| "grad_norm": 2.071026140631432, | |
| "kl": 0.0494384765625, | |
| "learning_rate": 7.06865232028808e-08, | |
| "loss": 0.0, | |
| "reward": 0.8162688612937927, | |
| "reward_std": 0.29165610671043396, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.46210217475891113, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 78.43750381469727, | |
| "epoch": 2.2831858407079646, | |
| "grad_norm": 0.7368071897691737, | |
| "kl": 0.027099609375, | |
| "learning_rate": 6.736252082953306e-08, | |
| "loss": 0.0, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.2903675436973572, | |
| "rewards/correct_code_reward_func": 0.3333333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 73.45833587646484, | |
| "epoch": 2.3008849557522124, | |
| "grad_norm": 2.1125816683606726, | |
| "kl": 0.0389404296875, | |
| "learning_rate": 6.410636082394771e-08, | |
| "loss": 0.0, | |
| "reward": 0.5384955406188965, | |
| "reward_std": 0.272975854575634, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.4551621973514557, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 101.02083587646484, | |
| "epoch": 2.3185840707964602, | |
| "grad_norm": 3.1749370092384415, | |
| "kl": 0.0380859375, | |
| "learning_rate": 6.091925271664156e-08, | |
| "loss": 0.0, | |
| "reward": 0.6666666716337204, | |
| "reward_std": 0.2903675250709057, | |
| "rewards/correct_code_reward_func": 0.1875, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 64.22916793823242, | |
| "epoch": 2.336283185840708, | |
| "grad_norm": 2.773648686745547, | |
| "kl": 0.11944580078125, | |
| "learning_rate": 5.7802380388170344e-08, | |
| "loss": 0.0001, | |
| "reward": 0.8125, | |
| "reward_std": 0.37034809589385986, | |
| "rewards/correct_code_reward_func": 0.3333333358168602, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 56.64583396911621, | |
| "epoch": 2.353982300884956, | |
| "grad_norm": 3.7650155223115793, | |
| "kl": 0.0516357421875, | |
| "learning_rate": 5.4756901629364886e-08, | |
| "loss": 0.0001, | |
| "reward": 1.129325270652771, | |
| "reward_std": 0.32795289158821106, | |
| "rewards/correct_code_reward_func": 0.6458333432674408, | |
| "rewards/len_reward_func": 0.4834919422864914, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 70.1875, | |
| "epoch": 2.3716814159292037, | |
| "grad_norm": 1.0135949886271995, | |
| "kl": 0.042236328125, | |
| "learning_rate": 5.178394771125968e-08, | |
| "loss": 0.0, | |
| "reward": 0.7218915224075317, | |
| "reward_std": 0.2874666675925255, | |
| "rewards/correct_code_reward_func": 0.2291666716337204, | |
| "rewards/len_reward_func": 0.49272486567497253, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 132.7916717529297, | |
| "epoch": 2.3893805309734515, | |
| "grad_norm": 1.3477438258822867, | |
| "kl": 0.03106689453125, | |
| "learning_rate": 4.888462296487128e-08, | |
| "loss": 0.0, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.42218445241451263, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 60.22916793823242, | |
| "epoch": 2.4070796460176993, | |
| "grad_norm": 1.2650991231847002, | |
| "kl": 0.058837890625, | |
| "learning_rate": 4.606000437098476e-08, | |
| "loss": 0.0001, | |
| "reward": 0.8958333730697632, | |
| "reward_std": 0.2041093409061432, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 64.6875, | |
| "epoch": 2.4247787610619467, | |
| "grad_norm": 2.901815806581681, | |
| "kl": 0.03662109375, | |
| "learning_rate": 4.3311141160099377e-08, | |
| "loss": 0.0, | |
| "reward": 0.9150572419166565, | |
| "reward_std": 0.3222523480653763, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.4983905553817749, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 92.79166984558105, | |
| "epoch": 2.442477876106195, | |
| "grad_norm": 0.6516445128630077, | |
| "kl": 0.03466796875, | |
| "learning_rate": 4.063905442268201e-08, | |
| "loss": 0.0, | |
| "reward": 0.6041666865348816, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 67.60416793823242, | |
| "epoch": 2.4601769911504423, | |
| "grad_norm": 1.8554639671156326, | |
| "kl": 0.0423583984375, | |
| "learning_rate": 3.8044736729873795e-08, | |
| "loss": 0.0, | |
| "reward": 0.9469756484031677, | |
| "reward_std": 0.43488872051239014, | |
| "rewards/correct_code_reward_func": 0.479166679084301, | |
| "rewards/len_reward_func": 0.46780893206596375, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 77.45833396911621, | |
| "epoch": 2.47787610619469, | |
| "grad_norm": 3.8300963552021754, | |
| "kl": 0.04541015625, | |
| "learning_rate": 3.552915176479071e-08, | |
| "loss": 0.0, | |
| "reward": 1.0416666865348816, | |
| "reward_std": 0.320542111992836, | |
| "rewards/correct_code_reward_func": 0.5416666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 58.708335876464844, | |
| "epoch": 2.495575221238938, | |
| "grad_norm": 1.6137020616307847, | |
| "kl": 0.06591796875, | |
| "learning_rate": 3.309323396455446e-08, | |
| "loss": 0.0001, | |
| "reward": 0.9375000596046448, | |
| "reward_std": 0.2658637687563896, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 86.12500381469727, | |
| "epoch": 2.5132743362831858, | |
| "grad_norm": 3.390075353900997, | |
| "kl": 0.0819091796875, | |
| "learning_rate": 3.073788817318707e-08, | |
| "loss": 0.0001, | |
| "reward": 0.8744058609008789, | |
| "reward_std": 0.429522842168808, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.4994058310985565, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 76.89583587646484, | |
| "epoch": 2.5309734513274336, | |
| "grad_norm": 17.278809046793295, | |
| "kl": 0.2576904296875, | |
| "learning_rate": 2.846398930549859e-08, | |
| "loss": 0.0003, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.1178511306643486, | |
| "rewards/correct_code_reward_func": 0.20833333395421505, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 86.625, | |
| "epoch": 2.5486725663716814, | |
| "grad_norm": 4.630135346925831, | |
| "kl": 0.0435791015625, | |
| "learning_rate": 2.62723820220917e-08, | |
| "loss": 0.0, | |
| "reward": 0.7428782880306244, | |
| "reward_std": 0.3244539424777031, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.4720449447631836, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 60.54166793823242, | |
| "epoch": 2.566371681415929, | |
| "grad_norm": 1.7403036344539455, | |
| "kl": 0.067626953125, | |
| "learning_rate": 2.416388041560491e-08, | |
| "loss": 0.0001, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.24056155234575272, | |
| "rewards/correct_code_reward_func": 0.520833358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 72.68750190734863, | |
| "epoch": 2.584070796460177, | |
| "grad_norm": 8.835725978147865, | |
| "kl": 0.03240966796875, | |
| "learning_rate": 2.2139267708310454e-08, | |
| "loss": 0.0, | |
| "reward": 1.0000000596046448, | |
| "reward_std": 0.42218445241451263, | |
| "rewards/correct_code_reward_func": 0.5000000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 65.31250381469727, | |
| "epoch": 2.601769911504425, | |
| "grad_norm": 21.996556093676393, | |
| "kl": 0.144287109375, | |
| "learning_rate": 2.019929596117889e-08, | |
| "loss": 0.0001, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.4446708858013153, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 54.270835876464844, | |
| "epoch": 2.6194690265486726, | |
| "grad_norm": 4.1936490058839695, | |
| "kl": 0.099609375, | |
| "learning_rate": 1.8344685794519504e-08, | |
| "loss": 0.0001, | |
| "reward": 1.036210298538208, | |
| "reward_std": 0.37449972331523895, | |
| "rewards/correct_code_reward_func": 0.5416666716337204, | |
| "rewards/len_reward_func": 0.49454365670681, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 99.29166793823242, | |
| "epoch": 2.6371681415929205, | |
| "grad_norm": 63.75890524377285, | |
| "kl": 0.2841796875, | |
| "learning_rate": 1.6576126120299043e-08, | |
| "loss": 0.0003, | |
| "reward": 0.8559730350971222, | |
| "reward_std": 0.3255860209465027, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.4809730350971222, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 107.31250381469727, | |
| "epoch": 2.6548672566371683, | |
| "grad_norm": 36.89829026803691, | |
| "kl": 0.7239990234375, | |
| "learning_rate": 1.4894273886239206e-08, | |
| "loss": 0.0007, | |
| "reward": 0.6250000298023224, | |
| "reward_std": 0.3382536321878433, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 64.35416793823242, | |
| "epoch": 2.672566371681416, | |
| "grad_norm": 2.096221752318607, | |
| "kl": 0.052490234375, | |
| "learning_rate": 1.3299753831787192e-08, | |
| "loss": 0.0001, | |
| "reward": 0.75, | |
| "reward_std": 0.38857419788837433, | |
| "rewards/correct_code_reward_func": 0.25, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 96.64583587646484, | |
| "epoch": 2.6902654867256635, | |
| "grad_norm": 0.6573939843127445, | |
| "kl": 0.032470703125, | |
| "learning_rate": 1.1793158256050706e-08, | |
| "loss": 0.0, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.3521219789981842, | |
| "rewards/correct_code_reward_func": 0.3125000074505806, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 57.45833396911621, | |
| "epoch": 2.7079646017699117, | |
| "grad_norm": 2.8347099035822936, | |
| "kl": 0.051513671875, | |
| "learning_rate": 1.0375046797782867e-08, | |
| "loss": 0.0001, | |
| "reward": 0.8958333730697632, | |
| "reward_std": 0.204109326004982, | |
| "rewards/correct_code_reward_func": 0.395833358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 97.02083587646484, | |
| "epoch": 2.725663716814159, | |
| "grad_norm": 89.9892277373503, | |
| "kl": 0.531982421875, | |
| "learning_rate": 9.045946227499297e-09, | |
| "loss": 0.0005, | |
| "reward": 0.9904971122741699, | |
| "reward_std": 0.4233323186635971, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.46966375410556793, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 75.37500381469727, | |
| "epoch": 2.7433628318584073, | |
| "grad_norm": 41.09603274193404, | |
| "kl": 0.0634765625, | |
| "learning_rate": 7.806350251804483e-09, | |
| "loss": 0.0001, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.3492931127548218, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 92.85417175292969, | |
| "epoch": 2.7610619469026547, | |
| "grad_norm": 1.148928120100101, | |
| "kl": 0.0374755859375, | |
| "learning_rate": 6.6567193299996996e-09, | |
| "loss": 0.0, | |
| "reward": 0.7610294222831726, | |
| "reward_std": 0.485231876373291, | |
| "rewards/correct_code_reward_func": 0.3125, | |
| "rewards/len_reward_func": 0.4485294073820114, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 98.41667175292969, | |
| "epoch": 2.7787610619469025, | |
| "grad_norm": 0.2382852510613484, | |
| "kl": 0.02642822265625, | |
| "learning_rate": 5.597480503041485e-09, | |
| "loss": 0.0, | |
| "reward": 0.5000000149011612, | |
| "reward_std": 0.1178511306643486, | |
| "rewards/correct_code_reward_func": 0.0416666679084301, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 77.20833587646484, | |
| "epoch": 2.7964601769911503, | |
| "grad_norm": 6.517708531460766, | |
| "kl": 0.0968017578125, | |
| "learning_rate": 4.629027234912986e-09, | |
| "loss": 0.0001, | |
| "reward": 0.9473089575767517, | |
| "reward_std": 0.46372126042842865, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.4264756143093109, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 80.27083396911621, | |
| "epoch": 2.814159292035398, | |
| "grad_norm": 3.3787178462046454, | |
| "kl": 0.0364990234375, | |
| "learning_rate": 3.751719266468584e-09, | |
| "loss": 0.0, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.13607724383473396, | |
| "rewards/correct_code_reward_func": 0.3541666679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 77.83333587646484, | |
| "epoch": 2.831858407079646, | |
| "grad_norm": 1.0432210255226626, | |
| "kl": 0.04864501953125, | |
| "learning_rate": 2.9658824818044325e-09, | |
| "loss": 0.0, | |
| "reward": 0.5659293830394745, | |
| "reward_std": 0.21801644191145897, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.4825960248708725, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 68.77083587646484, | |
| "epoch": 2.849557522123894, | |
| "grad_norm": 4.445236186593027, | |
| "kl": 0.061279296875, | |
| "learning_rate": 2.271808787206092e-09, | |
| "loss": 0.0001, | |
| "reward": 1.0243507027626038, | |
| "reward_std": 0.4195573776960373, | |
| "rewards/correct_code_reward_func": 0.6041666865348816, | |
| "rewards/len_reward_func": 0.4201839864253998, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 86.83333587646484, | |
| "epoch": 2.8672566371681416, | |
| "grad_norm": 4.1412440697628705, | |
| "kl": 0.5286865234375, | |
| "learning_rate": 1.6697560027171543e-09, | |
| "loss": 0.0005, | |
| "reward": 0.6875000149011612, | |
| "reward_std": 0.23144195601344109, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 72.6875, | |
| "epoch": 2.8849557522123894, | |
| "grad_norm": 7.176170560573175, | |
| "kl": 0.04962158203125, | |
| "learning_rate": 1.1599477663696843e-09, | |
| "loss": 0.0, | |
| "reward": 0.7519232630729675, | |
| "reward_std": 0.26646677777171135, | |
| "rewards/correct_code_reward_func": 0.27083333395421505, | |
| "rewards/len_reward_func": 0.4810899347066879, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 87.64583587646484, | |
| "epoch": 2.9026548672566372, | |
| "grad_norm": 1.9523625673288243, | |
| "kl": 0.05029296875, | |
| "learning_rate": 7.425734511116998e-10, | |
| "loss": 0.0001, | |
| "reward": 0.6250000298023224, | |
| "reward_std": 0.22233545035123825, | |
| "rewards/correct_code_reward_func": 0.1250000037252903, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 79.41666793823242, | |
| "epoch": 2.920353982300885, | |
| "grad_norm": 29.99743474150218, | |
| "kl": 0.346923828125, | |
| "learning_rate": 4.17788094463023e-10, | |
| "loss": 0.0003, | |
| "reward": 0.5346376597881317, | |
| "reward_std": 0.21488384157419205, | |
| "rewards/correct_code_reward_func": 0.06250000186264515, | |
| "rewards/len_reward_func": 0.4721376597881317, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 71.14583587646484, | |
| "epoch": 2.938053097345133, | |
| "grad_norm": 2.049279192654788, | |
| "kl": 0.037353515625, | |
| "learning_rate": 1.857123409250705e-10, | |
| "loss": 0.0, | |
| "reward": 0.8958333730697632, | |
| "reward_std": 0.24056155234575272, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 57.000003814697266, | |
| "epoch": 2.9557522123893807, | |
| "grad_norm": 11.198599869127536, | |
| "kl": 0.0416259765625, | |
| "learning_rate": 4.6432397166285e-11, | |
| "loss": 0.0, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.29602527618408203, | |
| "rewards/correct_code_reward_func": 0.4791666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 50.625, | |
| "epoch": 2.9734513274336285, | |
| "grad_norm": 26.983307381671835, | |
| "kl": 0.271484375, | |
| "learning_rate": 0.0, | |
| "loss": 0.0003, | |
| "reward": 0.9375, | |
| "reward_std": 0.39486490190029144, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 169 | |
| }, | |
| { | |
| "epoch": 2.9734513274336285, | |
| "step": 169, | |
| "total_flos": 0.0, | |
| "train_loss": 6.70051409522318e-05, | |
| "train_runtime": 7347.2678, | |
| "train_samples_per_second": 0.138, | |
| "train_steps_per_second": 0.023 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 169, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 4, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |