| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.11428571428571428, |
| "eval_steps": 500, |
| "global_step": 100, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 2700.4271850585938, |
| "cov_mean": -6.0587970438064076e-05, |
| "cov_std": 0.35307812318205833, |
| "entropy": 0.3681640625, |
| "epoch": 0.001142857142857143, |
| "grad_norm": 0.45492634177207947, |
| "kl": 0.0, |
| "learning_rate": 1e-07, |
| "loss": -0.0382, |
| "reward": 0.7604166893288493, |
| "reward_std": 0.4268697127699852, |
| "rewards/accuracy_reward": 0.25000001303851604, |
| "rewards/format_reward": 0.5104166669771075, |
| "step": 1, |
| "w_high_ratio": 0.2200421690940857, |
| "w_low_ratio": 0.03663695091381669, |
| "w_max": 2.1593789756298065, |
| "w_mean": 1.4711343348026276, |
| "w_min": 6.525355682266089e-35, |
| "w_std": 0.2659660503268242 |
| }, |
| { |
| "completion_length": 3127.3958435058594, |
| "cov_mean": -2.155053698515985e-05, |
| "cov_std": 0.310540571808815, |
| "entropy": 0.353515625, |
| "epoch": 0.002285714285714286, |
| "grad_norm": 0.5142282843589783, |
| "kl": 0.0, |
| "learning_rate": 2e-07, |
| "loss": 0.0049, |
| "reward": 0.6458333637565374, |
| "reward_std": 0.4249730706214905, |
| "rewards/accuracy_reward": 0.2812500102445483, |
| "rewards/format_reward": 0.3645833386108279, |
| "step": 2, |
| "w_high_ratio": 0.05183619633316994, |
| "w_low_ratio": 0.036958135198801756, |
| "w_max": 1.8325217366218567, |
| "w_mean": 1.2113382518291473, |
| "w_min": 0.0, |
| "w_std": 0.20957503467798233 |
| }, |
| { |
| "completion_length": 3733.9583740234375, |
| "cov_mean": -2.010253456319333e-05, |
| "cov_std": 0.2787674851715565, |
| "entropy": 0.451171875, |
| "epoch": 0.0034285714285714284, |
| "grad_norm": 0.28526851534843445, |
| "kl": 4.315376281738281e-05, |
| "learning_rate": 3e-07, |
| "loss": 0.0391, |
| "reward": 0.16666667070239782, |
| "reward_std": 0.31075509265065193, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/format_reward": 0.13541666697710752, |
| "step": 3, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03727753087878227, |
| "w_max": 1.426508367061615, |
| "w_mean": 1.069275826215744, |
| "w_min": 0.0, |
| "w_std": 0.17276715114712715 |
| }, |
| { |
| "completion_length": 2456.5104370117188, |
| "cov_mean": 6.050579213479068e-05, |
| "cov_std": 0.4746721386909485, |
| "entropy": 0.404296875, |
| "epoch": 0.004571428571428572, |
| "grad_norm": 0.4348958432674408, |
| "kl": 3.764033317565918e-05, |
| "learning_rate": 4e-07, |
| "loss": -0.0128, |
| "reward": 0.8125000223517418, |
| "reward_std": 0.4816601872444153, |
| "rewards/accuracy_reward": 0.1354166679084301, |
| "rewards/format_reward": 0.677083358168602, |
| "step": 4, |
| "w_high_ratio": 0.09904679656028748, |
| "w_low_ratio": 0.03708651335909963, |
| "w_max": 2.0615443289279938, |
| "w_mean": 1.430301010608673, |
| "w_min": 2.2357144385761694e-24, |
| "w_std": 0.25941959768533707 |
| }, |
| { |
| "completion_length": 3495.9584350585938, |
| "cov_mean": -0.00011635808687060489, |
| "cov_std": 0.43240153789520264, |
| "entropy": 0.4658203125, |
| "epoch": 0.005714285714285714, |
| "grad_norm": 0.44890686869621277, |
| "kl": 4.1797757148742676e-05, |
| "learning_rate": 5e-07, |
| "loss": -0.077, |
| "reward": 0.38541667722165585, |
| "reward_std": 0.41880372166633606, |
| "rewards/accuracy_reward": 0.0416666679084301, |
| "rewards/format_reward": 0.34375001303851604, |
| "step": 5, |
| "w_high_ratio": 0.057075776159763336, |
| "w_low_ratio": 0.04613090865314007, |
| "w_max": 1.8747033178806305, |
| "w_mean": 1.2025023698806763, |
| "w_min": 2.0185204017920427e-32, |
| "w_std": 0.24863697960972786 |
| }, |
| { |
| "completion_length": 3429.7604370117188, |
| "cov_mean": 7.305060944418074e-05, |
| "cov_std": 0.3747733160853386, |
| "entropy": 0.4658203125, |
| "epoch": 0.006857142857142857, |
| "grad_norm": 0.323794424533844, |
| "kl": 4.607439041137695e-05, |
| "learning_rate": 6e-07, |
| "loss": 0.0289, |
| "reward": 0.4791666828095913, |
| "reward_std": 0.4919559434056282, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/format_reward": 0.3125000111758709, |
| "step": 6, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.045196511782705784, |
| "w_max": 1.4718832075595856, |
| "w_mean": 1.11675626039505, |
| "w_min": 0.25, |
| "w_std": 0.20970172435045242 |
| }, |
| { |
| "completion_length": 3298.1875610351562, |
| "cov_mean": 1.544893893878907e-05, |
| "cov_std": 0.5214613378047943, |
| "entropy": 0.3837890625, |
| "epoch": 0.008, |
| "grad_norm": 0.3830799460411072, |
| "kl": 2.524256706237793e-05, |
| "learning_rate": 7e-07, |
| "loss": -0.0429, |
| "reward": 0.802083358168602, |
| "reward_std": 0.5995434895157814, |
| "rewards/accuracy_reward": 0.2083333432674408, |
| "rewards/format_reward": 0.5937500223517418, |
| "step": 7, |
| "w_high_ratio": 0.10538750886917114, |
| "w_low_ratio": 0.0529888030141592, |
| "w_max": 1.5837838649749756, |
| "w_mean": 1.266415923833847, |
| "w_min": 1.0439673559219887e-43, |
| "w_std": 0.28801921010017395 |
| }, |
| { |
| "completion_length": 2903.697998046875, |
| "cov_mean": -8.183626050595194e-06, |
| "cov_std": 0.2509700208902359, |
| "entropy": 0.355712890625, |
| "epoch": 0.009142857142857144, |
| "grad_norm": 0.2550622820854187, |
| "kl": 1.93864107131958e-05, |
| "learning_rate": 8e-07, |
| "loss": -0.0211, |
| "reward": 0.7812500298023224, |
| "reward_std": 0.3037422299385071, |
| "rewards/accuracy_reward": 0.3333333432674408, |
| "rewards/format_reward": 0.4479166865348816, |
| "step": 8, |
| "w_high_ratio": 0.152647502720356, |
| "w_low_ratio": 0.020135570783168077, |
| "w_max": 1.6893351376056671, |
| "w_mean": 1.325122356414795, |
| "w_min": 0.5, |
| "w_std": 0.13287343829870224 |
| }, |
| { |
| "completion_length": 3427.0001220703125, |
| "cov_mean": 2.1584107344096992e-05, |
| "cov_std": 0.41377831995487213, |
| "entropy": 0.4404296875, |
| "epoch": 0.010285714285714285, |
| "grad_norm": 0.36070960760116577, |
| "kl": 4.297494888305664e-05, |
| "learning_rate": 9e-07, |
| "loss": -0.0144, |
| "reward": 0.4375000149011612, |
| "reward_std": 0.5150812789797783, |
| "rewards/accuracy_reward": 0.11458333395421505, |
| "rewards/format_reward": 0.322916679084301, |
| "step": 9, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.053231239318847656, |
| "w_max": 1.665610283613205, |
| "w_mean": 1.168578028678894, |
| "w_min": 0.0, |
| "w_std": 0.2691279575228691 |
| }, |
| { |
| "completion_length": 2921.260498046875, |
| "cov_mean": -0.00011126142680950579, |
| "cov_std": 0.37009984999895096, |
| "entropy": 0.34130859375, |
| "epoch": 0.011428571428571429, |
| "grad_norm": 0.463008314371109, |
| "kl": 2.7805566787719727e-05, |
| "learning_rate": 1e-06, |
| "loss": -0.0304, |
| "reward": 0.604166679084301, |
| "reward_std": 0.49994633346796036, |
| "rewards/accuracy_reward": 0.19791667349636555, |
| "rewards/format_reward": 0.4062500111758709, |
| "step": 10, |
| "w_high_ratio": 0.10105682164430618, |
| "w_low_ratio": 0.039817056618630886, |
| "w_max": 2.203812062740326, |
| "w_mean": 1.3322690725326538, |
| "w_min": 0.0, |
| "w_std": 0.24996469356119633 |
| }, |
| { |
| "completion_length": 3739.4583740234375, |
| "cov_mean": 1.6300052948281518e-05, |
| "cov_std": 0.22599393874406815, |
| "entropy": 0.3876953125, |
| "epoch": 0.012571428571428572, |
| "grad_norm": 0.2399490475654602, |
| "kl": 2.893805503845215e-05, |
| "learning_rate": 9.997258721585931e-07, |
| "loss": 0.0375, |
| "reward": 0.1979166716337204, |
| "reward_std": 0.3496348336338997, |
| "rewards/accuracy_reward": 0.07291666697710752, |
| "rewards/format_reward": 0.12500000558793545, |
| "step": 11, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03310772683471441, |
| "w_max": 1.3216401040554047, |
| "w_mean": 1.05658358335495, |
| "w_min": 0.25, |
| "w_std": 0.13504411466419697 |
| }, |
| { |
| "completion_length": 2672.3959045410156, |
| "cov_mean": -1.72590916918125e-05, |
| "cov_std": 0.4432706832885742, |
| "entropy": 0.38525390625, |
| "epoch": 0.013714285714285714, |
| "grad_norm": 0.5923348665237427, |
| "kl": 3.87728214263916e-05, |
| "learning_rate": 9.989038226169207e-07, |
| "loss": -0.0398, |
| "reward": 0.7916666865348816, |
| "reward_std": 0.44830071553587914, |
| "rewards/accuracy_reward": 0.15625, |
| "rewards/format_reward": 0.6354166865348816, |
| "step": 12, |
| "w_high_ratio": 0.18827838450670242, |
| "w_low_ratio": 0.039877891540527344, |
| "w_max": 2.278617113828659, |
| "w_mean": 1.533039778470993, |
| "w_min": 0.0, |
| "w_std": 0.30229373276233673 |
| }, |
| { |
| "completion_length": 3129.9168090820312, |
| "cov_mean": 5.8586666455084924e-05, |
| "cov_std": 0.3728199526667595, |
| "entropy": 0.3896484375, |
| "epoch": 0.014857142857142857, |
| "grad_norm": 0.7794428467750549, |
| "kl": 2.4527311325073242e-05, |
| "learning_rate": 9.975348529157229e-07, |
| "loss": -0.0268, |
| "reward": 0.6145833507180214, |
| "reward_std": 0.3987155184149742, |
| "rewards/accuracy_reward": 0.1979166716337204, |
| "rewards/format_reward": 0.4166666716337204, |
| "step": 13, |
| "w_high_ratio": 0.1489243432879448, |
| "w_low_ratio": 0.03552113357000053, |
| "w_max": 1.913963943719864, |
| "w_mean": 1.3093808591365814, |
| "w_min": 0.0, |
| "w_std": 0.22065079398453236 |
| }, |
| { |
| "completion_length": 2925.8750610351562, |
| "cov_mean": 7.129182449716609e-06, |
| "cov_std": 0.3760656714439392, |
| "entropy": 0.353515625, |
| "epoch": 0.016, |
| "grad_norm": 0.6446069478988647, |
| "kl": 2.3916363716125488e-05, |
| "learning_rate": 9.956206309337066e-07, |
| "loss": -0.0123, |
| "reward": 0.6562500298023224, |
| "reward_std": 0.41464143246412277, |
| "rewards/accuracy_reward": 0.17708334233611822, |
| "rewards/format_reward": 0.4791666716337204, |
| "step": 14, |
| "w_high_ratio": 0.04337232932448387, |
| "w_low_ratio": 0.04529819544404745, |
| "w_max": 1.892830729484558, |
| "w_mean": 1.2675927579402924, |
| "w_min": 7.264961823368798e-41, |
| "w_std": 0.26389942318201065 |
| }, |
| { |
| "completion_length": 2977.6458740234375, |
| "cov_mean": 2.068816957034869e-05, |
| "cov_std": 0.2604588344693184, |
| "entropy": 0.3759765625, |
| "epoch": 0.017142857142857144, |
| "grad_norm": 0.4150949716567993, |
| "kl": 3.0860304832458496e-05, |
| "learning_rate": 9.931634888554935e-07, |
| "loss": -0.0032, |
| "reward": 0.6041666902601719, |
| "reward_std": 0.25453949719667435, |
| "rewards/accuracy_reward": 0.20833333861082792, |
| "rewards/format_reward": 0.3958333358168602, |
| "step": 15, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.027457581600174308, |
| "w_max": 1.3896198272705078, |
| "w_mean": 1.1120553016662598, |
| "w_min": 0.0, |
| "w_std": 0.1376371644437313 |
| }, |
| { |
| "completion_length": 3860.1771850585938, |
| "cov_mean": 3.589540312987083e-05, |
| "cov_std": 0.1900031939148903, |
| "entropy": 0.45751953125, |
| "epoch": 0.018285714285714287, |
| "grad_norm": 0.25400859117507935, |
| "kl": 3.0994415283203125e-05, |
| "learning_rate": 9.901664203302124e-07, |
| "loss": 0.0468, |
| "reward": 0.1458333395421505, |
| "reward_std": 0.24955713748931885, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/format_reward": 0.08333333674818277, |
| "step": 16, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.028868926223367453, |
| "w_max": 1.2315952479839325, |
| "w_mean": 1.0204393565654755, |
| "w_min": 0.25, |
| "w_std": 0.10705284960567951 |
| }, |
| { |
| "completion_length": 2540.4895935058594, |
| "cov_mean": -0.0001341435610129338, |
| "cov_std": 0.47027114033699036, |
| "entropy": 0.45166015625, |
| "epoch": 0.019428571428571427, |
| "grad_norm": 0.46505773067474365, |
| "kl": 5.486607551574707e-05, |
| "learning_rate": 9.866330768241983e-07, |
| "loss": -0.0001, |
| "reward": 0.812500037252903, |
| "reward_std": 0.49661170691251755, |
| "rewards/accuracy_reward": 0.2291666716337204, |
| "rewards/format_reward": 0.5833333507180214, |
| "step": 17, |
| "w_high_ratio": 0.15490177273750305, |
| "w_low_ratio": 0.04533949866890907, |
| "w_max": 2.4269833266735077, |
| "w_mean": 1.407887727022171, |
| "w_min": 6.344169701078957e-36, |
| "w_std": 0.2766948938369751 |
| }, |
| { |
| "completion_length": 3100.604248046875, |
| "cov_mean": 9.316997420683037e-06, |
| "cov_std": 0.345074113458395, |
| "entropy": 0.36865234375, |
| "epoch": 0.02057142857142857, |
| "grad_norm": 0.26780965924263, |
| "kl": 3.306567668914795e-05, |
| "learning_rate": 9.825677631722435e-07, |
| "loss": 0.0093, |
| "reward": 0.5208333507180214, |
| "reward_std": 0.4629998616874218, |
| "rewards/accuracy_reward": 0.11458333861082792, |
| "rewards/format_reward": 0.4062500074505806, |
| "step": 18, |
| "w_high_ratio": 0.12114312499761581, |
| "w_low_ratio": 0.046439004596322775, |
| "w_max": 2.0130829215049744, |
| "w_mean": 1.2991048991680145, |
| "w_min": 0.0, |
| "w_std": 0.22966066375374794 |
| }, |
| { |
| "completion_length": 3111.229248046875, |
| "cov_mean": -1.3287149158713873e-10, |
| "cov_std": 0.305859737098217, |
| "entropy": 0.37939453125, |
| "epoch": 0.021714285714285714, |
| "grad_norm": 0.2747480869293213, |
| "kl": 4.106760025024414e-05, |
| "learning_rate": 9.779754323328192e-07, |
| "loss": -0.0265, |
| "reward": 0.8437500251457095, |
| "reward_std": 0.44304458051919937, |
| "rewards/accuracy_reward": 0.3854166818782687, |
| "rewards/format_reward": 0.4583333535119891, |
| "step": 19, |
| "w_high_ratio": 0.007512019481509924, |
| "w_low_ratio": 0.03380460711196065, |
| "w_max": 1.7184137105941772, |
| "w_mean": 1.2587754130363464, |
| "w_min": 0.0, |
| "w_std": 0.17892248183488846 |
| }, |
| { |
| "completion_length": 2602.0000610351562, |
| "cov_mean": -4.431823981576599e-06, |
| "cov_std": 0.3394951745867729, |
| "entropy": 0.334716796875, |
| "epoch": 0.022857142857142857, |
| "grad_norm": 0.2722320854663849, |
| "kl": 7.30752944946289e-05, |
| "learning_rate": 9.728616793536587e-07, |
| "loss": -0.016, |
| "reward": 0.989583358168602, |
| "reward_std": 0.41970477253198624, |
| "rewards/accuracy_reward": 0.2812500074505806, |
| "rewards/format_reward": 0.708333358168602, |
| "step": 20, |
| "w_high_ratio": 0.053037188947200775, |
| "w_low_ratio": 0.03638318553566933, |
| "w_max": 1.8807734847068787, |
| "w_mean": 1.3341735899448395, |
| "w_min": 0.28736552596092224, |
| "w_std": 0.21230074390769005 |
| }, |
| { |
| "completion_length": 2938.9896240234375, |
| "cov_mean": 1.115940267482074e-05, |
| "cov_std": 0.3257220685482025, |
| "entropy": 0.42919921875, |
| "epoch": 0.024, |
| "grad_norm": 0.9565598964691162, |
| "kl": 0.00023573637008666992, |
| "learning_rate": 9.672327345550543e-07, |
| "loss": 0.0067, |
| "reward": 0.604166692122817, |
| "reward_std": 0.4190382584929466, |
| "rewards/accuracy_reward": 0.17708333674818277, |
| "rewards/format_reward": 0.4270833460614085, |
| "step": 21, |
| "w_high_ratio": 0.2210865318775177, |
| "w_low_ratio": 0.03242270671762526, |
| "w_max": 1.8820680975914001, |
| "w_mean": 1.4674345552921295, |
| "w_min": 2.9579121420725567e-37, |
| "w_std": 0.23061984032392502 |
| }, |
| { |
| "completion_length": 1817.8125305175781, |
| "cov_mean": 3.471795207588002e-05, |
| "cov_std": 0.4554404318332672, |
| "entropy": 0.37548828125, |
| "epoch": 0.025142857142857144, |
| "grad_norm": 0.6403908729553223, |
| "kl": 0.0001913309097290039, |
| "learning_rate": 9.610954559391704e-07, |
| "loss": -0.0476, |
| "reward": 1.1041667014360428, |
| "reward_std": 0.36877378076314926, |
| "rewards/accuracy_reward": 0.28125000558793545, |
| "rewards/format_reward": 0.8229166716337204, |
| "step": 22, |
| "w_high_ratio": 0.3237294889986515, |
| "w_low_ratio": 0.03283138480037451, |
| "w_max": 3.265183061361313, |
| "w_mean": 1.977644294500351, |
| "w_min": 5.22007475166266e-30, |
| "w_std": 0.34112248569726944 |
| }, |
| { |
| "completion_length": 2787.4896850585938, |
| "cov_mean": 3.710087821673369e-05, |
| "cov_std": 0.3974616304039955, |
| "entropy": 0.38525390625, |
| "epoch": 0.026285714285714287, |
| "grad_norm": 0.6307149529457092, |
| "kl": 0.00013697147369384766, |
| "learning_rate": 9.54457320834625e-07, |
| "loss": -0.0959, |
| "reward": 0.6041666716337204, |
| "reward_std": 0.4507869631052017, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/format_reward": 0.4583333358168602, |
| "step": 23, |
| "w_high_ratio": 0.18567950278520584, |
| "w_low_ratio": 0.042684963904321194, |
| "w_max": 2.490270048379898, |
| "w_mean": 1.4617709517478943, |
| "w_min": 1.9980297628321344e-34, |
| "w_std": 0.30704520270228386 |
| }, |
| { |
| "completion_length": 3070.3125610351562, |
| "cov_mean": 5.4191557865124196e-05, |
| "cov_std": 0.5094943791627884, |
| "entropy": 0.367431640625, |
| "epoch": 0.027428571428571427, |
| "grad_norm": 0.5826663374900818, |
| "kl": 0.0001633763313293457, |
| "learning_rate": 9.473264167865171e-07, |
| "loss": -0.02, |
| "reward": 0.8229166939854622, |
| "reward_std": 0.5960981249809265, |
| "rewards/accuracy_reward": 0.2708333358168602, |
| "rewards/format_reward": 0.5520833507180214, |
| "step": 24, |
| "w_high_ratio": 0.16660767421126366, |
| "w_low_ratio": 0.05264063738286495, |
| "w_max": 2.2747649252414703, |
| "w_mean": 1.4129657745361328, |
| "w_min": 0.0, |
| "w_std": 0.3425128310918808 |
| }, |
| { |
| "completion_length": 3037.8125, |
| "cov_mean": -3.0258611332101282e-05, |
| "cov_std": 0.2974156364798546, |
| "entropy": 0.4462890625, |
| "epoch": 0.02857142857142857, |
| "grad_norm": 0.33458662033081055, |
| "kl": 0.00022605061531066895, |
| "learning_rate": 9.397114317029974e-07, |
| "loss": 0.0202, |
| "reward": 0.6875000149011612, |
| "reward_std": 0.4009811282157898, |
| "rewards/accuracy_reward": 0.2395833432674408, |
| "rewards/format_reward": 0.4479166865348816, |
| "step": 25, |
| "w_high_ratio": 0.1122930608689785, |
| "w_low_ratio": 0.034513187827542424, |
| "w_max": 1.991903692483902, |
| "w_mean": 1.2860748171806335, |
| "w_min": 0.25, |
| "w_std": 0.2155727706849575 |
| }, |
| { |
| "completion_length": 3117.4584350585938, |
| "cov_mean": -8.455652073280362e-05, |
| "cov_std": 0.23132415860891342, |
| "entropy": 0.42626953125, |
| "epoch": 0.029714285714285714, |
| "grad_norm": 0.41856279969215393, |
| "kl": 6.198883056640625e-05, |
| "learning_rate": 9.316216432703916e-07, |
| "loss": -0.053, |
| "reward": 0.6562500074505806, |
| "reward_std": 0.20796867460012436, |
| "rewards/accuracy_reward": 0.1979166716337204, |
| "rewards/format_reward": 0.4583333432674408, |
| "step": 26, |
| "w_high_ratio": 0.04478337988257408, |
| "w_low_ratio": 0.011211750796064734, |
| "w_max": 1.7470227479934692, |
| "w_mean": 1.224214792251587, |
| "w_min": 0.25, |
| "w_std": 0.10643218830227852 |
| }, |
| { |
| "completion_length": 3314.1041870117188, |
| "cov_mean": 2.3687035081820795e-05, |
| "cov_std": 0.44449782371520996, |
| "entropy": 0.44921875, |
| "epoch": 0.030857142857142857, |
| "grad_norm": 0.3447716236114502, |
| "kl": 0.0005403757095336914, |
| "learning_rate": 9.230669076497687e-07, |
| "loss": -0.0255, |
| "reward": 0.6354166865348816, |
| "reward_std": 0.5543549209833145, |
| "rewards/accuracy_reward": 0.1875000037252903, |
| "rewards/format_reward": 0.447916679084301, |
| "step": 27, |
| "w_high_ratio": 0.04345417395234108, |
| "w_low_ratio": 0.047908838372677565, |
| "w_max": 1.680276095867157, |
| "w_mean": 1.1971288323402405, |
| "w_min": 0.0, |
| "w_std": 0.23072999715805054 |
| }, |
| { |
| "completion_length": 3026.5626220703125, |
| "cov_mean": 4.19466396124335e-05, |
| "cov_std": 0.4268927723169327, |
| "entropy": 0.4091796875, |
| "epoch": 0.032, |
| "grad_norm": 0.4165506958961487, |
| "kl": 0.00021666288375854492, |
| "learning_rate": 9.140576474687263e-07, |
| "loss": -0.0277, |
| "reward": 0.8437500298023224, |
| "reward_std": 0.4840777963399887, |
| "rewards/accuracy_reward": 0.3750000149011612, |
| "rewards/format_reward": 0.4687500149011612, |
| "step": 28, |
| "w_high_ratio": 0.09859117493033409, |
| "w_low_ratio": 0.04558967752382159, |
| "w_max": 2.2461937069892883, |
| "w_mean": 1.3320654332637787, |
| "w_min": 0.0, |
| "w_std": 0.2779119908809662 |
| }, |
| { |
| "completion_length": 3620.1563110351562, |
| "cov_mean": -3.5699086538443225e-05, |
| "cov_std": 0.34214310348033905, |
| "entropy": 0.43017578125, |
| "epoch": 0.03314285714285714, |
| "grad_norm": 0.3975992202758789, |
| "kl": 0.0004680156707763672, |
| "learning_rate": 9.046048391230247e-07, |
| "loss": 0.0814, |
| "reward": 0.2291666716337204, |
| "reward_std": 0.39027639478445053, |
| "rewards/accuracy_reward": 0.0625, |
| "rewards/format_reward": 0.1666666716337204, |
| "step": 29, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04788992367684841, |
| "w_max": 1.3650976717472076, |
| "w_mean": 1.0994165241718292, |
| "w_min": 0.25, |
| "w_std": 0.22099098190665245 |
| }, |
| { |
| "completion_length": 3238.17724609375, |
| "cov_mean": 5.147313459019642e-06, |
| "cov_std": 0.466504842042923, |
| "entropy": 0.38134765625, |
| "epoch": 0.03428571428571429, |
| "grad_norm": 0.3571697175502777, |
| "kl": 0.0008358955383300781, |
| "learning_rate": 8.9471999940354e-07, |
| "loss": 0.0066, |
| "reward": 0.7395833693444729, |
| "reward_std": 0.6347565650939941, |
| "rewards/accuracy_reward": 0.2708333423361182, |
| "rewards/format_reward": 0.4687500074505806, |
| "step": 30, |
| "w_high_ratio": 0.09309378266334534, |
| "w_low_ratio": 0.04863026458770037, |
| "w_max": 1.7810659110546112, |
| "w_mean": 1.2216298282146454, |
| "w_min": 0.0, |
| "w_std": 0.26985964179039 |
| }, |
| { |
| "completion_length": 3252.6771240234375, |
| "cov_mean": -2.1283684873196762e-05, |
| "cov_std": 0.31266890093684196, |
| "entropy": 0.376953125, |
| "epoch": 0.03542857142857143, |
| "grad_norm": 0.28267669677734375, |
| "kl": 0.00033402442932128906, |
| "learning_rate": 8.844151714648274e-07, |
| "loss": 0.0136, |
| "reward": 0.53125, |
| "reward_std": 0.3965660147368908, |
| "rewards/accuracy_reward": 0.1979166679084301, |
| "rewards/format_reward": 0.3333333358168602, |
| "step": 31, |
| "w_high_ratio": 0.13101918064057827, |
| "w_low_ratio": 0.03200887469574809, |
| "w_max": 1.9720379710197449, |
| "w_mean": 1.2918123006820679, |
| "w_min": 0.25, |
| "w_std": 0.22479910403490067 |
| }, |
| { |
| "completion_length": 3411.0208740234375, |
| "cov_mean": -2.1311116142896935e-06, |
| "cov_std": 0.37892643362283707, |
| "entropy": 0.439453125, |
| "epoch": 0.036571428571428574, |
| "grad_norm": 0.245382159948349, |
| "kl": 0.00054168701171875, |
| "learning_rate": 8.737029101523929e-07, |
| "loss": 0.0361, |
| "reward": 0.6666666716337204, |
| "reward_std": 0.5180360972881317, |
| "rewards/accuracy_reward": 0.2916666669771075, |
| "rewards/format_reward": 0.3750000037252903, |
| "step": 32, |
| "w_high_ratio": 0.0408792570233345, |
| "w_low_ratio": 0.0450265696272254, |
| "w_max": 1.5775762796401978, |
| "w_mean": 1.1778713762760162, |
| "w_min": 0.0, |
| "w_std": 0.22701822221279144 |
| }, |
| { |
| "completion_length": 3525.760498046875, |
| "cov_mean": -4.6630036194983404e-05, |
| "cov_std": 0.3494578003883362, |
| "entropy": 0.37939453125, |
| "epoch": 0.037714285714285714, |
| "grad_norm": 0.2607223689556122, |
| "kl": 0.0014134645462036133, |
| "learning_rate": 8.625962667065487e-07, |
| "loss": 0.0251, |
| "reward": 0.47916669212281704, |
| "reward_std": 0.5007270351052284, |
| "rewards/accuracy_reward": 0.18750001024454832, |
| "rewards/format_reward": 0.2916666818782687, |
| "step": 33, |
| "w_high_ratio": 0.038857411593198776, |
| "w_low_ratio": 0.04762307275086641, |
| "w_max": 1.5454095602035522, |
| "w_mean": 1.1393149495124817, |
| "w_min": 0.0, |
| "w_std": 0.22297161258757114 |
| }, |
| { |
| "completion_length": 2453.9791870117188, |
| "cov_mean": -9.226416841556784e-05, |
| "cov_std": 0.4420240595936775, |
| "entropy": 0.43701171875, |
| "epoch": 0.038857142857142854, |
| "grad_norm": 0.5145274996757507, |
| "kl": 0.0017969608306884766, |
| "learning_rate": 8.511087728614862e-07, |
| "loss": -0.018, |
| "reward": 0.9583333684131503, |
| "reward_std": 0.4463650956749916, |
| "rewards/accuracy_reward": 0.375000006519258, |
| "rewards/format_reward": 0.5833333432674408, |
| "step": 34, |
| "w_high_ratio": 0.13110801950097084, |
| "w_low_ratio": 0.052741190418601036, |
| "w_max": 2.1099403500556946, |
| "w_mean": 1.343121200799942, |
| "w_min": 0.0, |
| "w_std": 0.31580859050154686 |
| }, |
| { |
| "completion_length": 3289.4376220703125, |
| "cov_mean": 5.6672024584258907e-05, |
| "cov_std": 0.3820243999361992, |
| "entropy": 0.42529296875, |
| "epoch": 0.04, |
| "grad_norm": 0.35655274987220764, |
| "kl": 0.0010710954666137695, |
| "learning_rate": 8.392544243589427e-07, |
| "loss": 0.0185, |
| "reward": 0.6458333432674408, |
| "reward_std": 0.5987429097294807, |
| "rewards/accuracy_reward": 0.2708333395421505, |
| "rewards/format_reward": 0.3750000037252903, |
| "step": 35, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.05319366045296192, |
| "w_max": 1.4592802226543427, |
| "w_mean": 1.0992730259895325, |
| "w_min": 3.853570776893247e-45, |
| "w_std": 0.22482672333717346 |
| }, |
| { |
| "completion_length": 3621.78125, |
| "cov_mean": 7.363205986621324e-05, |
| "cov_std": 0.19353903084993362, |
| "entropy": 0.4765625, |
| "epoch": 0.04114285714285714, |
| "grad_norm": 0.30453404784202576, |
| "kl": 0.001312553882598877, |
| "learning_rate": 8.270476638965461e-07, |
| "loss": -0.0147, |
| "reward": 0.1666666716337204, |
| "reward_std": 0.17689276114106178, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/format_reward": 0.15625, |
| "step": 36, |
| "w_high_ratio": 0.006425275932997465, |
| "w_low_ratio": 0.022115232422947884, |
| "w_max": 1.4147455394268036, |
| "w_mean": 1.117747575044632, |
| "w_min": 0.5, |
| "w_std": 0.1310933344066143 |
| }, |
| { |
| "completion_length": 3511.2813110351562, |
| "cov_mean": -8.677186542627169e-06, |
| "cov_std": 0.2782720513641834, |
| "entropy": 0.43408203125, |
| "epoch": 0.04228571428571429, |
| "grad_norm": 0.357716828584671, |
| "kl": 0.0007710456848144531, |
| "learning_rate": 8.145033635316128e-07, |
| "loss": -0.0037, |
| "reward": 0.21875000279396772, |
| "reward_std": 0.20087094232439995, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/format_reward": 0.20833333861082792, |
| "step": 37, |
| "w_high_ratio": 0.07502041757106781, |
| "w_low_ratio": 0.02649637684226036, |
| "w_max": 1.848203331232071, |
| "w_mean": 1.2203471958637238, |
| "w_min": 0.25, |
| "w_std": 0.18464426696300507 |
| }, |
| { |
| "completion_length": 3627.8438110351562, |
| "cov_mean": -6.837155524408445e-05, |
| "cov_std": 0.14785446226596832, |
| "entropy": 0.44921875, |
| "epoch": 0.04342857142857143, |
| "grad_norm": 0.28619271516799927, |
| "kl": 0.0005115270614624023, |
| "learning_rate": 8.01636806561836e-07, |
| "loss": 0.0051, |
| "reward": 0.3645833358168602, |
| "reward_std": 0.25648824870586395, |
| "rewards/accuracy_reward": 0.1770833395421505, |
| "rewards/format_reward": 0.1875000111758709, |
| "step": 38, |
| "w_high_ratio": 0.051015257835388184, |
| "w_low_ratio": 0.01901705004274845, |
| "w_max": 1.441689670085907, |
| "w_mean": 1.1396986544132233, |
| "w_min": 0.5, |
| "w_std": 0.09881949052214622 |
| }, |
| { |
| "completion_length": 3050.979248046875, |
| "cov_mean": 4.009085705547477e-05, |
| "cov_std": 0.20474716648459435, |
| "entropy": 0.366943359375, |
| "epoch": 0.044571428571428574, |
| "grad_norm": 0.2579488158226013, |
| "kl": 0.0016760826110839844, |
| "learning_rate": 7.884636689049422e-07, |
| "loss": -0.0486, |
| "reward": 0.7083333358168602, |
| "reward_std": 0.2621734067797661, |
| "rewards/accuracy_reward": 0.21875, |
| "rewards/format_reward": 0.4895833358168602, |
| "step": 39, |
| "w_high_ratio": 0.04251348227262497, |
| "w_low_ratio": 0.022377901477739215, |
| "w_max": 1.5512709319591522, |
| "w_mean": 1.1656133532524109, |
| "w_min": 0.25, |
| "w_std": 0.12140309438109398 |
| }, |
| { |
| "completion_length": 2983.6876220703125, |
| "cov_mean": -2.4692089027666952e-05, |
| "cov_std": 0.3223262317478657, |
| "entropy": 0.39697265625, |
| "epoch": 0.045714285714285714, |
| "grad_norm": 0.3634110987186432, |
| "kl": 0.003978729248046875, |
| "learning_rate": 7.75e-07, |
| "loss": 0.0235, |
| "reward": 0.7083333507180214, |
| "reward_std": 0.39669927954673767, |
| "rewards/accuracy_reward": 0.20833334140479565, |
| "rewards/format_reward": 0.5000000149011612, |
| "step": 40, |
| "w_high_ratio": 0.023237863555550575, |
| "w_low_ratio": 0.03736424632370472, |
| "w_max": 1.7330412566661835, |
| "w_mean": 1.253759890794754, |
| "w_min": 0.0, |
| "w_std": 0.2265719212591648 |
| }, |
| { |
| "completion_length": 3342.916748046875, |
| "cov_mean": 7.446667950716801e-06, |
| "cov_std": 0.36464616656303406, |
| "entropy": 0.38232421875, |
| "epoch": 0.046857142857142854, |
| "grad_norm": 0.3088231682777405, |
| "kl": 0.0006549358367919922, |
| "learning_rate": 7.612622032536507e-07, |
| "loss": 0.01, |
| "reward": 0.416666679084301, |
| "reward_std": 0.49482717365026474, |
| "rewards/accuracy_reward": 0.10416667070239782, |
| "rewards/format_reward": 0.3125000037252903, |
| "step": 41, |
| "w_high_ratio": 0.02740619331598282, |
| "w_low_ratio": 0.045960213989019394, |
| "w_max": 1.735625982284546, |
| "w_mean": 1.2091110348701477, |
| "w_min": 0.0, |
| "w_std": 0.24962139129638672 |
| }, |
| { |
| "completion_length": 3026.229248046875, |
| "cov_mean": 2.7950623916694894e-05, |
| "cov_std": 0.22710799425840378, |
| "entropy": 0.46923828125, |
| "epoch": 0.048, |
| "grad_norm": 0.188913494348526, |
| "kl": 0.0005321502685546875, |
| "learning_rate": 7.472670160550848e-07, |
| "loss": 0.0293, |
| "reward": 0.36458334140479565, |
| "reward_std": 0.2087036520242691, |
| "rewards/accuracy_reward": 0.010416666977107525, |
| "rewards/format_reward": 0.3541666744276881, |
| "step": 42, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03042191034182906, |
| "w_max": 1.318351686000824, |
| "w_mean": 1.0878586322069168, |
| "w_min": 0.25, |
| "w_std": 0.1288738138973713 |
| }, |
| { |
| "completion_length": 3391.8333740234375, |
| "cov_mean": 1.0224231118627358e-05, |
| "cov_std": 0.3281352184712887, |
| "entropy": 0.421875, |
| "epoch": 0.04914285714285714, |
| "grad_norm": 0.3027113378047943, |
| "kl": 0.0014162063598632812, |
| "learning_rate": 7.330314893841101e-07, |
| "loss": 0.0293, |
| "reward": 0.4479166716337204, |
| "reward_std": 0.4176873341202736, |
| "rewards/accuracy_reward": 0.18750000558793545, |
| "rewards/format_reward": 0.2604166716337204, |
| "step": 43, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.0375382611528039, |
| "w_max": 1.406785786151886, |
| "w_mean": 1.0772785246372223, |
| "w_min": 0.0, |
| "w_std": 0.17778386548161507 |
| }, |
| { |
| "completion_length": 2956.250030517578, |
| "cov_mean": -4.8457440243510064e-05, |
| "cov_std": 0.4899379573762417, |
| "entropy": 0.3984375, |
| "epoch": 0.05028571428571429, |
| "grad_norm": 0.4807187020778656, |
| "kl": 0.001889944076538086, |
| "learning_rate": 7.185729670371604e-07, |
| "loss": 0.0382, |
| "reward": 0.7395833684131503, |
| "reward_std": 0.4934317544102669, |
| "rewards/accuracy_reward": 0.2395833432674408, |
| "rewards/format_reward": 0.5000000176951289, |
| "step": 44, |
| "w_high_ratio": 0.09932601824402809, |
| "w_low_ratio": 0.052049466874450445, |
| "w_max": 2.26826411485672, |
| "w_mean": 1.3322851359844208, |
| "w_min": 0.0, |
| "w_std": 0.30705559998750687 |
| }, |
| { |
| "completion_length": 3707.854248046875, |
| "cov_mean": -4.304830235923873e-05, |
| "cov_std": 0.290540874004364, |
| "entropy": 0.4033203125, |
| "epoch": 0.05142857142857143, |
| "grad_norm": 0.16171735525131226, |
| "kl": 0.002239227294921875, |
| "learning_rate": 7.039090644965509e-07, |
| "loss": 0.0065, |
| "reward": 0.5000000279396772, |
| "reward_std": 0.4716969132423401, |
| "rewards/accuracy_reward": 0.21875000279396772, |
| "rewards/format_reward": 0.2812500102445483, |
| "step": 45, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03661782294511795, |
| "w_max": 1.3389239609241486, |
| "w_mean": 1.0591736733913422, |
| "w_min": 0.0, |
| "w_std": 0.1518448255956173 |
| }, |
| { |
| "completion_length": 3539.2188110351562, |
| "cov_mean": 0.00013026446868025232, |
| "cov_std": 0.34402996301651, |
| "entropy": 0.5166015625, |
| "epoch": 0.052571428571428575, |
| "grad_norm": 0.28467056155204773, |
| "kl": 0.0012564659118652344, |
| "learning_rate": 6.890576474687263e-07, |
| "loss": 0.0542, |
| "reward": 0.2395833395421505, |
| "reward_std": 0.31017982587218285, |
| "rewards/accuracy_reward": 0.041666666977107525, |
| "rewards/format_reward": 0.1979166716337204, |
| "step": 46, |
| "w_high_ratio": 0.038347791880369186, |
| "w_low_ratio": 0.04659654130227864, |
| "w_max": 1.562936246395111, |
| "w_mean": 1.0923520624637604, |
| "w_min": 6.825020352992891e-31, |
| "w_std": 0.19784706830978394 |
| }, |
| { |
| "completion_length": 3168.5313110351562, |
| "cov_mean": -1.4116008060227614e-05, |
| "cov_std": 0.3409600518643856, |
| "entropy": 0.4072265625, |
| "epoch": 0.053714285714285714, |
| "grad_norm": 0.21553435921669006, |
| "kl": 0.0014462471008300781, |
| "learning_rate": 6.740368101176495e-07, |
| "loss": 0.0063, |
| "reward": 0.7083333432674408, |
| "reward_std": 0.505032941699028, |
| "rewards/accuracy_reward": 0.28125, |
| "rewards/format_reward": 0.4270833358168602, |
| "step": 47, |
| "w_high_ratio": 0.1824447140097618, |
| "w_low_ratio": 0.0349124651402235, |
| "w_max": 1.8283918797969818, |
| "w_mean": 1.3520298600196838, |
| "w_min": 0.25, |
| "w_std": 0.17590370774269104 |
| }, |
| { |
| "completion_length": 3045.5209350585938, |
| "cov_mean": 1.4324002222565468e-05, |
| "cov_std": 0.41333913803100586, |
| "entropy": 0.40966796875, |
| "epoch": 0.054857142857142854, |
| "grad_norm": 0.514603316783905, |
| "kl": 0.0032389163970947266, |
| "learning_rate": 6.588648530198504e-07, |
| "loss": -0.0291, |
| "reward": 0.5729166902601719, |
| "reward_std": 0.4265441298484802, |
| "rewards/accuracy_reward": 0.20833334419876337, |
| "rewards/format_reward": 0.3645833460614085, |
| "step": 48, |
| "w_high_ratio": 0.14455852285027504, |
| "w_low_ratio": 0.04336892580613494, |
| "w_max": 2.106689363718033, |
| "w_mean": 1.3398694097995758, |
| "w_min": 0.0, |
| "w_std": 0.29295507818460464 |
| }, |
| { |
| "completion_length": 2561.3646850585938, |
| "cov_mean": -5.681005814039963e-05, |
| "cov_std": 0.3279518857598305, |
| "entropy": 0.36865234375, |
| "epoch": 0.056, |
| "grad_norm": 0.25349730253219604, |
| "kl": 0.0052852630615234375, |
| "learning_rate": 6.435602608679916e-07, |
| "loss": 0.0033, |
| "reward": 0.833333358168602, |
| "reward_std": 0.46440815925598145, |
| "rewards/accuracy_reward": 0.21875001303851604, |
| "rewards/format_reward": 0.6145833432674408, |
| "step": 49, |
| "w_high_ratio": 0.02398517169058323, |
| "w_low_ratio": 0.04019799269735813, |
| "w_max": 1.7071016132831573, |
| "w_mean": 1.2629328966140747, |
| "w_min": 0.25, |
| "w_std": 0.19682494550943375 |
| }, |
| { |
| "completion_length": 3245.2083740234375, |
| "cov_mean": 4.621147024863603e-05, |
| "cov_std": 0.35216138139367104, |
| "entropy": 0.35205078125, |
| "epoch": 0.05714285714285714, |
| "grad_norm": 0.20161262154579163, |
| "kl": 0.0010981559753417969, |
| "learning_rate": 6.281416799501187e-07, |
| "loss": 0.014, |
| "reward": 0.6354166967794299, |
| "reward_std": 0.420846089720726, |
| "rewards/accuracy_reward": 0.2812500149011612, |
| "rewards/format_reward": 0.3541666818782687, |
| "step": 50, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.042036826722323895, |
| "w_max": 1.5023921430110931, |
| "w_mean": 1.146658033132553, |
| "w_min": 0.0, |
| "w_std": 0.206118730828166 |
| }, |
| { |
| "completion_length": 2383.6980590820312, |
| "cov_mean": -2.697939362406032e-05, |
| "cov_std": 0.38119756057858467, |
| "entropy": 0.42822265625, |
| "epoch": 0.05828571428571429, |
| "grad_norm": 0.4689660966396332, |
| "kl": 0.0037221908569335938, |
| "learning_rate": 6.126278954320294e-07, |
| "loss": 0.011, |
| "reward": 0.7604166865348816, |
| "reward_std": 0.39414726570248604, |
| "rewards/accuracy_reward": 0.20833333861082792, |
| "rewards/format_reward": 0.5520833432674408, |
| "step": 51, |
| "w_high_ratio": 0.0406239777803421, |
| "w_low_ratio": 0.04253132501617074, |
| "w_max": 1.5954174101352692, |
| "w_mean": 1.217557042837143, |
| "w_min": 0.0, |
| "w_std": 0.24302981421351433 |
| }, |
| { |
| "completion_length": 3188.3021240234375, |
| "cov_mean": -1.9340616745466832e-05, |
| "cov_std": 0.42421338707208633, |
| "entropy": 0.41259765625, |
| "epoch": 0.05942857142857143, |
| "grad_norm": 0.43333640694618225, |
| "kl": 0.0021028518676757812, |
| "learning_rate": 5.97037808470444e-07, |
| "loss": -0.0265, |
| "reward": 0.7500000298023224, |
| "reward_std": 0.56574647128582, |
| "rewards/accuracy_reward": 0.3229166716337204, |
| "rewards/format_reward": 0.4270833507180214, |
| "step": 52, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04618662968277931, |
| "w_max": 1.4080191552639008, |
| "w_mean": 1.1331122517585754, |
| "w_min": 0.25, |
| "w_std": 0.21432508528232574 |
| }, |
| { |
| "completion_length": 2943.291748046875, |
| "cov_mean": 1.8985145061378716e-05, |
| "cov_std": 0.42384568601846695, |
| "entropy": 0.41259765625, |
| "epoch": 0.060571428571428575, |
| "grad_norm": 0.28105950355529785, |
| "kl": 0.001844644546508789, |
| "learning_rate": 5.813904131848564e-07, |
| "loss": 0.0064, |
| "reward": 0.895833358168602, |
| "reward_std": 0.5318989902734756, |
| "rewards/accuracy_reward": 0.30208334885537624, |
| "rewards/format_reward": 0.5937500149011612, |
| "step": 53, |
| "w_high_ratio": 0.0625, |
| "w_low_ratio": 0.04191893618553877, |
| "w_max": 1.6293854713439941, |
| "w_mean": 1.2208082973957062, |
| "w_min": 0.25, |
| "w_std": 0.20713179931044579 |
| }, |
| { |
| "completion_length": 3035.8438110351562, |
| "cov_mean": 6.566583215317223e-05, |
| "cov_std": 0.39858507737517357, |
| "entropy": 0.3798828125, |
| "epoch": 0.061714285714285715, |
| "grad_norm": 0.38175392150878906, |
| "kl": 0.0011749267578125, |
| "learning_rate": 5.657047735161255e-07, |
| "loss": -0.0521, |
| "reward": 0.9791667088866234, |
| "reward_std": 0.5917666852474213, |
| "rewards/accuracy_reward": 0.42708334885537624, |
| "rewards/format_reward": 0.552083358168602, |
| "step": 54, |
| "w_high_ratio": 0.044216301292181015, |
| "w_low_ratio": 0.03513730876147747, |
| "w_max": 1.7533048391342163, |
| "w_mean": 1.3249602317810059, |
| "w_min": 0.0, |
| "w_std": 0.22004129365086555 |
| }, |
| { |
| "completion_length": 3354.666748046875, |
| "cov_mean": -3.56279870175058e-05, |
| "cov_std": 0.30405301228165627, |
| "entropy": 0.419921875, |
| "epoch": 0.06285714285714286, |
| "grad_norm": 0.3018040060997009, |
| "kl": 0.0012395381927490234, |
| "learning_rate": 5.5e-07, |
| "loss": -0.0185, |
| "reward": 0.5208333469927311, |
| "reward_std": 0.3702617585659027, |
| "rewards/accuracy_reward": 0.1979166716337204, |
| "rewards/format_reward": 0.3229166679084301, |
| "step": 55, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03356400621123612, |
| "w_max": 1.4872715771198273, |
| "w_mean": 1.1252524256706238, |
| "w_min": 0.0, |
| "w_std": 0.17813249677419662 |
| }, |
| { |
| "completion_length": 3199.7188720703125, |
| "cov_mean": -1.4941413155611372e-05, |
| "cov_std": 0.25671473518013954, |
| "entropy": 0.40185546875, |
| "epoch": 0.064, |
| "grad_norm": 0.21876569092273712, |
| "kl": 0.0006089210510253906, |
| "learning_rate": 5.342952264838747e-07, |
| "loss": -0.0325, |
| "reward": 0.6458333395421505, |
| "reward_std": 0.27983931452035904, |
| "rewards/accuracy_reward": 0.1979166716337204, |
| "rewards/format_reward": 0.4479166828095913, |
| "step": 56, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.028304541716352105, |
| "w_max": 1.453190177679062, |
| "w_mean": 1.1167612671852112, |
| "w_min": 1.400437147815929e-36, |
| "w_std": 0.14085076563060284 |
| }, |
| { |
| "completion_length": 3545.4271850585938, |
| "cov_mean": 1.6042108654801268e-05, |
| "cov_std": 0.19959648326039314, |
| "entropy": 0.31787109375, |
| "epoch": 0.06514285714285714, |
| "grad_norm": 0.15603993833065033, |
| "kl": 0.0003676414489746094, |
| "learning_rate": 5.186095868151436e-07, |
| "loss": 0.0067, |
| "reward": 0.4270833395421505, |
| "reward_std": 0.26762784644961357, |
| "rewards/accuracy_reward": 0.13541667256504297, |
| "rewards/format_reward": 0.29166667722165585, |
| "step": 57, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.028780631721019745, |
| "w_max": 1.3290077447891235, |
| "w_mean": 1.1172043681144714, |
| "w_min": 0.25, |
| "w_std": 0.12668126448988914 |
| }, |
| { |
| "completion_length": 2554.1458740234375, |
| "cov_mean": -8.6602381998091e-05, |
| "cov_std": 0.4178408235311508, |
| "entropy": 0.333740234375, |
| "epoch": 0.06628571428571428, |
| "grad_norm": 0.4292741119861603, |
| "kl": 0.0062389373779296875, |
| "learning_rate": 5.02962191529556e-07, |
| "loss": -0.045, |
| "reward": 0.9375000447034836, |
| "reward_std": 0.5022420063614845, |
| "rewards/accuracy_reward": 0.28125000558793545, |
| "rewards/format_reward": 0.6562500223517418, |
| "step": 58, |
| "w_high_ratio": 0.08225274085998535, |
| "w_low_ratio": 0.038829353637993336, |
| "w_max": 1.8886817693710327, |
| "w_mean": 1.3429239392280579, |
| "w_min": 0.0, |
| "w_std": 0.24752679839730263 |
| }, |
| { |
| "completion_length": 3169.0625610351562, |
| "cov_mean": 4.032192100567045e-05, |
| "cov_std": 0.30262381583452225, |
| "entropy": 0.34619140625, |
| "epoch": 0.06742857142857143, |
| "grad_norm": 0.38411930203437805, |
| "kl": 0.0004928112030029297, |
| "learning_rate": 4.873721045679706e-07, |
| "loss": 0.0209, |
| "reward": 0.5000000102445483, |
| "reward_std": 0.27465011551976204, |
| "rewards/accuracy_reward": 0.1770833358168602, |
| "rewards/format_reward": 0.3229166669771075, |
| "step": 59, |
| "w_high_ratio": 0.15833117440342903, |
| "w_low_ratio": 0.027412693481892347, |
| "w_max": 2.2743676006793976, |
| "w_mean": 1.3636261224746704, |
| "w_min": 0.25, |
| "w_std": 0.22530828416347504 |
| }, |
| { |
| "completion_length": 3205.885498046875, |
| "cov_mean": 5.204351282372954e-06, |
| "cov_std": 0.21868624165654182, |
| "entropy": 0.35986328125, |
| "epoch": 0.06857142857142857, |
| "grad_norm": 0.2159898579120636, |
| "kl": 0.0013399124145507812, |
| "learning_rate": 4.7185832004988133e-07, |
| "loss": 0.0012, |
| "reward": 0.552083358168602, |
| "reward_std": 0.3438131511211395, |
| "rewards/accuracy_reward": 0.13541667070239782, |
| "rewards/format_reward": 0.416666679084301, |
| "step": 60, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.0294471038505435, |
| "w_max": 1.4707800149917603, |
| "w_mean": 1.1660968363285065, |
| "w_min": 0.25, |
| "w_std": 0.13705550134181976 |
| }, |
| { |
| "completion_length": 3322.7084350585938, |
| "cov_mean": 3.926075260096695e-05, |
| "cov_std": 0.3601963371038437, |
| "entropy": 0.3564453125, |
| "epoch": 0.06971428571428571, |
| "grad_norm": 0.39628517627716064, |
| "kl": 0.0009188652038574219, |
| "learning_rate": 4.5643973913200837e-07, |
| "loss": -0.0714, |
| "reward": 0.5833333414047956, |
| "reward_std": 0.515319935977459, |
| "rewards/accuracy_reward": 0.1666666716337204, |
| "rewards/format_reward": 0.41666666977107525, |
| "step": 61, |
| "w_high_ratio": 0.03236403688788414, |
| "w_low_ratio": 0.03914234112016857, |
| "w_max": 1.7150523364543915, |
| "w_mean": 1.2098670601844788, |
| "w_min": 0.0, |
| "w_std": 0.2282575462013483 |
| }, |
| { |
| "completion_length": 3006.885498046875, |
| "cov_mean": 2.1906609163124813e-05, |
| "cov_std": 0.46718820184469223, |
| "entropy": 0.33935546875, |
| "epoch": 0.07085714285714285, |
| "grad_norm": 0.30309396982192993, |
| "kl": 0.0017642974853515625, |
| "learning_rate": 4.4113514698014953e-07, |
| "loss": -0.035, |
| "reward": 0.8229167014360428, |
| "reward_std": 0.6976551413536072, |
| "rewards/accuracy_reward": 0.27083334140479565, |
| "rewards/format_reward": 0.5520833432674408, |
| "step": 62, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.05965683329850435, |
| "w_max": 1.593172937631607, |
| "w_mean": 1.1934809386730194, |
| "w_min": 1.0754965713692971e-43, |
| "w_std": 0.28461313247680664 |
| }, |
| { |
| "completion_length": 2580.229248046875, |
| "cov_mean": 5.573407906922512e-05, |
| "cov_std": 0.3945640027523041, |
| "entropy": 0.39111328125, |
| "epoch": 0.072, |
| "grad_norm": 0.45718619227409363, |
| "kl": 0.001735687255859375, |
| "learning_rate": 4.2596318988235037e-07, |
| "loss": -0.0807, |
| "reward": 0.958333358168602, |
| "reward_std": 0.5150914192199707, |
| "rewards/accuracy_reward": 0.31250001303851604, |
| "rewards/format_reward": 0.6458333432674408, |
| "step": 63, |
| "w_high_ratio": 0.08918174356222153, |
| "w_low_ratio": 0.03547394182533026, |
| "w_max": 2.0218554735183716, |
| "w_mean": 1.3539214432239532, |
| "w_min": 0.0, |
| "w_std": 0.23917281255126 |
| }, |
| { |
| "completion_length": 3340.0313110351562, |
| "cov_mean": 7.548666690126993e-05, |
| "cov_std": 0.31813300400972366, |
| "entropy": 0.4248046875, |
| "epoch": 0.07314285714285715, |
| "grad_norm": 0.5389249920845032, |
| "kl": 0.0018744468688964844, |
| "learning_rate": 4.1094235253127374e-07, |
| "loss": -0.0757, |
| "reward": 0.5104166865348816, |
| "reward_std": 0.4255366772413254, |
| "rewards/accuracy_reward": 0.1979166753590107, |
| "rewards/format_reward": 0.3125000074505806, |
| "step": 64, |
| "w_high_ratio": 0.15633241832256317, |
| "w_low_ratio": 0.0320257218554616, |
| "w_max": 1.8200311660766602, |
| "w_mean": 1.3009037971496582, |
| "w_min": 0.25, |
| "w_std": 0.21954189240932465 |
| }, |
| { |
| "completion_length": 2969.541748046875, |
| "cov_mean": -2.4962878342194017e-05, |
| "cov_std": 0.2727944627404213, |
| "entropy": 0.353759765625, |
| "epoch": 0.07428571428571429, |
| "grad_norm": 0.2332458198070526, |
| "kl": 0.0015840530395507812, |
| "learning_rate": 3.9609093550344907e-07, |
| "loss": 0.012, |
| "reward": 0.666666679084301, |
| "reward_std": 0.357291292399168, |
| "rewards/accuracy_reward": 0.2083333358168602, |
| "rewards/format_reward": 0.4583333432674408, |
| "step": 65, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03699250426143408, |
| "w_max": 1.5260689854621887, |
| "w_mean": 1.164745956659317, |
| "w_min": 1.0748096548620851e-35, |
| "w_std": 0.14831626787781715 |
| }, |
| { |
| "completion_length": 2311.2709197998047, |
| "cov_mean": 2.157548601644521e-05, |
| "cov_std": 0.23550356552004814, |
| "entropy": 0.31787109375, |
| "epoch": 0.07542857142857143, |
| "grad_norm": 0.5557933449745178, |
| "kl": 0.0011758804321289062, |
| "learning_rate": 3.8142703296283953e-07, |
| "loss": -0.0116, |
| "reward": 0.9062500260770321, |
| "reward_std": 0.29818330332636833, |
| "rewards/accuracy_reward": 0.38541667722165585, |
| "rewards/format_reward": 0.520833333954215, |
| "step": 66, |
| "w_high_ratio": 0.10277381911873817, |
| "w_low_ratio": 0.03059569653123617, |
| "w_max": 1.97393798828125, |
| "w_mean": 1.278551459312439, |
| "w_min": 0.0, |
| "w_std": 0.20061488449573517 |
| }, |
| { |
| "completion_length": 3726.135498046875, |
| "cov_mean": 8.957304089562967e-07, |
| "cov_std": 0.12987565249204636, |
| "entropy": 0.35205078125, |
| "epoch": 0.07657142857142857, |
| "grad_norm": 0.11042757332324982, |
| "kl": 0.0013402700424194336, |
| "learning_rate": 3.6696851061588994e-07, |
| "loss": -0.0059, |
| "reward": 0.1770833358168602, |
| "reward_std": 0.15668470412492752, |
| "rewards/accuracy_reward": 0.03125, |
| "rewards/format_reward": 0.1458333358168602, |
| "step": 67, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.01588357798755169, |
| "w_max": 1.2971015870571136, |
| "w_mean": 1.0769366472959518, |
| "w_min": 0.5, |
| "w_std": 0.08244858682155609 |
| }, |
| { |
| "completion_length": 2420.979232788086, |
| "cov_mean": 1.4521759112540167e-05, |
| "cov_std": 0.2692869156599045, |
| "entropy": 0.37353515625, |
| "epoch": 0.07771428571428571, |
| "grad_norm": 0.3267475962638855, |
| "kl": 0.0064449310302734375, |
| "learning_rate": 3.5273298394491515e-07, |
| "loss": -0.0214, |
| "reward": 0.822916679084301, |
| "reward_std": 0.39656912535429, |
| "rewards/accuracy_reward": 0.2187500074505806, |
| "rewards/format_reward": 0.6041666716337204, |
| "step": 68, |
| "w_high_ratio": 0.028063589707016945, |
| "w_low_ratio": 0.029913824051618576, |
| "w_max": 1.8040810525417328, |
| "w_mean": 1.316351056098938, |
| "w_min": 0.25, |
| "w_std": 0.17742961645126343 |
| }, |
| { |
| "completion_length": 2790.5730590820312, |
| "cov_mean": 3.09187362290686e-05, |
| "cov_std": 0.42568420618772507, |
| "entropy": 0.4453125, |
| "epoch": 0.07885714285714286, |
| "grad_norm": 0.7317398190498352, |
| "kl": 0.006710052490234375, |
| "learning_rate": 3.387377967463493e-07, |
| "loss": -0.0192, |
| "reward": 0.4895833358168602, |
| "reward_std": 0.41857269406318665, |
| "rewards/accuracy_reward": 0.08333333674818277, |
| "rewards/format_reward": 0.4062500074505806, |
| "step": 69, |
| "w_high_ratio": 0.19593259878456593, |
| "w_low_ratio": 0.036022237967699766, |
| "w_max": 3.16190242767334, |
| "w_mean": 1.5170519053936005, |
| "w_min": 0.0, |
| "w_std": 0.3000602051615715 |
| }, |
| { |
| "completion_length": 3255.7709350585938, |
| "cov_mean": -6.321061573544284e-05, |
| "cov_std": 0.31040842086076736, |
| "entropy": 0.34765625, |
| "epoch": 0.08, |
| "grad_norm": 0.2253309190273285, |
| "kl": 0.0019707679748535156, |
| "learning_rate": 3.250000000000001e-07, |
| "loss": 0.038, |
| "reward": 0.5104166679084301, |
| "reward_std": 0.3943600133061409, |
| "rewards/accuracy_reward": 0.09375000093132257, |
| "rewards/format_reward": 0.4166666716337204, |
| "step": 70, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04103634599596262, |
| "w_max": 1.2795763909816742, |
| "w_mean": 1.0765459537506104, |
| "w_min": 0.0, |
| "w_std": 0.15424982458353043 |
| }, |
| { |
| "completion_length": 2893.093795776367, |
| "cov_mean": -5.907902482249483e-05, |
| "cov_std": 0.28848100267350674, |
| "entropy": 0.39111328125, |
| "epoch": 0.08114285714285714, |
| "grad_norm": 0.4800524115562439, |
| "kl": 0.0017595291137695312, |
| "learning_rate": 3.115363310950578e-07, |
| "loss": 0.0284, |
| "reward": 0.562500030733645, |
| "reward_std": 0.3276159390807152, |
| "rewards/accuracy_reward": 0.18750000651925802, |
| "rewards/format_reward": 0.37500000558793545, |
| "step": 71, |
| "w_high_ratio": 0.07317740470170975, |
| "w_low_ratio": 0.03526141308248043, |
| "w_max": 2.095757156610489, |
| "w_mean": 1.278771072626114, |
| "w_min": 4.3955273748636295e-36, |
| "w_std": 0.23910818807780743 |
| }, |
| { |
| "completion_length": 3460.354248046875, |
| "cov_mean": 6.12462517892709e-05, |
| "cov_std": 0.36787012219429016, |
| "entropy": 0.4931640625, |
| "epoch": 0.08228571428571428, |
| "grad_norm": 0.39541420340538025, |
| "kl": 0.0038938522338867188, |
| "learning_rate": 2.9836319343816397e-07, |
| "loss": -0.0245, |
| "reward": 0.3333333469927311, |
| "reward_std": 0.4593120887875557, |
| "rewards/accuracy_reward": 0.06250000093132257, |
| "rewards/format_reward": 0.2708333395421505, |
| "step": 72, |
| "w_high_ratio": 0.040276188403367996, |
| "w_low_ratio": 0.046780284494161606, |
| "w_max": 1.6123959422111511, |
| "w_mean": 1.1790215075016022, |
| "w_min": 0.0, |
| "w_std": 0.24864742159843445 |
| }, |
| { |
| "completion_length": 3808.166748046875, |
| "cov_mean": 2.382581669735373e-05, |
| "cov_std": 0.26704882457852364, |
| "entropy": 0.4677734375, |
| "epoch": 0.08342857142857144, |
| "grad_norm": 0.39250648021698, |
| "kl": 0.0005078315734863281, |
| "learning_rate": 2.854966364683872e-07, |
| "loss": 0.0333, |
| "reward": 0.21875001024454832, |
| "reward_std": 0.34505901485681534, |
| "rewards/accuracy_reward": 0.0729166679084301, |
| "rewards/format_reward": 0.14583333488553762, |
| "step": 73, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.031687673181295395, |
| "w_max": 1.3024969696998596, |
| "w_mean": 1.03611221909523, |
| "w_min": 0.0, |
| "w_std": 0.13818097487092018 |
| }, |
| { |
| "completion_length": 3584.8438110351562, |
| "cov_mean": -4.775809429702349e-05, |
| "cov_std": 0.3642261065542698, |
| "entropy": 0.42138671875, |
| "epoch": 0.08457142857142858, |
| "grad_norm": 0.22599299252033234, |
| "kl": 0.002000570297241211, |
| "learning_rate": 2.729523361034538e-07, |
| "loss": 0.0465, |
| "reward": 0.552083358168602, |
| "reward_std": 0.47496967762708664, |
| "rewards/accuracy_reward": 0.2604166679084301, |
| "rewards/format_reward": 0.2916666753590107, |
| "step": 74, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04971347749233246, |
| "w_max": 1.3916961252689362, |
| "w_mean": 1.079592913389206, |
| "w_min": 0.0, |
| "w_std": 0.20160867273807526 |
| }, |
| { |
| "completion_length": 3403.7188720703125, |
| "cov_mean": 4.775901288667228e-06, |
| "cov_std": 0.31741658970713615, |
| "entropy": 0.39208984375, |
| "epoch": 0.08571428571428572, |
| "grad_norm": 0.28842759132385254, |
| "kl": 0.002251148223876953, |
| "learning_rate": 2.6074557564105724e-07, |
| "loss": -0.0149, |
| "reward": 0.5312500074505806, |
| "reward_std": 0.3348959684371948, |
| "rewards/accuracy_reward": 0.1770833358168602, |
| "rewards/format_reward": 0.3541666744276881, |
| "step": 75, |
| "w_high_ratio": 0.05775141716003418, |
| "w_low_ratio": 0.04404309671372175, |
| "w_max": 1.7450213432312012, |
| "w_mean": 1.203758329153061, |
| "w_min": 0.25, |
| "w_std": 0.19945883378386497 |
| }, |
| { |
| "completion_length": 3129.885498046875, |
| "cov_mean": 1.2703612810582854e-05, |
| "cov_std": 0.2526189871132374, |
| "entropy": 0.396484375, |
| "epoch": 0.08685714285714285, |
| "grad_norm": 0.15439313650131226, |
| "kl": 0.0004563331604003906, |
| "learning_rate": 2.488912271385139e-07, |
| "loss": 0.0167, |
| "reward": 0.572916679084301, |
| "reward_std": 0.2542962171137333, |
| "rewards/accuracy_reward": 0.1041666716337204, |
| "rewards/format_reward": 0.4687500074505806, |
| "step": 76, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03176691988483071, |
| "w_max": 1.4051032662391663, |
| "w_mean": 1.1220950484275818, |
| "w_min": 0.25, |
| "w_std": 0.1384560540318489 |
| }, |
| { |
| "completion_length": 3342.510498046875, |
| "cov_mean": 4.0359717786486726e-05, |
| "cov_std": 0.2773597203195095, |
| "entropy": 0.3876953125, |
| "epoch": 0.088, |
| "grad_norm": 0.39251646399497986, |
| "kl": 0.0005764961242675781, |
| "learning_rate": 2.374037332934512e-07, |
| "loss": 0.0041, |
| "reward": 0.552083333954215, |
| "reward_std": 0.26390674337744713, |
| "rewards/accuracy_reward": 0.1458333432674408, |
| "rewards/format_reward": 0.40625000558793545, |
| "step": 77, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.035081148613244295, |
| "w_max": 1.4559779465198517, |
| "w_mean": 1.158086121082306, |
| "w_min": 0.0, |
| "w_std": 0.16358545422554016 |
| }, |
| { |
| "completion_length": 3630.3438720703125, |
| "cov_mean": 2.0063042029505596e-05, |
| "cov_std": 0.3748911917209625, |
| "entropy": 0.39599609375, |
| "epoch": 0.08914285714285715, |
| "grad_norm": 0.50286865234375, |
| "kl": 0.013364076614379883, |
| "learning_rate": 2.2629708984760706e-07, |
| "loss": 0.0002, |
| "reward": 0.4479166939854622, |
| "reward_std": 0.5592645406723022, |
| "rewards/accuracy_reward": 0.1770833432674408, |
| "rewards/format_reward": 0.2708333469927311, |
| "step": 78, |
| "w_high_ratio": 0.04053416848182678, |
| "w_low_ratio": 0.047788419760763645, |
| "w_max": 1.5594379305839539, |
| "w_mean": 1.1108087003231049, |
| "w_min": 0.0, |
| "w_std": 0.2227778285741806 |
| }, |
| { |
| "completion_length": 2405.6354598999023, |
| "cov_mean": 9.158174270851305e-05, |
| "cov_std": 0.34057875350117683, |
| "entropy": 0.326171875, |
| "epoch": 0.09028571428571429, |
| "grad_norm": 0.4699649214744568, |
| "kl": 0.0014543533325195312, |
| "learning_rate": 2.1558482853517253e-07, |
| "loss": 0.0159, |
| "reward": 0.8750000223517418, |
| "reward_std": 0.34952742606401443, |
| "rewards/accuracy_reward": 0.250000006519258, |
| "rewards/format_reward": 0.6250000298023224, |
| "step": 79, |
| "w_high_ratio": 0.11352726817131042, |
| "w_low_ratio": 0.03820635436568409, |
| "w_max": 1.81002277135849, |
| "w_mean": 1.309929370880127, |
| "w_min": 0.0, |
| "w_std": 0.2257107077166438 |
| }, |
| { |
| "completion_length": 3596.3750610351562, |
| "cov_mean": 7.016459858277813e-05, |
| "cov_std": 0.21266964823007584, |
| "entropy": 0.46337890625, |
| "epoch": 0.09142857142857143, |
| "grad_norm": 0.224221870303154, |
| "kl": 0.0014083385467529297, |
| "learning_rate": 2.0528000059645995e-07, |
| "loss": 0.0048, |
| "reward": 0.5520833348855376, |
| "reward_std": 0.33429770916700363, |
| "rewards/accuracy_reward": 0.20833333395421505, |
| "rewards/format_reward": 0.3437500009313226, |
| "step": 80, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.029426061548292637, |
| "w_max": 1.2386472523212433, |
| "w_mean": 1.0539521872997284, |
| "w_min": 0.25, |
| "w_std": 0.11237771809101105 |
| }, |
| { |
| "completion_length": 3377.5521240234375, |
| "cov_mean": 1.5553408047708217e-05, |
| "cov_std": 0.22893034294247627, |
| "entropy": 0.52197265625, |
| "epoch": 0.09257142857142857, |
| "grad_norm": 0.27431702613830566, |
| "kl": 0.0027017593383789062, |
| "learning_rate": 1.9539516087697517e-07, |
| "loss": 0.0442, |
| "reward": 0.3125000027939677, |
| "reward_std": 0.3139677047729492, |
| "rewards/accuracy_reward": 0.0833333358168602, |
| "rewards/format_reward": 0.22916666697710752, |
| "step": 81, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03366563096642494, |
| "w_max": 1.3624085187911987, |
| "w_mean": 1.1045191884040833, |
| "w_min": 0.25, |
| "w_std": 0.13904165104031563 |
| }, |
| { |
| "completion_length": 3133.4583740234375, |
| "cov_mean": -1.667316132625274e-05, |
| "cov_std": 0.27454316802322865, |
| "entropy": 0.408203125, |
| "epoch": 0.09371428571428571, |
| "grad_norm": 0.38514095544815063, |
| "kl": 0.0017423629760742188, |
| "learning_rate": 1.8594235253127372e-07, |
| "loss": -0.0213, |
| "reward": 0.5937500149011612, |
| "reward_std": 0.35102832317352295, |
| "rewards/accuracy_reward": 0.18750000465661287, |
| "rewards/format_reward": 0.4062500074505806, |
| "step": 82, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.034213229548186064, |
| "w_max": 1.6117319464683533, |
| "w_mean": 1.1832448840141296, |
| "w_min": 9.42250974980716e-31, |
| "w_std": 0.1813708432018757 |
| }, |
| { |
| "completion_length": 3142.1771240234375, |
| "cov_mean": 4.071597049914999e-05, |
| "cov_std": 0.17724395170807838, |
| "entropy": 0.45751953125, |
| "epoch": 0.09485714285714286, |
| "grad_norm": 0.3409688472747803, |
| "kl": 0.001071929931640625, |
| "learning_rate": 1.7693309235023127e-07, |
| "loss": 0.0102, |
| "reward": 0.38541667722165585, |
| "reward_std": 0.20087094232439995, |
| "rewards/accuracy_reward": 0.11458333395421505, |
| "rewards/format_reward": 0.2708333386108279, |
| "step": 83, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.023759393487125635, |
| "w_max": 1.2598195374011993, |
| "w_mean": 1.0758444666862488, |
| "w_min": 0.25, |
| "w_std": 0.10355101153254509 |
| }, |
| { |
| "completion_length": 3299.4271850585938, |
| "cov_mean": 1.610395520401653e-05, |
| "cov_std": 0.3381447046995163, |
| "entropy": 0.43505859375, |
| "epoch": 0.096, |
| "grad_norm": 0.3248639404773712, |
| "kl": 0.0011587142944335938, |
| "learning_rate": 1.6837835672960831e-07, |
| "loss": -0.0241, |
| "reward": 0.7500000298023224, |
| "reward_std": 0.4175455719232559, |
| "rewards/accuracy_reward": 0.3020833358168602, |
| "rewards/format_reward": 0.4479166865348816, |
| "step": 84, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.034003185108304024, |
| "w_max": 1.4201359450817108, |
| "w_mean": 1.1546694040298462, |
| "w_min": 0.25, |
| "w_std": 0.19938229769468307 |
| }, |
| { |
| "completion_length": 3368.1146240234375, |
| "cov_mean": 1.1272636584180873e-05, |
| "cov_std": 0.3433619923889637, |
| "entropy": 0.353515625, |
| "epoch": 0.09714285714285714, |
| "grad_norm": 0.2291695475578308, |
| "kl": 0.0006651878356933594, |
| "learning_rate": 1.6028856829700258e-07, |
| "loss": 0.0156, |
| "reward": 0.552083333954215, |
| "reward_std": 0.5098650455474854, |
| "rewards/accuracy_reward": 0.17708334233611822, |
| "rewards/format_reward": 0.3750000102445483, |
| "step": 85, |
| "w_high_ratio": 0.05620652809739113, |
| "w_low_ratio": 0.04038320202380419, |
| "w_max": 1.5647819340229034, |
| "w_mean": 1.2357110977172852, |
| "w_min": 1.9246834407501362e-42, |
| "w_std": 0.2121213562786579 |
| }, |
| { |
| "completion_length": 3338.4479370117188, |
| "cov_mean": 6.590973612219386e-06, |
| "cov_std": 0.2853589281439781, |
| "entropy": 0.45654296875, |
| "epoch": 0.09828571428571428, |
| "grad_norm": 0.3797212839126587, |
| "kl": 0.0021305084228515625, |
| "learning_rate": 1.5267358321348285e-07, |
| "loss": -0.0236, |
| "reward": 0.5208333432674408, |
| "reward_std": 0.299550324678421, |
| "rewards/accuracy_reward": 0.1770833395421505, |
| "rewards/format_reward": 0.34375, |
| "step": 86, |
| "w_high_ratio": 0.05717267841100693, |
| "w_low_ratio": 0.02882188046351075, |
| "w_max": 1.963057816028595, |
| "w_mean": 1.2316765785217285, |
| "w_min": 0.25, |
| "w_std": 0.1924368217587471 |
| }, |
| { |
| "completion_length": 3156.2188110351562, |
| "cov_mean": 8.500453259330243e-05, |
| "cov_std": 0.5130489468574524, |
| "entropy": 0.5185546875, |
| "epoch": 0.09942857142857142, |
| "grad_norm": 0.46258026361465454, |
| "kl": 0.00295257568359375, |
| "learning_rate": 1.4554267916537495e-07, |
| "loss": -0.0212, |
| "reward": 0.5833333432674408, |
| "reward_std": 0.5054471418261528, |
| "rewards/accuracy_reward": 0.14583333674818277, |
| "rewards/format_reward": 0.4375000074505806, |
| "step": 87, |
| "w_high_ratio": 0.13361193798482418, |
| "w_low_ratio": 0.05879375524818897, |
| "w_max": 1.8866381645202637, |
| "w_mean": 1.3355560600757599, |
| "w_min": 0.0, |
| "w_std": 0.33218564838171005 |
| }, |
| { |
| "completion_length": 2856.8959350585938, |
| "cov_mean": -5.945291923126206e-05, |
| "cov_std": 0.5394940003752708, |
| "entropy": 0.44140625, |
| "epoch": 0.10057142857142858, |
| "grad_norm": 0.890120804309845, |
| "kl": 0.02105236053466797, |
| "learning_rate": 1.3890454406082956e-07, |
| "loss": -0.0362, |
| "reward": 0.8958333730697632, |
| "reward_std": 0.7090653479099274, |
| "rewards/accuracy_reward": 0.3645833432674408, |
| "rewards/format_reward": 0.5312500149011612, |
| "step": 88, |
| "w_high_ratio": 0.21772565320134163, |
| "w_low_ratio": 0.049323974177241325, |
| "w_max": 2.266294479370117, |
| "w_mean": 1.4745216369628906, |
| "w_min": 0.0, |
| "w_std": 0.3573007620871067 |
| }, |
| { |
| "completion_length": 3663.3021850585938, |
| "cov_mean": -5.7599154160925536e-05, |
| "cov_std": 0.2993681877851486, |
| "entropy": 0.43505859375, |
| "epoch": 0.10171428571428572, |
| "grad_norm": 0.3626137971878052, |
| "kl": 0.00238800048828125, |
| "learning_rate": 1.3276726544494571e-07, |
| "loss": 0.0099, |
| "reward": 0.3645833469927311, |
| "reward_std": 0.47493190318346024, |
| "rewards/accuracy_reward": 0.12500000651925802, |
| "rewards/format_reward": 0.23958334513008595, |
| "step": 89, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.0442591137252748, |
| "w_max": 1.4542400240898132, |
| "w_mean": 1.079471230506897, |
| "w_min": 0.0, |
| "w_std": 0.18644139170646667 |
| }, |
| { |
| "completion_length": 2692.8125610351562, |
| "cov_mean": 4.647310163363727e-06, |
| "cov_std": 0.3419715017080307, |
| "entropy": 0.5390625, |
| "epoch": 0.10285714285714286, |
| "grad_norm": 0.36119693517684937, |
| "kl": 0.0059909820556640625, |
| "learning_rate": 1.2713832064634125e-07, |
| "loss": 0.0442, |
| "reward": 0.5729166697710752, |
| "reward_std": 0.3239624537527561, |
| "rewards/accuracy_reward": 0.06250000279396772, |
| "rewards/format_reward": 0.5104166744276881, |
| "step": 90, |
| "w_high_ratio": 0.14126155525445938, |
| "w_low_ratio": 0.02876184810884297, |
| "w_max": 2.2300167977809906, |
| "w_mean": 1.42653489112854, |
| "w_min": 1.471363387541058e-44, |
| "w_std": 0.21830591186881065 |
| }, |
| { |
| "completion_length": 3451.5313110351562, |
| "cov_mean": 1.8957917745865416e-05, |
| "cov_std": 0.4000442400574684, |
| "entropy": 0.45654296875, |
| "epoch": 0.104, |
| "grad_norm": 0.4046555459499359, |
| "kl": 0.0024518966674804688, |
| "learning_rate": 1.220245676671809e-07, |
| "loss": -0.0665, |
| "reward": 0.6770833730697632, |
| "reward_std": 0.40345774590969086, |
| "rewards/accuracy_reward": 0.260416679084301, |
| "rewards/format_reward": 0.4166666865348816, |
| "step": 91, |
| "w_high_ratio": 0.06697376817464828, |
| "w_low_ratio": 0.0342584103345871, |
| "w_max": 1.4452155828475952, |
| "w_mean": 1.1833300292491913, |
| "w_min": 0.25, |
| "w_std": 0.2148173823952675 |
| }, |
| { |
| "completion_length": 3021.2188110351562, |
| "cov_mean": -6.440309425670421e-05, |
| "cov_std": 0.45011676102876663, |
| "entropy": 0.38720703125, |
| "epoch": 0.10514285714285715, |
| "grad_norm": 0.44917768239974976, |
| "kl": 0.007047891616821289, |
| "learning_rate": 1.1743223682775649e-07, |
| "loss": 0.0015, |
| "reward": 0.7291666865348816, |
| "reward_std": 0.5440528020262718, |
| "rewards/accuracy_reward": 0.19791667256504297, |
| "rewards/format_reward": 0.5312500074505806, |
| "step": 92, |
| "w_high_ratio": 0.060439030872657895, |
| "w_low_ratio": 0.046312456484884024, |
| "w_max": 2.021807760000229, |
| "w_mean": 1.2982419431209564, |
| "w_min": 0.0, |
| "w_std": 0.2590548172593117 |
| }, |
| { |
| "completion_length": 3865.6250610351562, |
| "cov_mean": 1.7783165276341606e-05, |
| "cov_std": 0.23176611587405205, |
| "entropy": 0.52880859375, |
| "epoch": 0.10628571428571429, |
| "grad_norm": 0.2683483958244324, |
| "kl": 0.002590179443359375, |
| "learning_rate": 1.1336692317580158e-07, |
| "loss": 0.0487, |
| "reward": 0.1145833358168602, |
| "reward_std": 0.27467070147395134, |
| "rewards/accuracy_reward": 0.031250000931322575, |
| "rewards/format_reward": 0.08333333488553762, |
| "step": 93, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.03534638648852706, |
| "w_max": 1.232116162776947, |
| "w_mean": 1.0145132839679718, |
| "w_min": 4.369921683443164e-37, |
| "w_std": 0.12999964877963066 |
| }, |
| { |
| "completion_length": 3234.9271850585938, |
| "cov_mean": 1.053587311616866e-05, |
| "cov_std": 0.20603487268090248, |
| "entropy": 0.52490234375, |
| "epoch": 0.10742857142857143, |
| "grad_norm": 0.25439760088920593, |
| "kl": 0.0029878616333007812, |
| "learning_rate": 1.0983357966978745e-07, |
| "loss": 0.0214, |
| "reward": 0.4270833348855376, |
| "reward_std": 0.17300797998905182, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/format_reward": 0.281250006519258, |
| "step": 94, |
| "w_high_ratio": 0.0574149414896965, |
| "w_low_ratio": 0.021458564791828394, |
| "w_max": 1.955292284488678, |
| "w_mean": 1.259522944688797, |
| "w_min": 0.25, |
| "w_std": 0.14511701837182045 |
| }, |
| { |
| "completion_length": 3710.2396240234375, |
| "cov_mean": 1.0393607226433232e-05, |
| "cov_std": 0.3523143231868744, |
| "entropy": 0.42578125, |
| "epoch": 0.10857142857142857, |
| "grad_norm": 0.20859070122241974, |
| "kl": 0.0004982948303222656, |
| "learning_rate": 1.068365111445064e-07, |
| "loss": 0.0209, |
| "reward": 0.34375, |
| "reward_std": 0.45749304443597794, |
| "rewards/accuracy_reward": 0.11458333488553762, |
| "rewards/format_reward": 0.2291666716337204, |
| "step": 95, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.04463845491409302, |
| "w_max": 1.3612166047096252, |
| "w_mean": 1.0816101431846619, |
| "w_min": 0.25, |
| "w_std": 0.20336205884814262 |
| }, |
| { |
| "completion_length": 3114.041748046875, |
| "cov_mean": -6.73702224958106e-05, |
| "cov_std": 0.36486806720495224, |
| "entropy": 0.4013671875, |
| "epoch": 0.10971428571428571, |
| "grad_norm": 0.3484714925289154, |
| "kl": 0.002678394317626953, |
| "learning_rate": 1.0437936906629334e-07, |
| "loss": -0.0171, |
| "reward": 0.6979166865348816, |
| "reward_std": 0.46633855253458023, |
| "rewards/accuracy_reward": 0.26041666977107525, |
| "rewards/format_reward": 0.4375000074505806, |
| "step": 96, |
| "w_high_ratio": 0.1971779614686966, |
| "w_low_ratio": 0.04042420396581292, |
| "w_max": 2.1471228301525116, |
| "w_mean": 1.42051500082016, |
| "w_min": 1.9142914218395184e-30, |
| "w_std": 0.24729400128126144 |
| }, |
| { |
| "completion_length": 3457.760498046875, |
| "cov_mean": -1.2597635304700816e-05, |
| "cov_std": 0.2245815545320511, |
| "entropy": 0.41552734375, |
| "epoch": 0.11085714285714286, |
| "grad_norm": 0.24951718747615814, |
| "kl": 0.0011386871337890625, |
| "learning_rate": 1.0246514708427701e-07, |
| "loss": -0.0394, |
| "reward": 0.5625000055879354, |
| "reward_std": 0.30785489082336426, |
| "rewards/accuracy_reward": 0.23958333395421505, |
| "rewards/format_reward": 0.32291666977107525, |
| "step": 97, |
| "w_high_ratio": 0.0, |
| "w_low_ratio": 0.02458228124305606, |
| "w_max": 1.5016373097896576, |
| "w_mean": 1.1687426269054413, |
| "w_min": 0.25, |
| "w_std": 0.14521248266100883 |
| }, |
| { |
| "completion_length": 3134.1251220703125, |
| "cov_mean": 8.062847064138623e-05, |
| "cov_std": 0.3362714499235153, |
| "entropy": 0.384765625, |
| "epoch": 0.112, |
| "grad_norm": 0.3256186544895172, |
| "kl": 0.0006456375122070312, |
| "learning_rate": 1.0109617738307911e-07, |
| "loss": -0.0102, |
| "reward": 0.572916679084301, |
| "reward_std": 0.3148062378168106, |
| "rewards/accuracy_reward": 0.14583333395421505, |
| "rewards/format_reward": 0.4270833432674408, |
| "step": 98, |
| "w_high_ratio": 0.00467883562669158, |
| "w_low_ratio": 0.039873864501714706, |
| "w_max": 1.795419156551361, |
| "w_mean": 1.266855150461197, |
| "w_min": 0.25, |
| "w_std": 0.22055331617593765 |
| }, |
| { |
| "completion_length": 3086.7395935058594, |
| "cov_mean": 1.7768036741472315e-05, |
| "cov_std": 0.15420139580965042, |
| "entropy": 0.35205078125, |
| "epoch": 0.11314285714285714, |
| "grad_norm": 0.3351471722126007, |
| "kl": 0.0013794898986816406, |
| "learning_rate": 1.002741278414069e-07, |
| "loss": 0.0477, |
| "reward": 0.5000000223517418, |
| "reward_std": 0.22891659289598465, |
| "rewards/accuracy_reward": 0.1875000074505806, |
| "rewards/format_reward": 0.3125, |
| "step": 99, |
| "w_high_ratio": 0.055527687072753906, |
| "w_low_ratio": 0.02032439224421978, |
| "w_max": 1.390455573797226, |
| "w_mean": 1.1158559024333954, |
| "w_min": 0.5, |
| "w_std": 0.11466998234391212 |
| }, |
| { |
| "completion_length": 3004.6354370117188, |
| "cov_mean": 2.6663584918651395e-05, |
| "cov_std": 0.3363041579723358, |
| "entropy": 0.36181640625, |
| "epoch": 0.11428571428571428, |
| "grad_norm": 0.40786388516426086, |
| "kl": 0.002788543701171875, |
| "learning_rate": 1e-07, |
| "loss": -0.0227, |
| "reward": 0.7187500149011612, |
| "reward_std": 0.4770490266382694, |
| "rewards/accuracy_reward": 0.2500000074505806, |
| "rewards/format_reward": 0.4687500149011612, |
| "step": 100, |
| "w_high_ratio": 0.01440071314573288, |
| "w_low_ratio": 0.04455719655379653, |
| "w_max": 1.8046407103538513, |
| "w_mean": 1.2536596357822418, |
| "w_min": 2.0318827732709848e-44, |
| "w_std": 0.2585573047399521 |
| }, |
| { |
| "epoch": 0.11428571428571428, |
| "step": 100, |
| "total_flos": 0.0, |
| "train_loss": -0.004574703625403345, |
| "train_runtime": 8361.7209, |
| "train_samples_per_second": 1.148, |
| "train_steps_per_second": 0.012 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 100, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 6, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
