{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9990600241708071,
  "eval_steps": 400,
  "global_step": 465,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010742580905062441,
      "grad_norm": 97.31650637788948,
      "learning_rate": 8.51063829787234e-08,
      "logits/chosen": -10.386978149414062,
      "logits/rejected": -10.301819801330566,
      "logps/chosen": -0.9842015504837036,
      "logps/rejected": -0.9797419309616089,
      "loss": 6.1427,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -9.84201431274414,
      "rewards/margins": -0.044595133513212204,
      "rewards/rejected": -9.797419548034668,
      "step": 5
    },
    {
      "epoch": 0.021485161810124883,
      "grad_norm": 120.32296756999746,
      "learning_rate": 1.702127659574468e-07,
      "logits/chosen": -10.528864860534668,
      "logits/rejected": -10.550542831420898,
      "logps/chosen": -1.1053581237792969,
      "logps/rejected": -1.0148571729660034,
      "loss": 5.8489,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -11.053580284118652,
      "rewards/margins": -0.9050087928771973,
      "rewards/rejected": -10.148571968078613,
      "step": 10
    },
    {
      "epoch": 0.03222774271518732,
      "grad_norm": 63.03552892163084,
      "learning_rate": 2.553191489361702e-07,
      "logits/chosen": -10.048526763916016,
      "logits/rejected": -9.960186004638672,
      "logps/chosen": -1.2764005661010742,
      "logps/rejected": -1.1620838642120361,
      "loss": 6.2427,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -12.764005661010742,
      "rewards/margins": -1.1431667804718018,
      "rewards/rejected": -11.62083911895752,
      "step": 15
    },
    {
      "epoch": 0.042970323620249766,
      "grad_norm": 115.65630638049038,
      "learning_rate": 3.404255319148936e-07,
      "logits/chosen": -9.938735961914062,
      "logits/rejected": -10.032970428466797,
      "logps/chosen": -0.886467456817627,
      "logps/rejected": -0.9640073776245117,
      "loss": 5.5509,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -8.86467456817627,
      "rewards/margins": 0.775399386882782,
      "rewards/rejected": -9.640073776245117,
      "step": 20
    },
    {
      "epoch": 0.05371290452531221,
      "grad_norm": 96.51643381412,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -9.94281005859375,
      "logits/rejected": -9.81358814239502,
      "logps/chosen": -0.8847745060920715,
      "logps/rejected": -0.9943971633911133,
      "loss": 5.2621,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -8.84774398803711,
      "rewards/margins": 1.0962274074554443,
      "rewards/rejected": -9.943971633911133,
      "step": 25
    },
    {
      "epoch": 0.06445548543037465,
      "grad_norm": 79.98872357640575,
      "learning_rate": 5.106382978723404e-07,
      "logits/chosen": -9.344173431396484,
      "logits/rejected": -9.224719047546387,
      "logps/chosen": -0.6412209272384644,
      "logps/rejected": -0.6745115518569946,
      "loss": 5.0537,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -6.412209510803223,
      "rewards/margins": 0.3329065442085266,
      "rewards/rejected": -6.745115756988525,
      "step": 30
    },
    {
      "epoch": 0.07519806633543709,
      "grad_norm": 90.98736128158838,
      "learning_rate": 5.957446808510638e-07,
      "logits/chosen": -8.797516822814941,
      "logits/rejected": -8.775238037109375,
      "logps/chosen": -0.7022872567176819,
      "logps/rejected": -0.6628175973892212,
      "loss": 5.2242,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -7.0228729248046875,
      "rewards/margins": -0.3946966230869293,
      "rewards/rejected": -6.628176212310791,
      "step": 35
    },
    {
      "epoch": 0.08594064724049953,
      "grad_norm": 85.27765487288175,
      "learning_rate": 6.808510638297872e-07,
      "logits/chosen": -8.165349006652832,
      "logits/rejected": -7.939520835876465,
      "logps/chosen": -0.6976481676101685,
      "logps/rejected": -0.7118233442306519,
      "loss": 4.8954,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -6.976480960845947,
      "rewards/margins": 0.14175233244895935,
      "rewards/rejected": -7.118234157562256,
      "step": 40
    },
    {
      "epoch": 0.09668322814556198,
      "grad_norm": 60.0810198108563,
      "learning_rate": 7.659574468085107e-07,
      "logits/chosen": -8.638801574707031,
      "logits/rejected": -8.301128387451172,
      "logps/chosen": -0.5912588834762573,
      "logps/rejected": -0.643363356590271,
      "loss": 4.5924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -5.912589073181152,
      "rewards/margins": 0.5210448503494263,
      "rewards/rejected": -6.433633327484131,
      "step": 45
    },
    {
      "epoch": 0.10742580905062442,
      "grad_norm": 56.311443110110645,
      "learning_rate": 7.998983280184396e-07,
      "logits/chosen": -8.66537857055664,
      "logits/rejected": -8.455618858337402,
      "logps/chosen": -0.532910943031311,
      "logps/rejected": -0.5861265063285828,
      "loss": 4.6603,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -5.3291096687316895,
      "rewards/margins": 0.5321555733680725,
      "rewards/rejected": -5.861265659332275,
      "step": 50
    },
    {
      "epoch": 0.11816838995568685,
      "grad_norm": 48.22964437945852,
      "learning_rate": 7.992771864078597e-07,
      "logits/chosen": -8.251328468322754,
      "logits/rejected": -8.19440746307373,
      "logps/chosen": -0.6145357489585876,
      "logps/rejected": -0.7057743072509766,
      "loss": 4.6648,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -6.145357131958008,
      "rewards/margins": 0.9123857617378235,
      "rewards/rejected": -7.057743072509766,
      "step": 55
    },
    {
      "epoch": 0.1289109708607493,
      "grad_norm": 58.650484172589515,
      "learning_rate": 7.980922636120897e-07,
      "logits/chosen": -8.88660717010498,
      "logits/rejected": -8.566266059875488,
      "logps/chosen": -0.5707553625106812,
      "logps/rejected": -0.6651050448417664,
      "loss": 4.4872,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -5.707553863525391,
      "rewards/margins": 0.9434973001480103,
      "rewards/rejected": -6.6510515213012695,
      "step": 60
    },
    {
      "epoch": 0.13965355176581173,
      "grad_norm": 50.701519232478496,
      "learning_rate": 7.963452327474534e-07,
      "logits/chosen": -8.946069717407227,
      "logits/rejected": -8.80902099609375,
      "logps/chosen": -0.6769601702690125,
      "logps/rejected": -0.7762205600738525,
      "loss": 4.6697,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -6.769601345062256,
      "rewards/margins": 0.9926047325134277,
      "rewards/rejected": -7.762206077575684,
      "step": 65
    },
    {
      "epoch": 0.15039613267087418,
      "grad_norm": 55.73800060229348,
      "learning_rate": 7.940385606293987e-07,
      "logits/chosen": -8.857608795166016,
      "logits/rejected": -8.605030059814453,
      "logps/chosen": -0.6864951252937317,
      "logps/rejected": -0.7849037647247314,
      "loss": 4.579,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -6.864950656890869,
      "rewards/margins": 0.9840868711471558,
      "rewards/rejected": -7.849038124084473,
      "step": 70
    },
    {
      "epoch": 0.16113871357593662,
      "grad_norm": 61.45336329452104,
      "learning_rate": 7.911755042893434e-07,
      "logits/chosen": -9.510955810546875,
      "logits/rejected": -9.195249557495117,
      "logps/chosen": -0.6112648248672485,
      "logps/rejected": -0.7157038450241089,
      "loss": 4.4206,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -6.112648010253906,
      "rewards/margins": 1.044390320777893,
      "rewards/rejected": -7.157038688659668,
      "step": 75
    },
    {
      "epoch": 0.17188129448099906,
      "grad_norm": 63.05991232357819,
      "learning_rate": 7.877601063757321e-07,
      "logits/chosen": -8.919134140014648,
      "logits/rejected": -8.694350242614746,
      "logps/chosen": -0.6809953451156616,
      "logps/rejected": -0.8292111158370972,
      "loss": 4.3782,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -6.809953212738037,
      "rewards/margins": 1.4821574687957764,
      "rewards/rejected": -8.29211139678955,
      "step": 80
    },
    {
      "epoch": 0.1826238753860615,
      "grad_norm": 80.14192606614782,
      "learning_rate": 7.837971894457989e-07,
      "logits/chosen": -9.203702926635742,
      "logits/rejected": -8.941718101501465,
      "logps/chosen": -0.6751728653907776,
      "logps/rejected": -0.8182951807975769,
      "loss": 4.6537,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -6.751728057861328,
      "rewards/margins": 1.4312243461608887,
      "rewards/rejected": -8.182951927185059,
      "step": 85
    },
    {
      "epoch": 0.19336645629112395,
      "grad_norm": 57.0890392656848,
      "learning_rate": 7.792923491560942e-07,
      "logits/chosen": -8.762784004211426,
      "logits/rejected": -8.619606018066406,
      "logps/chosen": -0.6792052388191223,
      "logps/rejected": -0.782240092754364,
      "loss": 4.4488,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -6.792051792144775,
      "rewards/margins": 1.0303497314453125,
      "rewards/rejected": -7.822402000427246,
      "step": 90
    },
    {
      "epoch": 0.2041090371961864,
      "grad_norm": 62.86804177211711,
      "learning_rate": 7.742519463613926e-07,
      "logits/chosen": -9.201288223266602,
      "logits/rejected": -9.108887672424316,
      "logps/chosen": -0.6846643090248108,
      "logps/rejected": -0.8157285451889038,
      "loss": 4.2734,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -6.846642971038818,
      "rewards/margins": 1.3106427192687988,
      "rewards/rejected": -8.157285690307617,
      "step": 95
    },
    {
      "epoch": 0.21485161810124884,
      "grad_norm": 156.13974754079638,
      "learning_rate": 7.68683098133138e-07,
      "logits/chosen": -8.922677040100098,
      "logits/rejected": -8.721606254577637,
      "logps/chosen": -0.7626782655715942,
      "logps/rejected": -0.8957880139350891,
      "loss": 4.4059,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -7.626782417297363,
      "rewards/margins": 1.3310966491699219,
      "rewards/rejected": -8.957880020141602,
      "step": 100
    },
    {
      "epoch": 0.22559419900631128,
      "grad_norm": 105.18853752160656,
      "learning_rate": 7.625936677101051e-07,
      "logits/chosen": -9.266253471374512,
      "logits/rejected": -9.21663761138916,
      "logps/chosen": -0.7982575297355652,
      "logps/rejected": -0.824684739112854,
      "loss": 4.4496,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -7.9825758934021,
      "rewards/margins": 0.2642715573310852,
      "rewards/rejected": -8.246847152709961,
      "step": 105
    },
    {
      "epoch": 0.2363367799113737,
      "grad_norm": 55.591763777771575,
      "learning_rate": 7.559922533954731e-07,
      "logits/chosen": -9.533427238464355,
      "logits/rejected": -9.415986061096191,
      "logps/chosen": -0.760522723197937,
      "logps/rejected": -0.880321204662323,
      "loss": 4.2742,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -7.605226039886475,
      "rewards/margins": 1.1979866027832031,
      "rewards/rejected": -8.803213119506836,
      "step": 110
    },
    {
      "epoch": 0.24707936081643614,
      "grad_norm": 68.68376533729582,
      "learning_rate": 7.488881764159808e-07,
      "logits/chosen": -10.321605682373047,
      "logits/rejected": -10.032452583312988,
      "logps/chosen": -0.7515496015548706,
      "logps/rejected": -0.8424334526062012,
      "loss": 4.146,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -7.515496730804443,
      "rewards/margins": 0.9088379740715027,
      "rewards/rejected": -8.424333572387695,
      "step": 115
    },
    {
      "epoch": 0.2578219417214986,
      "grad_norm": 84.12336331826019,
      "learning_rate": 7.412914677603135e-07,
      "logits/chosen": -9.94493293762207,
      "logits/rejected": -9.746753692626953,
      "logps/chosen": -0.8637507557868958,
      "logps/rejected": -0.9913690686225891,
      "loss": 4.2196,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -8.637508392333984,
      "rewards/margins": 1.2761822938919067,
      "rewards/rejected": -9.913690567016602,
      "step": 120
    },
    {
      "epoch": 0.26856452262656105,
      "grad_norm": 65.56256589725982,
      "learning_rate": 7.332128540153017e-07,
      "logits/chosen": -11.014134407043457,
      "logits/rejected": -10.658555030822754,
      "logps/chosen": -0.7951505184173584,
      "logps/rejected": -0.9381753206253052,
      "loss": 4.0983,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -7.951504707336426,
      "rewards/margins": 1.430248498916626,
      "rewards/rejected": -9.381753921508789,
      "step": 125
    },
    {
      "epoch": 0.27930710353162347,
      "grad_norm": 65.20658081437091,
      "learning_rate": 7.246637422199322e-07,
      "logits/chosen": -10.825661659240723,
      "logits/rejected": -10.685798645019531,
      "logps/chosen": -0.867350697517395,
      "logps/rejected": -1.0283457040786743,
      "loss": 4.0614,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -8.673507690429688,
      "rewards/margins": 1.6099493503570557,
      "rewards/rejected": -10.283456802368164,
      "step": 130
    },
    {
      "epoch": 0.29004968443668594,
      "grad_norm": 71.60093594488256,
      "learning_rate": 7.156562037585574e-07,
      "logits/chosen": -11.534868240356445,
      "logits/rejected": -11.279884338378906,
      "logps/chosen": -0.8583852648735046,
      "logps/rejected": -1.0596177577972412,
      "loss": 3.9342,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -8.583852767944336,
      "rewards/margins": 2.012324810028076,
      "rewards/rejected": -10.59617805480957,
      "step": 135
    },
    {
      "epoch": 0.30079226534174835,
      "grad_norm": 90.84373325771121,
      "learning_rate": 7.062029573160467e-07,
      "logits/chosen": -12.217193603515625,
      "logits/rejected": -11.932929039001465,
      "logps/chosen": -0.9700864553451538,
      "logps/rejected": -1.183528184890747,
      "loss": 3.69,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -9.700864791870117,
      "rewards/margins": 2.1344172954559326,
      "rewards/rejected": -11.835283279418945,
      "step": 140
    },
    {
      "epoch": 0.31153484624681077,
      "grad_norm": 108.66029102385438,
      "learning_rate": 6.963173509189455e-07,
      "logits/chosen": -13.757959365844727,
      "logits/rejected": -13.603772163391113,
      "logps/chosen": -1.109348177909851,
      "logps/rejected": -1.3448617458343506,
      "loss": 3.808,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -11.093481063842773,
      "rewards/margins": 2.355137348175049,
      "rewards/rejected": -13.448617935180664,
      "step": 145
    },
    {
      "epoch": 0.32227742715187324,
      "grad_norm": 120.43718591165803,
      "learning_rate": 6.860133430880024e-07,
      "logits/chosen": -14.138008117675781,
      "logits/rejected": -13.760258674621582,
      "logps/chosen": -1.2876828908920288,
      "logps/rejected": -1.5249125957489014,
      "loss": 3.7404,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -12.876829147338867,
      "rewards/margins": 2.3722963333129883,
      "rewards/rejected": -15.249125480651855,
      "step": 150
    },
    {
      "epoch": 0.33302000805693566,
      "grad_norm": 110.47332053227805,
      "learning_rate": 6.753054831286747e-07,
      "logits/chosen": -14.456746101379395,
      "logits/rejected": -14.32475471496582,
      "logps/chosen": -1.3068922758102417,
      "logps/rejected": -1.516629934310913,
      "loss": 3.8826,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -13.06892204284668,
      "rewards/margins": 2.097377300262451,
      "rewards/rejected": -15.166299819946289,
      "step": 155
    },
    {
      "epoch": 0.34376258896199813,
      "grad_norm": 280.21180497264857,
      "learning_rate": 6.642088905874433e-07,
      "logits/chosen": -14.917425155639648,
      "logits/rejected": -14.61968994140625,
      "logps/chosen": -1.3311243057250977,
      "logps/rejected": -1.5419546365737915,
      "loss": 3.7542,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -13.311243057250977,
      "rewards/margins": 2.108302593231201,
      "rewards/rejected": -15.41954517364502,
      "step": 160
    },
    {
      "epoch": 0.35450516986706054,
      "grad_norm": 97.70985018206724,
      "learning_rate": 6.527392339029455e-07,
      "logits/chosen": -14.492767333984375,
      "logits/rejected": -14.438491821289062,
      "logps/chosen": -1.2500733137130737,
      "logps/rejected": -1.525674819946289,
      "loss": 3.4458,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -12.500734329223633,
      "rewards/margins": 2.756016254425049,
      "rewards/rejected": -15.256749153137207,
      "step": 165
    },
    {
      "epoch": 0.365247750772123,
      "grad_norm": 115.13192096245145,
      "learning_rate": 6.409127082820689e-07,
      "logits/chosen": -14.65284252166748,
      "logits/rejected": -14.798192977905273,
      "logps/chosen": -1.573756217956543,
      "logps/rejected": -1.85759699344635,
      "loss": 3.6649,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -15.73755931854248,
      "rewards/margins": 2.838409900665283,
      "rewards/rejected": -18.575969696044922,
      "step": 170
    },
    {
      "epoch": 0.37599033167718543,
      "grad_norm": 120.51325866275113,
      "learning_rate": 6.287460128322457e-07,
      "logits/chosen": -15.47950553894043,
      "logits/rejected": -15.095005989074707,
      "logps/chosen": -1.6204099655151367,
      "logps/rejected": -1.9557273387908936,
      "loss": 3.4241,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -16.2041015625,
      "rewards/margins": 3.3531746864318848,
      "rewards/rejected": -19.557273864746094,
      "step": 175
    },
    {
      "epoch": 0.3867329125822479,
      "grad_norm": 177.63227168183283,
      "learning_rate": 6.16256326982239e-07,
      "logits/chosen": -15.96112060546875,
      "logits/rejected": -15.974037170410156,
      "logps/chosen": -1.5117028951644897,
      "logps/rejected": -1.9151155948638916,
      "loss": 3.4091,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -15.117030143737793,
      "rewards/margins": 4.034125328063965,
      "rewards/rejected": -19.151155471801758,
      "step": 180
    },
    {
      "epoch": 0.3974754934873103,
      "grad_norm": 119.1370485384488,
      "learning_rate": 6.034612862247114e-07,
      "logits/chosen": -15.26576042175293,
      "logits/rejected": -15.074376106262207,
      "logps/chosen": -1.525244951248169,
      "logps/rejected": -1.833696722984314,
      "loss": 3.105,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -15.252447128295898,
      "rewards/margins": 3.0845184326171875,
      "rewards/rejected": -18.336965560913086,
      "step": 185
    },
    {
      "epoch": 0.4082180743923728,
      "grad_norm": 115.37836093687133,
      "learning_rate": 5.903789572148295e-07,
      "logits/chosen": -14.956746101379395,
      "logits/rejected": -14.613385200500488,
      "logps/chosen": -1.6238969564437866,
      "logps/rejected": -2.0761537551879883,
      "loss": 3.3782,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -16.238969802856445,
      "rewards/margins": 4.522566795349121,
      "rewards/rejected": -20.76153564453125,
      "step": 190
    },
    {
      "epoch": 0.4189606552974352,
      "grad_norm": 129.58691495445163,
      "learning_rate": 5.770278122600662e-07,
      "logits/chosen": -15.031437873840332,
      "logits/rejected": -14.86229133605957,
      "logps/chosen": -1.686156988143921,
      "logps/rejected": -2.002657890319824,
      "loss": 3.5752,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -16.861570358276367,
      "rewards/margins": 3.165009021759033,
      "rewards/rejected": -20.026578903198242,
      "step": 195
    },
    {
      "epoch": 0.4297032362024977,
      "grad_norm": 103.32755261822129,
      "learning_rate": 5.634267032372192e-07,
      "logits/chosen": -15.254351615905762,
      "logits/rejected": -15.037300109863281,
      "logps/chosen": -1.6978883743286133,
      "logps/rejected": -2.0401151180267334,
      "loss": 3.3862,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -16.978879928588867,
      "rewards/margins": 3.4222705364227295,
      "rewards/rejected": -20.401153564453125,
      "step": 200
    },
    {
      "epoch": 0.4404458171075601,
      "grad_norm": 128.218280047368,
      "learning_rate": 5.495948349734758e-07,
      "logits/chosen": -14.857444763183594,
      "logits/rejected": -14.940084457397461,
      "logps/chosen": -1.7693378925323486,
      "logps/rejected": -2.061185836791992,
      "loss": 3.1236,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -17.693378448486328,
      "rewards/margins": 2.91848087310791,
      "rewards/rejected": -20.611858367919922,
      "step": 205
    },
    {
      "epoch": 0.45118839801262256,
      "grad_norm": 107.84849265892646,
      "learning_rate": 5.355517381291105e-07,
      "logits/chosen": -16.1163330078125,
      "logits/rejected": -15.79603099822998,
      "logps/chosen": -1.7783616781234741,
      "logps/rejected": -2.141852378845215,
      "loss": 3.1842,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -17.783615112304688,
      "rewards/margins": 3.6349079608917236,
      "rewards/rejected": -21.41852378845215,
      "step": 210
    },
    {
      "epoch": 0.461930978917685,
      "grad_norm": 108.45122155813243,
      "learning_rate": 5.21317241620105e-07,
      "logits/chosen": -16.612058639526367,
      "logits/rejected": -16.40911293029785,
      "logps/chosen": -1.7623794078826904,
      "logps/rejected": -2.1722538471221924,
      "loss": 3.4158,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -17.623790740966797,
      "rewards/margins": 4.0987443923950195,
      "rewards/rejected": -21.722537994384766,
      "step": 215
    },
    {
      "epoch": 0.4726735598227474,
      "grad_norm": 115.68303627040952,
      "learning_rate": 5.069114446196291e-07,
      "logits/chosen": -14.818249702453613,
      "logits/rejected": -14.463000297546387,
      "logps/chosen": -1.832088828086853,
      "logps/rejected": -2.3377938270568848,
      "loss": 3.1208,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -18.32088851928711,
      "rewards/margins": 5.057046890258789,
      "rewards/rejected": -23.37793731689453,
      "step": 220
    },
    {
      "epoch": 0.48341614072780986,
      "grad_norm": 115.29167221000085,
      "learning_rate": 4.923546881779183e-07,
      "logits/chosen": -15.626416206359863,
      "logits/rejected": -15.386268615722656,
      "logps/chosen": -1.7103182077407837,
      "logps/rejected": -2.0725884437561035,
      "loss": 3.0943,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -17.103179931640625,
      "rewards/margins": 3.622704267501831,
      "rewards/rejected": -20.72588539123535,
      "step": 225
    },
    {
      "epoch": 0.4941587216328723,
      "grad_norm": 110.55749043958713,
      "learning_rate": 4.776675265006186e-07,
      "logits/chosen": -15.542366027832031,
      "logits/rejected": -15.470932006835938,
      "logps/chosen": -1.7697646617889404,
      "logps/rejected": -2.1862690448760986,
      "loss": 3.2388,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -17.69764518737793,
      "rewards/margins": 4.165043830871582,
      "rewards/rejected": -21.862689971923828,
      "step": 230
    },
    {
      "epoch": 0.5049013025379347,
      "grad_norm": 194.28560949567938,
      "learning_rate": 4.62870697926156e-07,
      "logits/chosen": -14.893994331359863,
      "logits/rejected": -15.048649787902832,
      "logps/chosen": -1.764272928237915,
      "logps/rejected": -2.2773966789245605,
      "loss": 3.2608,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -17.642728805541992,
      "rewards/margins": 5.13123893737793,
      "rewards/rejected": -22.773967742919922,
      "step": 235
    },
    {
      "epoch": 0.5156438834429972,
      "grad_norm": 118.02806259112292,
      "learning_rate": 4.479850956431092e-07,
      "logits/chosen": -15.161096572875977,
      "logits/rejected": -15.109652519226074,
      "logps/chosen": -1.7125499248504639,
      "logps/rejected": -2.1439616680145264,
      "loss": 3.0925,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -17.125499725341797,
      "rewards/margins": 4.31411600112915,
      "rewards/rejected": -21.43961524963379,
      "step": 240
    },
    {
      "epoch": 0.5263864643480596,
      "grad_norm": 94.41744321870145,
      "learning_rate": 4.33031738188933e-07,
      "logits/chosen": -15.63347053527832,
      "logits/rejected": -15.267684936523438,
      "logps/chosen": -1.6985079050064087,
      "logps/rejected": -2.109764575958252,
      "loss": 3.2248,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -16.98508071899414,
      "rewards/margins": 4.1125664710998535,
      "rewards/rejected": -21.097644805908203,
      "step": 245
    },
    {
      "epoch": 0.5371290452531221,
      "grad_norm": 132.41327050594964,
      "learning_rate": 4.180317397716889e-07,
      "logits/chosen": -15.816108703613281,
      "logits/rejected": -15.996536254882812,
      "logps/chosen": -1.7325900793075562,
      "logps/rejected": -2.2646021842956543,
      "loss": 3.1082,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -17.32590103149414,
      "rewards/margins": 5.320119857788086,
      "rewards/rejected": -22.646020889282227,
      "step": 250
    },
    {
      "epoch": 0.5478716261581845,
      "grad_norm": 123.8588013852755,
      "learning_rate": 4.030062804566888e-07,
      "logits/chosen": -15.66382122039795,
      "logits/rejected": -15.521852493286133,
      "logps/chosen": -1.703418493270874,
      "logps/rejected": -2.1070733070373535,
      "loss": 3.0582,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -17.03418731689453,
      "rewards/margins": 4.036545753479004,
      "rewards/rejected": -21.07073211669922,
      "step": 255
    },
    {
      "epoch": 0.5586142070632469,
      "grad_norm": 100.71973130430227,
      "learning_rate": 3.8797657626014614e-07,
      "logits/chosen": -15.945584297180176,
      "logits/rejected": -15.955207824707031,
      "logps/chosen": -1.7251708507537842,
      "logps/rejected": -2.1078107357025146,
      "loss": 3.1785,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -17.251708984375,
      "rewards/margins": 3.826399326324463,
      "rewards/rejected": -21.078109741210938,
      "step": 260
    },
    {
      "epoch": 0.5693567879683094,
      "grad_norm": 118.85717581252126,
      "learning_rate": 3.729638491920669e-07,
      "logits/chosen": -15.352853775024414,
      "logits/rejected": -15.46752643585205,
      "logps/chosen": -1.7334463596343994,
      "logps/rejected": -2.135951519012451,
      "loss": 3.1444,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -17.334461212158203,
      "rewards/margins": 4.025053977966309,
      "rewards/rejected": -21.359514236450195,
      "step": 265
    },
    {
      "epoch": 0.5800993688733719,
      "grad_norm": 85.82493905943672,
      "learning_rate": 3.5798929729067464e-07,
      "logits/chosen": -16.411659240722656,
      "logits/rejected": -16.199129104614258,
      "logps/chosen": -1.7729793787002563,
      "logps/rejected": -2.220877170562744,
      "loss": 2.7332,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -17.729793548583984,
      "rewards/margins": 4.478978157043457,
      "rewards/rejected": -22.208770751953125,
      "step": 270
    },
    {
      "epoch": 0.5908419497784343,
      "grad_norm": 106.03649240710887,
      "learning_rate": 3.4307406469068595e-07,
      "logits/chosen": -14.978113174438477,
      "logits/rejected": -15.032865524291992,
      "logps/chosen": -1.809618592262268,
      "logps/rejected": -2.3406805992126465,
      "loss": 2.9025,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -18.096187591552734,
      "rewards/margins": 5.3106207847595215,
      "rewards/rejected": -23.40680694580078,
      "step": 275
    },
    {
      "epoch": 0.6015845306834967,
      "grad_norm": 111.84746493906809,
      "learning_rate": 3.282392117676968e-07,
      "logits/chosen": -15.695220947265625,
      "logits/rejected": -15.893925666809082,
      "logps/chosen": -1.9052400588989258,
      "logps/rejected": -2.4178624153137207,
      "loss": 2.8972,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -19.052398681640625,
      "rewards/margins": 5.126224040985107,
      "rewards/rejected": -24.178625106811523,
      "step": 280
    },
    {
      "epoch": 0.6123271115885591,
      "grad_norm": 103.57050178439866,
      "learning_rate": 3.135056854008371e-07,
      "logits/chosen": -15.808944702148438,
      "logits/rejected": -15.778024673461914,
      "logps/chosen": -1.7815015316009521,
      "logps/rejected": -2.271728515625,
      "loss": 2.8431,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -17.81501579284668,
      "rewards/margins": 4.9022698402404785,
      "rewards/rejected": -22.717287063598633,
      "step": 285
    },
    {
      "epoch": 0.6230696924936215,
      "grad_norm": 108.56282250963646,
      "learning_rate": 2.988942893956833e-07,
      "logits/chosen": -15.920690536499023,
      "logits/rejected": -15.81785774230957,
      "logps/chosen": -1.8119595050811768,
      "logps/rejected": -2.2470898628234863,
      "loss": 3.0512,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -18.11959457397461,
      "rewards/margins": 4.351304054260254,
      "rewards/rejected": -22.470901489257812,
      "step": 290
    },
    {
      "epoch": 0.6338122733986841,
      "grad_norm": 116.17030678566253,
      "learning_rate": 2.844256551091911e-07,
      "logits/chosen": -16.46514320373535,
      "logits/rejected": -16.6345272064209,
      "logps/chosen": -1.7750489711761475,
      "logps/rejected": -2.3242697715759277,
      "loss": 2.8258,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -17.75048828125,
      "rewards/margins": 5.492205619812012,
      "rewards/rejected": -23.242694854736328,
      "step": 295
    },
    {
      "epoch": 0.6445548543037465,
      "grad_norm": 147.68751040852177,
      "learning_rate": 2.7012021231812664e-07,
      "logits/chosen": -16.046567916870117,
      "logits/rejected": -15.983491897583008,
      "logps/chosen": -1.897653579711914,
      "logps/rejected": -2.281114339828491,
      "loss": 3.2387,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -18.97653579711914,
      "rewards/margins": 3.834609270095825,
      "rewards/rejected": -22.811145782470703,
      "step": 300
    },
    {
      "epoch": 0.6552974352088089,
      "grad_norm": 124.9827211839036,
      "learning_rate": 2.5599816037212954e-07,
      "logits/chosen": -15.513954162597656,
      "logits/rejected": -15.608304023742676,
      "logps/chosen": -1.8266518115997314,
      "logps/rejected": -2.34920072555542,
      "loss": 2.8842,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -18.266515731811523,
      "rewards/margins": 5.225489616394043,
      "rewards/rejected": -23.492008209228516,
      "step": 305
    },
    {
      "epoch": 0.6660400161138713,
      "grad_norm": 141.73868750375132,
      "learning_rate": 2.4207943967214064e-07,
      "logits/chosen": -16.29332160949707,
      "logits/rejected": -16.148832321166992,
      "logps/chosen": -1.9530408382415771,
      "logps/rejected": -2.4292497634887695,
      "loss": 3.0684,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -19.530406951904297,
      "rewards/margins": 4.762091636657715,
      "rewards/rejected": -24.292499542236328,
      "step": 310
    },
    {
      "epoch": 0.6767825970189338,
      "grad_norm": 172.98236669812187,
      "learning_rate": 2.2838370351446547e-07,
      "logits/chosen": -15.80296802520752,
      "logits/rejected": -15.732707023620605,
      "logps/chosen": -1.7977100610733032,
      "logps/rejected": -2.3419437408447266,
      "loss": 2.7361,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -17.977100372314453,
      "rewards/margins": 5.4423370361328125,
      "rewards/rejected": -23.419437408447266,
      "step": 315
    },
    {
      "epoch": 0.6875251779239963,
      "grad_norm": 112.41993171792654,
      "learning_rate": 2.1493029034023188e-07,
      "logits/chosen": -15.622350692749023,
      "logits/rejected": -15.62098503112793,
      "logps/chosen": -1.784433126449585,
      "logps/rejected": -2.267364025115967,
      "loss": 2.8725,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -17.844331741333008,
      "rewards/margins": 4.829309940338135,
      "rewards/rejected": -22.673641204833984,
      "step": 320
    },
    {
      "epoch": 0.6982677588290587,
      "grad_norm": 116.72973302505498,
      "learning_rate": 2.0173819642942376e-07,
      "logits/chosen": -15.983564376831055,
      "logits/rejected": -15.785995483398438,
      "logps/chosen": -1.9481559991836548,
      "logps/rejected": -2.5938241481781006,
      "loss": 2.9776,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -19.48155975341797,
      "rewards/margins": 6.456681251525879,
      "rewards/rejected": -25.938241958618164,
      "step": 325
    },
    {
      "epoch": 0.7090103397341211,
      "grad_norm": 137.081227030888,
      "learning_rate": 1.888260490780485e-07,
      "logits/chosen": -15.733128547668457,
      "logits/rejected": -15.935976028442383,
      "logps/chosen": -1.8248291015625,
      "logps/rejected": -2.2822577953338623,
      "loss": 3.0336,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -18.248287200927734,
      "rewards/margins": 4.574288368225098,
      "rewards/rejected": -22.82257652282715,
      "step": 330
    },
    {
      "epoch": 0.7197529206391836,
      "grad_norm": 117.59237011545949,
      "learning_rate": 1.7621208029631078e-07,
      "logits/chosen": -15.677896499633789,
      "logits/rejected": -15.52332878112793,
      "logps/chosen": -1.8683006763458252,
      "logps/rejected": -2.448673725128174,
      "loss": 2.8842,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -18.68300437927246,
      "rewards/margins": 5.803731918334961,
      "rewards/rejected": -24.486736297607422,
      "step": 335
    },
    {
      "epoch": 0.730495501544246,
      "grad_norm": 122.52464571458833,
      "learning_rate": 1.6391410106493227e-07,
      "logits/chosen": -15.322489738464355,
      "logits/rejected": -15.156359672546387,
      "logps/chosen": -1.9856983423233032,
      "logps/rejected": -2.5366251468658447,
      "loss": 2.8321,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -19.856985092163086,
      "rewards/margins": 5.509265422821045,
      "rewards/rejected": -25.36625099182129,
      "step": 340
    },
    {
      "epoch": 0.7412380824493084,
      "grad_norm": 98.13439288099542,
      "learning_rate": 1.5194947618596673e-07,
      "logits/chosen": -16.58279037475586,
      "logits/rejected": -16.51984405517578,
      "logps/chosen": -1.9558055400848389,
      "logps/rejected": -2.502556324005127,
      "loss": 3.0276,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -19.558055877685547,
      "rewards/margins": 5.467510223388672,
      "rewards/rejected": -25.025564193725586,
      "step": 345
    },
    {
      "epoch": 0.7519806633543709,
      "grad_norm": 104.49522716837973,
      "learning_rate": 1.4033509976362083e-07,
      "logits/chosen": -15.616808891296387,
      "logits/rejected": -15.737129211425781,
      "logps/chosen": -1.9377790689468384,
      "logps/rejected": -2.3541884422302246,
      "loss": 2.9578,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -19.377790451049805,
      "rewards/margins": 4.164095401763916,
      "rewards/rejected": -23.541885375976562,
      "step": 350
    },
    {
      "epoch": 0.7627232442594333,
      "grad_norm": 107.6884489363759,
      "learning_rate": 1.2908737134970363e-07,
      "logits/chosen": -14.991783142089844,
      "logits/rejected": -15.030191421508789,
      "logps/chosen": -1.8485435247421265,
      "logps/rejected": -2.38105845451355,
      "loss": 3.0742,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -18.485435485839844,
      "rewards/margins": 5.325149059295654,
      "rewards/rejected": -23.810585021972656,
      "step": 355
    },
    {
      "epoch": 0.7734658251644958,
      "grad_norm": 105.82348414404971,
      "learning_rate": 1.1822217278738515e-07,
      "logits/chosen": -15.954935073852539,
      "logits/rejected": -16.074337005615234,
      "logps/chosen": -1.9491569995880127,
      "logps/rejected": -2.3941268920898438,
      "loss": 3.0422,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -19.4915714263916,
      "rewards/margins": 4.449699878692627,
      "rewards/rejected": -23.941268920898438,
      "step": 360
    },
    {
      "epoch": 0.7842084060695582,
      "grad_norm": 208.94228225878328,
      "learning_rate": 1.0775484578596241e-07,
      "logits/chosen": -16.27437973022461,
      "logits/rejected": -16.11089324951172,
      "logps/chosen": -1.8912540674209595,
      "logps/rejected": -2.4439680576324463,
      "loss": 2.7384,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -18.912538528442383,
      "rewards/margins": 5.5271430015563965,
      "rewards/rejected": -24.439682006835938,
      "step": 365
    },
    {
      "epoch": 0.7949509869746206,
      "grad_norm": 126.14591373204779,
      "learning_rate": 9.770017025829673e-08,
      "logits/chosen": -16.250118255615234,
      "logits/rejected": -16.38758087158203,
      "logps/chosen": -2.065514087677002,
      "logps/rejected": -2.72831654548645,
      "loss": 2.6227,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -20.655139923095703,
      "rewards/margins": 6.628026008605957,
      "rewards/rejected": -27.283166885375977,
      "step": 370
    },
    {
      "epoch": 0.805693567879683,
      "grad_norm": 121.32471246816698,
      "learning_rate": 8.807234345151027e-08,
      "logits/chosen": -16.10807991027832,
      "logits/rejected": -16.029443740844727,
      "logps/chosen": -1.9544318914413452,
      "logps/rejected": -2.565009355545044,
      "loss": 2.899,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -19.544321060180664,
      "rewards/margins": 6.105769634246826,
      "rewards/rejected": -25.65009117126465,
      "step": 375
    },
    {
      "epoch": 0.8164361487847456,
      "grad_norm": 201.08985911319357,
      "learning_rate": 7.888495990040924e-08,
      "logits/chosen": -14.932947158813477,
      "logits/rejected": -14.924430847167969,
      "logps/chosen": -1.965070128440857,
      "logps/rejected": -2.5968446731567383,
      "loss": 2.9375,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -19.650699615478516,
      "rewards/margins": 6.317748069763184,
      "rewards/rejected": -25.968448638916016,
      "step": 380
    },
    {
      "epoch": 0.827178729689808,
      "grad_norm": 132.40169960863122,
      "learning_rate": 7.015099223193943e-08,
      "logits/chosen": -16.522253036499023,
      "logits/rejected": -16.4668025970459,
      "logps/chosen": -2.0363919734954834,
      "logps/rejected": -2.5673530101776123,
      "loss": 2.8305,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -20.36391830444336,
      "rewards/margins": 5.309610366821289,
      "rewards/rejected": -25.67352867126465,
      "step": 385
    },
    {
      "epoch": 0.8379213105948704,
      "grad_norm": 113.91880111182708,
      "learning_rate": 6.188277284777857e-08,
      "logits/chosen": -15.97686767578125,
      "logits/rejected": -15.647547721862793,
      "logps/chosen": -2.0422708988189697,
      "logps/rejected": -2.549582004547119,
      "loss": 2.7972,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -20.42270851135254,
      "rewards/margins": 5.073108673095703,
      "rewards/rejected": -25.49581527709961,
      "step": 390
    },
    {
      "epoch": 0.8486638914999328,
      "grad_norm": 126.36653904434415,
      "learning_rate": 5.409197651092965e-08,
      "logits/chosen": -15.692411422729492,
      "logits/rejected": -15.747018814086914,
      "logps/chosen": -2.10833477973938,
      "logps/rejected": -2.676734685897827,
      "loss": 2.6959,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -21.083349227905273,
      "rewards/margins": 5.683998107910156,
      "rewards/rejected": -26.767345428466797,
      "step": 395
    },
    {
      "epoch": 0.8594064724049953,
      "grad_norm": 115.47783241869803,
      "learning_rate": 4.678960386090298e-08,
      "logits/chosen": -15.30811595916748,
      "logits/rejected": -15.540433883666992,
      "logps/chosen": -1.9201295375823975,
      "logps/rejected": -2.523301601409912,
      "loss": 2.7196,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -19.2012939453125,
      "rewards/margins": 6.03171968460083,
      "rewards/rejected": -25.233013153076172,
      "step": 400
    },
    {
      "epoch": 0.8594064724049953,
      "eval_logits/chosen": -14.502357482910156,
      "eval_logits/rejected": -14.432116508483887,
      "eval_logps/chosen": -1.8952556848526,
      "eval_logps/rejected": -2.3938686847686768,
      "eval_loss": 2.7580387592315674,
      "eval_rewards/accuracies": 0.7704917788505554,
      "eval_rewards/chosen": -18.952556610107422,
      "eval_rewards/margins": 4.986130237579346,
      "eval_rewards/rejected": -23.938688278198242,
      "eval_runtime": 86.7509,
      "eval_samples_per_second": 22.374,
      "eval_steps_per_second": 1.406,
      "step": 400
    },
    {
      "epoch": 0.8701490533100578,
      "grad_norm": 112.75479415537629,
      "learning_rate": 3.998596588076366e-08,
      "logits/chosen": -15.262601852416992,
      "logits/rejected": -14.926340103149414,
      "logps/chosen": -1.9843738079071045,
      "logps/rejected": -2.531123638153076,
      "loss": 3.0297,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -19.843738555908203,
      "rewards/margins": 5.467496871948242,
      "rewards/rejected": -25.311237335205078,
      "step": 405
    },
    {
      "epoch": 0.8808916342151202,
      "grad_norm": 124.72604723824772,
      "learning_rate": 3.3690669337976996e-08,
      "logits/chosen": -15.930806159973145,
      "logits/rejected": -15.931562423706055,
      "logps/chosen": -1.9564170837402344,
      "logps/rejected": -2.517925500869751,
      "loss": 2.6997,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -19.564170837402344,
      "rewards/margins": 5.615079402923584,
      "rewards/rejected": -25.17925262451172,
      "step": 410
    },
    {
      "epoch": 0.8916342151201826,
      "grad_norm": 124.6339007044252,
      "learning_rate": 2.7912603219609798e-08,
      "logits/chosen": -16.048858642578125,
      "logits/rejected": -15.944051742553711,
      "logps/chosen": -2.1005568504333496,
      "logps/rejected": -2.568063735961914,
      "loss": 2.6664,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -21.005565643310547,
      "rewards/margins": 4.675069332122803,
      "rewards/rejected": -25.680639266967773,
      "step": 415
    },
    {
      "epoch": 0.9023767960252451,
      "grad_norm": 121.69678463559275,
      "learning_rate": 2.265992618104029e-08,
      "logits/chosen": -16.162189483642578,
      "logits/rejected": -16.180706024169922,
      "logps/chosen": -2.018214702606201,
      "logps/rejected": -2.648829460144043,
      "loss": 2.7859,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -20.182147979736328,
      "rewards/margins": 6.306146621704102,
      "rewards/rejected": -26.488290786743164,
      "step": 420
    },
    {
      "epoch": 0.9131193769303075,
      "grad_norm": 267.04077532902875,
      "learning_rate": 1.7940055025900304e-08,
      "logits/chosen": -14.952715873718262,
      "logits/rejected": -14.8345308303833,
      "logps/chosen": -1.9776952266693115,
      "logps/rejected": -2.4361090660095215,
      "loss": 3.0349,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -19.77695083618164,
      "rewards/margins": 4.584140777587891,
      "rewards/rejected": -24.361093521118164,
      "step": 425
    },
    {
      "epoch": 0.92386195783537,
      "grad_norm": 115.027170497356,
      "learning_rate": 1.3759654233514817e-08,
      "logits/chosen": -15.242410659790039,
      "logits/rejected": -15.459531784057617,
      "logps/chosen": -1.8886226415634155,
      "logps/rejected": -2.344308853149414,
      "loss": 2.7946,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -18.8862247467041,
      "rewards/margins": 4.556865692138672,
      "rewards/rejected": -23.443092346191406,
      "step": 430
    },
    {
      "epoch": 0.9346045387404324,
      "grad_norm": 116.35861873175186,
      "learning_rate": 1.0124626548627402e-08,
      "logits/chosen": -15.868557929992676,
      "logits/rejected": -15.74053955078125,
      "logps/chosen": -1.9277899265289307,
      "logps/rejected": -2.5743651390075684,
      "loss": 2.7446,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -19.27789878845215,
      "rewards/margins": 6.46575403213501,
      "rewards/rejected": -25.743656158447266,
      "step": 435
    },
    {
      "epoch": 0.9453471196454948,
      "grad_norm": 101.71134576562835,
      "learning_rate": 7.040104646698042e-09,
      "logits/chosen": -15.439016342163086,
      "logits/rejected": -15.44897747039795,
      "logps/chosen": -1.959753394126892,
      "logps/rejected": -2.5274107456207275,
      "loss": 2.6695,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -19.597536087036133,
      "rewards/margins": 5.676573276519775,
      "rewards/rejected": -25.274110794067383,
      "step": 440
    },
    {
      "epoch": 0.9560897005505573,
      "grad_norm": 110.29407715801415,
      "learning_rate": 4.510443886542114e-09,
      "logits/chosen": -15.287280082702637,
      "logits/rejected": -15.34148120880127,
      "logps/chosen": -2.031892776489258,
      "logps/rejected": -2.656055450439453,
      "loss": 2.7797,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -20.318927764892578,
      "rewards/margins": 6.2416229248046875,
      "rewards/rejected": -26.560550689697266,
      "step": 445
    },
    {
      "epoch": 0.9668322814556197,
      "grad_norm": 122.67920649488762,
      "learning_rate": 2.539216160544333e-09,
      "logits/chosen": -15.329913139343262,
      "logits/rejected": -15.057653427124023,
      "logps/chosen": -2.1148476600646973,
      "logps/rejected": -2.682511806488037,
      "loss": 2.8577,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -21.14847755432129,
      "rewards/margins": 5.676642894744873,
      "rewards/rejected": -26.825122833251953,
      "step": 450
    },
    {
      "epoch": 0.9775748623606821,
      "grad_norm": 127.18334450658871,
      "learning_rate": 1.1292048511303054e-09,
      "logits/chosen": -15.279156684875488,
      "logits/rejected": -15.283819198608398,
      "logps/chosen": -2.0526576042175293,
      "logps/rejected": -2.5599799156188965,
      "loss": 2.9411,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -20.52657699584961,
      "rewards/margins": 5.0732221603393555,
      "rewards/rejected": -25.599796295166016,
      "step": 455
    },
    {
      "epoch": 0.9883174432657446,
      "grad_norm": 115.1645535617376,
      "learning_rate": 2.82400900618418e-10,
      "logits/chosen": -15.98609447479248,
      "logits/rejected": -15.959650993347168,
      "logps/chosen": -1.9472544193267822,
      "logps/rejected": -2.6097803115844727,
      "loss": 2.5164,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -19.47254180908203,
      "rewards/margins": 6.625257968902588,
      "rewards/rejected": -26.097803115844727,
      "step": 460
    },
    {
      "epoch": 0.9990600241708071,
      "grad_norm": 233.73237047558877,
      "learning_rate": 0.0,
      "logits/chosen": -15.412060737609863,
      "logits/rejected": -15.519399642944336,
      "logps/chosen": -1.9192397594451904,
      "logps/rejected": -2.4578652381896973,
      "loss": 2.5846,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -19.19239616394043,
      "rewards/margins": 5.386256217956543,
      "rewards/rejected": -24.57865333557129,
      "step": 465
    },
    {
      "epoch": 0.9990600241708071,
      "step": 465,
      "total_flos": 0.0,
      "train_loss": 3.541660060677477,
      "train_runtime": 9046.6245,
      "train_samples_per_second": 6.585,
      "train_steps_per_second": 0.051
    }
  ],
  "logging_steps": 5,
  "max_steps": 465,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}