{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 400,
  "global_step": 468,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010683760683760684,
      "grad_norm": 14.031576787271042,
      "learning_rate": 5.106382978723404e-08,
      "logits/chosen": -1.478906273841858,
      "logits/rejected": -1.4328124523162842,
      "logps/chosen": -0.817187488079071,
      "logps/rejected": -0.850390613079071,
      "loss": 1.4433,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.4703125059604645,
      "rewards/chosen": -1.634374976158142,
      "rewards/margins": 0.06545410305261612,
      "rewards/rejected": -1.700781226158142,
      "step": 5
    },
    {
      "epoch": 0.021367521367521368,
      "grad_norm": 23.294649241820373,
      "learning_rate": 1.1489361702127659e-07,
      "logits/chosen": -1.4249999523162842,
      "logits/rejected": -1.388281226158142,
      "logps/chosen": -0.7289062738418579,
      "logps/rejected": -0.725781261920929,
      "loss": 1.4406,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.4859375059604645,
      "rewards/chosen": -1.4578125476837158,
      "rewards/margins": -0.00665283203125,
      "rewards/rejected": -1.451562523841858,
      "step": 10
    },
    {
      "epoch": 0.03205128205128205,
      "grad_norm": 10.86773840315628,
      "learning_rate": 1.7872340425531914e-07,
      "logits/chosen": -1.4929687976837158,
      "logits/rejected": -1.466406226158142,
      "logps/chosen": -0.7378906011581421,
      "logps/rejected": -0.7523437738418579,
      "loss": 1.3816,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -1.4757812023162842,
      "rewards/margins": 0.02743835374712944,
      "rewards/rejected": -1.5046875476837158,
      "step": 15
    },
    {
      "epoch": 0.042735042735042736,
      "grad_norm": 6.43024052602064,
      "learning_rate": 2.425531914893617e-07,
      "logits/chosen": -1.411718726158142,
      "logits/rejected": -1.3984375,
      "logps/chosen": -0.750781238079071,
      "logps/rejected": -0.848437488079071,
      "loss": 1.3784,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -1.501562476158142,
      "rewards/margins": 0.19492188096046448,
      "rewards/rejected": -1.696874976158142,
      "step": 20
    },
    {
      "epoch": 0.053418803418803416,
      "grad_norm": 11.19925501261024,
      "learning_rate": 3.063829787234042e-07,
      "logits/chosen": -1.4601562023162842,
      "logits/rejected": -1.4375,
      "logps/chosen": -0.7308593988418579,
      "logps/rejected": -0.778124988079071,
      "loss": 1.3567,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.4617187976837158,
      "rewards/margins": 0.09446106106042862,
      "rewards/rejected": -1.556249976158142,
      "step": 25
    },
    {
      "epoch": 0.0641025641025641,
      "grad_norm": 10.155982563226914,
      "learning_rate": 3.702127659574468e-07,
      "logits/chosen": -1.4421875476837158,
      "logits/rejected": -1.4148437976837158,
      "logps/chosen": -0.6324218511581421,
      "logps/rejected": -0.66796875,
      "loss": 1.3238,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.4906249940395355,
      "rewards/chosen": -1.2648437023162842,
      "rewards/margins": 0.07158203423023224,
      "rewards/rejected": -1.3359375,
      "step": 30
    },
    {
      "epoch": 0.07478632478632478,
      "grad_norm": 12.236975012364756,
      "learning_rate": 4.3404255319148936e-07,
      "logits/chosen": -1.4679687023162842,
      "logits/rejected": -1.4406249523162842,
      "logps/chosen": -0.583203136920929,
      "logps/rejected": -0.599609375,
      "loss": 1.3457,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.166406273841858,
      "rewards/margins": 0.03232421725988388,
      "rewards/rejected": -1.19921875,
      "step": 35
    },
    {
      "epoch": 0.08547008547008547,
      "grad_norm": 15.99115276248814,
      "learning_rate": 4.978723404255318e-07,
      "logits/chosen": -1.497656226158142,
      "logits/rejected": -1.4500000476837158,
      "logps/chosen": -0.5269531011581421,
      "logps/rejected": -0.552734375,
      "loss": 1.3196,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.4984374940395355,
      "rewards/chosen": -1.0539062023162842,
      "rewards/margins": 0.05270995944738388,
      "rewards/rejected": -1.10546875,
      "step": 40
    },
    {
      "epoch": 0.09615384615384616,
      "grad_norm": 6.342347202429818,
      "learning_rate": 5.617021276595744e-07,
      "logits/chosen": -1.540624976158142,
      "logits/rejected": -1.510156273841858,
      "logps/chosen": -0.4320312440395355,
      "logps/rejected": -0.46113282442092896,
      "loss": 1.2917,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5171874761581421,
      "rewards/chosen": -0.864062488079071,
      "rewards/margins": 0.05752868577837944,
      "rewards/rejected": -0.9222656488418579,
      "step": 45
    },
    {
      "epoch": 0.10683760683760683,
      "grad_norm": 6.093826949661445,
      "learning_rate": 5.9996658982558e-07,
      "logits/chosen": -1.5,
      "logits/rejected": -1.471093773841858,
      "logps/chosen": -0.3994140625,
      "logps/rejected": -0.43671876192092896,
      "loss": 1.2759,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -0.798828125,
      "rewards/margins": 0.07478027045726776,
      "rewards/rejected": -0.8734375238418579,
      "step": 50
    },
    {
      "epoch": 0.11752136752136752,
      "grad_norm": 32.342746515822284,
      "learning_rate": 5.995908108202275e-07,
      "logits/chosen": -1.4992187023162842,
      "logits/rejected": -1.470312476158142,
      "logps/chosen": -0.3929687440395355,
      "logps/rejected": -0.42460936307907104,
      "loss": 1.2828,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": -0.785937488079071,
      "rewards/margins": 0.06329345703125,
      "rewards/rejected": -0.8492187261581421,
      "step": 55
    },
    {
      "epoch": 0.1282051282051282,
      "grad_norm": 7.8916210409345195,
      "learning_rate": 5.98798014903091e-07,
      "logits/chosen": -1.5226562023162842,
      "logits/rejected": -1.4874999523162842,
      "logps/chosen": -0.40058594942092896,
      "logps/rejected": -0.4244140684604645,
      "loss": 1.2934,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.526562511920929,
      "rewards/chosen": -0.8011718988418579,
      "rewards/margins": 0.04749755933880806,
      "rewards/rejected": -0.848828136920929,
      "step": 60
    },
    {
      "epoch": 0.1388888888888889,
      "grad_norm": 10.591326680588619,
      "learning_rate": 5.97589305610371e-07,
      "logits/chosen": -1.5109374523162842,
      "logits/rejected": -1.467187523841858,
      "logps/chosen": -0.3916015625,
      "logps/rejected": -0.4154296815395355,
      "loss": 1.2916,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.783203125,
      "rewards/margins": 0.04823913425207138,
      "rewards/rejected": -0.830859363079071,
      "step": 65
    },
    {
      "epoch": 0.14957264957264957,
      "grad_norm": 14.122292943634292,
      "learning_rate": 5.959663654109491e-07,
      "logits/chosen": -1.517187476158142,
      "logits/rejected": -1.478906273841858,
      "logps/chosen": -0.40234375,
      "logps/rejected": -0.4400390684604645,
      "loss": 1.2902,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.8046875,
      "rewards/margins": 0.07520751655101776,
      "rewards/rejected": -0.880078136920929,
      "step": 70
    },
    {
      "epoch": 0.16025641025641027,
      "grad_norm": 11.624864350308169,
      "learning_rate": 5.93931453364468e-07,
      "logits/chosen": -1.5617187023162842,
      "logits/rejected": -1.54296875,
      "logps/chosen": -0.40234375,
      "logps/rejected": -0.45429688692092896,
      "loss": 1.2626,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -0.8046875,
      "rewards/margins": 0.10439453274011612,
      "rewards/rejected": -0.9085937738418579,
      "step": 75
    },
    {
      "epoch": 0.17094017094017094,
      "grad_norm": 8.890913992364057,
      "learning_rate": 5.914874019768214e-07,
      "logits/chosen": -1.575781226158142,
      "logits/rejected": -1.5593750476837158,
      "logps/chosen": -0.4203124940395355,
      "logps/rejected": -0.4765625,
      "loss": 1.265,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.840624988079071,
      "rewards/margins": 0.11260070651769638,
      "rewards/rejected": -0.953125,
      "step": 80
    },
    {
      "epoch": 0.18162393162393162,
      "grad_norm": 9.053037451143402,
      "learning_rate": 5.886376132574339e-07,
      "logits/chosen": -1.517968773841858,
      "logits/rejected": -1.489843726158142,
      "logps/chosen": -0.44257813692092896,
      "logps/rejected": -0.5015624761581421,
      "loss": 1.2794,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5171874761581421,
      "rewards/chosen": -0.8851562738418579,
      "rewards/margins": 0.11624755710363388,
      "rewards/rejected": -1.0031249523162842,
      "step": 85
    },
    {
      "epoch": 0.19230769230769232,
      "grad_norm": 7.52173535900888,
      "learning_rate": 5.853860539838169e-07,
      "logits/chosen": -1.5085937976837158,
      "logits/rejected": -1.4734375476837158,
      "logps/chosen": -0.4273437559604645,
      "logps/rejected": -0.4761718809604645,
      "loss": 1.2881,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.528124988079071,
      "rewards/chosen": -0.854687511920929,
      "rewards/margins": 0.09850464016199112,
      "rewards/rejected": -0.952343761920929,
      "step": 90
    },
    {
      "epoch": 0.202991452991453,
      "grad_norm": 7.370453197913243,
      "learning_rate": 5.817372501799931e-07,
      "logits/chosen": -1.55859375,
      "logits/rejected": -1.5499999523162842,
      "logps/chosen": -0.38457030057907104,
      "logps/rejected": -0.44902342557907104,
      "loss": 1.2621,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.7691406011581421,
      "rewards/margins": 0.12803955376148224,
      "rewards/rejected": -0.8980468511581421,
      "step": 95
    },
    {
      "epoch": 0.21367521367521367,
      "grad_norm": 13.697273867527963,
      "learning_rate": 5.776962808164751e-07,
      "logits/chosen": -1.560156226158142,
      "logits/rejected": -1.513281226158142,
      "logps/chosen": -0.39531248807907104,
      "logps/rejected": -0.4423828125,
      "loss": 1.2896,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -0.7906249761581421,
      "rewards/margins": 0.09405517578125,
      "rewards/rejected": -0.884765625,
      "step": 100
    },
    {
      "epoch": 0.22435897435897437,
      "grad_norm": 14.90977337445018,
      "learning_rate": 5.73268770740567e-07,
      "logits/chosen": -1.521875023841858,
      "logits/rejected": -1.4874999523162842,
      "logps/chosen": -0.38300782442092896,
      "logps/rejected": -0.43476563692092896,
      "loss": 1.275,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.526562511920929,
      "rewards/chosen": -0.7660156488418579,
      "rewards/margins": 0.10239257663488388,
      "rewards/rejected": -0.8695312738418579,
      "step": 105
    },
    {
      "epoch": 0.23504273504273504,
      "grad_norm": 12.70136475909842,
      "learning_rate": 5.684608828468305e-07,
      "logits/chosen": -1.544531226158142,
      "logits/rejected": -1.501562476158142,
      "logps/chosen": -0.3892578184604645,
      "logps/rejected": -0.42597657442092896,
      "loss": 1.2849,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.778515636920929,
      "rewards/margins": 0.07318115234375,
      "rewards/rejected": -0.8519531488418579,
      "step": 110
    },
    {
      "epoch": 0.24572649572649571,
      "grad_norm": 10.809074159755266,
      "learning_rate": 5.632793094986134e-07,
      "logits/chosen": -1.5304687023162842,
      "logits/rejected": -1.5109374523162842,
      "logps/chosen": -0.39667969942092896,
      "logps/rejected": -0.4609375,
      "loss": 1.2658,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": -0.7933593988418579,
      "rewards/margins": 0.12885741889476776,
      "rewards/rejected": -0.921875,
      "step": 115
    },
    {
      "epoch": 0.2564102564102564,
      "grad_norm": 7.310341755161344,
      "learning_rate": 5.577312632125811e-07,
      "logits/chosen": -1.5437500476837158,
      "logits/rejected": -1.532812476158142,
      "logps/chosen": -0.40058594942092896,
      "logps/rejected": -0.4585937559604645,
      "loss": 1.261,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": -0.8011718988418579,
      "rewards/margins": 0.11567382514476776,
      "rewards/rejected": -0.917187511920929,
      "step": 120
    },
    {
      "epoch": 0.2670940170940171,
      "grad_norm": 8.26904210261963,
      "learning_rate": 5.518244666192185e-07,
      "logits/chosen": -1.552343726158142,
      "logits/rejected": -1.5281250476837158,
      "logps/chosen": -0.45878905057907104,
      "logps/rejected": -0.5210937261581421,
      "loss": 1.2867,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.9175781011581421,
      "rewards/margins": 0.12409820407629013,
      "rewards/rejected": -1.0421874523162842,
      "step": 125
    },
    {
      "epoch": 0.2777777777777778,
      "grad_norm": 10.250708094357067,
      "learning_rate": 5.455671417132762e-07,
      "logits/chosen": -1.5671875476837158,
      "logits/rejected": -1.5164062976837158,
      "logps/chosen": -0.40839844942092896,
      "logps/rejected": -0.49785155057907104,
      "loss": 1.2559,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.526562511920929,
      "rewards/chosen": -0.8167968988418579,
      "rewards/margins": 0.18012695014476776,
      "rewards/rejected": -0.9957031011581421,
      "step": 130
    },
    {
      "epoch": 0.28846153846153844,
      "grad_norm": 9.573630775951374,
      "learning_rate": 5.389679984091244e-07,
      "logits/chosen": -1.5554687976837158,
      "logits/rejected": -1.521093726158142,
      "logps/chosen": -0.40156251192092896,
      "logps/rejected": -0.4853515625,
      "loss": 1.2483,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": -0.8031250238418579,
      "rewards/margins": 0.16811522841453552,
      "rewards/rejected": -0.970703125,
      "step": 135
    },
    {
      "epoch": 0.29914529914529914,
      "grad_norm": 15.240586201261362,
      "learning_rate": 5.320362224169441e-07,
      "logits/chosen": -1.5382812023162842,
      "logits/rejected": -1.5125000476837158,
      "logps/chosen": -0.4029296934604645,
      "logps/rejected": -0.4652343690395355,
      "loss": 1.2741,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": -0.805859386920929,
      "rewards/margins": 0.12409667670726776,
      "rewards/rejected": -0.930468738079071,
      "step": 140
    },
    {
      "epoch": 0.30982905982905984,
      "grad_norm": 11.28111450928389,
      "learning_rate": 5.247814624566326e-07,
      "logits/chosen": -1.525781273841858,
      "logits/rejected": -1.49609375,
      "logps/chosen": -0.3916015625,
      "logps/rejected": -0.46757811307907104,
      "loss": 1.2728,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.783203125,
      "rewards/margins": 0.15163573622703552,
      "rewards/rejected": -0.9351562261581421,
      "step": 145
    },
    {
      "epoch": 0.32051282051282054,
      "grad_norm": 13.169292810618147,
      "learning_rate": 5.172138168272204e-07,
      "logits/chosen": -1.5265624523162842,
      "logits/rejected": -1.497656226158142,
      "logps/chosen": -0.40546876192092896,
      "logps/rejected": -0.4759765565395355,
      "loss": 1.2561,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.559374988079071,
      "rewards/chosen": -0.8109375238418579,
      "rewards/margins": 0.13974609971046448,
      "rewards/rejected": -0.951953113079071,
      "step": 150
    },
    {
      "epoch": 0.3311965811965812,
      "grad_norm": 10.242809753624087,
      "learning_rate": 5.093438193504931e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.4914062023162842,
      "logps/chosen": -0.38749998807907104,
      "logps/rejected": -0.4957031309604645,
      "loss": 1.234,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": -0.7749999761581421,
      "rewards/margins": 0.21766357123851776,
      "rewards/rejected": -0.991406261920929,
      "step": 155
    },
    {
      "epoch": 0.3418803418803419,
      "grad_norm": 10.439751676379643,
      "learning_rate": 5.011824247083868e-07,
      "logits/chosen": -1.52734375,
      "logits/rejected": -1.4968750476837158,
      "logps/chosen": -0.4423828125,
      "logps/rejected": -0.5267578363418579,
      "loss": 1.2656,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.884765625,
      "rewards/margins": 0.16884765028953552,
      "rewards/rejected": -1.0535156726837158,
      "step": 160
    },
    {
      "epoch": 0.3525641025641026,
      "grad_norm": 14.3284012618235,
      "learning_rate": 4.927409931945641e-07,
      "logits/chosen": -1.5789062976837158,
      "logits/rejected": -1.529687523841858,
      "logps/chosen": -0.4091796875,
      "logps/rejected": -0.5054687261581421,
      "loss": 1.2416,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.818359375,
      "rewards/margins": 0.19272461533546448,
      "rewards/rejected": -1.0109374523162842,
      "step": 165
    },
    {
      "epoch": 0.36324786324786323,
      "grad_norm": 14.80755642701636,
      "learning_rate": 4.840312749013969e-07,
      "logits/chosen": -1.5320312976837158,
      "logits/rejected": -1.502343773841858,
      "logps/chosen": -0.4046874940395355,
      "logps/rejected": -0.508984386920929,
      "loss": 1.2361,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.809374988079071,
      "rewards/margins": 0.20823974907398224,
      "rewards/rejected": -1.017968773841858,
      "step": 170
    },
    {
      "epoch": 0.37393162393162394,
      "grad_norm": 14.25663733412322,
      "learning_rate": 4.750653933643674e-07,
      "logits/chosen": -1.529687523841858,
      "logits/rejected": -1.5046875476837158,
      "logps/chosen": -0.4306640625,
      "logps/rejected": -0.591015636920929,
      "loss": 1.2129,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.559374988079071,
      "rewards/chosen": -0.861328125,
      "rewards/margins": 0.3213867247104645,
      "rewards/rejected": -1.182031273841858,
      "step": 175
    },
    {
      "epoch": 0.38461538461538464,
      "grad_norm": 15.795370678351055,
      "learning_rate": 4.6585582868665284e-07,
      "logits/chosen": -1.564062476158142,
      "logits/rejected": -1.5281250476837158,
      "logps/chosen": -0.42753905057907104,
      "logps/rejected": -0.5181640386581421,
      "loss": 1.252,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": -0.8550781011581421,
      "rewards/margins": 0.18134765326976776,
      "rewards/rejected": -1.0363280773162842,
      "step": 180
    },
    {
      "epoch": 0.3952991452991453,
      "grad_norm": 14.851431627716563,
      "learning_rate": 4.5641540016738414e-07,
      "logits/chosen": -1.5078125,
      "logits/rejected": -1.486718773841858,
      "logps/chosen": -0.4833984375,
      "logps/rejected": -0.573437511920929,
      "loss": 1.2683,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5296875238418579,
      "rewards/chosen": -0.966796875,
      "rewards/margins": 0.18000487983226776,
      "rewards/rejected": -1.146875023841858,
      "step": 185
    },
    {
      "epoch": 0.405982905982906,
      "grad_norm": 16.66852729151085,
      "learning_rate": 4.467572484577589e-07,
      "logits/chosen": -1.5695312023162842,
      "logits/rejected": -1.5398437976837158,
      "logps/chosen": -0.4722656309604645,
      "logps/rejected": -0.636523425579071,
      "loss": 1.2018,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -0.944531261920929,
      "rewards/margins": 0.327392578125,
      "rewards/rejected": -1.273046851158142,
      "step": 190
    },
    {
      "epoch": 0.4166666666666667,
      "grad_norm": 15.67537550174711,
      "learning_rate": 4.368948172698474e-07,
      "logits/chosen": -1.5234375,
      "logits/rejected": -1.4929687976837158,
      "logps/chosen": -0.47441405057907104,
      "logps/rejected": -0.5902343988418579,
      "loss": 1.2406,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5171874761581421,
      "rewards/chosen": -0.9488281011581421,
      "rewards/margins": 0.23137207329273224,
      "rewards/rejected": -1.1804687976837158,
      "step": 195
    },
    {
      "epoch": 0.42735042735042733,
      "grad_norm": 15.569695653680345,
      "learning_rate": 4.2684183466355006e-07,
      "logits/chosen": -1.524999976158142,
      "logits/rejected": -1.486718773841858,
      "logps/chosen": -0.4554687440395355,
      "logps/rejected": -0.570117175579071,
      "loss": 1.2451,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": -0.910937488079071,
      "rewards/margins": 0.22930297255516052,
      "rewards/rejected": -1.140234351158142,
      "step": 200
    },
    {
      "epoch": 0.43803418803418803,
      "grad_norm": 15.479616500843248,
      "learning_rate": 4.1661229393775783e-07,
      "logits/chosen": -1.5539062023162842,
      "logits/rejected": -1.528906226158142,
      "logps/chosen": -0.43828123807907104,
      "logps/rejected": -0.5511718988418579,
      "loss": 1.2274,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.565625011920929,
      "rewards/chosen": -0.8765624761581421,
      "rewards/margins": 0.22568359971046448,
      "rewards/rejected": -1.1023437976837158,
      "step": 205
    },
    {
      "epoch": 0.44871794871794873,
      "grad_norm": 14.30449666703877,
      "learning_rate": 4.062204341523094e-07,
      "logits/chosen": -1.5859375,
      "logits/rejected": -1.5390625,
      "logps/chosen": -0.530468761920929,
      "logps/rejected": -0.673828125,
      "loss": 1.25,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5609375238418579,
      "rewards/chosen": -1.060937523841858,
      "rewards/margins": 0.2860351502895355,
      "rewards/rejected": -1.34765625,
      "step": 210
    },
    {
      "epoch": 0.4594017094017094,
      "grad_norm": 12.308222317827216,
      "learning_rate": 3.9568072030786214e-07,
      "logits/chosen": -1.548437476158142,
      "logits/rejected": -1.5148437023162842,
      "logps/chosen": -0.5416015386581421,
      "logps/rejected": -0.7269531488418579,
      "loss": 1.2141,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5796874761581421,
      "rewards/chosen": -1.0832030773162842,
      "rewards/margins": 0.3740234375,
      "rewards/rejected": -1.4539062976837158,
      "step": 215
    },
    {
      "epoch": 0.4700854700854701,
      "grad_norm": 17.12922996772855,
      "learning_rate": 3.8500782321126227e-07,
      "logits/chosen": -1.5398437976837158,
      "logits/rejected": -1.501562476158142,
      "logps/chosen": -0.559374988079071,
      "logps/rejected": -0.66015625,
      "loss": 1.2812,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5640624761581421,
      "rewards/chosen": -1.118749976158142,
      "rewards/margins": 0.20118407905101776,
      "rewards/rejected": -1.3203125,
      "step": 220
    },
    {
      "epoch": 0.4807692307692308,
      "grad_norm": 11.416683434770386,
      "learning_rate": 3.742165990544432e-07,
      "logits/chosen": -1.49609375,
      "logits/rejected": -1.46484375,
      "logps/chosen": -0.4810546934604645,
      "logps/rejected": -0.5923827886581421,
      "loss": 1.2637,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -0.962109386920929,
      "rewards/margins": 0.22169189155101776,
      "rewards/rejected": -1.1847655773162842,
      "step": 225
    },
    {
      "epoch": 0.49145299145299143,
      "grad_norm": 11.009026957181359,
      "learning_rate": 3.633220687352755e-07,
      "logits/chosen": -1.528906226158142,
      "logits/rejected": -1.506250023841858,
      "logps/chosen": -0.451171875,
      "logps/rejected": -0.558789074420929,
      "loss": 1.2326,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.90234375,
      "rewards/margins": 0.21640625596046448,
      "rewards/rejected": -1.117578148841858,
      "step": 230
    },
    {
      "epoch": 0.5021367521367521,
      "grad_norm": 9.790833316230884,
      "learning_rate": 3.523393969491533e-07,
      "logits/chosen": -1.537500023841858,
      "logits/rejected": -1.49609375,
      "logps/chosen": -0.44843751192092896,
      "logps/rejected": -0.553906261920929,
      "loss": 1.2396,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.565625011920929,
      "rewards/chosen": -0.8968750238418579,
      "rewards/margins": 0.21157225966453552,
      "rewards/rejected": -1.107812523841858,
      "step": 235
    },
    {
      "epoch": 0.5128205128205128,
      "grad_norm": 11.337317668384426,
      "learning_rate": 3.4128387108042213e-07,
      "logits/chosen": -1.521093726158142,
      "logits/rejected": -1.5046875476837158,
      "logps/chosen": -0.47050780057907104,
      "logps/rejected": -0.603320300579071,
      "loss": 1.2046,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.9410156011581421,
      "rewards/margins": 0.2652343809604645,
      "rewards/rejected": -1.206640601158142,
      "step": 240
    },
    {
      "epoch": 0.5235042735042735,
      "grad_norm": 7.811613503865328,
      "learning_rate": 3.301708799230284e-07,
      "logits/chosen": -1.548437476158142,
      "logits/rejected": -1.5031249523162842,
      "logps/chosen": -0.484375,
      "logps/rejected": -0.628125011920929,
      "loss": 1.2204,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5765625238418579,
      "rewards/chosen": -0.96875,
      "rewards/margins": 0.2870117127895355,
      "rewards/rejected": -1.256250023841858,
      "step": 245
    },
    {
      "epoch": 0.5341880341880342,
      "grad_norm": 12.75427190985336,
      "learning_rate": 3.1901589226001136e-07,
      "logits/chosen": -1.560156226158142,
      "logits/rejected": -1.5242187976837158,
      "logps/chosen": -0.509570300579071,
      "logps/rejected": -0.595703125,
      "loss": 1.2453,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -1.019140601158142,
      "rewards/margins": 0.17375488579273224,
      "rewards/rejected": -1.19140625,
      "step": 250
    },
    {
      "epoch": 0.5448717948717948,
      "grad_norm": 14.20386765941155,
      "learning_rate": 3.0783443533165413e-07,
      "logits/chosen": -1.5554687976837158,
      "logits/rejected": -1.5226562023162842,
      "logps/chosen": -0.501953125,
      "logps/rejected": -0.643359363079071,
      "loss": 1.2107,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": -1.00390625,
      "rewards/margins": 0.283447265625,
      "rewards/rejected": -1.286718726158142,
      "step": 255
    },
    {
      "epoch": 0.5555555555555556,
      "grad_norm": 12.99228285452067,
      "learning_rate": 2.9664207322226467e-07,
      "logits/chosen": -1.55078125,
      "logits/rejected": -1.533593773841858,
      "logps/chosen": -0.5103515386581421,
      "logps/rejected": -0.6527343988418579,
      "loss": 1.2294,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.598437488079071,
      "rewards/chosen": -1.0207030773162842,
      "rewards/margins": 0.2850585877895355,
      "rewards/rejected": -1.3054687976837158,
      "step": 260
    },
    {
      "epoch": 0.5662393162393162,
      "grad_norm": 13.73274782795775,
      "learning_rate": 2.8545438519567173e-07,
      "logits/chosen": -1.521875023841858,
      "logits/rejected": -1.4968750476837158,
      "logps/chosen": -0.47832030057907104,
      "logps/rejected": -0.582226574420929,
      "loss": 1.2485,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.567187488079071,
      "rewards/chosen": -0.9566406011581421,
      "rewards/margins": 0.20791015028953552,
      "rewards/rejected": -1.164453148841858,
      "step": 265
    },
    {
      "epoch": 0.5769230769230769,
      "grad_norm": 12.89822692411219,
      "learning_rate": 2.742869440095914e-07,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.505468726158142,
      "logps/chosen": -0.5,
      "logps/rejected": -0.6136718988418579,
      "loss": 1.2508,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": -1.0,
      "rewards/margins": 0.22727051377296448,
      "rewards/rejected": -1.2273437976837158,
      "step": 270
    },
    {
      "epoch": 0.5876068376068376,
      "grad_norm": 15.122528311539122,
      "learning_rate": 2.6315529423904975e-07,
      "logits/chosen": -1.556249976158142,
      "logits/rejected": -1.5242187976837158,
      "logps/chosen": -0.48945313692092896,
      "logps/rejected": -0.6246093511581421,
      "loss": 1.2162,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -0.9789062738418579,
      "rewards/margins": 0.2700439393520355,
      "rewards/rejected": -1.2492187023162842,
      "step": 275
    },
    {
      "epoch": 0.5982905982905983,
      "grad_norm": 10.081247455678685,
      "learning_rate": 2.5207493063903506e-07,
      "logits/chosen": -1.524999976158142,
      "logits/rejected": -1.4874999523162842,
      "logps/chosen": -0.48261719942092896,
      "logps/rejected": -0.626953125,
      "loss": 1.2118,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5859375,
      "rewards/chosen": -0.9652343988418579,
      "rewards/margins": 0.28764647245407104,
      "rewards/rejected": -1.25390625,
      "step": 280
    },
    {
      "epoch": 0.6089743589743589,
      "grad_norm": 10.004059372308623,
      "learning_rate": 2.410612765764965e-07,
      "logits/chosen": -1.5476562976837158,
      "logits/rejected": -1.493749976158142,
      "logps/chosen": -0.508593738079071,
      "logps/rejected": -0.637499988079071,
      "loss": 1.2313,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6015625,
      "rewards/chosen": -1.017187476158142,
      "rewards/margins": 0.2581420838832855,
      "rewards/rejected": -1.274999976158142,
      "step": 285
    },
    {
      "epoch": 0.6196581196581197,
      "grad_norm": 13.343918364038107,
      "learning_rate": 2.301296625617126e-07,
      "logits/chosen": -1.56640625,
      "logits/rejected": -1.5421874523162842,
      "logps/chosen": -0.5296875238418579,
      "logps/rejected": -0.6832031011581421,
      "loss": 1.2206,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5953124761581421,
      "rewards/chosen": -1.0593750476837158,
      "rewards/margins": 0.30791014432907104,
      "rewards/rejected": -1.3664062023162842,
      "step": 290
    },
    {
      "epoch": 0.6303418803418803,
      "grad_norm": 11.405690420163953,
      "learning_rate": 2.1929530490891143e-07,
      "logits/chosen": -1.54296875,
      "logits/rejected": -1.5203125476837158,
      "logps/chosen": -0.513867199420929,
      "logps/rejected": -0.6363281011581421,
      "loss": 1.24,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5796874761581421,
      "rewards/chosen": -1.027734398841858,
      "rewards/margins": 0.24458007514476776,
      "rewards/rejected": -1.2726562023162842,
      "step": 295
    },
    {
      "epoch": 0.6410256410256411,
      "grad_norm": 11.838110328694215,
      "learning_rate": 2.0857328455584657e-07,
      "logits/chosen": -1.5398437976837158,
      "logits/rejected": -1.514062523841858,
      "logps/chosen": -0.5025390386581421,
      "logps/rejected": -0.637499988079071,
      "loss": 1.2237,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -1.0050780773162842,
      "rewards/margins": 0.270263671875,
      "rewards/rejected": -1.274999976158142,
      "step": 300
    },
    {
      "epoch": 0.6517094017094017,
      "grad_norm": 9.470686552249557,
      "learning_rate": 1.9797852607181124e-07,
      "logits/chosen": -1.576562523841858,
      "logits/rejected": -1.545312523841858,
      "logps/chosen": -0.47382813692092896,
      "logps/rejected": -0.627734363079071,
      "loss": 1.2092,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5921875238418579,
      "rewards/chosen": -0.9476562738418579,
      "rewards/margins": 0.3075195252895355,
      "rewards/rejected": -1.255468726158142,
      "step": 305
    },
    {
      "epoch": 0.6623931623931624,
      "grad_norm": 28.661639533450387,
      "learning_rate": 1.875257768833105e-07,
      "logits/chosen": -1.509374976158142,
      "logits/rejected": -1.493749976158142,
      "logps/chosen": -0.5083984136581421,
      "logps/rejected": -0.6343749761581421,
      "loss": 1.2552,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -1.0167968273162842,
      "rewards/margins": 0.25190430879592896,
      "rewards/rejected": -1.2687499523162842,
      "step": 310
    },
    {
      "epoch": 0.6730769230769231,
      "grad_norm": 12.529058191151396,
      "learning_rate": 1.7722958674630727e-07,
      "logits/chosen": -1.5234375,
      "logits/rejected": -1.5031249523162842,
      "logps/chosen": -0.521289050579071,
      "logps/rejected": -0.6499999761581421,
      "loss": 1.2271,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.042578101158142,
      "rewards/margins": 0.2585205137729645,
      "rewards/rejected": -1.2999999523162842,
      "step": 315
    },
    {
      "epoch": 0.6837606837606838,
      "grad_norm": 13.436410257147102,
      "learning_rate": 1.6710428749361832e-07,
      "logits/chosen": -1.525781273841858,
      "logits/rejected": -1.490625023841858,
      "logps/chosen": -0.5423828363418579,
      "logps/rejected": -0.701953113079071,
      "loss": 1.2103,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0847656726837158,
      "rewards/margins": 0.3174804747104645,
      "rewards/rejected": -1.403906226158142,
      "step": 320
    },
    {
      "epoch": 0.6944444444444444,
      "grad_norm": 15.554250562987784,
      "learning_rate": 1.571639730856482e-07,
      "logits/chosen": -1.5203125476837158,
      "logits/rejected": -1.4914062023162842,
      "logps/chosen": -0.5386718511581421,
      "logps/rejected": -0.6773437261581421,
      "loss": 1.2158,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -1.0773437023162842,
      "rewards/margins": 0.27617186307907104,
      "rewards/rejected": -1.3546874523162842,
      "step": 325
    },
    {
      "epoch": 0.7051282051282052,
      "grad_norm": 11.275430224004403,
      "learning_rate": 1.4742247999223112e-07,
      "logits/chosen": -1.5398437976837158,
      "logits/rejected": -1.51953125,
      "logps/chosen": -0.5498046875,
      "logps/rejected": -0.710156261920929,
      "loss": 1.2059,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5953124761581421,
      "rewards/chosen": -1.099609375,
      "rewards/margins": 0.3193359375,
      "rewards/rejected": -1.420312523841858,
      "step": 330
    },
    {
      "epoch": 0.7158119658119658,
      "grad_norm": 10.324498809414596,
      "learning_rate": 1.3789336793288877e-07,
      "logits/chosen": -1.489843726158142,
      "logits/rejected": -1.482031226158142,
      "logps/chosen": -0.538281261920929,
      "logps/rejected": -0.717578113079071,
      "loss": 1.187,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6078125238418579,
      "rewards/chosen": -1.076562523841858,
      "rewards/margins": 0.357666015625,
      "rewards/rejected": -1.435156226158142,
      "step": 335
    },
    {
      "epoch": 0.7264957264957265,
      "grad_norm": 14.78536991969488,
      "learning_rate": 1.285899010023115e-07,
      "logits/chosen": -1.5421874523162842,
      "logits/rejected": -1.517187476158142,
      "logps/chosen": -0.540820300579071,
      "logps/rejected": -0.678515613079071,
      "loss": 1.2288,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5921875238418579,
      "rewards/chosen": -1.081640601158142,
      "rewards/margins": 0.2734375,
      "rewards/rejected": -1.357031226158142,
      "step": 340
    },
    {
      "epoch": 0.7371794871794872,
      "grad_norm": 11.083012202046339,
      "learning_rate": 1.1952502920733554e-07,
      "logits/chosen": -1.509374976158142,
      "logits/rejected": -1.4968750476837158,
      "logps/chosen": -0.49921876192092896,
      "logps/rejected": -0.645703136920929,
      "loss": 1.2018,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.9984375238418579,
      "rewards/margins": 0.2920898497104645,
      "rewards/rejected": -1.291406273841858,
      "step": 345
    },
    {
      "epoch": 0.7478632478632479,
      "grad_norm": 13.711430399043142,
      "learning_rate": 1.1071137044111784e-07,
      "logits/chosen": -1.514062523841858,
      "logits/rejected": -1.4921875,
      "logps/chosen": -0.48417967557907104,
      "logps/rejected": -0.6484375,
      "loss": 1.2034,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.9683593511581421,
      "rewards/margins": 0.3277343809604645,
      "rewards/rejected": -1.296875,
      "step": 350
    },
    {
      "epoch": 0.7585470085470085,
      "grad_norm": 17.091136083884738,
      "learning_rate": 1.0216119291959624e-07,
      "logits/chosen": -1.5234375,
      "logits/rejected": -1.4874999523162842,
      "logps/chosen": -0.5474609136581421,
      "logps/rejected": -0.705859363079071,
      "loss": 1.2126,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.589062511920929,
      "rewards/chosen": -1.0949218273162842,
      "rewards/margins": 0.3169921934604645,
      "rewards/rejected": -1.411718726158142,
      "step": 355
    },
    {
      "epoch": 0.7692307692307693,
      "grad_norm": 11.69714980581959,
      "learning_rate": 9.388639810468644e-08,
      "logits/chosen": -1.528906226158142,
      "logits/rejected": -1.498437523841858,
      "logps/chosen": -0.516406238079071,
      "logps/rejected": -0.6910156011581421,
      "loss": 1.1951,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.573437511920929,
      "rewards/chosen": -1.032812476158142,
      "rewards/margins": 0.3477539122104645,
      "rewards/rejected": -1.3820312023162842,
      "step": 360
    },
    {
      "epoch": 0.7799145299145299,
      "grad_norm": 9.769420151980366,
      "learning_rate": 8.589850413798264e-08,
      "logits/chosen": -1.5578124523162842,
      "logits/rejected": -1.5226562023162842,
      "logps/chosen": -0.513476550579071,
      "logps/rejected": -0.6685546636581421,
      "loss": 1.1922,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.026953101158142,
      "rewards/margins": 0.3106933534145355,
      "rewards/rejected": -1.3371093273162842,
      "step": 365
    },
    {
      "epoch": 0.7905982905982906,
      "grad_norm": 16.620091820620416,
      "learning_rate": 7.82086298080236e-08,
      "logits/chosen": -1.514062523841858,
      "logits/rejected": -1.474218726158142,
      "logps/chosen": -0.5093749761581421,
      "logps/rejected": -0.6572265625,
      "loss": 1.2093,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.0187499523162842,
      "rewards/margins": 0.29389649629592896,
      "rewards/rejected": -1.314453125,
      "step": 370
    },
    {
      "epoch": 0.8012820512820513,
      "grad_norm": 13.721108190371638,
      "learning_rate": 7.082747907343972e-08,
      "logits/chosen": -1.51171875,
      "logits/rejected": -1.484375,
      "logps/chosen": -0.528515636920929,
      "logps/rejected": -0.6703125238418579,
      "loss": 1.2263,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.057031273841858,
      "rewards/margins": 0.283203125,
      "rewards/rejected": -1.3406250476837158,
      "step": 375
    },
    {
      "epoch": 0.811965811965812,
      "grad_norm": 10.832579603627439,
      "learning_rate": 6.37653261635252e-08,
      "logits/chosen": -1.560156226158142,
      "logits/rejected": -1.524999976158142,
      "logps/chosen": -0.5396484136581421,
      "logps/rejected": -0.745312511920929,
      "loss": 1.1876,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5921875238418579,
      "rewards/chosen": -1.0792968273162842,
      "rewards/margins": 0.41064453125,
      "rewards/rejected": -1.490625023841858,
      "step": 380
    },
    {
      "epoch": 0.8226495726495726,
      "grad_norm": 12.325756153692222,
      "learning_rate": 5.703200127697302e-08,
      "logits/chosen": -1.52734375,
      "logits/rejected": -1.5070312023162842,
      "logps/chosen": -0.526171863079071,
      "logps/rejected": -0.6773437261581421,
      "loss": 1.2089,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.052343726158142,
      "rewards/margins": 0.3001953065395355,
      "rewards/rejected": -1.3546874523162842,
      "step": 385
    },
    {
      "epoch": 0.8333333333333334,
      "grad_norm": 18.975074009248868,
      "learning_rate": 5.063687689868209e-08,
      "logits/chosen": -1.536718726158142,
      "logits/rejected": -1.498437523841858,
      "logps/chosen": -0.5103515386581421,
      "logps/rejected": -0.698437511920929,
      "loss": 1.188,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.596875011920929,
      "rewards/chosen": -1.0207030773162842,
      "rewards/margins": 0.3758789002895355,
      "rewards/rejected": -1.396875023841858,
      "step": 390
    },
    {
      "epoch": 0.844017094017094,
      "grad_norm": 10.381047057509468,
      "learning_rate": 4.4588854753680106e-08,
      "logits/chosen": -1.517187476158142,
      "logits/rejected": -1.490625023841858,
      "logps/chosen": -0.504101574420929,
      "logps/rejected": -0.62890625,
      "loss": 1.2251,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.008203148841858,
      "rewards/margins": 0.24863281846046448,
      "rewards/rejected": -1.2578125,
      "step": 395
    },
    {
      "epoch": 0.8547008547008547,
      "grad_norm": 10.580168425323338,
      "learning_rate": 3.889635341632302e-08,
      "logits/chosen": -1.53125,
      "logits/rejected": -1.5085937976837158,
      "logps/chosen": -0.535937488079071,
      "logps/rejected": -0.703125,
      "loss": 1.2028,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.598437488079071,
      "rewards/chosen": -1.071874976158142,
      "rewards/margins": 0.33247071504592896,
      "rewards/rejected": -1.40625,
      "step": 400
    },
    {
      "epoch": 0.8547008547008547,
      "eval_logits/chosen": -1.384198546409607,
      "eval_logits/rejected": -1.3550907373428345,
      "eval_logps/chosen": -0.5326991081237793,
      "eval_logps/rejected": -0.6943044066429138,
      "eval_loss": 1.197023630142212,
      "eval_nll_loss": 0.0,
      "eval_rewards/accuracies": 0.6236559152603149,
      "eval_rewards/chosen": -1.0653982162475586,
      "eval_rewards/margins": 0.3231083154678345,
      "eval_rewards/rejected": -1.3886088132858276,
      "eval_runtime": 26.9387,
      "eval_samples_per_second": 72.795,
      "eval_steps_per_second": 2.302,
      "step": 400
    },
    {
      "epoch": 0.8653846153846154,
      "grad_norm": 10.102504447281174,
      "learning_rate": 3.356729659201946e-08,
      "logits/chosen": -1.5578124523162842,
      "logits/rejected": -1.5125000476837158,
      "logps/chosen": -0.568359375,
      "logps/rejected": -0.699999988079071,
      "loss": 1.2354,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.5843750238418579,
      "rewards/chosen": -1.13671875,
      "rewards/margins": 0.26372069120407104,
      "rewards/rejected": -1.399999976158142,
      "step": 405
    },
    {
      "epoch": 0.8760683760683761,
      "grad_norm": 18.80658014166381,
      "learning_rate": 2.8609102087789262e-08,
      "logits/chosen": -1.51953125,
      "logits/rejected": -1.4835937023162842,
      "logps/chosen": -0.532031238079071,
      "logps/rejected": -0.6910156011581421,
      "loss": 1.2005,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.064062476158142,
      "rewards/margins": 0.31684571504592896,
      "rewards/rejected": -1.3820312023162842,
      "step": 410
    },
    {
      "epoch": 0.8867521367521367,
      "grad_norm": 12.388053747492433,
      "learning_rate": 2.4028671487010464e-08,
      "logits/chosen": -1.509374976158142,
      "logits/rejected": -1.489843726158142,
      "logps/chosen": -0.525585949420929,
      "logps/rejected": -0.6937500238418579,
      "loss": 1.19,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.598437488079071,
      "rewards/chosen": -1.051171898841858,
      "rewards/margins": 0.3369140625,
      "rewards/rejected": -1.3875000476837158,
      "step": 415
    },
    {
      "epoch": 0.8974358974358975,
      "grad_norm": 14.094598355889069,
      "learning_rate": 1.9832380542726224e-08,
      "logits/chosen": -1.5460937023162842,
      "logits/rejected": -1.5265624523162842,
      "logps/chosen": -0.535937488079071,
      "logps/rejected": -0.703125,
      "loss": 1.2081,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.071874976158142,
      "rewards/margins": 0.33330076932907104,
      "rewards/rejected": -1.40625,
      "step": 420
    },
    {
      "epoch": 0.9081196581196581,
      "grad_norm": 16.07539311926432,
      "learning_rate": 1.6026070302883656e-08,
      "logits/chosen": -1.5,
      "logits/rejected": -1.463281273841858,
      "logps/chosen": -0.521484375,
      "logps/rejected": -0.6792968511581421,
      "loss": 1.1997,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -1.04296875,
      "rewards/margins": 0.31560057401657104,
      "rewards/rejected": -1.3585937023162842,
      "step": 425
    },
    {
      "epoch": 0.9188034188034188,
      "grad_norm": 14.515898324276503,
      "learning_rate": 1.2615038979858417e-08,
      "logits/chosen": -1.540624976158142,
      "logits/rejected": -1.5187499523162842,
      "logps/chosen": -0.5269531011581421,
      "logps/rejected": -0.716796875,
      "loss": 1.1778,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -1.0539062023162842,
      "rewards/margins": 0.3780273497104645,
      "rewards/rejected": -1.43359375,
      "step": 430
    },
    {
      "epoch": 0.9294871794871795,
      "grad_norm": 11.346931084998822,
      "learning_rate": 9.604034575581698e-09,
      "logits/chosen": -1.5398437976837158,
      "logits/rejected": -1.5265624523162842,
      "logps/chosen": -0.5445312261581421,
      "logps/rejected": -0.7339843511581421,
      "loss": 1.1779,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -1.0890624523162842,
      "rewards/margins": 0.3792968690395355,
      "rewards/rejected": -1.4679687023162842,
      "step": 435
    },
    {
      "epoch": 0.9401709401709402,
      "grad_norm": 9.442038079746466,
      "learning_rate": 6.997248272535305e-09,
      "logits/chosen": -1.5046875476837158,
      "logits/rejected": -1.475000023841858,
      "logps/chosen": -0.5355468988418579,
      "logps/rejected": -0.689453125,
      "loss": 1.1997,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -1.0710937976837158,
      "rewards/margins": 0.3072265684604645,
      "rewards/rejected": -1.37890625,
      "step": 440
    },
    {
      "epoch": 0.9508547008547008,
      "grad_norm": 15.251874082935148,
      "learning_rate": 4.7983085998146596e-09,
      "logits/chosen": -1.5398437976837158,
      "logits/rejected": -1.5234375,
      "logps/chosen": -0.5464843511581421,
      "logps/rejected": -0.7046874761581421,
      "loss": 1.2014,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.0929687023162842,
      "rewards/margins": 0.3182617127895355,
      "rewards/rejected": -1.4093749523162842,
      "step": 445
    },
    {
      "epoch": 0.9615384615384616,
      "grad_norm": 16.8382443963138,
      "learning_rate": 3.0102763823796506e-09,
      "logits/chosen": -1.5203125476837158,
      "logits/rejected": -1.4812500476837158,
      "logps/chosen": -0.5503906011581421,
      "logps/rejected": -0.7093750238418579,
      "loss": 1.2004,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.620312511920929,
      "rewards/chosen": -1.1007812023162842,
      "rewards/margins": 0.3185058534145355,
      "rewards/rejected": -1.4187500476837158,
      "step": 450
    },
    {
      "epoch": 0.9722222222222222,
      "grad_norm": 17.584637370123247,
      "learning_rate": 1.6356404805242696e-09,
      "logits/chosen": -1.5546875,
      "logits/rejected": -1.51953125,
      "logps/chosen": -0.5498046875,
      "logps/rejected": -0.6880859136581421,
      "loss": 1.2288,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.589062511920929,
      "rewards/chosen": -1.099609375,
      "rewards/margins": 0.27663880586624146,
      "rewards/rejected": -1.3761718273162842,
      "step": 455
    },
    {
      "epoch": 0.9829059829059829,
      "grad_norm": 14.609935824322315,
      "learning_rate": 6.763143254950976e-10,
      "logits/chosen": -1.506250023841858,
      "logits/rejected": -1.4929687976837158,
      "logps/chosen": -0.5513671636581421,
      "logps/rejected": -0.719921886920929,
      "loss": 1.2021,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.1027343273162842,
      "rewards/margins": 0.33662110567092896,
      "rewards/rejected": -1.439843773841858,
      "step": 460
    },
    {
      "epoch": 0.9935897435897436,
      "grad_norm": 14.658436552359875,
      "learning_rate": 1.3363325608131136e-10,
      "logits/chosen": -1.5382812023162842,
      "logits/rejected": -1.5085937976837158,
      "logps/chosen": -0.5210937261581421,
      "logps/rejected": -0.701953113079071,
      "loss": 1.1787,
      "nll_loss": 0.0,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -1.0421874523162842,
      "rewards/margins": 0.3624023497104645,
      "rewards/rejected": -1.403906226158142,
      "step": 465
    },
    {
      "epoch": 1.0,
      "step": 468,
      "total_flos": 0.0,
      "train_loss": 1.2459888947315705,
      "train_runtime": 3192.5713,
      "train_samples_per_second": 18.755,
      "train_steps_per_second": 0.147
    }
  ],
  "logging_steps": 5,
  "max_steps": 468,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}