Model save

README.md

@@ -3,31 +3,20 @@ library_name: transformers
 license: other
 base_model: trl-lib/qwen1.5-0.5b-sft
 tags:
-- alignment-handbook
 - trl
 - simpo
 - generated_from_trainer
 model-index:
-- name: trl-lib/qwen1.5-0.5b-sft
+- name: simpo-exps_qwen05b
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# trl-lib/qwen1.5-0.5b-sft
+# simpo-exps_qwen05b
 
 This model is a fine-tuned version of [trl-lib/qwen1.5-0.5b-sft](https://huggingface.co/trl-lib/qwen1.5-0.5b-sft) on an unknown dataset.
-It achieves the following results on the evaluation set:
-- Loss: 5.8350
-- Rewards/chosen: -16.6895
-- Rewards/rejected: -17.6490
-- Rewards/accuracies: 0.5245
-- Rewards/margins: 0.9596
-- Logps/rejected: -1.7649
-- Logps/chosen: -1.6689
-- Logits/rejected: -0.3783
-- Logits/chosen: -0.4409
 
 ## Model description
 
@@ -47,12 +36,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 8e-08
-- train_batch_size: 2
+- train_batch_size: 1
 - eval_batch_size: 4
 - seed: 42
 - gradient_accumulation_steps: 16
-- total_train_batch_size: 32
-- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
 - training_steps: 20
@@ -63,7 +52,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- Transformers 4.48.3
+- Transformers 4.44.2
 - Pytorch 2.4.0+cu121
-- Datasets 2.18.0
-- Tokenizers 0.21.0
+- Datasets 3.0.1
+- Tokenizers 0.19.1

all_results.json

@@ -1,21 +1,8 @@
 {
-    "epoch": 0.64,
-    "eval_logits/chosen": -0.4409240782260895,
-    "eval_logits/rejected": -0.3783179819583893,
-    "eval_logps/chosen": -1.6689454317092896,
-    "eval_logps/rejected": -1.7649023532867432,
-    "eval_loss": 5.835046768188477,
-    "eval_rewards/accuracies": 0.5245413780212402,
-    "eval_rewards/chosen": -16.689455032348633,
-    "eval_rewards/margins": 0.9595676064491272,
-    "eval_rewards/rejected": -17.649023056030273,
-    "eval_runtime": 294.9751,
-    "eval_samples": 5595,
-    "eval_samples_per_second": 18.968,
-    "eval_steps_per_second": 4.743,
+    "epoch": 0.005352065562803144,
     "total_flos": 0.0,
-    "train_loss": 98.40630645751953,
-    "train_runtime": 107.1933,
-    "train_samples_per_second": 5.971,
-    "train_steps_per_second": 0.187
+    "train_loss": 0.7357763648033142,
+    "train_runtime": 51.0237,
+    "train_samples_per_second": 6.272,
+    "train_steps_per_second": 0.392
 }

generation_config.json

@@ -3,5 +3,5 @@
   "eos_token_id": 151645,
   "max_new_tokens": 2048,
   "pad_token_id": 151645,
-  "transformers_version": "4.48.3"
+  "transformers_version": "4.44.2"
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 0.64,
+    "epoch": 0.005352065562803144,
     "total_flos": 0.0,
-    "train_loss": 98.40630645751953,
-    "train_runtime": 107.1933,
-    "train_samples_per_second": 5.971,
-    "train_steps_per_second": 0.187
+    "train_loss": 0.7357763648033142,
+    "train_runtime": 51.0237,
+    "train_samples_per_second": 6.272,
+    "train_steps_per_second": 0.392
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.64,
+  "epoch": 0.005352065562803144,
   "eval_steps": 400,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -9,73 +9,77 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.16,
-      "grad_norm": 3744.0,
+      "epoch": 0.001338016390700786,
+      "grad_norm": 6.0,
       "learning_rate": 7.464101615137755e-08,
-      "logits/chosen": -0.5232290029525757,
-      "logits/rejected": -0.4501457214355469,
-      "logps/chosen": -1.740447998046875,
-      "logps/rejected": -1.7914221286773682,
-      "loss": 105.9691,
-      "rewards/accuracies": 0.45625001192092896,
-      "rewards/chosen": -17.404481887817383,
-      "rewards/margins": 0.5097407698631287,
-      "rewards/rejected": -17.91421890258789,
+      "logits/chosen": -0.8066712617874146,
+      "logits/rejected": -0.6942030787467957,
+      "logps/chosen": -1.797594428062439,
+      "logps/rejected": -1.9124763011932373,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.797594428062439,
+      "rewards/margins": 0.11488206684589386,
+      "rewards/rejected": -1.9124763011932373,
+      "semantic_entropy": 0.6421751379966736,
       "step": 5
     },
     {
-      "epoch": 0.32,
-      "grad_norm": 1224.0,
+      "epoch": 0.002676032781401572,
+      "grad_norm": 6.53125,
       "learning_rate": 4.6945927106677224e-08,
-      "logits/chosen": -0.4762292802333832,
-      "logits/rejected": -0.37752610445022583,
-      "logps/chosen": -1.6145172119140625,
-      "logps/rejected": -1.8529552221298218,
-      "loss": 87.966,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -16.145172119140625,
-      "rewards/margins": 2.384378671646118,
-      "rewards/rejected": -18.529550552368164,
+      "logits/chosen": -0.6600304841995239,
+      "logits/rejected": -0.5782949328422546,
+      "logps/chosen": -1.6283073425292969,
+      "logps/rejected": -1.860073447227478,
+      "loss": 0.6998,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.6283073425292969,
+      "rewards/margins": 0.23176583647727966,
+      "rewards/rejected": -1.860073447227478,
+      "semantic_entropy": 0.6761103868484497,
       "step": 10
     },
     {
-      "epoch": 0.48,
-      "grad_norm": 1464.0,
+      "epoch": 0.004014049172102358,
+      "grad_norm": 10.9375,
       "learning_rate": 1.4288495612538426e-08,
-      "logits/chosen": -0.5665463209152222,
-      "logits/rejected": -0.5252197980880737,
-      "logps/chosen": -1.716834306716919,
-      "logps/rejected": -1.7479000091552734,
-      "loss": 98.5938,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -17.16834259033203,
-      "rewards/margins": 0.310658723115921,
-      "rewards/rejected": -17.479000091552734,
+      "logits/chosen": -0.7018337249755859,
+      "logits/rejected": -0.6630809903144836,
+      "logps/chosen": -1.814099907875061,
+      "logps/rejected": -1.7862392663955688,
+      "loss": 0.8097,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.814099907875061,
+      "rewards/margins": -0.027860689908266068,
+      "rewards/rejected": -1.7862392663955688,
+      "semantic_entropy": 0.6300801038742065,
       "step": 15
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 4800.0,
+      "epoch": 0.005352065562803144,
+      "grad_norm": 12.5625,
       "learning_rate": 0.0,
-      "logits/chosen": -0.5845416784286499,
-      "logits/rejected": -0.5405458211898804,
-      "logps/chosen": -1.8133652210235596,
-      "logps/rejected": -1.8326694965362549,
-      "loss": 101.0963,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -18.133655548095703,
-      "rewards/margins": 0.19304139912128448,
-      "rewards/rejected": -18.326696395874023,
+      "logits/chosen": -0.6895834803581238,
+      "logits/rejected": -0.6498485207557678,
+      "logps/chosen": -1.7860157489776611,
+      "logps/rejected": -1.9019441604614258,
+      "loss": 0.7433,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -1.7860157489776611,
+      "rewards/margins": 0.11592836678028107,
+      "rewards/rejected": -1.9019441604614258,
+      "semantic_entropy": 0.6502264738082886,
       "step": 20
     },
     {
-      "epoch": 0.64,
+      "epoch": 0.005352065562803144,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 98.40630645751953,
-      "train_runtime": 107.1933,
-      "train_samples_per_second": 5.971,
-      "train_steps_per_second": 0.187
+      "train_loss": 0.7357763648033142,
+      "train_runtime": 51.0237,
+      "train_samples_per_second": 6.272,
+      "train_steps_per_second": 0.392
     }
   ],
   "logging_steps": 5,
@@ -96,7 +100,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }