Training in progress, epoch 1

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr4-Project/runs/n0ajdo2h) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/yoon307-kaist/medgemma-27b-it-dr4-Project/runs/md2qaah5) 
 
 
 This model was trained with SFT.

adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
+    "out_proj",
     "q_proj",
-    "v_proj",
-    "o_proj",
+    "k_proj",
     "fc1",
     "fc2",
-    "down_proj",
-    "k_proj",
-    "out_proj",
-    "up_proj"
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "trainable_token_indices": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3970a91ed296e722a0a69c51d3f666501769b1d90a32776d014bef682f3881de
+oid sha256:1e7416daa15f9e3546c7acc2cc8d97c53a83ba8f13cc7d919d2a14b5098fb268
 size 6127553104

runs/Jul20_11-55-00_meedgxh100a/events.out.tfevents.1753026902.meedgxh100a.1324727.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45f7558c6db5183cc387904438acb08fb8bc0ae7b66c3ef1af40a4f394dee7c1
+size 9917

train_medgemma_ft_copy.py

@@ -215,9 +215,9 @@ if __name__ == '__main__':
             if train:
                 return random.sample(neg, 5*num_sample), random.sample(pos, num_sample)
             else:
-                # return random.sample(neg, 5*num_sample), pos
+                return random.sample(neg, num_sample), pos
                 # return random.sample(neg, 15), random.sample(pos, 15)
-                return neg, pos
+                # return neg, pos
         train_dataset = sum(subset(train_dataset,train=True), [])
         val_dataset_raw = sum(subset(val_dataset_raw,train=False), [])
 
@@ -286,7 +286,7 @@ if __name__ == '__main__':
         from peft import PeftModel
         print("🔁 Loading trained PEFT weights...")
         # model = PeftModel.from_pretrained(model, exp_name)
-        model = PeftModel.from_pretrained(model, exp_name+"/checkpoint-598")
+        model = PeftModel.from_pretrained(model, exp_name+"/checkpoint-368")
         # model = PeftModel.from_pretrained(model, "llava-1.5-7b-hf-dr-all/checkpoint-80")
         phase= "val"
     else:
@@ -309,7 +309,7 @@ if __name__ == '__main__':
         save_strategy="epoch",                                   # Save checkpoint every epoch
         eval_strategy="steps",                                   # Evaluate every `eval_steps`
         eval_steps=10000,                                           # Number of steps between evaluations
-        learning_rate=1e-3,                             # Learning rate based on QLoRA paper
+        learning_rate=5e-4,                             # Learning rate based on QLoRA paper
         bf16=True,                                               # Use bfloat16 precision
         max_grad_norm=0.3,                                       # Max gradient norm based on QLoRA paper
         warmup_ratio=0.03,                                       # Warmup ratio based on QLoRA paper

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f5982150528a15b0c3a536efe19c87a8fe24952006c13cabac89f7509a65aa7
+oid sha256:d6872e82880b94ff7074d3aa909ee437acad559e5263eff6ee7d85664a176db4
 size 5816