File size: 15,928 Bytes
f44ef3c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 |
2025-08-29 01:51:16 - pico-train - INFO - Step 0 -- 📊 Evaluation Results
2025-08-29 01:51:16 - pico-train - INFO - └── paloma: inf
2025-08-29 01:51:17 - pico-train - INFO - ==================================================
2025-08-29 01:51:17 - pico-train - INFO - ✨ Training Configuration
2025-08-29 01:51:17 - pico-train - INFO - ==================================================
2025-08-29 01:51:17 - pico-train - INFO - ╭─────────────────────────────────────────────────────╮
2025-08-29 01:51:17 - pico-train - INFO - │ checkpointing: │
2025-08-29 01:51:17 - pico-train - INFO - │ checkpoints_dir: checkpoints │
2025-08-29 01:51:17 - pico-train - INFO - │ evaluation: │
2025-08-29 01:51:17 - pico-train - INFO - │ eval_results_dir: eval_results │
2025-08-29 01:51:17 - pico-train - INFO - │ fabric_checkpoint_dir: fabric_state │
2025-08-29 01:51:17 - pico-train - INFO - │ fabric_checkpoint_filename: checkpoint.pt │
2025-08-29 01:51:17 - pico-train - INFO - │ hf_checkpoint: │
2025-08-29 01:51:17 - pico-train - INFO - │ collection_slug: null │
2025-08-29 01:51:17 - pico-train - INFO - │ repo_id: ThomasTheMaker/pico-decoder-tiny │
2025-08-29 01:51:17 - pico-train - INFO - │ learning_dynamics: │
2025-08-29 01:51:17 - pico-train - INFO - │ batch_size: 1 │
2025-08-29 01:51:17 - pico-train - INFO - │ eval_data: null │
2025-08-29 01:51:17 - pico-train - INFO - │ layer_suffixes: │
2025-08-29 01:51:17 - pico-train - INFO - │ - attention.v_proj │
2025-08-29 01:51:17 - pico-train - INFO - │ - attention.o_proj │
2025-08-29 01:51:17 - pico-train - INFO - │ - swiglu.w_2 │
2025-08-29 01:51:17 - pico-train - INFO - │ sequence_idx: -1 │
2025-08-29 01:51:17 - pico-train - INFO - │ learning_dynamics_dir: learning_dynamics │
2025-08-29 01:51:17 - pico-train - INFO - │ logs_dir: logs │
2025-08-29 01:51:17 - pico-train - INFO - │ run_name: pico-decoder-tiny-dolma29k-v3 │
2025-08-29 01:51:17 - pico-train - INFO - │ runs_dir: runs │
2025-08-29 01:51:17 - pico-train - INFO - │ save_every_n_steps: 500 │
2025-08-29 01:51:17 - pico-train - INFO - │ save_to_hf: true │
2025-08-29 01:51:17 - pico-train - INFO - │ training: │
2025-08-29 01:51:17 - pico-train - INFO - │ auto_resume: true │
2025-08-29 01:51:17 - pico-train - INFO - │ data: │
2025-08-29 01:51:17 - pico-train - INFO - │ dataloader: │
2025-08-29 01:51:17 - pico-train - INFO - │ batch_size: 4 │
2025-08-29 01:51:17 - pico-train - INFO - │ dataset: │
2025-08-29 01:51:17 - pico-train - INFO - │ name: pico-lm/pretokenized-dolma │
2025-08-29 01:51:17 - pico-train - INFO - │ tokenizer: │
2025-08-29 01:51:17 - pico-train - INFO - │ name: allenai/OLMo-7B-0724-hf │
2025-08-29 01:51:17 - pico-train - INFO - │ vocab_size: 50304 │
2025-08-29 01:51:17 - pico-train - INFO - │ evaluation: │
2025-08-29 01:51:17 - pico-train - INFO - │ metrics: │
2025-08-29 01:51:17 - pico-train - INFO - │ - paloma │
2025-08-29 01:51:17 - pico-train - INFO - │ paloma: │
2025-08-29 01:51:17 - pico-train - INFO - │ batch_size: 1 │
2025-08-29 01:51:17 - pico-train - INFO - │ dataset_name: pico-lm/pretokenized-paloma-tinsy │
2025-08-29 01:51:17 - pico-train - INFO - │ dataset_split: val │
2025-08-29 01:51:17 - pico-train - INFO - │ max_length: 2048 │
2025-08-29 01:51:17 - pico-train - INFO - │ model: │
2025-08-29 01:51:17 - pico-train - INFO - │ activation_hidden_dim: 384 │
2025-08-29 01:51:17 - pico-train - INFO - │ attention_n_heads: 12 │
2025-08-29 01:51:17 - pico-train - INFO - │ attention_n_kv_heads: 4 │
2025-08-29 01:51:17 - pico-train - INFO - │ batch_size: 1024 │
2025-08-29 01:51:17 - pico-train - INFO - │ d_model: 96 │
2025-08-29 01:51:17 - pico-train - INFO - │ max_seq_len: 2048 │
2025-08-29 01:51:17 - pico-train - INFO - │ model_type: pico_decoder │
2025-08-29 01:51:17 - pico-train - INFO - │ n_layers: 12 │
2025-08-29 01:51:17 - pico-train - INFO - │ norm_eps: 1.0e-06 │
2025-08-29 01:51:17 - pico-train - INFO - │ position_emb_theta: 10000.0 │
2025-08-29 01:51:17 - pico-train - INFO - │ vocab_size: 50304 │
2025-08-29 01:51:17 - pico-train - INFO - │ monitoring: │
2025-08-29 01:51:17 - pico-train - INFO - │ logging: │
2025-08-29 01:51:17 - pico-train - INFO - │ log_every_n_steps: 25 │
2025-08-29 01:51:17 - pico-train - INFO - │ log_level: INFO │
2025-08-29 01:51:17 - pico-train - INFO - │ save_to_wandb: false │
2025-08-29 01:51:17 - pico-train - INFO - │ wandb: │
2025-08-29 01:51:17 - pico-train - INFO - │ entity: boymyc │
2025-08-29 01:51:17 - pico-train - INFO - │ project: pico-decoder-tiny │
2025-08-29 01:51:17 - pico-train - INFO - │ training: │
2025-08-29 01:51:17 - pico-train - INFO - │ fabric: │
2025-08-29 01:51:17 - pico-train - INFO - │ accelerator: cuda │
2025-08-29 01:51:17 - pico-train - INFO - │ num_devices: 1 │
2025-08-29 01:51:17 - pico-train - INFO - │ num_nodes: 1 │
2025-08-29 01:51:17 - pico-train - INFO - │ precision: bf16-mixed │
2025-08-29 01:51:17 - pico-train - INFO - │ max_steps: 20000 │
2025-08-29 01:51:17 - pico-train - INFO - │ optimization: │
2025-08-29 01:51:17 - pico-train - INFO - │ gradient_accumulation_steps: 4 │
2025-08-29 01:51:17 - pico-train - INFO - │ lr: 5.0e-05 │
2025-08-29 01:51:17 - pico-train - INFO - │ lr_scheduler: linear_with_warmup │
2025-08-29 01:51:17 - pico-train - INFO - │ lr_warmup_steps: 8000 │
2025-08-29 01:51:17 - pico-train - INFO - │ optimizer: adamw │
2025-08-29 01:51:17 - pico-train - INFO - │ │
2025-08-29 01:51:17 - pico-train - INFO - ╰─────────────────────────────────────────────────────╯
2025-08-29 01:51:17 - pico-train - INFO - ==================================================
2025-08-29 01:51:17 - pico-train - INFO - ⛭ Runtime Summary:
2025-08-29 01:51:17 - pico-train - INFO - ==================================================
2025-08-29 01:51:17 - pico-train - INFO - Starting from step: 0
2025-08-29 01:51:17 - pico-train - INFO - Model Setup:
2025-08-29 01:51:17 - pico-train - INFO - └─ Total Parameters: 11,282,784
2025-08-29 01:51:17 - pico-train - INFO - └─ Trainable Parameters: 11,282,784
2025-08-29 01:51:17 - pico-train - INFO - Distributed Setup:
2025-08-29 01:51:17 - pico-train - INFO - └─ Number of Devices: 1
2025-08-29 01:51:17 - pico-train - INFO - └─ Device Type: NVIDIA GeForce RTX 5090
2025-08-29 01:51:17 - pico-train - INFO - └─ Available Memory: 33.68 GB
2025-08-29 01:51:17 - pico-train - INFO - Software Setup:
2025-08-29 01:51:17 - pico-train - INFO - └─ Python Version: 3.10.12
2025-08-29 01:51:17 - pico-train - INFO - └─ PyTorch Version: 2.8.0+cu128
2025-08-29 01:51:17 - pico-train - INFO - └─ CUDA Version: 12.8
2025-08-29 01:51:17 - pico-train - INFO - └─ Operating System: Linux 6.8.0-63-generic
2025-08-29 01:51:17 - pico-train - INFO - Batch Size Configuration:
2025-08-29 01:51:17 - pico-train - INFO - └─ Global Batch Size: 4
2025-08-29 01:51:17 - pico-train - INFO - └─ Per Device Batch Size: 1
2025-08-29 01:51:17 - pico-train - INFO - └─ Gradient Accumulation Steps: 4
2025-08-29 01:51:17 - pico-train - INFO - ==================================================
2025-08-29 01:51:18 - pico-train - INFO - Step 0 -- 🔄 Training Metrics
2025-08-29 01:51:18 - pico-train - INFO - ├── Loss: 10.9975
2025-08-29 01:51:18 - pico-train - INFO - ├── Learning Rate: 0.00e+00
2025-08-29 01:51:18 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:51:18 - pico-train - INFO - Step 0 -- 📈 Saving Learning Dynamics
2025-08-29 01:51:33 - pico-train - INFO - Step 25 -- 🔄 Training Metrics
2025-08-29 01:51:33 - pico-train - INFO - ├── Loss: 10.9972
2025-08-29 01:51:33 - pico-train - INFO - ├── Learning Rate: 1.56e-07
2025-08-29 01:51:33 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:51:46 - pico-train - INFO - Step 50 -- 🔄 Training Metrics
2025-08-29 01:51:46 - pico-train - INFO - ├── Loss: 11.0030
2025-08-29 01:51:46 - pico-train - INFO - ├── Learning Rate: 3.13e-07
2025-08-29 01:51:46 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:51:58 - pico-train - INFO - Step 75 -- 🔄 Training Metrics
2025-08-29 01:51:58 - pico-train - INFO - ├── Loss: 11.0034
2025-08-29 01:51:58 - pico-train - INFO - ├── Learning Rate: 4.69e-07
2025-08-29 01:51:58 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:52:11 - pico-train - INFO - Step 100 -- 🔄 Training Metrics
2025-08-29 01:52:11 - pico-train - INFO - ├── Loss: 10.9962
2025-08-29 01:52:11 - pico-train - INFO - ├── Learning Rate: 6.25e-07
2025-08-29 01:52:11 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:52:24 - pico-train - INFO - Step 125 -- 🔄 Training Metrics
2025-08-29 01:52:24 - pico-train - INFO - ├── Loss: 10.9973
2025-08-29 01:52:24 - pico-train - INFO - ├── Learning Rate: 7.81e-07
2025-08-29 01:52:24 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:52:36 - pico-train - INFO - Step 150 -- 🔄 Training Metrics
2025-08-29 01:52:36 - pico-train - INFO - ├── Loss: 10.9943
2025-08-29 01:52:36 - pico-train - INFO - ├── Learning Rate: 9.38e-07
2025-08-29 01:52:36 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:52:49 - pico-train - INFO - Step 175 -- 🔄 Training Metrics
2025-08-29 01:52:49 - pico-train - INFO - ├── Loss: 10.9860
2025-08-29 01:52:49 - pico-train - INFO - ├── Learning Rate: 1.09e-06
2025-08-29 01:52:49 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:53:02 - pico-train - INFO - Step 200 -- 🔄 Training Metrics
2025-08-29 01:53:02 - pico-train - INFO - ├── Loss: 10.9885
2025-08-29 01:53:02 - pico-train - INFO - ├── Learning Rate: 1.25e-06
2025-08-29 01:53:02 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:53:14 - pico-train - INFO - Step 225 -- 🔄 Training Metrics
2025-08-29 01:53:14 - pico-train - INFO - ├── Loss: 10.9816
2025-08-29 01:53:14 - pico-train - INFO - ├── Learning Rate: 1.41e-06
2025-08-29 01:53:14 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:53:27 - pico-train - INFO - Step 250 -- 🔄 Training Metrics
2025-08-29 01:53:27 - pico-train - INFO - ├── Loss: 10.9786
2025-08-29 01:53:27 - pico-train - INFO - ├── Learning Rate: 1.56e-06
2025-08-29 01:53:27 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:53:40 - pico-train - INFO - Step 275 -- 🔄 Training Metrics
2025-08-29 01:53:40 - pico-train - INFO - ├── Loss: 10.9707
2025-08-29 01:53:40 - pico-train - INFO - ├── Learning Rate: 1.72e-06
2025-08-29 01:53:40 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:53:53 - pico-train - INFO - Step 300 -- 🔄 Training Metrics
2025-08-29 01:53:53 - pico-train - INFO - ├── Loss: 10.9700
2025-08-29 01:53:53 - pico-train - INFO - ├── Learning Rate: 1.88e-06
2025-08-29 01:53:53 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:54:05 - pico-train - INFO - Step 325 -- 🔄 Training Metrics
2025-08-29 01:54:05 - pico-train - INFO - ├── Loss: 10.9626
2025-08-29 01:54:05 - pico-train - INFO - ├── Learning Rate: 2.03e-06
2025-08-29 01:54:05 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:54:18 - pico-train - INFO - Step 350 -- 🔄 Training Metrics
2025-08-29 01:54:18 - pico-train - INFO - ├── Loss: 10.9580
2025-08-29 01:54:18 - pico-train - INFO - ├── Learning Rate: 2.19e-06
2025-08-29 01:54:18 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:54:31 - pico-train - INFO - Step 375 -- 🔄 Training Metrics
2025-08-29 01:54:31 - pico-train - INFO - ├── Loss: 10.9486
2025-08-29 01:54:31 - pico-train - INFO - ├── Learning Rate: 2.34e-06
2025-08-29 01:54:31 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:54:44 - pico-train - INFO - Step 400 -- 🔄 Training Metrics
2025-08-29 01:54:44 - pico-train - INFO - ├── Loss: 10.9417
2025-08-29 01:54:44 - pico-train - INFO - ├── Learning Rate: 2.50e-06
2025-08-29 01:54:44 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:54:57 - pico-train - INFO - Step 425 -- 🔄 Training Metrics
2025-08-29 01:54:57 - pico-train - INFO - ├── Loss: 10.9328
2025-08-29 01:54:57 - pico-train - INFO - ├── Learning Rate: 2.66e-06
2025-08-29 01:54:57 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:55:10 - pico-train - INFO - Step 450 -- 🔄 Training Metrics
2025-08-29 01:55:10 - pico-train - INFO - ├── Loss: 10.9242
2025-08-29 01:55:10 - pico-train - INFO - ├── Learning Rate: 2.81e-06
2025-08-29 01:55:10 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:55:22 - pico-train - INFO - Step 475 -- 🔄 Training Metrics
2025-08-29 01:55:22 - pico-train - INFO - ├── Loss: 10.9170
2025-08-29 01:55:22 - pico-train - INFO - ├── Learning Rate: 2.97e-06
2025-08-29 01:55:22 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-29 01:55:35 - pico-train - INFO - Step 500 -- 💾 Saving Checkpoint
|