File size: 15,985 Bytes
ce2c393 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 |
2025-08-30 01:43:03 - pico-train - INFO - Step 32000 -- 📊 Evaluation Results
2025-08-30 01:43:03 - pico-train - INFO - └── paloma: 2.977755235898109e+26
2025-08-30 01:43:05 - pico-train - INFO - ==================================================
2025-08-30 01:43:05 - pico-train - INFO - ✨ Training Configuration
2025-08-30 01:43:05 - pico-train - INFO - ==================================================
2025-08-30 01:43:05 - pico-train - INFO - ╭─────────────────────────────────────────────────────╮
2025-08-30 01:43:05 - pico-train - INFO - │ checkpointing: │
2025-08-30 01:43:05 - pico-train - INFO - │ checkpoints_dir: checkpoints │
2025-08-30 01:43:05 - pico-train - INFO - │ evaluation: │
2025-08-30 01:43:05 - pico-train - INFO - │ eval_results_dir: eval_results │
2025-08-30 01:43:05 - pico-train - INFO - │ fabric_checkpoint_dir: fabric_state │
2025-08-30 01:43:05 - pico-train - INFO - │ fabric_checkpoint_filename: checkpoint.pt │
2025-08-30 01:43:05 - pico-train - INFO - │ hf_checkpoint: │
2025-08-30 01:43:05 - pico-train - INFO - │ collection_slug: null │
2025-08-30 01:43:05 - pico-train - INFO - │ repo_id: ThomasTheMaker/pico-decoder-tiny │
2025-08-30 01:43:05 - pico-train - INFO - │ learning_dynamics: │
2025-08-30 01:43:05 - pico-train - INFO - │ batch_size: 1 │
2025-08-30 01:43:05 - pico-train - INFO - │ eval_data: null │
2025-08-30 01:43:05 - pico-train - INFO - │ layer_suffixes: │
2025-08-30 01:43:05 - pico-train - INFO - │ - attention.v_proj │
2025-08-30 01:43:05 - pico-train - INFO - │ - attention.o_proj │
2025-08-30 01:43:05 - pico-train - INFO - │ - swiglu.w_2 │
2025-08-30 01:43:05 - pico-train - INFO - │ sequence_idx: -1 │
2025-08-30 01:43:05 - pico-train - INFO - │ learning_dynamics_dir: learning_dynamics │
2025-08-30 01:43:05 - pico-train - INFO - │ logs_dir: logs │
2025-08-30 01:43:05 - pico-train - INFO - │ run_name: pico-decoder-tiny-dolma5M-v1 │
2025-08-30 01:43:05 - pico-train - INFO - │ runs_dir: runs │
2025-08-30 01:43:05 - pico-train - INFO - │ save_every_n_steps: 500 │
2025-08-30 01:43:05 - pico-train - INFO - │ save_to_hf: true │
2025-08-30 01:43:05 - pico-train - INFO - │ training: │
2025-08-30 01:43:05 - pico-train - INFO - │ auto_resume: true │
2025-08-30 01:43:05 - pico-train - INFO - │ data: │
2025-08-30 01:43:05 - pico-train - INFO - │ dataloader: │
2025-08-30 01:43:05 - pico-train - INFO - │ batch_size: 4 │
2025-08-30 01:43:05 - pico-train - INFO - │ dataset: │
2025-08-30 01:43:05 - pico-train - INFO - │ name: ThomasTheMaker/pretokenized-dolma-5M │
2025-08-30 01:43:05 - pico-train - INFO - │ tokenizer: │
2025-08-30 01:43:05 - pico-train - INFO - │ name: allenai/OLMo-7B-0724-hf │
2025-08-30 01:43:05 - pico-train - INFO - │ vocab_size: 50304 │
2025-08-30 01:43:05 - pico-train - INFO - │ evaluation: │
2025-08-30 01:43:05 - pico-train - INFO - │ metrics: │
2025-08-30 01:43:05 - pico-train - INFO - │ - paloma │
2025-08-30 01:43:05 - pico-train - INFO - │ paloma: │
2025-08-30 01:43:05 - pico-train - INFO - │ batch_size: 1 │
2025-08-30 01:43:05 - pico-train - INFO - │ dataset_name: pico-lm/pretokenized-paloma-tinsy │
2025-08-30 01:43:05 - pico-train - INFO - │ dataset_split: val │
2025-08-30 01:43:05 - pico-train - INFO - │ max_length: 2048 │
2025-08-30 01:43:05 - pico-train - INFO - │ model: │
2025-08-30 01:43:05 - pico-train - INFO - │ activation_hidden_dim: 384 │
2025-08-30 01:43:05 - pico-train - INFO - │ attention_n_heads: 12 │
2025-08-30 01:43:05 - pico-train - INFO - │ attention_n_kv_heads: 4 │
2025-08-30 01:43:05 - pico-train - INFO - │ batch_size: 1024 │
2025-08-30 01:43:05 - pico-train - INFO - │ d_model: 96 │
2025-08-30 01:43:05 - pico-train - INFO - │ max_seq_len: 2048 │
2025-08-30 01:43:05 - pico-train - INFO - │ model_type: pico_decoder │
2025-08-30 01:43:05 - pico-train - INFO - │ n_layers: 12 │
2025-08-30 01:43:05 - pico-train - INFO - │ norm_eps: 1.0e-06 │
2025-08-30 01:43:05 - pico-train - INFO - │ position_emb_theta: 10000.0 │
2025-08-30 01:43:05 - pico-train - INFO - │ vocab_size: 50304 │
2025-08-30 01:43:05 - pico-train - INFO - │ monitoring: │
2025-08-30 01:43:05 - pico-train - INFO - │ logging: │
2025-08-30 01:43:05 - pico-train - INFO - │ log_every_n_steps: 25 │
2025-08-30 01:43:05 - pico-train - INFO - │ log_level: INFO │
2025-08-30 01:43:05 - pico-train - INFO - │ save_to_wandb: false │
2025-08-30 01:43:05 - pico-train - INFO - │ wandb: │
2025-08-30 01:43:05 - pico-train - INFO - │ entity: boymyc │
2025-08-30 01:43:05 - pico-train - INFO - │ project: pico-decoder-tiny │
2025-08-30 01:43:05 - pico-train - INFO - │ training: │
2025-08-30 01:43:05 - pico-train - INFO - │ fabric: │
2025-08-30 01:43:05 - pico-train - INFO - │ accelerator: cuda │
2025-08-30 01:43:05 - pico-train - INFO - │ num_devices: 1 │
2025-08-30 01:43:05 - pico-train - INFO - │ num_nodes: 1 │
2025-08-30 01:43:05 - pico-train - INFO - │ precision: bf16-mixed │
2025-08-30 01:43:05 - pico-train - INFO - │ max_steps: 20000 │
2025-08-30 01:43:05 - pico-train - INFO - │ optimization: │
2025-08-30 01:43:05 - pico-train - INFO - │ gradient_accumulation_steps: 4 │
2025-08-30 01:43:05 - pico-train - INFO - │ lr: 5.0e-05 │
2025-08-30 01:43:05 - pico-train - INFO - │ lr_scheduler: cosine │
2025-08-30 01:43:05 - pico-train - INFO - │ lr_warmup_steps: 8000 │
2025-08-30 01:43:05 - pico-train - INFO - │ optimizer: adamw │
2025-08-30 01:43:05 - pico-train - INFO - │ │
2025-08-30 01:43:05 - pico-train - INFO - ╰─────────────────────────────────────────────────────╯
2025-08-30 01:43:05 - pico-train - INFO - ==================================================
2025-08-30 01:43:05 - pico-train - INFO - ⛭ Runtime Summary:
2025-08-30 01:43:05 - pico-train - INFO - ==================================================
2025-08-30 01:43:05 - pico-train - INFO - Starting from step: 32000
2025-08-30 01:43:05 - pico-train - INFO - Model Setup:
2025-08-30 01:43:05 - pico-train - INFO - └─ Total Parameters: 11,282,784
2025-08-30 01:43:05 - pico-train - INFO - └─ Trainable Parameters: 11,282,784
2025-08-30 01:43:05 - pico-train - INFO - Distributed Setup:
2025-08-30 01:43:05 - pico-train - INFO - └─ Number of Devices: 1
2025-08-30 01:43:05 - pico-train - INFO - └─ Device Type: NVIDIA GeForce RTX 5090
2025-08-30 01:43:05 - pico-train - INFO - └─ Available Memory: 33.68 GB
2025-08-30 01:43:05 - pico-train - INFO - Software Setup:
2025-08-30 01:43:05 - pico-train - INFO - └─ Python Version: 3.10.12
2025-08-30 01:43:05 - pico-train - INFO - └─ PyTorch Version: 2.8.0+cu128
2025-08-30 01:43:05 - pico-train - INFO - └─ CUDA Version: 12.8
2025-08-30 01:43:05 - pico-train - INFO - └─ Operating System: Linux 6.8.0-63-generic
2025-08-30 01:43:05 - pico-train - INFO - Batch Size Configuration:
2025-08-30 01:43:05 - pico-train - INFO - └─ Global Batch Size: 4
2025-08-30 01:43:05 - pico-train - INFO - └─ Per Device Batch Size: 1
2025-08-30 01:43:05 - pico-train - INFO - └─ Gradient Accumulation Steps: 4
2025-08-30 01:43:05 - pico-train - INFO - ==================================================
2025-08-30 01:43:06 - pico-train - INFO - Step 32000 -- 🔄 Training Metrics
2025-08-30 01:43:06 - pico-train - INFO - ├── Loss: 6.3376
2025-08-30 01:43:06 - pico-train - INFO - ├── Learning Rate: 7.32e-06
2025-08-30 01:43:06 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:43:06 - pico-train - INFO - Step 32000 -- 📈 Saving Learning Dynamics
2025-08-30 01:43:20 - pico-train - INFO - Step 32025 -- 🔄 Training Metrics
2025-08-30 01:43:20 - pico-train - INFO - ├── Loss: 6.1999
2025-08-30 01:43:20 - pico-train - INFO - ├── Learning Rate: 7.28e-06
2025-08-30 01:43:20 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:43:33 - pico-train - INFO - Step 32050 -- 🔄 Training Metrics
2025-08-30 01:43:33 - pico-train - INFO - ├── Loss: 6.1488
2025-08-30 01:43:33 - pico-train - INFO - ├── Learning Rate: 7.24e-06
2025-08-30 01:43:33 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:43:45 - pico-train - INFO - Step 32075 -- 🔄 Training Metrics
2025-08-30 01:43:45 - pico-train - INFO - ├── Loss: 6.0460
2025-08-30 01:43:45 - pico-train - INFO - ├── Learning Rate: 7.19e-06
2025-08-30 01:43:45 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:43:58 - pico-train - INFO - Step 32100 -- 🔄 Training Metrics
2025-08-30 01:43:58 - pico-train - INFO - ├── Loss: 6.1627
2025-08-30 01:43:58 - pico-train - INFO - ├── Learning Rate: 7.15e-06
2025-08-30 01:43:58 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:44:11 - pico-train - INFO - Step 32125 -- 🔄 Training Metrics
2025-08-30 01:44:11 - pico-train - INFO - ├── Loss: 6.2085
2025-08-30 01:44:11 - pico-train - INFO - ├── Learning Rate: 7.11e-06
2025-08-30 01:44:11 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:44:23 - pico-train - INFO - Step 32150 -- 🔄 Training Metrics
2025-08-30 01:44:23 - pico-train - INFO - ├── Loss: 6.1659
2025-08-30 01:44:23 - pico-train - INFO - ├── Learning Rate: 7.06e-06
2025-08-30 01:44:23 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:44:36 - pico-train - INFO - Step 32175 -- 🔄 Training Metrics
2025-08-30 01:44:36 - pico-train - INFO - ├── Loss: 6.1719
2025-08-30 01:44:36 - pico-train - INFO - ├── Learning Rate: 7.02e-06
2025-08-30 01:44:36 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:44:48 - pico-train - INFO - Step 32200 -- 🔄 Training Metrics
2025-08-30 01:44:48 - pico-train - INFO - ├── Loss: 6.2081
2025-08-30 01:44:48 - pico-train - INFO - ├── Learning Rate: 6.98e-06
2025-08-30 01:44:48 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:45:01 - pico-train - INFO - Step 32225 -- 🔄 Training Metrics
2025-08-30 01:45:01 - pico-train - INFO - ├── Loss: 6.1955
2025-08-30 01:45:01 - pico-train - INFO - ├── Learning Rate: 6.94e-06
2025-08-30 01:45:01 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:45:14 - pico-train - INFO - Step 32250 -- 🔄 Training Metrics
2025-08-30 01:45:14 - pico-train - INFO - ├── Loss: 6.1139
2025-08-30 01:45:14 - pico-train - INFO - ├── Learning Rate: 6.89e-06
2025-08-30 01:45:14 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:45:26 - pico-train - INFO - Step 32275 -- 🔄 Training Metrics
2025-08-30 01:45:26 - pico-train - INFO - ├── Loss: 6.1075
2025-08-30 01:45:26 - pico-train - INFO - ├── Learning Rate: 6.85e-06
2025-08-30 01:45:26 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:45:39 - pico-train - INFO - Step 32300 -- 🔄 Training Metrics
2025-08-30 01:45:39 - pico-train - INFO - ├── Loss: 6.0814
2025-08-30 01:45:39 - pico-train - INFO - ├── Learning Rate: 6.81e-06
2025-08-30 01:45:39 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:45:51 - pico-train - INFO - Step 32325 -- 🔄 Training Metrics
2025-08-30 01:45:51 - pico-train - INFO - ├── Loss: 6.0880
2025-08-30 01:45:51 - pico-train - INFO - ├── Learning Rate: 6.77e-06
2025-08-30 01:45:51 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:46:04 - pico-train - INFO - Step 32350 -- 🔄 Training Metrics
2025-08-30 01:46:04 - pico-train - INFO - ├── Loss: 6.1997
2025-08-30 01:46:04 - pico-train - INFO - ├── Learning Rate: 6.73e-06
2025-08-30 01:46:04 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:46:16 - pico-train - INFO - Step 32375 -- 🔄 Training Metrics
2025-08-30 01:46:16 - pico-train - INFO - ├── Loss: 6.1376
2025-08-30 01:46:16 - pico-train - INFO - ├── Learning Rate: 6.68e-06
2025-08-30 01:46:16 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:46:29 - pico-train - INFO - Step 32400 -- 🔄 Training Metrics
2025-08-30 01:46:29 - pico-train - INFO - ├── Loss: 6.1077
2025-08-30 01:46:29 - pico-train - INFO - ├── Learning Rate: 6.64e-06
2025-08-30 01:46:29 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:46:42 - pico-train - INFO - Step 32425 -- 🔄 Training Metrics
2025-08-30 01:46:42 - pico-train - INFO - ├── Loss: 6.2641
2025-08-30 01:46:42 - pico-train - INFO - ├── Learning Rate: 6.60e-06
2025-08-30 01:46:42 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:46:54 - pico-train - INFO - Step 32450 -- 🔄 Training Metrics
2025-08-30 01:46:54 - pico-train - INFO - ├── Loss: 6.1020
2025-08-30 01:46:54 - pico-train - INFO - ├── Learning Rate: 6.56e-06
2025-08-30 01:46:54 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:47:07 - pico-train - INFO - Step 32475 -- 🔄 Training Metrics
2025-08-30 01:47:07 - pico-train - INFO - ├── Loss: 6.2170
2025-08-30 01:47:07 - pico-train - INFO - ├── Learning Rate: 6.52e-06
2025-08-30 01:47:07 - pico-train - INFO - └── Inf/NaN count: 0
2025-08-30 01:47:19 - pico-train - INFO - Step 32500 -- 💾 Saving Checkpoint
|