|
|
|
|
|
model_name_or_path: Qwen/Qwen2.5-1.5B |
|
|
model_revision: main |
|
|
torch_dtype: bfloat16 |
|
|
attn_implementation: flash_attention_2 |
|
|
|
|
|
|
|
|
dataset_name: trl-lib/tldr |
|
|
dataset_num_proc: 4 |
|
|
|
|
|
|
|
|
bf16: true |
|
|
do_eval: false |
|
|
eval_strategy: 'no' |
|
|
gradient_accumulation_steps: 8 |
|
|
gradient_checkpointing: true |
|
|
gradient_checkpointing_kwargs: |
|
|
use_reentrant: false |
|
|
|
|
|
hub_strategy: every_save |
|
|
learning_rate: 1.0e-05 |
|
|
log_level: info |
|
|
logging_steps: 1 |
|
|
logging_strategy: steps |
|
|
lr_scheduler_type: cosine_with_min_lr |
|
|
lr_scheduler_kwargs: |
|
|
min_lr_rate: 0.1 |
|
|
packing: false |
|
|
max_grad_norm: 0.2 |
|
|
max_length: 512 |
|
|
max_steps: -1 |
|
|
num_train_epochs: 10 |
|
|
output_dir: data/Qwen2.5-0.5B-SFT |
|
|
overwrite_output_dir: true |
|
|
per_device_eval_batch_size: 1 |
|
|
per_device_train_batch_size: 2 |
|
|
push_to_hub: true |
|
|
report_to: |
|
|
- wandb |
|
|
save_strategy: epoch |
|
|
save_total_limit: 1 |
|
|
seed: 42 |
|
|
use_liger_kernel: true |
|
|
warmup_ratio: 0.03 |
