| lm_eval --model hf \ | |
| --model_args pretrained=masakhane/African-ultrachat-alpaca \ | |
| --tasks afrimmlu_direct_amh,afrimmlu_direct_eng,afrimmlu_direct_ewe,afrimmlu_direct_fra,afrimmlu_direct_hau,afrimmlu_direct_ibo,afrimmlu_direct_kin,afrimmlu_direct_lin,afrimmlu_direct_lug,afrimmlu_direct_orm,afrimmlu_direct_sna,afrimmlu_direct_sot,afrimmlu_direct_twi,afrimmlu_direct_wol,afrimmlu_direct_xho,afrimmlu_direct_yor,afrimmlu_direct_zul \ | |
| --device cuda:0 \ | |
| --batch_size 1 \ | |
| --num_fewshot 0 \ | |
| --verbosity DEBUG \ | |
| --wandb_args project=afrimmlu | |