| group: multimedqa | |
| task: | |
| - pubmedqa | |
| - medmcqa | |
| - medqa_4options | |
| - task: mmlu_anatomy | |
| task_alias: "anatomy (mmlu)" | |
| - task: mmlu_clinical_knowledge | |
| task_alias: "clinical_knowledge (mmlu)" | |
| - task: mmlu_college_medicine | |
| task_alias: "college_medicine (mmlu)" | |
| - task: mmlu_medical_genetics | |
| task_alias: "medical_genetics (mmlu)" | |
| - task: mmlu_professional_medicine | |
| task_alias: "professional_medicine (mmlu)" | |
| - task: mmlu_college_biology | |
| task_alias: "college_biology (mmlu)" | |
| aggregate_metric_list: | |
| - metric: acc | |
| aggregation: mean | |
| weight_by_size: True | |