Update Dataset Groups and add Circumflex Dataset

data.py

@@ -52,6 +52,7 @@ DATASET_TASK_DICT = {
     'turkce_atasozleri': Tasks.MULTIPLE_CHOICE,
     'turkishmmlu':Tasks.MULTIPLE_CHOICE,
     'bilmecebench':Tasks.MULTIPLE_CHOICE,
+    'circumflex_tr':Tasks.MULTIPLE_CHOICE,
 
     # fact-checking, not sure whether these are multi-choice
     # 'trclaim19': Tasks.MULTIPLE_CHOICE,
@@ -93,8 +94,8 @@ DATASET_GROUPS = {
         'description': 'Turkish splits of SQuAD-like datasets XQuAD and TQUAD.',
     },
     'MCQA': {
-        'datasets': ['xcopa_tr', 'exams_tr', 'belebele_tr', 'turkce_atasozleri'] + [x for x in DATASET_TASK_DICT.keys() if x.startswith('turkish_plu')],
-        'description': 'Multiple Choice Question Answering datasets: XCOPA, Exams, Belebele, Turkish PLU, and Turkce-Atasozleri.'
+        'datasets': ['xcopa_tr', 'exams_tr', 'belebele_tr', 'turkce_atasozleri', 'bilmecebench', 'turkishmmlu', 'circumflex_tr'] + [x for x in DATASET_TASK_DICT.keys() if x.startswith('turkish_plu')],
+        'description': 'Multiple Choice Question Answering datasets: XCOPA, Exams, Belebele, Turkish PLU, Turkce-Atasozleri, BilmeceBench, TurkishMMLU, and CircumflexTR'
     },
     'TC': {
         'datasets': ['sts_tr', 'offenseval_tr', 'news_cat', 'ironytr', ],

results/zero-shot/CerebrumTech__cere-llama-3-8b-tr.json

@@ -186,6 +186,12 @@
             "task": "multiple_choice",
             "acc": 0.3393665158371041,
             "acc_norm": 0.3393665158371041
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.6142857142857143,
+            "acc_norm": 0.6142857142857143
         }
     ]
 }

results/zero-shot/Llama-3.3-70B-Instruct.json

@@ -188,6 +188,12 @@
             "task": "multiple_choice",
             "acc": 0.7262443438914027,
             "acc_norm": 0.7262443438914027
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.6,
+            "acc_norm": 0.6
         }
     ]
 }

results/zero-shot/Ministral-8B-Instruct.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.248868778280543,
             "acc_norm": 0.248868778280543
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5571428571428572,
+            "acc_norm": 0.5571428571428572
         }
     ]
 }

results/zero-shot/Mistral-7B-Instruct-v0.3.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.2149321266968326,
             "acc_norm": 0.2149321266968326
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5142857142857142,
+            "acc_norm": 0.5142857142857142
         }
     ]
 }

results/zero-shot/Mistral-7B-v0.3.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.23529411764705882,
             "acc_norm": 0.23529411764705882
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5142857142857142,
+            "acc_norm": 0.5142857142857142
         }
     ]
 }

results/zero-shot/Mixtral-8x7B-Instruct-v0.1.json

@@ -187,6 +187,12 @@
             "task": "multiple_choice",
             "acc": 0.3416289592760181,
             "acc_norm": 0.3416289592760181
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5714285714285714,
+            "acc_norm": 0.5714285714285714
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B-Instruct.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.2420814479638009,
             "acc_norm": 0.2420814479638009
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5857142857142857,
+            "acc_norm": 0.5857142857142857
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.251131221719457,
             "acc_norm": 0.251131221719457
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.4714285714285714,
+            "acc_norm": 0.4714285714285714
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B-Instruct.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.2918552036199095,
             "acc_norm": 0.2918552036199095
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5142857142857142,
+            "acc_norm": 0.5142857142857142
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.2986425339366516,
             "acc_norm": 0.2986425339366516
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5142857142857142,
+            "acc_norm": 0.5142857142857142
         }
     ]
 }

results/zero-shot/Qwen2.5-14B-Instruct.json

@@ -187,6 +187,12 @@
             "task": "multiple_choice",
             "acc": 0.5701357466063348,
             "acc_norm": 0.5701357466063348
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5428571428571428,
+            "acc_norm": 0.5428571428571428
         }
     ]
 }

results/zero-shot/Qwen2.5-14B.json

@@ -187,6 +187,12 @@
             "task": "multiple_choice",
             "acc": 0.4751131221719457,
             "acc_norm": 0.4751131221719457
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5714285714285714,
+            "acc_norm": 0.5714285714285714
         }
     ]
 }

results/zero-shot/Qwen2.5-3B-Instruct.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.33031674208144796,
             "acc_norm": 0.33031674208144796
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.6285714285714286,
+            "acc_norm": 0.6285714285714286
         }
     ]
 }

results/zero-shot/Qwen2.5-3B.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.24434389140271492,
             "acc_norm": 0.24434389140271492
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.6285714285714286,
+            "acc_norm": 0.6285714285714286
         }
     ]
 }

results/zero-shot/Qwen2.5-7B-Instruct.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.5203619909502263,
             "acc_norm": 0.5203619909502263
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5142857142857142,
+            "acc_norm": 0.5142857142857142
         }
     ]
 }

results/zero-shot/Qwen2.5-7B.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.4841628959276018,
             "acc_norm": 0.4841628959276018
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5428571428571428,
+            "acc_norm": 0.5428571428571428
         }
     ]
 }

results/zero-shot/aya-23-35B.json

@@ -187,6 +187,12 @@
             "task": "multiple_choice",
             "acc": 0.34841628959276016,
             "acc_norm": 0.34841628959276016
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5285714285714286,
+            "acc_norm": 0.5285714285714286
         }
     ]
 }

results/zero-shot/aya-23-8b.json

@@ -181,6 +181,12 @@
             "task": "multiple_choice",
             "acc": 0.3438914027149321,
             "acc_norm": 0.3438914027149321
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.4857142857142857,
+            "acc_norm": 0.4857142857142857
         }
     ]
 }

results/zero-shot/aya-expanse-32b.json

@@ -186,6 +186,12 @@
             "task": "multiple_choice",
             "acc": 0.4117647058823529,
             "acc_norm": 0.4117647058823529
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.6285714285714286,
+            "acc_norm": 0.6285714285714286
         }
     ]
 }

results/zero-shot/aya-expanse-8b.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.48868778280542985,
             "acc_norm": 0.48868778280542985
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5571428571428572,
+            "acc_norm": 0.5571428571428572
         }
     ]
 }

results/zero-shot/aya101.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.47058823529411764,
             "acc_norm": 0.47058823529411764
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5571428571428572,
+            "acc_norm": 0.5571428571428572
         }
     ]
 }

results/zero-shot/commencis-7b.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.2420814479638009,
             "acc_norm": 0.2420814479638009
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.44285714285714284,
+            "acc_norm": 0.44285714285714284
         }
     ]
 }

results/zero-shot/kanarya-2b.json

@@ -184,6 +184,12 @@
             "task": "multiple_choice",
             "acc": 0.27149321266968324,
             "acc_norm": 0.27149321266968324
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.4857142857142857,
+            "acc_norm": 0.4857142857142857
         }
     ]
 }

results/zero-shot/llama-3-8b-instruct.json

@@ -180,6 +180,12 @@
             "task": "multiple_choice",
             "acc": 0.38461538461538464,
             "acc_norm": 0.38461538461538464
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5571428571428572,
+            "acc_norm": 0.5571428571428572
         }
     ]
 }

results/zero-shot/llama-3-8b.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.29638009049773756,
             "acc_norm": 0.29638009049773756
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.4857142857142857,
+            "acc_norm": 0.4857142857142857
         }
     ]
 }

results/zero-shot/llama-3.1-8b-instruct.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.416289592760181,
             "acc_norm": 0.416289592760181
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5714285714285714,
+            "acc_norm": 0.5714285714285714
         }
     ]
 }

results/zero-shot/llama-3.1-8b.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.3212669683257919,
             "acc_norm": 0.3212669683257919
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5571428571428572,
+            "acc_norm": 0.5571428571428572
         }
     ]
 }

results/zero-shot/llama-3.2-1b.json

@@ -211,6 +211,12 @@
             "task": "multiple_choice",
             "acc": 0.2239819004524887,
             "acc_norm": 0.2239819004524887
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5,
+            "acc_norm": 0.5
         }
     ]
 }

results/zero-shot/llama-3.2-3b-instruct.json

@@ -211,6 +211,12 @@
             "task": "multiple_choice",
             "acc": 0.30995475113122173,
             "acc_norm": 0.30995475113122173
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5142857142857142,
+            "acc_norm": 0.5142857142857142
         }
     ]
 }

results/zero-shot/llama-3.2-3b.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.2895927601809955,
             "acc_norm": 0.2895927601809955
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5714285714285714,
+            "acc_norm": 0.5714285714285714
         }
     ]
 }

results/zero-shot/mistral-7b.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.25339366515837103,
             "acc_norm": 0.25339366515837103
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5571428571428572,
+            "acc_norm": 0.5571428571428572
         }
     ]
 }

results/zero-shot/trendyol-7b.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.23076923076923078,
             "acc_norm": 0.23076923076923078
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.5428571428571428,
+            "acc_norm": 0.5428571428571428
         }
     ]
 }

results/zero-shot/turna.json

@@ -185,6 +185,12 @@
             "task": "multiple_choice",
             "acc": 0.2420814479638009,
             "acc_norm": 0.2420814479638009
+        },
+        {
+            "name": "circumflex_tr",
+            "task": "multiple_choice",
+            "acc": 0.4714285714285714,
+            "acc_norm": 0.4714285714285714
         }
     ]
 }