Add selimc/bilmecebench dataset and model results

data.py

@@ -49,6 +49,9 @@ DATASET_TASK_DICT = {
     'turkish_plu_next_event_prediction': Tasks.MULTIPLE_CHOICE,
     'turkish_plu_step_inference': Tasks.MULTIPLE_CHOICE,
     'turkish_plu_step_ordering': Tasks.MULTIPLE_CHOICE,
+    'turkce_atasozleri': Tasks.MULTIPLE_CHOICE,
+    'turkishmmlu':Tasks.MULTIPLE_CHOICE,
+    'bilmecebench':Tasks.MULTIPLE_CHOICE,
 
     # fact-checking, not sure whether these are multi-choice
     # 'trclaim19': Tasks.MULTIPLE_CHOICE,
@@ -64,8 +67,6 @@ DATASET_TASK_DICT = {
     # other generation
     'wmt-tr-en-prompt': Tasks.MACHINE_TRANSLATION,
     'gecturk_generation': Tasks.GRAMMATICAL_ERROR_CORRECTION,
-    'turkce_atasozleri': Tasks.MULTIPLE_CHOICE,
-    'turkishmmlu':Tasks.MULTIPLE_CHOICE,
 }
 
 

data/datasets.json

@@ -197,5 +197,13 @@
         "url": "https://huggingface.co/datasets/AYueksel/TurkishMMLU",
         "hf_name": "AYueksel/TurkishMMLU",
         "generative": false
+    },
+    "bilmecebench": {
+        "name": "Riddles",
+        "task": "multiple_choice",
+        "description": "A dataset for Turkish riddles and their answers.",
+        "url": "https://huggingface.co/datasets/selimc/bilmecebench",
+        "hf_name": "selimc/bilmecebench",
+        "generative": false
     }
 }

results/zero-shot/CerebrumTech__cere-llama-3-8b-tr.json

@@ -180,6 +180,12 @@
             "task": "multiple_choice",
             "acc": 0.25555555555555554,
             "acc_norm": 0.25555555555555554
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.3393665158371041,
+            "acc_norm": 0.3393665158371041
         }
     ]
 }

results/zero-shot/Llama-3.3-70B-Instruct.json

@@ -182,6 +182,12 @@
             "task": "multiple_choice",
             "acc": 0.646,
             "acc_norm": 0.646
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.7262443438914027,
+            "acc_norm": 0.7262443438914027
         }
     ]
 }

results/zero-shot/Ministral-8B-Instruct.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.2644444444444444,
             "acc_norm": 0.2644444444444444
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.248868778280543,
+            "acc_norm": 0.248868778280543
         }
     ]
 }

results/zero-shot/Mistral-7B-Instruct-v0.3.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.19555555555555557,
             "acc_norm": 0.19555555555555557
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2149321266968326,
+            "acc_norm": 0.2149321266968326
         }
     ]
 }

results/zero-shot/Mistral-7B-v0.3.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.2688888888888889,
             "acc_norm": 0.2688888888888889
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.23529411764705882,
+            "acc_norm": 0.23529411764705882
         }
     ]
 }

results/zero-shot/Mixtral-8x7B-Instruct-v0.1.json

@@ -181,6 +181,12 @@
             "task": "multiple_choice",
             "acc": 0.358,
             "acc_norm": 0.358
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.3416289592760181,
+            "acc_norm": 0.3416289592760181
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B-Instruct.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.2111111111111111,
             "acc_norm": 0.2111111111111111
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2420814479638009,
+            "acc_norm": 0.2420814479638009
         }
     ]
 }

results/zero-shot/Qwen2.5-0.5B.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.17888888888888888,
             "acc_norm": 0.17888888888888888
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.251131221719457,
+            "acc_norm": 0.251131221719457
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B-Instruct.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.28888888888888886,
             "acc_norm": 0.28888888888888886
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2918552036199095,
+            "acc_norm": 0.2918552036199095
         }
     ]
 }

results/zero-shot/Qwen2.5-1.5B.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.23,
             "acc_norm": 0.23
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2986425339366516,
+            "acc_norm": 0.2986425339366516
         }
     ]
 }

results/zero-shot/Qwen2.5-14B-Instruct.json

@@ -181,6 +181,12 @@
             "task": "multiple_choice",
             "acc": 0.5944444444444444,
             "acc_norm": 0.5944444444444444
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.5701357466063348,
+            "acc_norm": 0.5701357466063348
         }
     ]
 }

results/zero-shot/Qwen2.5-14B.json

@@ -181,6 +181,12 @@
             "task": "multiple_choice",
             "acc": 0.5622222222222222,
             "acc_norm": 0.5622222222222222
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.4751131221719457,
+            "acc_norm": 0.4751131221719457
         }
     ]
 }

results/zero-shot/Qwen2.5-3B-Instruct.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.37777777777777777,
             "acc_norm": 0.37777777777777777
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.33031674208144796,
+            "acc_norm": 0.33031674208144796
         }
     ]
 }

results/zero-shot/Qwen2.5-3B.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.22555555555555556,
             "acc_norm": 0.22555555555555556
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.24434389140271492,
+            "acc_norm": 0.24434389140271492
         }
     ]
 }

results/zero-shot/Qwen2.5-7B-Instruct.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.47555555555555556,
             "acc_norm": 0.47555555555555556
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.5203619909502263,
+            "acc_norm": 0.5203619909502263
         }
     ]
 }

results/zero-shot/Qwen2.5-7B.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.49333333333333335,
             "acc_norm": 0.49333333333333335
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.4841628959276018,
+            "acc_norm": 0.4841628959276018
         }
     ]
 }

results/zero-shot/aya-23-35B.json

@@ -181,6 +181,12 @@
             "task": "multiple_choice",
             "acc": 0.4533333333333333,
             "acc_norm": 0.4533333333333333
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.34841628959276016,
+            "acc_norm": 0.34841628959276016
         }
     ]
 }

results/zero-shot/aya-23-8b.json

@@ -175,6 +175,12 @@
             "task": "multiple_choice",
             "acc": 0.33,
             "acc_norm": 0.33
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.3438914027149321,
+            "acc_norm": 0.3438914027149321
         }
     ]
 }

results/zero-shot/aya-expanse-32b.json

@@ -180,6 +180,12 @@
             "task": "multiple_choice",
             "acc": 0.5688888888888889,
             "acc_norm": 0.5688888888888889
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.4117647058823529,
+            "acc_norm": 0.4117647058823529
         }
     ]
 }

results/zero-shot/aya-expanse-8b.json

@@ -173,6 +173,12 @@
             "rouge1": 0.011791589892292625,
             "rouge2": 0.005495865074665877,
             "rougeL": 0.009523063183214307
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.48868778280542985,
+            "acc_norm": 0.48868778280542985
         }
     ]
 }

results/zero-shot/aya101.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.374,
             "acc_norm": 0.374
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.47058823529411764,
+            "acc_norm": 0.47058823529411764
         }
     ]
 }

results/zero-shot/commencis-7b.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.24666666666666667,
             "acc_norm": 0.24666666666666667
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2420814479638009,
+            "acc_norm": 0.2420814479638009
         }
     ]
 }

results/zero-shot/kanarya-2b.json

@@ -178,6 +178,12 @@
             "task": "multiple_choice",
             "acc": 0.18,
             "acc_norm": 0.18
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.27149321266968324,
+            "acc_norm": 0.27149321266968324
         }
     ]
 }

results/zero-shot/llama-3-8b-instruct.json

@@ -174,7 +174,12 @@
             "task": "multiple_choice",
             "acc": 0.3811111111111111,
             "acc_norm": 0.3811111111111111
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.38461538461538464,
+            "acc_norm": 0.38461538461538464
         }
-
     ]
 }

results/zero-shot/llama-3-8b.json

@@ -173,7 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.2544444444444444,
             "acc_norm": 0.2544444444444444
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.29638009049773756,
+            "acc_norm": 0.29638009049773756
         }
-
     ]
 }

results/zero-shot/llama-3.1-8b-instruct.json

@@ -173,7 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.3811111111111111,
             "acc_norm": 0.3811111111111111
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.416289592760181,
+            "acc_norm": 0.416289592760181
         }
-
     ]
 }

results/zero-shot/llama-3.1-8b.json

@@ -173,6 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.3055555555555556,
             "acc_norm": 0.3055555555555556
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.3212669683257919,
+            "acc_norm": 0.3212669683257919
         }
     ]
 }

results/zero-shot/llama-3.2-1b.json

@@ -205,7 +205,12 @@
             "task": "multiple_choice",
             "acc": 0.18888888888888888,
             "acc_norm": 0.18888888888888888
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2239819004524887,
+            "acc_norm": 0.2239819004524887
         }
-
     ]
 }

results/zero-shot/llama-3.2-3b-instruct.json

@@ -205,6 +205,12 @@
             "rouge1": 0.20895844439423625,
             "rouge2": 0.06600479924922747,
             "rougeL": 0.15296503277602666
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.30995475113122173,
+            "acc_norm": 0.30995475113122173
         }
     ]
 }

results/zero-shot/llama-3.2-3b.json

@@ -173,7 +173,12 @@
             "task": "multiple_choice",
             "acc": 0.29,
             "acc_norm": 0.29
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2895927601809955,
+            "acc_norm": 0.2895927601809955
         }
- 
     ]
 }

results/zero-shot/mistral-7b.json

@@ -179,6 +179,12 @@
             "rouge1": 0.03270500689456488,
             "rouge2": 0.011127489805662754,
             "rougeL": 0.025124899949616367
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.25339366515837103,
+            "acc_norm": 0.25339366515837103
         }
     ]
 }

results/zero-shot/trendyol-7b.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.2477777777777778,
             "acc_norm": 0.2477777777777778
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.23076923076923078,
+            "acc_norm": 0.23076923076923078
         }
     ]
 }

results/zero-shot/turna.json

@@ -179,6 +179,12 @@
             "task": "multiple_choice",
             "acc": 0.19333333333333333,
             "acc_norm": 0.19333333333333333
+        },
+        {
+            "name": "bilmecebench",
+            "task": "multiple_choice",
+            "acc": 0.2420814479638009,
+            "acc_norm": 0.2420814479638009
         }
     ]
 }