[ { "language_name": "English", "bcp_47": "en", "speakers": 1636485840, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.4351349353198866, "chrf": 54.9504915580248 } ], "bleu": 0.4351349353198866, "chrf": 54.9504915580248, "commonvoice_hours": 2649.0, "commonvoice_locale": "en", "population": { "AC": 931, "AE": 4996040, "AG": 84434, "AI": 17186, "AQ": 300, "AR": 3183537, "AS": 47954, "AT": 6467398, "AU": 24447840, "AW": 2986, "BA": 1726016, "BB": 294560, "BD": 29277180, "BE": 6915213, "BG": 1741725, "BI": 6289, "BM": 66010, "BN": 7896, "BR": 16937280, "BS": 337721, "BT": 86055, "BV": 1, "BW": 1876956, "BZ": 399598, "CA": 32416926, "CC": 101, "CH": 5126434, "CK": 8574, "CL": 1727746, "CM": 10543100, "CN": 62731, "CP": 1, "CQ": 482, "CX": 1389, "CY": 924676, "CZ": 2889675, "DE": 51302208, "DG": 495, "DK": 5047693, "DM": 69788, "DO": 7980, "DZ": 3008103, "EE": 614310, "EG": 36443400, "ER": 3587908, "ES": 12003792, "ET": 46488590, "FI": 3900169, "FJ": 879816, "FK": 2814, "FM": 58389, "FR": 26460798, "GB": 64445878, "GD": 108570, "GG": 67052, "GH": 6161442, "GI": 23665, "GM": 869600, "GR": 5409621, "GS": 20, "GU": 153321, "GY": 750204, "HK": 3697454, "HM": 1, "HN": 40635, "HR": 2071598, "HU": 1954366, "IE": 5073039, "IL": 7374158, "IM": 90499, "IN": 251957100, "IO": 3500, "IQ": 13605445, "IT": 21216918, "JE": 96019, "JM": 2752399, "JO": 4869270, "KE": 10170301, "KI": 111796, "KN": 52745, "KY": 60705, "KZ": 2863785, "LB": 2187844, "LC": 149838, "LK": 2288920, "LR": 4210839, "LS": 531719, "LT": 1037955, "LU": 351893, "LV": 865366, "MA": 4978638, "MG": 4852026, "MH": 72463, "MO": 14133, "MP": 49890, "MS": 3492, "MT": 402395, "MU": 993146, "MV": 293928, "MW": 13353858, "MX": 16724500, "MY": 6856941, "NA": 184105, "NF": 1678, "NG": 113434840, "NL": 15552360, "NP": 909837, "NR": 9350, "NU": 1120, "NZ": 4826970, "PA": 545171, "PG": 3629730, "PH": 69875840, "PK": 116750500, "PL": 12633159, "PM": 187, "PN": 46, "PR": 1562644, "PT": 2781729, "PW": 1887, "RO": 6603899, "RW": 1906860, "SB": 685097, "SC": 36473, "SD": 27792576, "SE": 8774150, "SG": 5774984, "SH": 5425, "SI": 1240581, "SK": 1414556, "SL": 2318726, "SS": 2851524, "SX": 29816, "SZ": 883584, "TA": 272, "TC": 54807, "TH": 18623898, "TK": 1285, "TO": 29707, "TR": 13942975, "TT": 1063735, "TV": 1066, "TZ": 40401432, "UG": 1686867, "UM": 316, "US": 319333440, "VC": 97334, "VG": 36633, "VI": 79676, "VU": 247616, "WS": 4279, "YE": 2689596, "ZA": 17503716, "ZM": 2788256, "ZW": 6109446 } }, { "language_name": "Chinese", "bcp_47": "zh", "speakers": 1304678914, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3977775857451761, "chrf": 57.672913792439125 } ], "bleu": 0.3977775857451761, "chrf": 57.672913792439125, "commonvoice_hours": 422.0, "commonvoice_locale": "zh-TW", "population": { "AU": 534796, "BN": 51093, "CA": 678494, "CN": 1254618000, "GB": 197283, "GF": 4988, "HK": 7249910, "ID": 2456639, "MN": 44352, "MO": 632892, "MY": 5550857, "PA": 5841, "PF": 23019, "PH": 797021, "SG": 4781438, "SR": 6705, "TH": 1241593, "TW": 22422850, "US": 2295209, "VN": 1085934 } }, { "language_name": "Hindi", "bcp_47": "hi", "speakers": 546882144, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.333521621016373, "chrf": 50.48364584189306 } ], "bleu": 0.333521621016373, "chrf": 50.48364584189306, "commonvoice_hours": 16.0, "commonvoice_locale": "hi-IN", "population": { "CA": 188470, "FJ": 411829, "IN": 545022990, "NP": 127377, "UG": 2206, "ZA": 1129272 } }, { "language_name": "Spanish", "bcp_47": "es", "speakers": 493528077, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.29160032861883095, "chrf": 47.668399832701844 } ], "bleu": 0.29160032861883095, "chrf": 47.668399832701844, "commonvoice_hours": 446.0, "commonvoice_locale": "es", "population": { "AD": 33110, "AR": 45479100, "BO": 7100339, "BR": 76218, "BZ": 111887, "CA": 603106, "CL": 17823064, "CO": 45648864, "CR": 4843090, "CU": 11059100, "CW": 5751, "DE": 4809582, "DO": 8189766, "EA": 147000, "EC": 16228704, "ES": 49515642, "FR": 8820266, "GB": 5260888, "GI": 14790, "GQ": 727475, "GT": 15952569, "HN": 7203565, "IC": 2056618, "MA": 23115, "MX": 106779500, "NI": 4838683, "PA": 2686915, "PE": 23297950, "PH": 33846110, "PR": 2774491, "PT": 1030270, "PY": 230134, "RO": 2130290, "SV": 5768179, "SX": 4823, "TT": 4110, "US": 31933344, "UY": 2981097, "VE": 23488572 } }, { "language_name": "Arabic", "bcp_47": "ar", "speakers": 351664197, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.277257629790728, "chrf": 46.62779335380641 } ], "bleu": 0.277257629790728, "chrf": 46.62779335380641, "commonvoice_hours": 91.0, "commonvoice_locale": "ar", "population": { "AE": 7793822, "BH": 1309350, "CA": 565412, "CM": 108206, "CY": 1267, "DJ": 67292, "DZ": 31799946, "EG": 97876560, "EH": 652271, "ER": 297979, "GB": 197283, "IL": 1735096, "IQ": 26433436, "IR": 1698466, "JO": 10820600, "KE": 24623, "KM": 558545, "KW": 2993710, "LB": 4703865, "LY": 5099000, "MA": 22048254, "ML": 175981, "MR": 3404658, "NE": 47822, "NG": 151960, "OM": 3778520, "PS": 4818260, "QA": 2175311, "SA": 34173500, "SD": 27792576, "SO": 3997414, "SS": 2851524, "SY": 15518720, "TD": 2869158, "TJ": 976, "TN": 10549080, "TR": 459298, "YE": 22114456 } }, { "language_name": "Urdu", "bcp_47": "ur", "speakers": 290790290, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2659144372728079, "chrf": 44.14831240898717 } ], "bleu": 0.2659144372728079, "chrf": 44.14831240898717, "commonvoice_hours": 76.0, "commonvoice_locale": "ur", "population": { "CA": 286475, "GB": 2301638, "IN": 66304500, "MU": 71727, "PK": 221825950 } }, { "language_name": "French", "bcp_47": "fr", "speakers": 278611507, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.315663773358301, "chrf": 49.253978669350964 } ], "bleu": 0.315663773358301, "chrf": 49.253978669350964, "commonvoice_hours": 1052.0, "commonvoice_locale": "fr", "population": { "AD": 5775, "AT": 974540, "BE": 4453866, "BF": 4583788, "BI": 7000822, "BJ": 4502610, "BL": 6837, "CA": 11308230, "CD": 3867640, "CF": 2935521, "CG": 4446179, "CH": 1764838, "CI": 13465739, "CM": 18866600, "CY": 88668, "DE": 14428746, "DJ": 19358, "DZ": 8594580, "FR": 67169718, "GA": 1405473, "GB": 15125053, "GF": 153622, "GN": 3632946, "GP": 407498, "GQ": 73584, "GR": 954639, "HT": 520187, "HU": 293155, "IE": 880017, "IT": 3931370, "KM": 473917, "LB": 20238, "LU": 546691, "MA": 7112340, "MC": 38610, "MF": 32556, "MG": 18599433, "ML": 8994564, "MQ": 427408, "MR": 680932, "MT": 50299, "MU": 41381, "NC": 278409, "NE": 6603996, "NL": 5011316, "PF": 180024, "PM": 5133, "PT": 1545405, "RE": 700950, "RO": 3621493, "RW": 2288, "SC": 57589, "SN": 6137196, "SY": 1144506, "TD": 4388124, "TF": 140, "TG": 5251148, "TN": 8673688, "US": 1862778, "VU": 149166, "WF": 7610, "YT": 110580 } }, { "language_name": "Bangla", "bcp_47": "bn", "speakers": 267193288, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21265887286151353, "chrf": 41.501657722373686 } ], "bleu": 0.21265887286151353, "chrf": 41.501657722373686, "commonvoice_hours": 49.0, "commonvoice_locale": "bn", "population": { "BD": 159397980, "CA": 90466, "GB": 263044, "IN": 107413290, "NP": 28508 } }, { "language_name": "Portuguese", "bcp_47": "pt", "speakers": 237496885, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.27514792195783394, "chrf": 45.901248962808694 } ], "bleu": 0.27514792195783394, "chrf": 45.901248962808694, "commonvoice_hours": 176.0, "commonvoice_locale": "pt", "population": { "AG": 1571, "AO": 21789941, "BR": 192661560, "CA": 229934, "CH": 285736, "CV": 443274, "FR": 882027, "GB": 131522, "GQ": 1, "GW": 1927100, "LU": 100541, "MO": 30723, "MZ": 8126514, "PT": 9890592, "ST": 179454, "TL": 816395 } }, { "language_name": "Punjabi", "bcp_47": "pa", "speakers": 203571210, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.32250608979374484, "chrf": 48.43043110055342 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3048037308116852, "chrf": 48.4304965568793 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.2314793285308931, "chrf": 37.53667069805556 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.37468960333154994, "chrf": 53.381398796420086 }, { "model": "deepseek/deepseek-chat", "bleu": 0.359056244961623, "chrf": 52.278269268214366 }, { "model": "microsoft/phi-4", "bleu": 0.27331521881795146, "chrf": 45.15087304059057 } ], "bleu": 0.31097503604124127, "chrf": 47.53468991011888, "commonvoice_hours": 2.3, "commonvoice_locale": "pa-IN", "population": { "CA": 603106, "GB": 2367400, "IN": 37130520, "KE": 10170, "PK": 163450700, "SG": 9314 } }, { "language_name": "Russian", "bcp_47": "ru", "speakers": 195841151, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.26108507692625094, "chrf": 45.063308940468154 } ], "bleu": 0.26108507692625094, "chrf": 45.063308940468154, "commonvoice_hours": 241.0, "commonvoice_locale": "ru", "population": { "BG": 1602387, "BY": 1137350, "CA": 211087, "CN": 13940, "DE": 4809582, "EE": 688027, "FI": 45131, "GE": 359730, "IL": 954303, "KG": 2147364, "KZ": 13746168, "LT": 2185168, "LV": 714867, "MD": 100935, "MN": 4118, "PL": 6890814, "RU": 133218680, "SJ": 1200, "TJ": 1064840, "TM": 663436, "UA": 20204534, "US": 798334, "UZ": 4279156 } }, { "language_name": "Swahili", "bcp_47": "sw", "speakers": 171610296, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2709203338132304, "chrf": 44.36399636969686 } ], "bleu": 0.2709203338132304, "chrf": 44.36399636969686, "commonvoice_hours": 411.0, "commonvoice_locale": "sw", "population": { "BI": 6408, "CD": 50890000, "KE": 35328414, "MZ": 9330, "SO": 235142, "TZ": 52697520, "UG": 32439750, "YT": 2716, "ZA": 1016 } }, { "language_name": "Indonesian", "bcp_47": "id", "speakers": 171207687, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.27441353638286026, "chrf": 46.025445629112156 } ], "bleu": 0.27441353638286026, "chrf": 46.025445629112156, "commonvoice_hours": 33.0, "commonvoice_locale": "id", "population": { "ID": 170896640, "NL": 311047 } }, { "language_name": "German", "bcp_47": "de", "speakers": 136350226, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3338682761061998, "chrf": 50.216731068308064 } ], "bleu": 0.3338682761061998, "chrf": 50.216731068308064, "commonvoice_hours": 1357.0, "commonvoice_locale": "de", "population": { "AT": 8593666, "BE": 2578554, "BG": 557352, "BR": 1778414, "CA": 294014, "CH": 6134913, "CZ": 1605375, "DE": 72945327, "DK": 2758623, "FI": 1002901, "FR": 3392410, "GB": 5918499, "GR": 530355, "HU": 1758929, "IT": 998443, "KZ": 1221882, "LI": 39137, "LT": 382404, "LU": 395880, "NA": 23671, "NL": 12269084, "PL": 7273637, "PY": 208559, "RO": 44736, "SI": 883126, "SK": 1196932, "US": 1563403 } }, { "language_name": "Japanese", "bcp_47": "ja", "speakers": 119729026, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2940100667664714, "chrf": 46.403097021492236 } ], "bleu": 0.2940100667664714, "chrf": 46.403097021492236, "commonvoice_hours": 222.0, "commonvoice_locale": "ja", "population": { "BR": 444604, "CA": 52772, "JP": 119231650 } }, { "language_name": "Telugu", "bcp_47": "te", "speakers": 95478480, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.28964452051612244, "chrf": 45.75290973782886 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2750887189010237, "chrf": 46.31463752811596 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.1314237858560668, "chrf": 27.750433857144273 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.35187073123584545, "chrf": 52.04190885735586 }, { "model": "deepseek/deepseek-chat", "bleu": 0.2808664068178743, "chrf": 48.221979700718 }, { "model": "microsoft/phi-4", "bleu": 0.16468103557770178, "chrf": 36.43981433605732 } ], "bleu": 0.24892919981743908, "chrf": 42.75361400287005, "commonvoice_hours": 0.3, "commonvoice_locale": "te", "population": { "IN": 95478480 } }, { "language_name": "Marathi", "bcp_47": "mr", "speakers": 92826300, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.235200323237626, "chrf": 39.43789667749676 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2584800238292114, "chrf": 44.69889855306244 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.1158656438579424, "chrf": 27.118712681006436 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3039098126596327, "chrf": 48.26657289026638 }, { "model": "deepseek/deepseek-chat", "bleu": 0.23702154369195902, "chrf": 43.039588362590955 }, { "model": "microsoft/phi-4", "bleu": 0.14770612974379574, "chrf": 35.63558588907277 } ], "bleu": 0.21636391283669454, "chrf": 39.69954250891596, "commonvoice_hours": 20.0, "commonvoice_locale": "mr", "population": { "IN": 92826300 } }, { "language_name": "Javanese", "bcp_47": "jv", "speakers": 91180665, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23082586428104943, "chrf": 41.42591471734489 } ], "bleu": 0.23082586428104943, "chrf": 41.42591471734489, "commonvoice_hours": 0.0, "commonvoice_locale": "jv", "population": { "ID": 90788840, "MY": 391825 } }, { "language_name": "Vietnamese", "bcp_47": "vi", "speakers": 86222962, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.2790022403255029, "chrf": 44.76577365559692 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.252552287345529, "chrf": 43.351007120897606 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.17142561681893811, "chrf": 34.23876708175754 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.32076560886563743, "chrf": 49.46269756420099 }, { "model": "deepseek/deepseek-chat", "bleu": 0.269842412561934, "chrf": 44.17252674642975 }, { "model": "microsoft/phi-4", "bleu": 0.19118351096728373, "chrf": 37.642090247027234 } ], "bleu": 0.24746194614747083, "chrf": 42.272143735985004, "commonvoice_hours": 5.9, "commonvoice_locale": "vi", "population": { "CA": 184701, "CN": 6970, "US": 1130973, "VN": 84900318 } }, { "language_name": "Tamil", "bcp_47": "ta", "speakers": 85616159, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.2159676106476219, "chrf": 38.592451568525966 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.24147470924504938, "chrf": 41.98664320436057 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.11055269618146167, "chrf": 26.929660798631293 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.2993653070835946, "chrf": 46.760725728832945 }, { "model": "deepseek/deepseek-chat", "bleu": 0.22772498517043588, "chrf": 40.963440857462984 }, { "model": "microsoft/phi-4", "bleu": 0.14949134449145374, "chrf": 33.188135588116566 } ], "bleu": 0.2074294421366029, "chrf": 38.07017629098839, "commonvoice_hours": 234.0, "commonvoice_locale": "ta", "population": { "CA": 184701, "GB": 2104355, "IN": 78239310, "LK": 3433380, "MU": 34484, "MY": 1371388, "RE": 118138, "SG": 130403 } }, { "language_name": "Persian", "bcp_47": "fa", "speakers": 84710459, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2543274430452483, "chrf": 44.798186786819564 } ], "bleu": 0.2543274430452483, "chrf": 44.798186786819564, "commonvoice_hours": 370.0, "commonvoice_locale": "fa", "population": { "AE": 189850, "AF": 18321900, "CA": 245012, "IQ": 338192, "IR": 63692475, "OM": 43849, "PK": 1541107, "QA": 268859, "TJ": 69215 } }, { "language_name": "Turkish", "bcp_47": "tr", "speakers": 80360704, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2848412390293461, "chrf": 45.023438128876236 } ], "bleu": 0.2848412390293461, "chrf": 45.023438128876236, "commonvoice_hours": 127.0, "commonvoice_locale": "tr", "population": { "BG": 766359, "CA": 37694, "CY": 291336, "DE": 2003992, "GB": 131522, "GR": 127285, "MK": 74409, "NL": 207365, "RO": 27694, "TR": 76276275, "UA": 184476, "UZ": 232297 } }, { "language_name": "Cantonese", "bcp_47": "yue", "speakers": 79654759, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2745324223697869, "chrf": 46.18505717482203 } ], "bleu": 0.2745324223697869, "chrf": 46.18505717482203, "commonvoice_hours": 203.0, "commonvoice_locale": "yue", "population": { "CA": 640800, "CN": 72489040, "HK": 6524919 } }, { "language_name": "Korean", "bcp_47": "ko", "speakers": 78357046, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21423807187509414, "chrf": 42.31955622695572 } ], "bleu": 0.21423807187509414, "chrf": 42.31955622695572, "commonvoice_hours": 1.7, "commonvoice_locale": "ko", "population": { "BR": 44460, "CA": 169623, "CN": 2091030, "JP": 652636, "KP": 22566280, "KR": 51835100, "US": 997917 } }, { "language_name": "Italian", "bcp_47": "it", "speakers": 70247060, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.29744196180619636, "chrf": 46.58884190721562 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2947459489130278, "chrf": 48.23578446556176 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.26709213193768344, "chrf": 43.42719841351614 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.34518602347709243, "chrf": 51.45922256128821 }, { "model": "deepseek/deepseek-chat", "bleu": 0.3136120219290237, "chrf": 49.456527591725454 }, { "model": "microsoft/phi-4", "bleu": 0.26629405288011837, "chrf": 44.95821657057411 } ], "bleu": 0.29739535682385704, "chrf": 47.35429858498022, "commonvoice_hours": 362.0, "commonvoice_locale": "it", "population": { "AT": 797350, "AU": 483864, "BR": 592805, "CA": 343016, "CH": 361372, "DE": 5611179, "FR": 1153419, "GB": 131522, "HR": 67644, "IT": 59282565, "MT": 256070, "SI": 3995, "SM": 30466, "US": 1130973, "VA": 820 } }, { "language_name": "Filipino", "bcp_47": "fil", "speakers": 67471096, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3020075248846111, "chrf": 45.148692233744825 } ], "bleu": 0.3020075248846111, "chrf": 45.148692233744825, "commonvoice_hours": 0.0, "commonvoice_locale": "tl", "population": { "CA": 565412, "PH": 65508600, "US": 1397084 } }, { "language_name": "Egyptian Arabic", "bcp_47": "arz", "speakers": 66639360, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.18353324091064518, "chrf": 37.67362087042454 } ], "bleu": 0.18353324091064518, "chrf": 37.67362087042454, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "EG": 66639360 } }, { "language_name": "Gujarati", "bcp_47": "gu", "speakers": 61721799, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.27618486571036477, "chrf": 45.30697533607047 } ], "bleu": 0.27618486571036477, "chrf": 45.30697533607047, "commonvoice_hours": 0.0, "commonvoice_locale": "gu-IN", "population": { "CA": 135699, "GB": 1907072, "IN": 59674050, "KE": 4978 } }, { "language_name": "Thai", "bcp_47": "th", "speakers": 55181920, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2461561900137243, "chrf": 43.886996888191135 } ], "bleu": 0.2461561900137243, "chrf": 43.886996888191135, "commonvoice_hours": 172.0, "commonvoice_locale": "th", "population": { "TH": 55181920 } }, { "language_name": "Kannada", "bcp_47": "kn", "speakers": 49065330, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.25650866519447973, "chrf": 43.53706086839864 } ], "bleu": 0.25650866519447973, "chrf": 43.53706086839864, "commonvoice_hours": 0.0, "commonvoice_locale": "kn", "population": { "IN": 49065330 } }, { "language_name": "Malayalam", "bcp_47": "ml", "speakers": 43257484, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.23073727076678055, "chrf": 41.53402359471923 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21782657144614825, "chrf": 41.630569782738704 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.1695641998407403, "chrf": 33.20242503336964 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3195014249623395, "chrf": 50.399254960139395 }, { "model": "deepseek/deepseek-chat", "bleu": 0.2633223158501049, "chrf": 45.473592535604965 }, { "model": "microsoft/phi-4", "bleu": 0.19162873119255258, "chrf": 38.21485785002488 } ], "bleu": 0.23209675234311103, "chrf": 41.7424539594328, "commonvoice_hours": 2.8, "commonvoice_locale": "ml", "population": { "AE": 699446, "BH": 49665, "IL": 7981, "IN": 42434880, "MY": 48978, "QA": 6599, "SG": 9935 } }, { "language_name": "Odia", "bcp_47": "or", "speakers": 42434880, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2616054244059909, "chrf": 44.81898318236423 } ], "bleu": 0.2616054244059909, "chrf": 44.81898318236423, "commonvoice_hours": 2.8, "commonvoice_locale": "or", "population": { "IN": 42434880 } }, { "language_name": "Polish", "bcp_47": "pl", "speakers": 41077399, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.24382878885531348, "chrf": 43.88802125000008 } ], "bleu": 0.24382878885531348, "chrf": 43.88802125000008, "commonvoice_hours": 174.0, "commonvoice_locale": "pl", "population": { "CA": 173393, "CZ": 52442, "DE": 232463, "GB": 2630444, "IL": 130132, "PL": 36751008, "RO": 2769, "SK": 50598, "UA": 1054150 } }, { "language_name": "Hausa", "bcp_47": "ha", "speakers": 40411882, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1493358875548207, "chrf": 31.734228520521885 } ], "bleu": 0.1493358875548207, "chrf": 31.734228520521885, "commonvoice_hours": 4.1, "commonvoice_locale": "ha", "population": { "CM": 38843, "GH": 252326, "NE": 9336684, "NG": 29963920, "SD": 820109 } }, { "language_name": "Sindhi", "bcp_47": "sd", "speakers": 40329510, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.22524971121549384, "chrf": 41.35718488130492 } ], "bleu": 0.22524971121549384, "chrf": 41.35718488130492, "commonvoice_hours": 0.4, "commonvoice_locale": "sd", "population": { "IN": 5304360, "PK": 35025150 } }, { "language_name": "Malay", "bcp_47": "ms", "speakers": 38097307, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.27545115634664297, "chrf": 46.5366529956061 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2445459295400275, "chrf": 43.683725288188164 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.23380594556884363, "chrf": 38.09288562825641 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3521510571182875, "chrf": 51.6107732437805 }, { "model": "deepseek/deepseek-chat", "bleu": 0.2984537737499322, "chrf": 47.07139912930323 }, { "model": "microsoft/phi-4", "bleu": 0.20974647653543713, "chrf": 40.94603982712764 } ], "bleu": 0.2690257231431951, "chrf": 44.656912685377016, "commonvoice_hours": 0.0, "commonvoice_locale": "ms", "population": { "BN": 455189, "CC": 495, "ID": 12283196, "MY": 24489075, "SG": 869352 } }, { "language_name": "Burmese", "bcp_47": "my", "speakers": 36559231, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.17529594258449108, "chrf": 38.09396067803311 } ], "bleu": 0.17529594258449108, "chrf": 38.09396067803311, "commonvoice_hours": 0.0, "commonvoice_locale": "my", "population": { "BD": 341567, "MM": 36217664 } }, { "language_name": "Amharic", "bcp_47": "am", "speakers": 35728475, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.16453710162389373, "chrf": 34.89886094698235 } ], "bleu": 0.16453710162389373, "chrf": 34.89886094698235, "commonvoice_hours": 1.8, "commonvoice_locale": "am", "population": { "ET": 35677290, "IL": 51185 } }, { "language_name": "Oromo", "bcp_47": "om", "speakers": 34897121, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.0691897827221633, "chrf": 22.166735299684706 } ], "bleu": 0.0691897827221633, "chrf": 22.166735299684706, "commonvoice_hours": 0.0, "commonvoice_locale": "om", "population": { "ET": 34596160, "KE": 251581, "SO": 49380 } }, { "language_name": "Bhojpuri", "bcp_47": "bho", "speakers": 32934797, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23176648838308359, "chrf": 42.06617216025931 } ], "bleu": 0.23176648838308359, "chrf": 42.06617216025931, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 30500070, "MU": 372430, "NP": 2062297 } }, { "language_name": "Uzbek", "bcp_47": "uz", "speakers": 32792780, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2038544554531401, "chrf": 39.8364082393657 } ], "bleu": 0.2038544554531401, "chrf": 39.8364082393657, "commonvoice_hours": 100.0, "commonvoice_locale": "uz", "population": { "AF": 1722259, "CN": 5576, "TM": 497577, "TR": 1968, "UZ": 30565400 } }, { "language_name": "Azerbaijani", "bcp_47": "az", "speakers": 32446682, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1911853993562902, "chrf": 38.284553121320506 } ], "bleu": 0.1911853993562902, "chrf": 38.284553121320506, "commonvoice_hours": 0.5, "commonvoice_locale": "az", "population": { "AM": 0, "AZ": 10093536, "IQ": 699709, "IR": 20381592, "RU": 131801, "TR": 1140044 } }, { "language_name": "Sundanese", "bcp_47": "su", "speakers": 32043120, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.18638464691782505, "chrf": 36.390067106438586 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.22413489641063433, "chrf": 39.08607675576149 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.12396443500389862, "chrf": 27.44605100630464 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.31100961611618916, "chrf": 48.684591434160616 }, { "model": "deepseek/deepseek-chat", "bleu": 0.23467068441612768, "chrf": 40.54860164135514 }, { "model": "microsoft/phi-4", "bleu": 0.1315217916431994, "chrf": 32.71394871178255 } ], "bleu": 0.20194767841797903, "chrf": 37.47822277596717, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "ID": 32043120 } }, { "language_name": "Dutch", "bcp_47": "nl", "speakers": 31765645, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2592795402705898, "chrf": 44.356739380497835 } ], "bleu": 0.2592795402705898, "chrf": 44.356739380497835, "commonvoice_hours": 114.0, "commonvoice_locale": "nl", "population": { "AW": 115845, "BE": 6446385, "BQ": 1600, "CA": 56541, "CW": 12108, "DE": 7214373, "FR": 88203, "NL": 17280400, "SR": 548612, "SX": 1578 } }, { "language_name": "Moroccan Arabic", "bcp_47": "ary", "speakers": 30938679, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1795140543637709, "chrf": 37.426822543484725 } ], "bleu": 0.1795140543637709, "chrf": 37.426822543484725, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "MA": 30938679 } }, { "language_name": "Ukrainian", "bcp_47": "uk", "speakers": 29348975, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.2564463888571809, "chrf": 41.5181715644979 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2922812040972885, "chrf": 46.82127627710616 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.24225796102393954, "chrf": 39.42058485499366 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3452563778145038, "chrf": 50.96771482708702 }, { "model": "deepseek/deepseek-chat", "bleu": 0.3292127494173498, "chrf": 48.580427643849056 }, { "model": "microsoft/phi-4", "bleu": 0.2452825737163755, "chrf": 42.05493360532406 } ], "bleu": 0.28512287582110635, "chrf": 44.893851462142976, "commonvoice_hours": 98.0, "commonvoice_locale": "uk", "population": { "CA": 75388, "MD": 471030, "PL": 149301, "RS": 0, "SK": 103371, "UA": 28549885 } }, { "language_name": "Yoruba", "bcp_47": "yo", "speakers": 28685568, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.09852676389614487, "chrf": 25.656950226425916 } ], "bleu": 0.09852676389614487, "chrf": 25.656950226425916, "commonvoice_hours": 5.9, "commonvoice_locale": "yo", "population": { "BJ": 861928, "NG": 27823640 } }, { "language_name": "Igbo", "bcp_47": "ig", "speakers": 27823640, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1549827013842116, "chrf": 35.431035675871804 } ], "bleu": 0.1549827013842116, "chrf": 35.431035675871804, "commonvoice_hours": 0.0, "commonvoice_locale": "ig", "population": { "NG": 27823640 } }, { "language_name": "Cebuano", "bcp_47": "ceb", "speakers": 26203440, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.27783470672988303, "chrf": 43.53549785827671 } ], "bleu": 0.27783470672988303, "chrf": 43.53549785827671, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "PH": 26203440 } }, { "language_name": "Awadhi", "bcp_47": "awa", "speakers": 25862924, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2554810263222905, "chrf": 42.35795046006446 } ], "bleu": 0.2554810263222905, "chrf": 42.35795046006446, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 25195710, "NP": 667214 } }, { "language_name": "Malagasy", "bcp_47": "mg", "speakers": 24260130, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.15163299980391426, "chrf": 32.41299083831688 } ], "bleu": 0.15163299980391426, "chrf": 32.41299083831688, "commonvoice_hours": 0.0, "commonvoice_locale": "mg", "population": { "MG": 24260130 } }, { "language_name": "Romanian", "bcp_47": "ro", "speakers": 22187408, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.33899025568959984, "chrf": 49.419389839471826 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.26666997541189236, "chrf": 44.76525386460237 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.24172488724041316, "chrf": 37.98045602918644 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.37365302832845815, "chrf": 52.777299654432 }, { "model": "deepseek/deepseek-chat", "bleu": 0.332600965807992, "chrf": 49.14582652136321 }, { "model": "microsoft/phi-4", "bleu": 0.2510789925018768, "chrf": 42.27877315264307 } ], "bleu": 0.30078635083003874, "chrf": 46.06116651028316, "commonvoice_hours": 21.0, "commonvoice_locale": "ro", "population": { "CA": 101774, "HU": 96741, "IL": 320993, "MD": 2119635, "RO": 19172610, "RS": 147256, "UA": 228399 } }, { "language_name": "Nepali", "bcp_47": "ne", "speakers": 20903374, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.26199282928489126, "chrf": 44.7560662712792 } ], "bleu": 0.26199282928489126, "chrf": 44.7560662712792, "commonvoice_hours": 1.3, "commonvoice_locale": "ne-NP", "population": { "BT": 132994, "IN": 7426104, "NP": 13344276 } }, { "language_name": "Maithili", "bcp_47": "mai", "speakers": 19249149, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23975507119180453, "chrf": 41.8894484718934 } ], "bleu": 0.23975507119180453, "chrf": 41.8894484718934, "commonvoice_hours": 0.0, "commonvoice_locale": "mai", "population": { "IN": 15913080, "NP": 3336069 } }, { "language_name": "Assamese", "bcp_47": "as", "speakers": 17239170, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.19363225565136952, "chrf": 38.96237165376663 } ], "bleu": 0.19363225565136952, "chrf": 38.96237165376663, "commonvoice_hours": 2.8, "commonvoice_locale": "as", "population": { "IN": 17239170 } }, { "language_name": "Nyanja", "bcp_47": "ny", "speakers": 17026781, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.09504458945778768, "chrf": 27.576030002164906 } ], "bleu": 0.09504458945778768, "chrf": 27.576030002164906, "commonvoice_hours": 0.0, "commonvoice_locale": "ny", "population": { "MW": 13353858, "MZ": 782553, "ZM": 2613990, "ZW": 276380 } }, { "language_name": "Somali", "bcp_47": "so", "speakers": 16911645, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.2024994684991584, "chrf": 37.37281822856629 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1532133716194419, "chrf": 32.05620028647162 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.043408496427191995, "chrf": 15.267715935014895 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3048371831537258, "chrf": 46.24092673305936 }, { "model": "deepseek/deepseek-chat", "bleu": 0.21360545410615966, "chrf": 38.32357547187653 }, { "model": "microsoft/phi-4", "bleu": 0.06484340154849859, "chrf": 22.781181465233722 } ], "bleu": 0.16373456255902938, "chrf": 32.00706968670374, "commonvoice_hours": 0.0, "commonvoice_locale": "so", "population": { "CA": 49002, "DJ": 377940, "ET": 6486780, "GB": 131522, "KE": 695863, "SO": 9170538 } }, { "language_name": "Magahi", "bcp_47": "mag", "speakers": 15913080, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.29925558767802407, "chrf": 47.539870710677974 } ], "bleu": 0.29925558767802407, "chrf": 47.539870710677974, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 15913080 } }, { "language_name": "Serbian", "bcp_47": "sr", "speakers": 15602410, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.26029402164210574, "chrf": 44.75391848259974 } ], "bleu": 0.26029402164210574, "chrf": 44.75391848259974, "commonvoice_hours": 7.4, "commonvoice_locale": "sr", "population": { "BA": 767118, "CA": 64080, "ME": 640352, "RO": 25563, "RS": 13884096, "RU": 4960, "TR": 22965, "XK": 193276 } }, { "language_name": "Sinhala", "bcp_47": "si", "speakers": 15564656, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.20259734060180434, "chrf": 39.00188422851495 } ], "bleu": 0.20259734060180434, "chrf": 39.00188422851495, "commonvoice_hours": 0.0, "commonvoice_locale": "si", "population": { "LK": 15564656 } }, { "language_name": "Khmer", "bcp_47": "km", "speakers": 15065030, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.21699232146684352, "chrf": 41.99979148031644 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21417349432612984, "chrf": 41.35771196976409 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.03287369352293625, "chrf": 14.140423341647319 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.34264229339556035, "chrf": 51.55826045210756 }, { "model": "deepseek/deepseek-chat", "bleu": 0.24630515818736093, "chrf": 44.60963216433486 }, { "model": "microsoft/phi-4", "bleu": 0.11830648687368288, "chrf": 31.534012306448343 } ], "bleu": 0.1952155746287523, "chrf": 37.53330528576977, "commonvoice_hours": 0.0, "commonvoice_locale": "km", "population": { "KH": 15065030 } }, { "language_name": "Chhattisgarhi", "bcp_47": "hne", "speakers": 14586990, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2212498883000727, "chrf": 39.102114067893005 } ], "bleu": 0.2212498883000727, "chrf": 39.102114067893005, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 14586990 } }, { "language_name": "Nigerian Fulfulde", "bcp_47": "fuv", "speakers": 14339876, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.08190470208193343, "chrf": 23.041107899884107 } ], "bleu": 0.08190470208193343, "chrf": 23.041107899884107, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "NG": 14339876 } }, { "language_name": "Zulu", "bcp_47": "zu", "speakers": 13973830, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.15449146502209737, "chrf": 33.087212745906356 } ], "bleu": 0.15449146502209737, "chrf": 33.087212745906356, "commonvoice_hours": 0.0, "commonvoice_locale": "zu", "population": { "LS": 275706, "MW": 69949, "MZ": 1806, "SZ": 75105, "ZA": 13551264 } }, { "language_name": "Kazakh", "bcp_47": "kk", "speakers": 13637392, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2392246097188628, "chrf": 42.67763456362536 } ], "bleu": 0.2392246097188628, "chrf": 42.67763456362536, "commonvoice_hours": 2.1, "commonvoice_locale": "kk", "population": { "AF": 2015, "CN": 1184917, "IR": 2972, "KZ": 12218816, "MN": 228098, "TR": 574 } }, { "language_name": "Czech", "bcp_47": "cs", "speakers": 13045532, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2844520855192069, "chrf": 47.327010269160255 } ], "bleu": 0.2844520855192069, "chrf": 47.327010269160255, "commonvoice_hours": 74.0, "commonvoice_locale": "cs", "population": { "CZ": 10488450, "SK": 2557082 } }, { "language_name": "Swedish", "bcp_47": "sv", "speakers": 12932871, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.31838456223051165, "chrf": 48.08041424667649 } ], "bleu": 0.31838456223051165, "chrf": 48.08041424667649, "commonvoice_hours": 47.0, "commonvoice_locale": "sv-SE", "population": { "AX": 25938, "DK": 763023, "FI": 2451535, "SE": 9692375 } }, { "language_name": "Hungarian", "bcp_47": "hu", "speakers": 12443430, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2517614908428288, "chrf": 42.77520440394925 } ], "bleu": 0.2517614908428288, "chrf": 42.77520440394925, "commonvoice_hours": 92.0, "commonvoice_locale": "hu", "population": { "AT": 23035, "CA": 49002, "HU": 9771830, "IL": 86755, "RO": 1405991, "RS": 336584, "SI": 9252, "SK": 598466, "UA": 162515 } }, { "language_name": "Greek", "bcp_47": "el", "speakers": 12292242, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.24888370153898132, "chrf": 42.743109839531535 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.27269196827838943, "chrf": 45.53483489961114 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.21351544070708506, "chrf": 39.64425368969459 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3500489758234636, "chrf": 51.1729677922222 }, { "model": "deepseek/deepseek-chat", "bleu": 0.32858632704534785, "chrf": 49.70348294319061 }, { "model": "microsoft/phi-4", "bleu": 0.1903000734693107, "chrf": 39.41911092527115 } ], "bleu": 0.2673377478104297, "chrf": 44.7029600149202, "commonvoice_hours": 20.0, "commonvoice_locale": "el", "population": { "AL": 58417, "CA": 124391, "CY": 1203346, "DE": 304607, "EG": 63516, "GR": 10501029, "IT": 21841, "RO": 4048, "TR": 4019, "UA": 7028 } }, { "language_name": "Shona", "bcp_47": "sn", "speakers": 11782503, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.10438047654339373, "chrf": 28.358507205354343 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1054140213254438, "chrf": 28.275522446025796 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.049580893458705456, "chrf": 18.458313551880636 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.20979752102494492, "chrf": 38.7737276002067 }, { "model": "deepseek/deepseek-chat", "bleu": 0.14066476436038525, "chrf": 32.050887533047465 }, { "model": "microsoft/phi-4", "bleu": 0.060530921002659346, "chrf": 22.35054590384825 } ], "bleu": 0.11172809961925541, "chrf": 28.04458404006053, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "ZW": 11782503 } }, { "language_name": "Central Kurdish", "bcp_47": "ckb", "speakers": 11086549, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.20654412682006296, "chrf": 38.66802600490074 } ], "bleu": 0.20654412682006296, "chrf": 38.66802600490074, "commonvoice_hours": 135.0, "commonvoice_locale": "ckb", "population": { "IQ": 7774540, "IR": 3312009 } }, { "language_name": "Kinyarwanda", "bcp_47": "rw", "speakers": 11083625, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.14006909985587948, "chrf": 31.739340000779823 } ], "bleu": 0.14006909985587948, "chrf": 31.739340000779823, "commonvoice_hours": 2002.0, "commonvoice_locale": "rw", "population": { "CD": 386764, "RW": 9788548, "UG": 908313 } }, { "language_name": "Wolof", "bcp_47": "wo", "speakers": 11025494, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.08408628490585719, "chrf": 23.767561072306325 } ], "bleu": 0.08408628490585719, "chrf": 23.767561072306325, "commonvoice_hours": 0.0, "commonvoice_locale": "wo", "population": { "MR": 10014, "SN": 11015480 } }, { "language_name": "Tunisian Arabic", "bcp_47": "aeb", "speakers": 10549080, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23738824104522893, "chrf": 42.85642541970995 } ], "bleu": 0.23738824104522893, "chrf": 42.85642541970995, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "TN": 10549080 } }, { "language_name": "Iloko", "bcp_47": "ilo", "speakers": 10481376, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.20685666710862224, "chrf": 37.066073786659494 } ], "bleu": 0.20685666710862224, "chrf": 37.066073786659494, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "PH": 10481376 } }, { "language_name": "Xhosa", "bcp_47": "xh", "speakers": 10182944, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1143143326923908, "chrf": 28.93119601652647 } ], "bleu": 0.1143143326923908, "chrf": 28.93119601652647, "commonvoice_hours": 0.0, "commonvoice_locale": "xh", "population": { "LS": 19496, "ZA": 10163448 } }, { "language_name": "Tigrinya", "bcp_47": "ti", "speakers": 10145911, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.08532345270447181, "chrf": 25.148121650983146 } ], "bleu": 0.08532345270447181, "chrf": 25.148121650983146, "commonvoice_hours": 0.0, "commonvoice_locale": "ti", "population": { "ER": 3648720, "ET": 6486780, "IL": 10411 } }, { "language_name": "Belarusian", "bcp_47": "be", "speakers": 10064517, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.18341973561481445, "chrf": 39.828398360117035 } ], "bleu": 0.18341973561481445, "chrf": 39.828398360117035, "commonvoice_hours": 1804.0, "commonvoice_locale": "be", "population": { "BY": 9477920, "PL": 222037, "UA": 364560 } }, { "language_name": "Luba-Lulua", "bcp_47": "lua", "speakers": 9770880, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.11581604983636683, "chrf": 28.08252510561598 } ], "bleu": 0.11581604983636683, "chrf": 28.08252510561598, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "CD": 9770880 } }, { "language_name": "Tajik", "bcp_47": "tg", "speakers": 9644223, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1846236171638531, "chrf": 39.50004300248175 } ], "bleu": 0.1846236171638531, "chrf": 39.50004300248175, "commonvoice_hours": 0.0, "commonvoice_locale": "tg", "population": { "PK": 770553, "TJ": 8873670 } }, { "language_name": "Umbundu", "bcp_47": "umb", "speakers": 9431467, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.05520341910203098, "chrf": 20.443280736080066 } ], "bleu": 0.05520341910203098, "chrf": 20.443280736080066, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "AO": 9431467 } }, { "language_name": "Bambara", "bcp_47": "bm", "speakers": 9385632, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.07227674667013836, "chrf": 22.136582910055218 } ], "bleu": 0.07227674667013836, "chrf": 22.136582910055218, "commonvoice_hours": 0.0, "commonvoice_locale": "bm", "population": { "ML": 9385632 } }, { "language_name": "Afrikaans", "bcp_47": "af", "speakers": 9318845, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.3277177864074156, "chrf": 48.61873150516328 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.31538459755111, "chrf": 47.75260784921264 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.26710030799119333, "chrf": 38.869810330084285 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.4361740814378139, "chrf": 56.524736028951835 }, { "model": "deepseek/deepseek-chat", "bleu": 0.37149647257024515, "chrf": 53.20003536968165 }, { "model": "microsoft/phi-4", "bleu": 0.2883662842075808, "chrf": 45.203083386764426 } ], "bleu": 0.3343732550275598, "chrf": 48.361500744976354, "commonvoice_hours": 0.5, "commonvoice_locale": "af", "population": { "BW": 6025, "NA": 1972552, "ZA": 7340268 } }, { "language_name": "Kikuyu", "bcp_47": "ki", "speakers": 9099743, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.0883556207236924, "chrf": 24.868277910094278 } ], "bleu": 0.0883556207236924, "chrf": 24.868277910094278, "commonvoice_hours": 0.0, "commonvoice_locale": "ki", "population": { "KE": 9099743 } }, { "language_name": "Haitian Creole", "bcp_47": "ht", "speakers": 8964918, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1872609836464467, "chrf": 37.00166091804026 } ], "bleu": 0.1872609836464467, "chrf": 37.00166091804026, "commonvoice_hours": 0.0, "commonvoice_locale": "ht", "population": { "HT": 8964918 } }, { "language_name": "Catalan", "bcp_47": "ca", "speakers": 8679139, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.29445274007068095, "chrf": 47.46074814103581 } ], "bleu": 0.29445274007068095, "chrf": 47.46074814103581, "commonvoice_hours": 2842.0, "commonvoice_locale": "ca", "population": { "AD": 39270, "ES": 8502686, "FR": 115342, "IT": 21841 } }, { "language_name": "Hebrew", "bcp_47": "he", "speakers": 8675480, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2824719214447976, "chrf": 46.279771845770604 } ], "bleu": 0.2824719214447976, "chrf": 46.279771845770604, "commonvoice_hours": 1.1, "commonvoice_locale": "he", "population": { "IL": 8675480 } }, { "language_name": "Mossi", "bcp_47": "mos", "speakers": 8334160, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.08102496244147746, "chrf": 22.184797797745208 } ], "bleu": 0.08102496244147746, "chrf": 22.184797797745208, "commonvoice_hours": 0.0, "commonvoice_locale": "mos", "population": { "BF": 8334160 } }, { "language_name": "Kimbundu", "bcp_47": "kmb", "speakers": 8130575, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.07329500673809967, "chrf": 22.838222030254723 } ], "bleu": 0.07329500673809967, "chrf": 22.838222030254723, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "AO": 8130575 } }, { "language_name": "Uyghur", "bcp_47": "ug", "speakers": 8052967, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.18397910035998616, "chrf": 35.50154017198535 } ], "bleu": 0.18397910035998616, "chrf": 35.50154017198535, "commonvoice_hours": 361.0, "commonvoice_locale": "ug", "population": { "AF": 3005, "CN": 7667110, "KZ": 381838, "MN": 1014 } }, { "language_name": "Minangkabau", "bcp_47": "min", "speakers": 8010780, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.22401269807719826, "chrf": 40.34678123737912 } ], "bleu": 0.22401269807719826, "chrf": 40.34678123737912, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "ID": 8010780 } }, { "language_name": "Bulgarian", "bcp_47": "bg", "speakers": 7878315, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.24723779163445408, "chrf": 45.55403997487483 } ], "bleu": 0.24723779163445408, "chrf": 45.55403997487483, "commonvoice_hours": 16.0, "commonvoice_locale": "bg", "population": { "BG": 6966900, "GR": 28639, "MD": 316263, "RO": 6817, "TR": 344474, "UA": 215222 } }, { "language_name": "Standard Moroccan Tamazight", "bcp_47": "zgh", "speakers": 7823574, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.0366272802298245, "chrf": 15.463910171449278 } ], "bleu": 0.0366272802298245, "chrf": 15.463910171449278, "commonvoice_hours": 1.3, "commonvoice_locale": "zgh", "population": { "MA": 7823574 } }, { "language_name": "Bosnian", "bcp_47": "bs", "speakers": 7594468, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3051247921441283, "chrf": 48.3731481201238 } ], "bleu": 0.3051247921441283, "chrf": 48.3731481201238, "commonvoice_hours": 0.0, "commonvoice_locale": "bs", "population": { "BA": 7594468 } }, { "language_name": "Rundi", "bcp_47": "rn", "speakers": 7475454, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.0957054530624, "chrf": 28.269164219007447 } ], "bleu": 0.0957054530624, "chrf": 28.269164219007447, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "BI": 7475454 } }, { "language_name": "Santali", "bcp_47": "sat", "speakers": 7293495, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.11554870024021023, "chrf": 29.549934477174254 } ], "bleu": 0.11554870024021023, "chrf": 29.549934477174254, "commonvoice_hours": 0.5, "commonvoice_locale": "sat", "population": { "IN": 7293495 } }, { "language_name": "Danish", "bcp_47": "da", "speakers": 7072056, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3512857581168584, "chrf": 51.12453594191334 } ], "bleu": 0.3512857581168584, "chrf": 51.12453594191334, "commonvoice_hours": 13.0, "commonvoice_locale": "da", "population": { "DE": 1603194, "DK": 5458551, "GL": 8066, "IS": 2245 } }, { "language_name": "Turkmen", "bcp_47": "tk", "speakers": 6870838, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.15410064596625964, "chrf": 36.3602620147462 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.14820890318014426, "chrf": 34.78567738931005 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.07938993687949465, "chrf": 24.385063397578556 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.2761834512123037, "chrf": 47.41646574422384 }, { "model": "deepseek/deepseek-chat", "bleu": 0.1718597543270264, "chrf": 38.84762953076257 }, { "model": "microsoft/phi-4", "bleu": 0.10118596975980092, "chrf": 28.31840639855478 } ], "bleu": 0.15515477688750492, "chrf": 35.018917412529326, "commonvoice_hours": 2.8, "commonvoice_locale": "tk", "population": { "AF": 622945, "IR": 2377852, "TM": 3870041 } }, { "language_name": "Kurdish", "bcp_47": "ku", "speakers": 6866757, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.14440915289810186, "chrf": 31.3622869278075 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.15987085387022903, "chrf": 35.18523232721762 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.10987778830152085, "chrf": 25.91454653951599 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.28985769410441137, "chrf": 46.64309582634758 }, { "model": "deepseek/deepseek-chat", "bleu": 0.1865343501300658, "chrf": 37.28483871763787 }, { "model": "microsoft/phi-4", "bleu": 0.10000019378200214, "chrf": 26.406217887319293 } ], "bleu": 0.16509167218105517, "chrf": 33.79936970430764, "commonvoice_hours": 69.0, "commonvoice_locale": "kmr", "population": { "AM": 99704, "AZ": 24494, "DE": 529054, "GE": 35573, "LB": 92983, "SY": 1551872, "TM": 22115, "TR": 4510962 } }, { "language_name": "Croatian", "bcp_47": "hr", "speakers": 6813164, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2290484937313612, "chrf": 41.79307139114272 } ], "bleu": 0.2290484937313612, "chrf": 41.79307139114272, "commonvoice_hours": 0.0, "commonvoice_locale": "hr", "population": { "AT": 106313, "BA": 460271, "CA": 45233, "DE": 633262, "HR": 4185472, "HU": 31270, "IT": 3495, "RS": 65213, "SI": 1282635 } }, { "language_name": "Albanian", "bcp_47": "sq", "speakers": 6791906, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.26490711574268994, "chrf": 44.54318851690701 } ], "bleu": 0.26490711574268994, "chrf": 44.54318851690701, "commonvoice_hours": 8.8, "commonvoice_locale": "sq", "population": { "AL": 3074580, "GR": 9971, "ME": 48179, "MK": 531492, "RS": 1332312, "TR": 17224, "XK": 1778148 } }, { "language_name": "Slovak", "bcp_47": "sk", "speakers": 6680269, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2826836020834733, "chrf": 45.73110764547564 } ], "bleu": 0.2826836020834733, "chrf": 45.73110764547564, "commonvoice_hours": 40.0, "commonvoice_locale": "sk", "population": { "CZ": 1712400, "HU": 11726, "RS": 59603, "SK": 4896540 } }, { "language_name": "Dyula", "bcp_47": "dyu", "speakers": 6667328, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.0633017924291756, "chrf": 21.71247042480193 } ], "bleu": 0.0633017924291756, "chrf": 21.71247042480193, "commonvoice_hours": 0.3, "commonvoice_locale": "dyu", "population": { "BF": 6667328 } }, { "language_name": "Mongolian", "bcp_47": "mn", "speakers": 6572846, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1950781841033538, "chrf": 41.09779803278993 } ], "bleu": 0.1950781841033538, "chrf": 41.09779803278993, "commonvoice_hours": 46.0, "commonvoice_locale": "mn", "population": { "CN": 3624452, "MN": 2946268, "RU": 2126 } }, { "language_name": "Southern Sotho", "bcp_47": "st", "speakers": 6390567, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.12381413258013083, "chrf": 28.651371806370722 } ], "bleu": 0.12381413258013083, "chrf": 28.651371806370722, "commonvoice_hours": 0.0, "commonvoice_locale": "st", "population": { "LS": 1929943, "ZA": 4460624 } }, { "language_name": "Tswana", "bcp_47": "tn", "speakers": 6113428, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.09139131060492443, "chrf": 25.97258334431281 } ], "bleu": 0.09139131060492443, "chrf": 25.97258334431281, "commonvoice_hours": 4.2, "commonvoice_locale": "tn", "population": { "BW": 1436683, "NA": 14728, "ZA": 4630015, "ZW": 32002 } }, { "language_name": "Guarani", "bcp_47": "gn", "speakers": 5827107, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.12296923497272805, "chrf": 29.040238903610113 } ], "bleu": 0.12296923497272805, "chrf": 29.040238903610113, "commonvoice_hours": 3.7, "commonvoice_locale": "gn", "population": { "AR": 21375, "BO": 52380, "PY": 5753352 } }, { "language_name": "Finnish", "bcp_47": "fi", "speakers": 5736842, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2306868672081301, "chrf": 42.094559446408105 } ], "bleu": 0.2306868672081301, "chrf": 42.094559446408105, "commonvoice_hours": 15.0, "commonvoice_locale": "fi", "population": { "EE": 258010, "FI": 5237370, "RU": 17007, "SE": 224455 } }, { "language_name": "Ganda", "bcp_47": "lg", "speakers": 5622890, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.09865217050437662, "chrf": 25.926064361947446 } ], "bleu": 0.09865217050437662, "chrf": 25.926064361947446, "commonvoice_hours": 437.0, "commonvoice_locale": "lg", "population": { "UG": 5622890 } }, { "language_name": "Kashmiri", "bcp_47": "ks", "speakers": 5598085, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1344939664526747, "chrf": 32.68903932403509 } ], "bleu": 0.1344939664526747, "chrf": 32.68903932403509, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 5436969, "PK": 161116 } }, { "language_name": "Norwegian Bokmål", "bcp_47": "nb", "speakers": 5468932, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3568538739752233, "chrf": 50.18863565152686 } ], "bleu": 0.3568538739752233, "chrf": 50.18863565152686, "commonvoice_hours": 0.1, "commonvoice_locale": "nb-NO", "population": { "NO": 5467440, "SJ": 1492 } }, { "language_name": "Bemba", "bcp_47": "bem", "speakers": 5402246, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.07496563614353445, "chrf": 23.431580504923282 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.10425825663987873, "chrf": 26.156513526794825 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.048552315311727906, "chrf": 17.052116746764433 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.20820762589055672, "chrf": 37.899164894105404 }, { "model": "deepseek/deepseek-chat", "bleu": 0.13037194233770932, "chrf": 29.069219847377738 }, { "model": "microsoft/phi-4", "bleu": 0.07919429950933718, "chrf": 22.45526755754465 } ], "bleu": 0.10759167930545738, "chrf": 26.01064384625172, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "ZM": 5402246 } }, { "language_name": "Armenian", "bcp_47": "hy", "speakers": 5317273, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2525113198548088, "chrf": 44.115149170486895 } ], "bleu": 0.2525113198548088, "chrf": 44.115149170486895, "commonvoice_hours": 31.0, "commonvoice_locale": "hy-AM", "population": { "AM": 2960894, "CY": 2787, "GE": 279790, "IR": 203816, "LB": 284420, "RU": 1190465, "SY": 349171, "TR": 45930 } }, { "language_name": "Northern Sotho", "bcp_47": "nso", "speakers": 5307578, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.1287306186367617, "chrf": 28.538499437787404 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.11431860079235977, "chrf": 29.789689541768464 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.048032427671766596, "chrf": 16.511000736562355 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.277532484522071, "chrf": 43.725160462861595 }, { "model": "deepseek/deepseek-chat", "bleu": 0.1559013863573944, "chrf": 31.344526297642712 }, { "model": "microsoft/phi-4", "bleu": 0.08683694629684643, "chrf": 24.176752802670837 } ], "bleu": 0.13522541071286664, "chrf": 29.014271546548898, "commonvoice_hours": 0.0, "commonvoice_locale": "nso", "population": { "ZA": 5307578 } }, { "language_name": "Luo (Kenya and Tanzania)", "bcp_47": "luo", "speakers": 5245734, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.07123028733548639, "chrf": 21.84956735049654 } ], "bleu": 0.07123028733548639, "chrf": 21.84956735049654, "commonvoice_hours": 30.0, "commonvoice_locale": "luo", "population": { "KE": 5245734 } }, { "language_name": "Tok Pisin", "bcp_47": "tpi", "speakers": 5154217, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.17665711931817996, "chrf": 35.46692292627831 } ], "bleu": 0.17665711931817996, "chrf": 35.46692292627831, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "PG": 5154217 } }, { "language_name": "Lao", "bcp_47": "lo", "speakers": 5138706, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.17291556794348653, "chrf": 35.37210957380201 } ], "bleu": 0.17291556794348653, "chrf": 35.37210957380201, "commonvoice_hours": 0.2, "commonvoice_locale": "lo", "population": { "LA": 5138706 } }, { "language_name": "Tsonga", "bcp_47": "ts", "speakers": 4880932, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.1264498146181144, "chrf": 26.535888048377064 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.09614725376527729, "chrf": 26.35974604226704 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.043920591728788254, "chrf": 15.171343812958735 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.2843690426617385, "chrf": 43.51952547276378 }, { "model": "deepseek/deepseek-chat", "bleu": 0.10072669531344912, "chrf": 24.778318092682298 }, { "model": "microsoft/phi-4", "bleu": 0.0708900783780892, "chrf": 24.114324582460767 } ], "bleu": 0.12041724607757613, "chrf": 26.746524341918285, "commonvoice_hours": 0.0, "commonvoice_locale": "ts", "population": { "MZ": 2377758, "SZ": 18776, "ZA": 2484398 } }, { "language_name": "Balinese", "bcp_47": "ban", "speakers": 4806468, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.20937766416587725, "chrf": 37.802202729584685 } ], "bleu": 0.20937766416587725, "chrf": 37.802202729584685, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "ID": 4806468 } }, { "language_name": "Ewe", "bcp_47": "ee", "speakers": 4690857, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.06328122760447334, "chrf": 21.930232101484705 } ], "bleu": 0.06328122760447334, "chrf": 21.930232101484705, "commonvoice_hours": 0.0, "commonvoice_locale": "ee", "population": { "GH": 3227422, "TG": 1463435 } }, { "language_name": "Buginese", "bcp_47": "bug", "speakers": 4298211, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.11888625287150432, "chrf": 26.566035803697112 } ], "bleu": 0.11888625287150432, "chrf": 26.566035803697112, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "ID": 4272416, "MY": 25795 } }, { "language_name": "Goan Konkani", "bcp_47": "gom", "speakers": 4243488, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.17517997036818814, "chrf": 35.662782661637515 } ], "bleu": 0.17517997036818814, "chrf": 35.662782661637515, "commonvoice_hours": 0.0, "commonvoice_locale": "gom", "population": { "IN": 4243488 } }, { "language_name": "Kamba", "bcp_47": "kam", "speakers": 4068120, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.09766297423802607, "chrf": 24.328994687372596 } ], "bleu": 0.09766297423802607, "chrf": 24.328994687372596, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "KE": 4068120 } }, { "language_name": "Banjar", "bcp_47": "bjn", "speakers": 4010288, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21429523594040997, "chrf": 37.997802850700054 } ], "bleu": 0.21429523594040997, "chrf": 37.997802850700054, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "ID": 4005390, "MY": 4898 } }, { "language_name": "Lombard", "bcp_47": "lmo", "speakers": 3901518, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.19986098660959015, "chrf": 38.27348032712485 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21159778572935684, "chrf": 40.44461475492081 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.1588300738463149, "chrf": 31.614789846388156 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.30563834118855027, "chrf": 47.111364119506355 }, { "model": "deepseek/deepseek-chat", "bleu": 0.22666325208418955, "chrf": 42.07703156302399 }, { "model": "microsoft/phi-4", "bleu": 0.18882590620933629, "chrf": 37.12550972377164 } ], "bleu": 0.215236057611223, "chrf": 39.44113172245597, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "CH": 344564, "IT": 3556954 } }, { "language_name": "Achinese", "bcp_47": "ace", "speakers": 3738364, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.16911632683538352, "chrf": 35.055560798896856 } ], "bleu": 0.16911632683538352, "chrf": 35.055560798896856, "commonvoice_hours": 0.0, "commonvoice_locale": "ace", "population": { "ID": 3738364 } }, { "language_name": "Shan", "bcp_47": "shn", "speakers": 3687984, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.03567194702202585, "chrf": 15.061253284595894 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.14589621017705648, "chrf": 32.36640913542123 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.011114664716630177, "chrf": 9.54345606166579 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.24688742301342204, "chrf": 43.47153430068834 }, { "model": "deepseek/deepseek-chat", "bleu": 0.11534595629433392, "chrf": 32.685789872044445 }, { "model": "microsoft/phi-4", "bleu": 0.06564720827517354, "chrf": 21.409820368208937 } ], "bleu": 0.10342723491644035, "chrf": 25.756377170437435, "commonvoice_hours": 0.0, "commonvoice_locale": "shn", "population": { "MM": 3621766, "TH": 66218 } }, { "language_name": "Georgian", "bcp_47": "ka", "speakers": 3543646, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.22489436376782782, "chrf": 42.05355918136569 } ], "bleu": 0.22489436376782782, "chrf": 42.05355918136569, "commonvoice_hours": 158.0, "commonvoice_locale": "ka", "population": { "GE": 3437420, "IR": 60296, "TR": 45930 } }, { "language_name": "Galician", "bcp_47": "gl", "speakers": 3515530, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2463790593991139, "chrf": 43.16984633200989 } ], "bleu": 0.2463790593991139, "chrf": 43.16984633200989, "commonvoice_hours": 109.0, "commonvoice_locale": "gl", "population": { "ES": 3501106, "PT": 14424 } }, { "language_name": "Lingala", "bcp_47": "ln", "speakers": 3514491, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.10115588577551943, "chrf": 26.041173534129616 } ], "bleu": 0.10115588577551943, "chrf": 26.041173534129616, "commonvoice_hours": 0.0, "commonvoice_locale": "ln", "population": { "AO": 217899, "CD": 3155180, "CF": 14378, "CG": 127034 } }, { "language_name": "Kabyle", "bcp_47": "kab", "speakers": 3351886, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.03368838568519845, "chrf": 17.84430440143716 } ], "bleu": 0.03368838568519845, "chrf": 17.84430440143716, "commonvoice_hours": 571.0, "commonvoice_locale": "kab", "population": { "DZ": 3351886 } }, { "language_name": "Kyrgyz", "bcp_47": "ky", "speakers": 3338267, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21002212869070494, "chrf": 39.3726736717899 } ], "bleu": 0.21002212869070494, "chrf": 39.3726736717899, "commonvoice_hours": 39.0, "commonvoice_locale": "ky", "population": { "CN": 473967, "KG": 2863152, "TR": 1148 } }, { "language_name": "Fon", "bcp_47": "fon", "speakers": 3216150, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.0832678269022026, "chrf": 21.627442109362057 } ], "bleu": 0.0832678269022026, "chrf": 21.627442109362057, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "BJ": 3216150 } }, { "language_name": "Waray", "bcp_47": "war", "speakers": 3166927, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2156899984074879, "chrf": 38.73293471051519 } ], "bleu": 0.2156899984074879, "chrf": 38.73293471051519, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "CA": 678, "PH": 3166249 } }, { "language_name": "Tibetan", "bcp_47": "bo", "speakers": 3006697, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.11883469874802492, "chrf": 29.188000714511094 } ], "bleu": 0.11883469874802492, "chrf": 29.188000714511094, "commonvoice_hours": 0.0, "commonvoice_locale": "bo", "population": { "CN": 2788040, "IN": 145870, "NP": 72787 } }, { "language_name": "Sango", "bcp_47": "sg", "speakers": 2935521, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.052708457503892185, "chrf": 21.077342933150366 } ], "bleu": 0.052708457503892185, "chrf": 21.077342933150366, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "CF": 2935521 } }, { "language_name": "Aymara", "bcp_47": "ay", "speakers": 2838620, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.07563412710891973, "chrf": 23.042901816075858 } ], "bleu": 0.07563412710891973, "chrf": 23.042901816075858, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "BO": 2327980, "PE": 510640 } }, { "language_name": "Dogri", "bcp_47": "doi", "speakers": 2652180, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.18698274115592, "chrf": 36.17811444987121 } ], "bleu": 0.18698274115592, "chrf": 36.17811444987121, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 2652180 } }, { "language_name": "Lithuanian", "bcp_47": "lt", "speakers": 2488617, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23629191535308328, "chrf": 41.457632804056466 } ], "bleu": 0.23629191535308328, "chrf": 41.457632804056466, "commonvoice_hours": 25.0, "commonvoice_locale": "lt", "population": { "GB": 131522, "LT": 2349056, "PL": 8039 } }, { "language_name": "Swati", "bcp_47": "ss", "speakers": 2212379, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.10571792263190831, "chrf": 26.72182646364123 } ], "bleu": 0.10571792263190831, "chrf": 26.72182646364123, "commonvoice_hours": 0.0, "commonvoice_locale": "ss", "population": { "LS": 47264, "SZ": 640598, "ZA": 1524517 } }, { "language_name": "Occitan", "bcp_47": "oc", "speakers": 2040398, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3116700967049491, "chrf": 49.33859359679982 } ], "bleu": 0.3116700967049491, "chrf": 49.33859359679982, "commonvoice_hours": 1.8, "commonvoice_locale": "oc", "population": { "ES": 4952, "FR": 2035446 } }, { "language_name": "Tatar", "bcp_47": "tt", "speakers": 1984108, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.20199966692246552, "chrf": 38.969447814905855 } ], "bleu": 0.20199966692246552, "chrf": 38.969447814905855, "commonvoice_hours": 32.0, "commonvoice_locale": "tt", "population": { "RU": 1984108 } }, { "language_name": "Slovenian", "bcp_47": "sl", "speakers": 1973181, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.25710751649810404, "chrf": 43.97419502631293 } ], "bleu": 0.25710751649810404, "chrf": 43.97419502631293, "commonvoice_hours": 17.0, "commonvoice_locale": "sl", "population": { "AT": 32780, "HU": 4984, "IT": 106085, "SI": 1829332 } }, { "language_name": "Bodo", "bcp_47": "brx", "speakers": 1856526, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.07193315161893905, "chrf": 21.198050861825372 } ], "bleu": 0.07193315161893905, "chrf": 21.198050861825372, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 1856526 } }, { "language_name": "Chuvash", "bcp_47": "cv", "speakers": 1842386, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.16319209573807847, "chrf": 35.5605244185656 } ], "bleu": 0.16319209573807847, "chrf": 35.5605244185656, "commonvoice_hours": 27.0, "commonvoice_locale": "cv", "population": { "RU": 1842386 } }, { "language_name": "Bashkir", "bcp_47": "ba", "speakers": 1842386, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23494956875272427, "chrf": 43.58361096793536 } ], "bleu": 0.23494956875272427, "chrf": 43.58361096793536, "commonvoice_hours": 259.0, "commonvoice_locale": "ba", "population": { "RU": 1842386 } }, { "language_name": "Tumbuka", "bcp_47": "tum", "speakers": 1780514, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.09211959148198216, "chrf": 25.078974624106454 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.08953119623294435, "chrf": 26.60406076382521 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.036353192983993324, "chrf": 15.247949668242475 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.21852974820220555, "chrf": 40.790711915222424 }, { "model": "deepseek/deepseek-chat", "bleu": 0.13308678184347988, "chrf": 31.89595169117875 }, { "model": "microsoft/phi-4", "bleu": 0.06252197708878435, "chrf": 23.417269345311365 } ], "bleu": 0.10535708130556494, "chrf": 27.172486334647783, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "MW": 1780514 } }, { "language_name": "Macedonian", "bcp_47": "mk", "speakers": 1608565, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2635416107541368, "chrf": 46.15461303306508 } ], "bleu": 0.2635416107541368, "chrf": 46.15461303306508, "commonvoice_hours": 18.0, "commonvoice_locale": "mk", "population": { "AL": 14451, "GR": 169714, "MK": 1424400 } }, { "language_name": "Pangasinan", "bcp_47": "pag", "speakers": 1528534, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.14637588345836686, "chrf": 33.27328635596556 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.17061619096272593, "chrf": 35.63408741214199 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.11313843155080379, "chrf": 26.507505375137484 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.27369890360254523, "chrf": 44.34141738332121 }, { "model": "deepseek/deepseek-chat", "bleu": 0.19116528491340065, "chrf": 36.672507303893106 }, { "model": "microsoft/phi-4", "bleu": 0.11054650956119119, "chrf": 30.67803234997905 } ], "bleu": 0.16759020067483896, "chrf": 34.51780603007307, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "PH": 1528534 } }, { "language_name": "Manipuri", "bcp_47": "mni", "speakers": 1476591, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.061702153982680315, "chrf": 20.627265799751633 } ], "bleu": 0.061702153982680315, "chrf": 20.627265799751633, "commonvoice_hours": 0.0, "commonvoice_locale": "mni", "population": { "BD": 17892, "IN": 1458699 } }, { "language_name": "Venetian", "bcp_47": "vec", "speakers": 1380829, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.24764447442173138, "chrf": 41.97335861313842 } ], "bleu": 0.24764447442173138, "chrf": 41.97335861313842, "commonvoice_hours": 0.0, "commonvoice_locale": "vec", "population": { "BR": 508118, "HR": 29594, "IT": 811235, "MX": 2444, "SI": 29438 } }, { "language_name": "Norwegian Nynorsk", "bcp_47": "nn", "speakers": 1366860, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.31661912673403325, "chrf": 48.792392911852595 } ], "bleu": 0.31661912673403325, "chrf": 48.792392911852595, "commonvoice_hours": 1.5, "commonvoice_locale": "nn-NO", "population": { "NO": 1366860 } }, { "language_name": "Irish", "bcp_47": "ga", "speakers": 1237487, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2768559181644857, "chrf": 46.47151564403362 } ], "bleu": 0.2768559181644857, "chrf": 46.47151564403362, "commonvoice_hours": 5.8, "commonvoice_locale": "ga-IE", "population": { "GB": 98642, "IE": 1138845 } }, { "language_name": "Latvian", "bcp_47": "lv", "speakers": 1147550, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23200427142275887, "chrf": 44.00679383253316 } ], "bleu": 0.23200427142275887, "chrf": 44.00679383253316, "commonvoice_hours": 260.0, "commonvoice_locale": "lv", "population": { "LV": 1147550 } }, { "language_name": "Basque", "bcp_47": "eu", "speakers": 1088519, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1907277513380933, "chrf": 40.633564870804214 } ], "bleu": 0.1907277513380933, "chrf": 40.633564870804214, "commonvoice_hours": 335.0, "commonvoice_locale": "eu", "population": { "ES": 1000316, "FR": 88203 } }, { "language_name": "Sardinian", "bcp_47": "sc", "speakers": 1060846, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2003666163856343, "chrf": 39.97859549386356 } ], "bleu": 0.2003666163856343, "chrf": 39.97859549386356, "commonvoice_hours": 2.9, "commonvoice_locale": "sc", "population": { "IT": 1060846 } }, { "language_name": "Najdi Arabic", "bcp_47": "ars", "speakers": 1025205, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.26325866988203733, "chrf": 45.79452460253912 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.25411630061861235, "chrf": 45.68081123321704 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.19634428413472024, "chrf": 37.402576382999925 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3267312117229826, "chrf": 50.07524798517934 }, { "model": "deepseek/deepseek-chat", "bleu": 0.27947088689796734, "chrf": 47.70370329275568 }, { "model": "microsoft/phi-4", "bleu": 0.23043700347741075, "chrf": 40.64509062227617 } ], "bleu": 0.2583930594556218, "chrf": 44.550325686494546, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "SA": 1025205 } }, { "language_name": "Yiddish", "bcp_47": "yi", "speakers": 997214, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2748989006328114, "chrf": 44.151717001541904 } ], "bleu": 0.2748989006328114, "chrf": 44.151717001541904, "commonvoice_hours": 0.5, "commonvoice_locale": "yi", "population": { "IL": 260264, "SE": 2959, "UA": 570998, "US": 162993 } }, { "language_name": "Kachin", "bcp_47": "kac", "speakers": 962032, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.06343642810657522, "chrf": 21.96116119019238 } ], "bleu": 0.06343642810657522, "chrf": 21.96116119019238, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "MM": 962032 } }, { "language_name": "Limburgish", "bcp_47": "li", "speakers": 950422, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.28017358847160223, "chrf": 45.82764538788154 } ], "bleu": 0.28017358847160223, "chrf": 45.82764538788154, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "NL": 950422 } }, { "language_name": "Welsh", "bcp_47": "cy", "speakers": 884910, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.31667961925197524, "chrf": 48.49344578612579 } ], "bleu": 0.31667961925197524, "chrf": 48.49344578612579, "commonvoice_hours": 124.0, "commonvoice_locale": "cy", "population": { "AR": 30016, "GB": 854894 } }, { "language_name": "Estonian", "bcp_47": "et", "speakers": 878449, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.23762161272302187, "chrf": 42.90409268311042 } ], "bleu": 0.23762161272302187, "chrf": 42.90409268311042, "commonvoice_hours": 58.0, "commonvoice_locale": "et", "population": { "EE": 872320, "FI": 6129 } }, { "language_name": "Asturian", "bcp_47": "ast", "speakers": 650205, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.3066166431048003, "chrf": 47.792858053998366 } ], "bleu": 0.3066166431048003, "chrf": 47.792858053998366, "commonvoice_hours": 0.7, "commonvoice_locale": "ast", "population": { "ES": 650205 } }, { "language_name": "N’Ko", "bcp_47": "nqo", "speakers": 626370, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.0026829540009563496, "chrf": 8.366238228343608 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.0028810767141941676, "chrf": 11.711522538883516 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.002244809403558117, "chrf": 6.87154254960649 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.1373860974763018, "chrf": 29.63649387292498 }, { "model": "deepseek/deepseek-chat", "bleu": 0.005449384832055512, "chrf": 13.63303465097306 }, { "model": "microsoft/phi-4", "bleu": 0.001220306675003964, "chrf": 11.614232149229839 } ], "bleu": 0.02531077151701165, "chrf": 13.638843998326914, "commonvoice_hours": 0.0, "commonvoice_locale": "nqo", "population": { "GN": 626370 } }, { "language_name": "Nuer", "bcp_47": "nus", "speakers": 591427, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.04422064781985695, "chrf": 18.101658717442856 } ], "bleu": 0.04422064781985695, "chrf": 18.101658717442856, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "SS": 591427 } }, { "language_name": "Ligurian", "bcp_47": "lij", "speakers": 536663, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.2433180508520944, "chrf": 42.16628456571689 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2730358021257564, "chrf": 43.59770121161605 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.1659569541464764, "chrf": 32.04882604989477 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.3868854055493315, "chrf": 52.982923067584665 }, { "model": "deepseek/deepseek-chat", "bleu": 0.30131335750773747, "chrf": 47.80059076162273 }, { "model": "microsoft/phi-4", "bleu": 0.22953082347299453, "chrf": 39.89195612874355 } ], "bleu": 0.26667339894239844, "chrf": 43.08138029752978, "commonvoice_hours": 5.1, "commonvoice_locale": "lij", "population": { "IT": 536663 } }, { "language_name": "Kabuverdianu", "bcp_47": "kea", "speakers": 530762, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2049568393036302, "chrf": 39.12735936405683 } ], "bleu": 0.2049568393036302, "chrf": 39.12735936405683, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "CV": 530762 } }, { "language_name": "Mari", "bcp_47": "chm", "speakers": 524371, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.1477332953533076, "chrf": 33.28969144955911 } ], "bleu": 0.1477332953533076, "chrf": 33.28969144955911, "commonvoice_hours": 282.0, "commonvoice_locale": "mhr", "population": { "RU": 524371 } }, { "language_name": "Sicilian", "bcp_47": "scn", "speakers": 511702, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.26991391704955275, "chrf": 44.10988575231252 } ], "bleu": 0.26991391704955275, "chrf": 44.10988575231252, "commonvoice_hours": 0.0, "commonvoice_locale": "scn", "population": { "IT": 511702 } }, { "language_name": "Silesian", "bcp_47": "szl", "speakers": 497670, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2478859256009672, "chrf": 42.936549641246145 } ], "bleu": 0.2478859256009672, "chrf": 42.936549641246145, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "PL": 497670 } }, { "language_name": "Maltese", "bcp_47": "mt", "speakers": 457267, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.27335683193570975, "chrf": 45.920903610737895 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.28654849898846085, "chrf": 46.936205424540766 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.15248109554681186, "chrf": 30.516627088468166 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.38954095833662916, "chrf": 54.945196672005885 }, { "model": "deepseek/deepseek-chat", "bleu": 0.3175303995061197, "chrf": 51.04064318379729 }, { "model": "microsoft/phi-4", "bleu": 0.11179045198515461, "chrf": 30.191257026189298 } ], "bleu": 0.2552080393831477, "chrf": 43.25847216762322, "commonvoice_hours": 8.7, "commonvoice_locale": "mt", "population": { "MT": 457267 } }, { "language_name": "Luxembourgish", "bcp_47": "lb", "speakers": 421015, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2835765541228824, "chrf": 44.80618475111259 } ], "bleu": 0.2835765541228824, "chrf": 44.80618475111259, "commonvoice_hours": 0.0, "commonvoice_locale": "lb", "population": { "LU": 421015 } }, { "language_name": "Dzongkha", "bcp_47": "dz", "speakers": 370341, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.05723633975042216, "chrf": 23.461346449303786 } ], "bleu": 0.05723633975042216, "chrf": 23.461346449303786, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "BT": 367689, "IN": 2652 } }, { "language_name": "Fijian", "bcp_47": "fj", "speakers": 365030, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.10451524271139898, "chrf": 24.713018515709646 } ], "bleu": 0.10451524271139898, "chrf": 24.713018515709646, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "FJ": 365030 } }, { "language_name": "Icelandic", "bcp_47": "is", "speakers": 350734, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2142644347281729, "chrf": 40.04812999157868 } ], "bleu": 0.2142644347281729, "chrf": 40.04812999157868, "commonvoice_hours": 0.1, "commonvoice_locale": "is", "population": { "IS": 350734 } }, { "language_name": "Samoan", "bcp_47": "sm", "speakers": 252717, "scores": [ { "model": "openai/gpt-4o-mini", "bleu": 0.13327372905795537, "chrf": 28.9583532166856 }, { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.11602460228698847, "chrf": 29.156949243212015 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "bleu": 0.05771585788755527, "chrf": 16.752178728973906 }, { "model": "google/gemini-2.0-flash-001", "bleu": 0.2585259997356889, "chrf": 45.18784342425295 }, { "model": "deepseek/deepseek-chat", "bleu": 0.22221137013078898, "chrf": 39.7496231353589 }, { "model": "microsoft/phi-4", "bleu": 0.04756300118196289, "chrf": 21.060764155029197 } ], "bleu": 0.13921909338015664, "chrf": 30.144285317252095, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "AS": 48943, "WS": 203774 } }, { "language_name": "Crimean Tatar", "bcp_47": "crh", "speakers": 245968, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.22725256040555009, "chrf": 42.07005703444819 } ], "bleu": 0.22725256040555009, "chrf": 42.07005703444819, "commonvoice_hours": 0.0, "commonvoice_locale": "crh", "population": { "UA": 245968 } }, { "language_name": "Papiamento", "bcp_47": "pap", "speakers": 211640, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21648792499796674, "chrf": 41.57061175824069 } ], "bleu": 0.21648792499796674, "chrf": 41.57061175824069, "commonvoice_hours": 0.0, "commonvoice_locale": "pap-AW", "population": { "AW": 72851, "BQ": 16200, "CW": 122589 } }, { "language_name": "Latgalian", "bcp_47": "ltg", "speakers": 167429, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.13475273241606922, "chrf": 32.044009672423776 } ], "bleu": 0.13475273241606922, "chrf": 32.044009672423776, "commonvoice_hours": 28.0, "commonvoice_locale": "ltg", "population": { "LV": 167429 } }, { "language_name": "Māori", "bcp_47": "mi", "speakers": 137913, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.17610729049259877, "chrf": 35.01961886760811 } ], "bleu": 0.17610729049259877, "chrf": 35.01961886760811, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "NZ": 137913 } }, { "language_name": "Scottish Gaelic", "bcp_47": "gd", "speakers": 72337, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2154279041570466, "chrf": 37.40630914857015 } ], "bleu": 0.2154279041570466, "chrf": 37.40630914857015, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "GB": 72337 } }, { "language_name": "Faroese", "bcp_47": "fo", "speakers": 71351, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.21180629663838063, "chrf": 39.28673819745006 } ], "bleu": 0.21180629663838063, "chrf": 39.28673819745006, "commonvoice_hours": 0.0, "commonvoice_locale": "fo", "population": { "DK": 22304, "FO": 49047 } }, { "language_name": "Friulian", "bcp_47": "fur", "speakers": 37442, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.2255876860328074, "chrf": 40.6191779521821 } ], "bleu": 0.2255876860328074, "chrf": 40.6191779521821, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IT": 37442 } }, { "language_name": "Sanskrit", "bcp_47": "sa", "speakers": 15913, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.14313294345831834, "chrf": 32.44676491537583 } ], "bleu": 0.14313294345831834, "chrf": 32.44676491537583, "commonvoice_hours": null, "commonvoice_locale": null, "population": { "IN": 15913 } }, { "language_name": "Esperanto", "bcp_47": "eo", "speakers": 301, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "bleu": 0.27440987441620224, "chrf": 44.40715599582661 } ], "bleu": 0.27440987441620224, "chrf": 44.40715599582661, "commonvoice_hours": 1436.0, "commonvoice_locale": "eo", "population": { "SM": 301 } } ]