Update README.md
Browse files
README.md
CHANGED
|
@@ -21,10 +21,10 @@ model-index:
|
|
| 21 |
metrics:
|
| 22 |
- name: Accuracy (0-shot)
|
| 23 |
type: Accuracy (0-shot)
|
| 24 |
-
value:
|
| 25 |
source:
|
| 26 |
name: Paper
|
| 27 |
-
url: https://
|
| 28 |
- task:
|
| 29 |
type: multiple-choice
|
| 30 |
dataset:
|
|
@@ -33,10 +33,10 @@ model-index:
|
|
| 33 |
metrics:
|
| 34 |
- name: Accuracy (5-shot)
|
| 35 |
type: Accuracy (5-shot)
|
| 36 |
-
value:
|
| 37 |
source:
|
| 38 |
name: Paper
|
| 39 |
-
url: https://
|
| 40 |
- task:
|
| 41 |
type: mix
|
| 42 |
dataset:
|
|
@@ -45,10 +45,10 @@ model-index:
|
|
| 45 |
metrics:
|
| 46 |
- name: Average scores (5-shot)
|
| 47 |
type: Average scores (5-shot)
|
| 48 |
-
value:
|
| 49 |
source:
|
| 50 |
name: Paper
|
| 51 |
-
url: https://
|
| 52 |
- task:
|
| 53 |
type: multiple_choice
|
| 54 |
dataset:
|
|
@@ -57,10 +57,10 @@ model-index:
|
|
| 57 |
metrics:
|
| 58 |
- name: Accuracy (5-shot)
|
| 59 |
type: Accuracy (5-shot)
|
| 60 |
-
value:
|
| 61 |
source:
|
| 62 |
name: Paper
|
| 63 |
-
url: https://
|
| 64 |
- task:
|
| 65 |
type: multiple_choice
|
| 66 |
dataset:
|
|
@@ -69,10 +69,10 @@ model-index:
|
|
| 69 |
metrics:
|
| 70 |
- name: Accuracy (5-shot)
|
| 71 |
type: Accuracy (5-shot)
|
| 72 |
-
value:
|
| 73 |
source:
|
| 74 |
name: Paper
|
| 75 |
-
url: https://
|
| 76 |
- task:
|
| 77 |
type: multiple_choice
|
| 78 |
dataset:
|
|
@@ -81,10 +81,10 @@ model-index:
|
|
| 81 |
metrics:
|
| 82 |
- name: Accuracy (5-shot)
|
| 83 |
type: Accuracy (5-shot)
|
| 84 |
-
value: 56.
|
| 85 |
source:
|
| 86 |
name: Paper
|
| 87 |
-
url: https://
|
| 88 |
- task:
|
| 89 |
type: multiple_choice
|
| 90 |
dataset:
|
|
@@ -93,10 +93,10 @@ model-index:
|
|
| 93 |
metrics:
|
| 94 |
- name: Accuracy (5-shot)
|
| 95 |
type: Accuracy (5-shot)
|
| 96 |
-
value: 43.
|
| 97 |
source:
|
| 98 |
name: Paper
|
| 99 |
-
url: https://
|
| 100 |
---
|
| 101 |
|
| 102 |
# **Model Card for Latxa 13b**
|
|
@@ -251,21 +251,18 @@ In order to reproduce our results please follow the instructions in Latxa's [Git
|
|
| 251 |
|
|
| 252 |
| XGLM | 7B | 57.71 | 23.88 | 41.47 | 22.96 | 24.43 | 26.53 | 24.59 | 32.51 |
|
| 253 |
| BLOOM | 7B | 57.18 | 27.00 | 40.17 | 25.34 | 28.41 | 27.17 | 25.07 | 33.86 |
|
| 254 |
-
| Mistral | 7B | 51.09 | **38.89**| 39.22 | 25.01 | 29.26 | 34.58 | 32.15 | 35.94 |
|
| 255 |
| Llama 2 | 7B | 50.43 | 26.22 | 38.20 | 24.09 | 27.27 | 29.50 | 28.84 | 32.51 |
|
| 256 |
-
| **Latxa v1**
|
| 257 |
-
| **Latxa v1.1** | 7B | **65.72**| 36.89 | **51.78**| **32.44**| **30.40**| **44.37** | **34.20**| **42.26** |
|
| 258 |
|
|
| 259 |
| mGPT | 13B | 55.39 | 25.00 | 37.56 | 25.00 | 24.15 | 27.17 | 25.73 | 32.14 |
|
| 260 |
| Llama 2 | 13B | 50.63 | 32.00 | 38.98 | 25.90 | 28.98 | 33.53 | 29.66 | 34.36 |
|
| 261 |
-
| **Latxa v1**
|
| 262 |
-
| **Latxa v1.1** | 13B | **67.24**| 51.56 | 54.04 | **45.02**| 29.83 | **56.44** | **43.18**| 49.62 |
|
| 263 |
|
|
| 264 |
| Mixtral | 8x7B | 52.55 | 50.44 | 45.00 | 26.43 | 37.50 | 42.51 | 39.87 | 41.97 |
|
| 265 |
| Yi | 34B | 52.22 | 54.56 | 43.90 | 27.30 | 34.66 | 42.57 | 39.68 | 42.05 |
|
| 266 |
| Llama 2 | 70B | 51.62 | 33.56 | 42.55 | 24.16 | 27.84 | 38.43 | 33.08 | 35.47 |
|
| 267 |
-
| **Latxa v1**
|
| 268 |
-
| **Latxa v1.1** | 70B | **69.76**| 64.89| **61.66**| **60.61**| **53.69**| **61.52** | **54.48**| **60.94** |
|
| 269 |
|
| 270 |
|
| 271 |
# **Environmental Impact**
|
|
|
|
| 21 |
metrics:
|
| 22 |
- name: Accuracy (0-shot)
|
| 23 |
type: Accuracy (0-shot)
|
| 24 |
+
value: 65.51
|
| 25 |
source:
|
| 26 |
name: Paper
|
| 27 |
+
url: https://arxiv.org/abs/2403.20266
|
| 28 |
- task:
|
| 29 |
type: multiple-choice
|
| 30 |
dataset:
|
|
|
|
| 33 |
metrics:
|
| 34 |
- name: Accuracy (5-shot)
|
| 35 |
type: Accuracy (5-shot)
|
| 36 |
+
value: 53.89
|
| 37 |
source:
|
| 38 |
name: Paper
|
| 39 |
+
url: https://arxiv.org/abs/2403.20266
|
| 40 |
- task:
|
| 41 |
type: mix
|
| 42 |
dataset:
|
|
|
|
| 45 |
metrics:
|
| 46 |
- name: Average scores (5-shot)
|
| 47 |
type: Average scores (5-shot)
|
| 48 |
+
value: 53.56
|
| 49 |
source:
|
| 50 |
name: Paper
|
| 51 |
+
url: https://arxiv.org/abs/2403.20266
|
| 52 |
- task:
|
| 53 |
type: multiple_choice
|
| 54 |
dataset:
|
|
|
|
| 57 |
metrics:
|
| 58 |
- name: Accuracy (5-shot)
|
| 59 |
type: Accuracy (5-shot)
|
| 60 |
+
value: 44.11
|
| 61 |
source:
|
| 62 |
name: Paper
|
| 63 |
+
url: https://arxiv.org/abs/2403.20266
|
| 64 |
- task:
|
| 65 |
type: multiple_choice
|
| 66 |
dataset:
|
|
|
|
| 69 |
metrics:
|
| 70 |
- name: Accuracy (5-shot)
|
| 71 |
type: Accuracy (5-shot)
|
| 72 |
+
value: 32.67
|
| 73 |
source:
|
| 74 |
name: Paper
|
| 75 |
+
url: https://arxiv.org/abs/2403.20266
|
| 76 |
- task:
|
| 77 |
type: multiple_choice
|
| 78 |
dataset:
|
|
|
|
| 81 |
metrics:
|
| 82 |
- name: Accuracy (5-shot)
|
| 83 |
type: Accuracy (5-shot)
|
| 84 |
+
value: 56.38
|
| 85 |
source:
|
| 86 |
name: Paper
|
| 87 |
+
url: https://arxiv.org/abs/2403.20266
|
| 88 |
- task:
|
| 89 |
type: multiple_choice
|
| 90 |
dataset:
|
|
|
|
| 93 |
metrics:
|
| 94 |
- name: Accuracy (5-shot)
|
| 95 |
type: Accuracy (5-shot)
|
| 96 |
+
value: 43.66
|
| 97 |
source:
|
| 98 |
name: Paper
|
| 99 |
+
url: https://arxiv.org/abs/2403.20266
|
| 100 |
---
|
| 101 |
|
| 102 |
# **Model Card for Latxa 13b**
|
|
|
|
| 251 |
|
|
| 252 |
| XGLM | 7B | 57.71 | 23.88 | 41.47 | 22.96 | 24.43 | 26.53 | 24.59 | 32.51 |
|
| 253 |
| BLOOM | 7B | 57.18 | 27.00 | 40.17 | 25.34 | 28.41 | 27.17 | 25.07 | 33.86 |
|
| 254 |
+
| Mistral | 7B | 51.09 | **38.89**| 39.22 | 25.01 | **29.26** | 34.58 | 32.15 | 35.94 |
|
| 255 |
| Llama 2 | 7B | 50.43 | 26.22 | 38.20 | 24.09 | 27.27 | 29.50 | 28.84 | 32.51 |
|
| 256 |
+
| **Latxa v1.1** | 7B | **65.45**| 37.33 | **52.56**| **30.26**| 25.00| **42.16** | **33.82**| **40.94** |
|
|
|
|
| 257 |
|
|
| 258 |
| mGPT | 13B | 55.39 | 25.00 | 37.56 | 25.00 | 24.15 | 27.17 | 25.73 | 32.14 |
|
| 259 |
| Llama 2 | 13B | 50.63 | 32.00 | 38.98 | 25.90 | 28.98 | 33.53 | 29.66 | 34.36 |
|
| 260 |
+
| **Latxa v1.1** | 13B | **66.51**| **53.89** | **53.36** | **44.11**| **32.67** | **56.38** | **43.66**| **50.08** |
|
|
|
|
| 261 |
|
|
| 262 |
| Mixtral | 8x7B | 52.55 | 50.44 | 45.00 | 26.43 | 37.50 | 42.51 | 39.87 | 41.97 |
|
| 263 |
| Yi | 34B | 52.22 | 54.56 | 43.90 | 27.30 | 34.66 | 42.57 | 39.68 | 42.05 |
|
| 264 |
| Llama 2 | 70B | 51.62 | 33.56 | 42.55 | 24.16 | 27.84 | 38.43 | 33.08 | 35.47 |
|
| 265 |
+
| **Latxa v1.1** | 70B | **70.55**| **71.67** | **59.74**| **60.65**| **50.57**| **62.45** | **51.90**| **61.08** |
|
|
|
|
| 266 |
|
| 267 |
|
| 268 |
# **Environmental Impact**
|