Remove 'auto' from Enum and add handling for submissions with 'auto'.

app.py

@@ -579,7 +579,7 @@ with gr.Blocks() as demo_submission:
         with gr.Column():
             precision = gr.Dropdown(
                 label="Precision",
-                choices=[i.value.name for i in Precision],
+                choices=[i.value.name for i in Precision] + ["auto"],
                 multiselect=False,
                 value="auto",
             )

src/display/utils.py

@@ -129,15 +129,12 @@ class WeightType(Enum):
 
 
 class Precision(Enum):
-    auto = ModelDetails("auto")
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     float32 = ModelDetails("float32")
 
     @staticmethod
     def from_str(precision: str) -> "Precision":
-        if precision == "auto":
-            return Precision.auto
         if precision == "float16":
             return Precision.float16
         if precision == "bfloat16":

src/submission/submit.py

@@ -1,6 +1,8 @@
 import json
 from datetime import datetime, timezone
 
+import torch
+
 from src.display.formatting import styled_error, styled_message, styled_warning
 from src.display.utils import EvalQueuedModel, LLMJpEvalVersion, VllmVersion
 from src.envs import API, EVAL_REQUESTS_PATH, HF_TOKEN, QUEUE_REPO
@@ -25,6 +27,29 @@ def add_new_eval(
 
     revision = revision or "main"
 
+    # Is the model on the hub?
+    model_on_hub, error, config = is_model_on_hub(
+        model_name=model_id, revision=revision, token=HF_TOKEN, test_tokenizer=True
+    )
+    if not model_on_hub:
+        return styled_error(f'Model "{model_id}" {error}')
+    if precision == "auto":
+        dtype = ""
+        if hasattr(config, "dtype"):
+            dtype = config.dtype
+        elif hasattr(config, "torch_dtype"):
+            dtype = config.torch_dtype
+        if dtype == torch.float16:
+            precision = "float16"
+        elif dtype in torch.bfloat16:
+            precision = "bfloat16"
+        elif dtype in torch.float32:
+            precision = "float32"
+        else:
+            return styled_error(
+                "Unable to retrieve a valid dtype from config.json. Please select an appropriate one from fp16/fp32/bf16 and resubmit."
+            )
+
     model_data = EvalQueuedModel(
         model=model_id,
         revision=revision,
@@ -47,13 +72,6 @@ def add_new_eval(
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 
-    # Is the model on the hub?
-    model_on_hub, error, _ = is_model_on_hub(
-        model_name=model_id, revision=revision, token=HF_TOKEN, test_tokenizer=True
-    )
-    if not model_on_hub:
-        return styled_error(f'Model "{model_id}" {error}')
-
     # Is the model info correctly filled?
     try:
         model_info = API.model_info(repo_id=model_id, revision=revision)