from dataclasses import dataclass from enum import Enum def fields(raw_class): return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] # These classes are for user facing column names, to avoid having to change them # all around the code when a modification is needed. @dataclass class ColumnContent: name: str type: str displayed_by_default: bool hidden: bool = False never_hidden: bool = False @dataclass(frozen=True) class AutoEvalColumn: system = ColumnContent("System Name", "markdown", True, never_hidden=True) organization = ColumnContent("Organization", "str", True, never_hidden=True) success_rate_overall = ColumnContent("Overall Success (%)", "number", True) success_rate_tier1 = ColumnContent("Deeper Tier Success (%)", "number", True) success_rate_tier2 = ColumnContent("Deepest Tier Success (%)", "number", True) submitted_on = ColumnContent("Submitted On", "datetime", True) # For the queue columns in the submission tab @dataclass(frozen=True) class EvalQueueColumn: # Queue column model = ColumnContent("model", "markdown", True) revision = ColumnContent("revision", "str", True) private = ColumnContent("private", "bool", True) precision = ColumnContent("precision", "str", True) weight_type = ColumnContent("weight_type", "str", True) status = ColumnContent("status", "str", True) # All the model information that we might need @dataclass class ModelDetails: name: str display_name: str = "" class ModelType(Enum): LLM = ModelDetails(name="LLM") AgenticLLM = ModelDetails(name="AgenticLLM") Other = ModelDetails(name="Other") def to_str(self): return self.value.name @staticmethod def from_str(type: str): if type == "AgenticLLM": return ModelType.AgenticLLM if type == "LLM": return ModelType.LLM return ModelType.Other class Precision(Enum): float16 = ModelDetails("float16") bfloat16 = ModelDetails("bfloat16") Unknown = ModelDetails("?") COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] EVAL_COLS = [c.name for c in fields(EvalQueueColumn)] EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]