Init the leaderboard

.gitignore

@@ -0,0 +1,212 @@
+.DS_Store
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -1,13 +1,46 @@
 ---
-title: Mmlongbench Doc
-emoji: 💻
-colorFrom: gray
-colorTo: green
+title: MMLongBench Doc
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
 sdk: gradio
-sdk_version: 5.35.0
 app_file: app.py
-pinned: false
-license: mit
+pinned: true
+license: apache-2.0
+short_description: A long-context, multimodal document understanding benchmark
+sdk_version: 5.19.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Start the configuration
+
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -0,0 +1,85 @@
+import gradio as gr
+import pandas as pd
+from pathlib import Path
+
+from src.json_leaderboard import create_leaderboard_df, get_leaderboard_stats
+from src.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    LINKS_AND_INFO,
+    TITLE,
+)
+from src.display.css_html_js import custom_css
+
+
+def create_simple_leaderboard():
+    """Create a simple leaderboard from JSON data"""
+    json_path = Path(__file__).parent / "leaderboard_data.json"
+    df = create_leaderboard_df(str(json_path))
+    
+    if df.empty:
+        return gr.Dataframe(value=pd.DataFrame({"Error": ["No data available"]}))
+    
+    return gr.Dataframe(
+        value=df,
+        headers=list(df.columns),
+        datatype=["html", "str", "html", "str", "str", "str", "str"],
+        interactive=False,
+        wrap=True
+    )
+
+
+def get_stats_display():
+    """Get formatted statistics for display"""
+    json_path = Path(__file__).parent / "leaderboard_data.json"
+    stats = get_leaderboard_stats(str(json_path))
+    
+    if not stats:
+        return "No statistics available"
+    
+    stats_text = f"""
+    ### 📊 Leaderboard Statistics
+    - **Total Models**: {stats['total_models']}
+    - **Best Score**: {stats['max_acc']:.1f}
+    - **Lowest Score**: {stats['min_acc']:.1f}
+    """
+    
+    return stats_text
+
+
+# Create the Gradio interface
+demo = gr.Blocks(css=custom_css, title="MMLongBench-Doc Leaderboard")
+
+with demo:
+    gr.HTML(TITLE)
+    gr.HTML(LINKS_AND_INFO)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
+            # Statistics display
+            stats_display = gr.Markdown(get_stats_display())
+            
+            # Leaderboard table
+            leaderboard_table = create_simple_leaderboard()
+            
+        
+    
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("## 📙 Citation")
+            gr.Markdown("If you use [MMLongBench-Doc](https://arxiv.org/abs/2407.01523) in your research, please cite our work:")
+            citation_textbox = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                elem_id="citation-textbox",
+                show_label=False,
+                interactive=False,
+                lines=8,
+                show_copy_button=True
+            )
+
+
+if __name__ == "__main__":
+    demo.launch()

leaderboard_data.json

@@ -0,0 +1,94 @@
+{
+  "leaderboard": [
+    {
+      "model": "GLM-4.1V-Thinking",
+      "link": "https://arxiv.org/pdf/2507.01006",
+      "hf": "https://huggingface.co/THUDM/GLM-4.1V-9B-Thinking",
+      "params": "9B",
+      "open_source": true,
+      "acc": 42.4,
+      "release_date": "2025-07",
+      "moe": false
+    },
+    {
+      "model": "GPT-4.1 2025-04-14 detail high",
+      "link": "https://openai.com/index/gpt-4-1/",
+      "hf": "-",
+      "params": "-",
+      "open_source": false,
+      "acc": 49.7,
+      "release_date": "2025-04",
+      "moe": "-"
+    },
+    {
+      "model": "GPT-4o 2024-11-20 detail high",
+      "link": "https://arxiv.org/abs/2410.21276",
+      "hf": "-",
+      "params": "-",
+      "open_source": false,
+      "acc": 46.3,
+      "release_date": "2024-11",
+      "moe": "-"
+    },
+    {
+      "model": "Kimi-VL-Thinking-2506",
+      "link": "https://arxiv.org/pdf/2504.07491",
+      "hf": "https://huggingface.co/moonshotai/Kimi-VL-A3B-Thinking-2506",
+      "params": "2.8B activated (16B total)",
+      "open_source": true,
+      "acc": 42.1,
+      "release_date": "2025-06",
+      "moe": true
+    },
+    {
+      "model": "Kimi-VL-A3B-Instruct",
+      "link": "https://arxiv.org/pdf/2504.07491",
+      "hf": "https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct",
+      "params": "2.8B activated (16B total)",
+      "open_source": true,
+      "acc": 35.1,
+      "release_date": "2025-04",
+      "moe": true
+    },
+    {
+      "model": "Qwen2.5-VL-72B",
+      "link": "https://arxiv.org/abs/2502.13923",
+      "hf": "https://huggingface.co/Qwen/Qwen2.5-VL-72B-Instruct",
+      "params": "72B",
+      "open_source": true,
+      "acc": 35.2,
+      "release_date": "2025-02",
+      "moe": false
+    },
+    {
+      "model": "Qwen2.5-VL-7B",
+      "link": "https://arxiv.org/abs/2502.13923",
+      "hf": "https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct",
+      "params": "7B",
+      "open_source": true,
+      "acc": 25.1,
+      "release_date": "2025-02",
+      "moe": false
+    },
+    {
+      "model": "MiniMax-VL-01",
+      "link": "https://arxiv.org/pdf/2501.08313",
+      "hf": "https://huggingface.co/MiniMaxAI/MiniMax-VL-01",
+      "params": "45.9B activated (456B total)",
+      "open_source": true,
+      "acc": 32.5,
+      "release_date": "2025-01",
+      "moe": true
+    },
+    {
+      "model": "Aria",
+      "link": "https://arxiv.org/pdf/2410.05993",
+      "hf": "https://huggingface.co/rhymes-ai/Aria",
+      "params": "3.9B activated (25.3B total)",
+      "open_source": true,
+      "acc": 28.3,
+      "release_date": "2024-10",
+      "moe": true
+    }
+  ]
+}

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,16 @@
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.13
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece

src/about.py

@@ -0,0 +1,103 @@
+from dataclasses import dataclass
+from enum import Enum
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    # For MMLongBench-Doc (https://arxiv.org/abs/2407.01523), we use ACC as the main metric
+    task0 = Task("mmlongbench_doc", "acc", "ACC")
+
+NUM_FEWSHOT = 0 # Change with your few shot
+# ---------------------------------------------------
+
+
+
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">🥇 <a href="https://arxiv.org/abs/2407.01523" target="_blank">MMLongBench-Doc</a> Leaderboard</h1>"""
+
+# Links and conference info
+LINKS_AND_INFO = """
+<div align="center">
+<p><strong>NeurIPS 2024 Datasets and Benchmarks Track (Spotlight)</strong></p>
+<p>
+<a href="https://mayubo2333.github.io/MMLongBench-Doc/" target="_blank">🏠 Homepage</a> | 
+<a href="https://arxiv.org/abs/2407.01523" target="_blank">📄 arXiv Paper</a> | 
+<a href="https://huggingface.co/datasets/yubo2333/MMLongBench-Doc" target="_blank">🤗 Dataset</a>
+</p>
+</div>
+"""
+
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+📚 [MMLongBench-Doc](https://arxiv.org/abs/2407.01523) is a long-context, multimodal document understanding benchmark designed to evaluate the performance of large multimodal models on complex document understanding tasks.
+
+📊 This leaderboard tracks the performance of various models on the [MMLongBench-Doc](https://arxiv.org/abs/2407.01523) benchmark, focusing on their ability to understand and process long documents with both text and visual elements.
+
+🔧 You can use the official [GitHub repo](https://github.com/mayubo2333/MMLongBench-Doc) or [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) to evaluate your model on [MMLongBench-Doc](https://arxiv.org/abs/2407.01523). We provide the [official evaluation results](https://huggingface.co/OpenIXCLab/mmlongbench-doc-results) of GPT-4.1 and GPT-4o.
+
+📝 To add your own model to the leaderboard, please send an Email to yubo001@e.ntu.edu.sg or zangyuhang@pjlab.org.cn then we will help with the evaluation and updating the leaderboard.
+"""
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = f"""
+## How it works
+
+[MMLongBench-Doc](https://arxiv.org/abs/2407.01523) evaluates multimodal models on their ability to understand long documents containing both text and visual elements. The benchmark includes various document understanding tasks that require models to process and reason over extended contexts.
+
+## Evaluation Metrics
+
+- **ACC (Accuracy)**: The primary metric measuring the overall accuracy of model predictions on document understanding tasks.
+- **Parameters**: Model size in billions of parameters
+- **Open Source**: Whether the model weights are publicly available
+
+## Reproducibility
+
+To reproduce our results, please refer to the official [MMLongBench-Doc](https://arxiv.org/abs/2407.01523) repository for evaluation scripts and detailed instructions.
+"""
+
+EVALUATION_QUEUE_TEXT = """
+## Some good practices before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@inproceedings{ma2024mmlongbenchdoc,
+title={{MMLONGBENCH}-{DOC}: Benchmarking Long-context Document Understanding with Visualizations},
+author={Yubo Ma and Yuhang Zang and Liangyu Chen and Meiqi Chen and Yizhu Jiao and Xinze Li and Xinyuan Lu and Ziyu Liu and Yan Ma and Xiaoyi Dong and Pan Zhang and Liangming Pan and Yu-Gang Jiang and Jiaqi Wang and Yixin Cao and Aixin Sun},
+booktitle={NeurIPS Datasets and Benchmarks Track},
+year={2024},
+url={https://openreview.net/forum?id=loJM1acwzf}
+}"""

src/display/css_html_js.py

@@ -0,0 +1,270 @@
+custom_css = """
+
+.markdown-text {
+    font-size: 16px !important;
+    line-height: 1.6 !important;
+}
+
+/* Enhanced Leaderboard table styling */
+.dataframe {
+    background: white !important;
+    border-radius: 12px !important;
+    box-shadow: 0 4px 16px rgba(0, 0, 0, 0.08) !important;
+    overflow: hidden !important;
+    border: 1px solid #e8ecef !important;
+    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif !important;
+}
+
+.dataframe th {
+    background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%) !important;
+    color: #2c3e50 !important;
+    font-weight: 600 !important;
+    font-size: 11px !important;
+    padding: 16px 12px !important;
+    text-align: center !important;
+    border-bottom: 2px solid #dee2e6 !important;
+    letter-spacing: 0.025em !important;
+    text-transform: uppercase !important;
+}
+
+/* Override any conflicting styles */
+.dataframe thead th {
+    font-size: 11px !important;
+}
+
+.dataframe th span {
+    font-size: 11px !important;
+}
+
+.dataframe td {
+    padding: 14px 12px !important;
+    border-bottom: 1px solid #f0f2f5 !important;
+    text-align: center !important;
+    vertical-align: middle !important;
+    font-size: 15px !important;
+    color: #2c3e50 !important;
+    line-height: 1.4 !important;
+}
+
+.dataframe tr:hover td {
+    background-color: #f8f9fa !important;
+    transition: background-color 0.2s ease !important;
+}
+
+.dataframe tr:nth-child(even) td {
+    background-color: #fdfdfd !important;
+}
+
+/* Enhanced hyperlinks in table */
+.dataframe a {
+    color: #0066cc !important;
+    text-decoration: underline !important;
+    font-weight: 500 !important;
+    transition: all 0.2s ease !important;
+    border-radius: 4px !important;
+    padding: 2px 6px !important;
+    display: inline-block !important;
+}
+
+.dataframe a:hover {
+    color: #004499 !important;
+    background-color: rgba(0, 102, 204, 0.1) !important;
+    text-decoration: underline !important;
+    transform: translateY(-1px) !important;
+}
+
+.dataframe a:visited {
+    color: #5a6c7d !important;
+}
+
+/* Model name styling (assuming first column contains model names) */
+.dataframe td:first-child {
+    font-weight: 600 !important;
+    color: #1a202c !important;
+    text-align: left !important;
+    padding-left: 16px !important;
+}
+
+/* Score highlighting */
+.dataframe td:last-child {
+    font-weight: 600 !important;
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+/* Citation section styling */
+#citation-textbox textarea {
+    background: linear-gradient(135deg, #f8f9fa 0%, #e9ecef 100%) !important;
+    border: 2px solid #dee2e6 !important;
+    border-radius: 12px !important;
+    font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace !important;
+    font-size: 12px !important;
+    padding: 20px !important;
+    line-height: 1.6 !important;
+    box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1) !important;
+}
+
+#citation-textbox label > button {
+    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%) !important;
+    border: none !important;
+    border-radius: 8px !important;
+    color: white !important;
+    padding: 8px 16px !important;
+    margin: 8px !important;
+    transform: scale(1.1);
+    transition: transform 0.2s ease;
+}
+
+#citation-textbox label > button:hover {
+    transform: scale(1.15) !important;
+}
+
+#leaderboard-table {
+    margin-top: 25px !important;
+}
+
+#leaderboard-table-lite {
+    margin-top: 25px !important;
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+
+/* Enhanced table column widths and responsiveness */
+#leaderboard-table td:nth-child(2),
+#leaderboard-table th:nth-child(2) {
+    max-width: 400px;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+}
+
+/* Responsive table improvements */
+.dataframe {
+    width: 100% !important;
+    margin: 16px 0 !important;
+}
+
+/* Better mobile responsiveness */
+@media (max-width: 768px) {
+    .dataframe th,
+    .dataframe td {
+        padding: 8px 6px !important;
+        font-size: 13px !important;
+    }
+    
+    .dataframe th {
+        font-size: 12px !important;
+    }
+    
+    #leaderboard-table td:nth-child(2),
+    #leaderboard-table th:nth-child(2) {
+        max-width: 200px;
+    }
+}
+
+/* Rank column special styling */
+.dataframe td:nth-child(1) {
+    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%) !important;
+    color: white !important;
+    font-weight: 700 !important;
+    font-size: 16px !important;
+    border-radius: 8px !important;
+    margin: 4px !important;
+    min-width: 40px !important;
+}
+
+.dataframe tr:nth-child(1) td:nth-child(1) {
+    background: linear-gradient(135deg, #ffd700 0%, #ffed4e 100%) !important;
+    color: #8b4513 !important;
+}
+
+.dataframe tr:nth-child(2) td:nth-child(1) {
+    background: linear-gradient(135deg, #c0c0c0 0%, #e8e8e8 100%) !important;
+    color: #4a4a4a !important;
+}
+
+.dataframe tr:nth-child(3) td:nth-child(1) {
+    background: linear-gradient(135deg, #cd7f32 0%, #daa520 100%) !important;
+    color: white !important;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,27 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,110 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+from src.about import Tasks
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+

src/envs.py

@@ -0,0 +1,25 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+
+OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/leaderboard"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/json_leaderboard.py

@@ -0,0 +1,76 @@
+import json
+import pandas as pd
+from pathlib import Path
+
+
+def load_leaderboard_from_json(json_path="leaderboard_data.json"):
+    """Load leaderboard data from JSON file"""
+    try:
+        with open(json_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return data['leaderboard']
+    except FileNotFoundError:
+        print(f"JSON file {json_path} not found")
+        return []
+    except json.JSONDecodeError:
+        print(f"Error decoding JSON file {json_path}")
+        return []
+
+
+def create_leaderboard_df(json_path="leaderboard_data.json"):
+    """Create a pandas DataFrame from JSON leaderboard data"""
+    leaderboard_data = load_leaderboard_from_json(json_path)
+    
+    if not leaderboard_data:
+        return pd.DataFrame()
+    
+    # Convert to DataFrame
+    df = pd.DataFrame(leaderboard_data)
+    
+    # Sort by ACC score (descending)
+    df = df.sort_values('acc', ascending=False).reset_index(drop=True)
+    
+    # Add ranking icons and make model names clickable links to papers
+    def add_ranking_icon_and_link(index, model_name, paper_link):
+        if index == 0:
+            return f'🥇 <a href="{paper_link}" target="_blank">{model_name}</a>'
+        elif index == 1:
+            return f'🥈 <a href="{paper_link}" target="_blank">{model_name}</a>'
+        elif index == 2:
+            return f'🥉 <a href="{paper_link}" target="_blank">{model_name}</a>'
+        else:
+            return f'<a href="{paper_link}" target="_blank">{model_name}</a>'
+    
+    # Format the DataFrame for display
+    display_df = pd.DataFrame({
+        'Model': [add_ranking_icon_and_link(i, model, link) for i, (model, link) in enumerate(zip(df['model'], df['link']))],
+        'Release Date': df['release_date'],
+        'HF Model': df['hf'].apply(lambda x: f'<a href="{x}" target="_blank">🤗</a>' if x != "-" else "-"),
+        'MoE': df['moe'].apply(lambda x: '-' if x == '-' else ('✓' if x else '✗')),
+        'Parameters': df['params'],
+        'Open Source': df['open_source'].apply(lambda x: '✓' if x else '✗'),
+        'ACC Score': df['acc'].apply(lambda x: f"{x:.1f}")
+    })
+    
+    return display_df
+
+
+def get_leaderboard_stats(json_path="leaderboard_data.json"):
+    """Get statistics about the leaderboard"""
+    leaderboard_data = load_leaderboard_from_json(json_path)
+    
+    if not leaderboard_data:
+        return {}
+    
+    df = pd.DataFrame(leaderboard_data)
+    
+    stats = {
+        'total_models': len(df),
+        'open_source_models': df['open_source'].sum(),
+        'moe_models': df['moe'].apply(lambda x: 1 if x is True else 0).sum(),
+        'avg_acc': df['acc'].mean(),
+        'max_acc': df['acc'].max(),
+        'min_acc': df['acc'].min()
+    }
+    
+    return stats

src/leaderboard/read_evals.py

@@ -0,0 +1,196 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" 
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -0,0 +1,117 @@
+import json
+import os
+
+import pandas as pd
+
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, ModelType, Precision, WeightType
+from src.leaderboard.read_evals import get_raw_eval_results
+from src.about import Tasks
+
+
+def load_csv_results():
+    """Load results from main-results.csv file"""
+    csv_path = "main-results.csv"
+    if not os.path.exists(csv_path):
+        return []
+    
+    df = pd.read_csv(csv_path)
+    results = []
+    
+    for _, row in df.iterrows():
+        # Parse parameters - handle different formats
+        param_str = str(row['Param'])
+        if 'activated' in param_str:
+            # Extract the activated parameter count (e.g., "2.8B activated (16B total)")
+            param_value = float(param_str.split('B')[0])
+        elif 'B' in param_str:
+            # Simple format (e.g., "9B")
+            param_value = float(param_str.replace('B', ''))
+        else:
+            param_value = 0
+            
+        # Convert CSV data to the format expected by the leaderboard
+        data_dict = {
+            AutoEvalColumn.model.name: make_clickable_model(row['Model']),
+            AutoEvalColumn.average.name: row['ACC'],  # Using ACC as the average score
+            AutoEvalColumn.params.name: param_value,
+            AutoEvalColumn.license.name: "Open Source" if row['Open Source?'] == 'Yes' else "Proprietary",
+            AutoEvalColumn.model_type.name: ModelType.FT.value.name,  # Default to fine-tuned
+            AutoEvalColumn.precision.name: Precision.float16.value.name,  # Default precision
+            AutoEvalColumn.weight_type.name: WeightType.Original.value.name,
+            AutoEvalColumn.architecture.name: "Unknown",
+            AutoEvalColumn.still_on_hub.name: True,
+            AutoEvalColumn.revision.name: "",
+            AutoEvalColumn.likes.name: 0,
+            AutoEvalColumn.model_type_symbol.name: ModelType.FT.value.symbol,
+        }
+        
+        # Add task-specific scores (required by the leaderboard)
+        for task in Tasks:
+            data_dict[task.name] = row['ACC']  # Use the same ACC score for all tasks
+            
+        results.append(data_dict)
+    
+    return results
+
+
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+    
+    # If no JSON data found, try loading from CSV
+    if not all_data_json:
+        all_data_json = load_csv_results()
+    
+    if not all_data_json:
+        # Return empty dataframe if no data found
+        return pd.DataFrame(columns=cols)
+
+    df = pd.DataFrame.from_records(all_data_json)
+    
+    # Only include columns that exist in the dataframe
+    existing_cols = [col for col in cols if col in df.columns]
+    
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[existing_cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -0,0 +1,99 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,119 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+        "private": False,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )