app.py

import gradio as gr
import pandas as pd
import requests
from huggingface_hub import HfApi, create_repo, upload_file, hf_hub_download
from apscheduler.schedulers.background import BackgroundScheduler
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
import os

# Hugging Face API setup
HF_TOKEN = os.getenv("HF_TOKEN")  # This should be set as an environment variable

api = HfApi(token=HF_TOKEN)

# Deepfake detector data
data_avg_performance = {
    "Detector": ["NPR", "UCF", "CAMO"],
    "Accuracy": [0.7169, 0.7229, 0.7555],
    "Precision": [0.9193, 0.9436, 0.9442],
    "Recall": [0.5996, 0.592, 0.647],
    "F1-Score": [0.7258, 0.7275, 0.7679],
    "MCC": [0.5044, 0.5285, 0.5707],
}

data_dataset_accuracy = {
    "Detector": ["NPR", "UCF", "CAMO"],
    "CelebA-HQ": [0.987, 0.995, 0.999],
    "Flickr30k": [0.916, 0.981, 0.979],
    "ImageNet": [0.834, 0.847, 0.831],
    "DiffusionDB": [0.876, 0.85, 0.961],
    "CelebA-HQ-SDXL": [0.386, 0.484, 0.682],
    "CelebA-HQ-Flux": [0.846, 0.794, 0.722],
    "Flickr30k-SDXL": [0.302, 0.256, 0.28],
    "MS-COCO-Flux": [0.588, 0.576, 0.59],
}

# Convert data to DataFrames
df_avg_performance = pd.DataFrame(data_avg_performance)
df_dataset_accuracy = pd.DataFrame(data_dataset_accuracy)

# Ensure correct data types
df_avg_performance['Detector'] = df_avg_performance['Detector'].astype(str)
df_avg_performance[['Accuracy', 'Precision', 'Recall', 'F1-Score', 'MCC']] = df_avg_performance[
    ['Accuracy', 'Precision', 'Recall', 'F1-Score', 'MCC']
].apply(pd.to_numeric)

def init_leaderboard():
    if df_avg_performance.empty:
        raise ValueError("Leaderboard DataFrame is empty.")
    
    # Display average performance leaderboard only
    leaderboard = Leaderboard(
        value=df_avg_performance,
        datatype=['str', 'number', 'number', 'number', 'number', 'number'],
        select_columns=SelectColumns(
            default_selection=["Detector", "Accuracy", "Precision", "Recall", "F1-Score", "MCC"],
            label="Select Columns to Display:"
        ),
        search_columns=["Detector"],
        filter_columns=[
            ColumnFilter("Detector", type="checkboxgroup", label="Detectors"),
        ],
        bool_checkboxgroup_label="Hide detectors",
        interactive=False,
    )
    return leaderboard

# Function to validate if a file exists in the Hugging Face repo
def check_file_in_repo(repo_id, file_path):
    try:
        # List the repo contents and check if the file exists
        contents = api.list_repo_files(repo_id)
        return file_path in contents
    except Exception as e:
        return False

# Function to handle form submission and validate inputs
def submit_detector(detector_name, model_repo, detector_file_path, configs_file_path):
    # Check if the Hugging Face repository exists
    try:
        repo_info = api.repo_info(model_repo)
    except Exception as e:
        return f"Repository '{model_repo}' not found. Error: {e}"

    # Validate the existence of the .py file and weights file in the repo
    if not check_file_in_repo(model_repo, detector_file_path):
        return f"The file '{detector_file_path}' was not found in the repository."
    
    if not check_file_in_repo(model_repo, configs_file_path):
        return f"The configs file '{configs_file_path}' was not found in the repository."

    # Prepare the data to push to the Hugging Face dataset
    submission_data = {
        "detector_name": detector_name,
        "model_repo": model_repo,
        "detector_file_path": detector_file_path,
        "configs_file_path": configs_file_path,
        "passed_invocation_test": "Pending",
        "evaluation_status": "Under Review"  # Set evaluated to False by default
    }
    df_submission = pd.DataFrame([submission_data])
    
    dataset_repo_id = "caliangandrew/dfd-arena-detectors"
    # Push to the dataset or create it if it doesn't exist
    try:
        # Create the dataset if it doesn't exist
        create_repo(repo_id=dataset_repo_id,
                    token=HF_TOKEN,
                    repo_type="dataset",
                    exist_ok=True)

        # Try to download the existing dataset
        existing_dataset_path = hf_hub_download(
            repo_id=dataset_repo_id,
            filename="submissions/submission.csv",
            token=HF_TOKEN,
            repo_type="dataset"
        )
        # If the file is found, load the existing dataset into a DataFrame
        df_existing = pd.read_csv(existing_dataset_path)
        print(df_existing)
        print(df_submission)
        # Check if the submission is already in the dataset (exact match)
        is_in_submissions = (df_existing == df_submission.iloc[0]).all(axis=1).any()
        if not is_in_submissions:
            # If no duplicate is found, append the new submission
            df_updated = pd.concat([df_existing, df_submission], ignore_index=True)
        else:
            return "Duplicate submission: This submission already exists in the dataset."
            
        # Append the new submission data
        df_updated = pd.concat([df_existing, df_submission], ignore_index=True)
    except Exception as e:
        # If the dataset doesn't exist, create a new DataFrame with the submission data
        df_updated = pd.DataFrame([submission_data])
    
    # Save the updated (or new) DataFrame as a CSV file
    submission_file = "submission.csv"
    df_updated.to_csv(submission_file, index=False)
    
    # Upload the updated (or new) file to the Hugging Face repository
    try:
        upload_file(
            path_or_fileobj=submission_file,
            path_in_repo="submissions/submission.csv",  # Location in the repo
            repo_id=dataset_repo_id,
            token=HF_TOKEN,
            repo_type="dataset",
            commit_message="Submitted for review: "+detector_name
        )
        return "Submission successful!"
    except Exception as e:
        return f"Failed to push submission: {str(e)}"

# Gradio demo
with gr.Blocks(theme=gr.themes.Base()) as demo:
    # Title
    gr.HTML("""
        <div style="text-align:center;">
            <h1> Deepfake Detection Arena (DFD) Leaderboard</h1>
        </div>
    """)
    
    # Description/Intro Section
    gr.Markdown("""
    ## 🎯 The Open Benchmark for Detecting AI-Generated Images
    [DFD-Arena](https://github.com/BitMind-AI/dfd-arena) is the first benchmark to address the open-source computer vision community's need for a *comprehensive evaluation framework* for state-of-the-art (SOTA) detection of AI-generated images.
    
    While [previous studies](https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9721302) have focused on benchmarking the SOTA on content-specific subsets of the deepfake detection problem, e.g. human face deepfake benchmarking via [DeepfakeBench](https://github.com/SCLBD/DeepfakeBench), these benchmarks do not adequately account for the broad spectrum of real and generated image types seen in everyday scenarios.
    
    ### 🔍 Explore DFD-Arena
    
    Learn how the framework evaluates on diverse, content-rich images with semantic balance between real and generated data:
    
    - 📂 [Code: DFD-Arena GitHub Repository](https://github.com/BitMind-AI/dfd-arena)
    
    - 📝 [Blog: Technical Write-Up](https://bitmindlabs.notion.site/BitMind-Deepfake-Detection-Arena-106af85402838007830ece5a6f3f35a8?pvs=25)
    ### ✍️ Authorship
    
    Both DFD-Arena and novel synthetic image datasets used for evaluation are created by [BitMind](https://www.bitmind.ca/).
    
    - 🐦 [X/Twitter: @BitMindAI](https://x.com/BitMindAI)
    """)
    
    with gr.Tabs():
        with gr.TabItem("🏅 Deepfake Detection Arena"):
            # Add text for Average Performance Metrics
            gr.Markdown("## Average Performance Metrics")
            leaderboard = init_leaderboard()

            # Add dataset-specific accuracy
            gr.Markdown("## Dataset-specific Accuracy")
            gr.DataFrame(value=df_dataset_accuracy)

        # # Submit Detector Results Form
        with gr.TabItem("🚀 Submit Detector Results"):
            gr.Markdown("Submit your detector results for evaluation.")
            
            with gr.Row():
                detector_name = gr.Textbox(label="Detector Name", placeholder="e.g., MyDetector")
                model_repo = gr.Textbox(label="Hugging Face Model Repo", placeholder="e.g., username/repo-name")
                detector_file_path = gr.Textbox(label="Path to detector .py", placeholder="e.g., my_detector.py")
                configs_file_path = gr.Textbox(label="Path to config .YAML", placeholder="e.g., my_config.yaml")
            
            submit_button = gr.Button("Submit",interactive=True)
            output = gr.Textbox(label="Submission Result",interactive=True)
            
            submit_button.click(fn=submit_detector, inputs=[detector_name, model_repo, detector_file_path, configs_file_path], outputs=output)

# Launch the Gradio app
demo.queue(default_concurrency_limit=40).launch()