src/leaderboard/read_evals.py

import json
import os
from dataclasses import dataclass
from typing import List
from src.display.utils import AutoEvalColumn

@dataclass
class EvalResult:
    eval_name: str
    result_name: str
    date: str
    accuracy: float
    # precision_score: float
    # recall: float
    # f1: float
    miou: float
    # producer_accuracy: List[float]
    # user_accuracy: List[float]
    # confusion_matrix: List[List[float]]

    @classmethod
    def init_from_json_file(cls, json_filepath):
        try:
            with open(json_filepath) as fp:
                data = json.load(fp)
        except json.decoder.JSONDecodeError:
            return None

        # Get the directory containing the file
        directory = os.path.dirname(json_filepath)
        # The username is assumed to be the last folder in the path
        username = os.path.basename(directory)

        result_name = data.get("result_name", "unknown_result")

        result_key = f"{username}_{result_name}"

        return cls(
            eval_name=result_key,
            result_name=result_name,
            date=data.get("submitted_time", ""),
            accuracy=data.get("accuracy", 0)*100,
            # precision_score=data.get("precision")*100,
            # recall=data.get("recall")*100,
            # f1=data.get("f1")*100,
            miou=data.get("miou", 0)*100,
            # producer_accuracy=data.get("producer_accuracy", []),
            # user_accuracy=data.get("user_accuracy", []),
            # confusion_matrix=data.get("confusion_matrix", []),
        )


    def to_dict(self):
        """Converts the EvalResult to a dict compatible with dataframe or leaderboard display."""

        data_dict = {
            AutoEvalColumn.eval_name.name: self.eval_name,
            AutoEvalColumn.result_name.name: self.result_name,
            AutoEvalColumn.date.name: self.date,
            AutoEvalColumn.accuracy.name: self.accuracy,
            # AutoEvalColumn.precision_score.name: self.precision_score,
            # AutoEvalColumn.recall.name: self.recall,
            # AutoEvalColumn.f1.name: self.f1,
            # AutoEvalColumn.producer_accuracy.name: self.producer_accuracy,
            # AutoEvalColumn.user_accuracy.name: self.user_accuracy,
            # AutoEvalColumn.confusion_matrix.name: self.confusion_matrix,
            AutoEvalColumn.miou.name: self.miou,
            # AutoEvalColumn.num_classes.name: len(self.num_classes) if self.num_classes else 0,
        }

        return data_dict


def get_raw_eval_results(results_path: str) -> list[EvalResult]:
    """From the path of the results folder root, extract all needed info for results"""
    result_filepaths = []

    for root, _, files in os.walk(results_path):
        if len(files) == 0 or all(not f.endswith(".json") for f in files):
            continue

        for file in files:
            if file.endswith(".json"):
                result_filepaths.append(os.path.join(root, file))

    results = []
    for result_filepath in result_filepaths:
        eval_result = EvalResult.init_from_json_file(result_filepath)
        if eval_result is None:
            continue

        try:
            eval_result.to_dict()  # Validate fields
            results.append(eval_result)
        except KeyError:
            continue

    return results