import json import os from dataclasses import dataclass from typing import List from src.display.utils import AutoEvalColumn @dataclass class EvalResult: eval_name: str result_name: str date: str accuracy: float # precision_score: float # recall: float # f1: float miou: float # producer_accuracy: List[float] # user_accuracy: List[float] # confusion_matrix: List[List[float]] @classmethod def init_from_json_file(cls, json_filepath): try: with open(json_filepath) as fp: data = json.load(fp) except json.decoder.JSONDecodeError: return None # Get the directory containing the file directory = os.path.dirname(json_filepath) # The username is assumed to be the last folder in the path username = os.path.basename(directory) result_name = data.get("result_name", "unknown_result") result_key = f"{username}_{result_name}" return cls( eval_name=result_key, result_name=result_name, date=data.get("submitted_time", ""), accuracy=data.get("accuracy", 0)*100, # precision_score=data.get("precision")*100, # recall=data.get("recall")*100, # f1=data.get("f1")*100, miou=data.get("miou", 0)*100, # producer_accuracy=data.get("producer_accuracy", []), # user_accuracy=data.get("user_accuracy", []), # confusion_matrix=data.get("confusion_matrix", []), ) def to_dict(self): """Converts the EvalResult to a dict compatible with dataframe or leaderboard display.""" data_dict = { AutoEvalColumn.eval_name.name: self.eval_name, AutoEvalColumn.result_name.name: self.result_name, AutoEvalColumn.date.name: self.date, AutoEvalColumn.accuracy.name: self.accuracy, # AutoEvalColumn.precision_score.name: self.precision_score, # AutoEvalColumn.recall.name: self.recall, # AutoEvalColumn.f1.name: self.f1, # AutoEvalColumn.producer_accuracy.name: self.producer_accuracy, # AutoEvalColumn.user_accuracy.name: self.user_accuracy, # AutoEvalColumn.confusion_matrix.name: self.confusion_matrix, AutoEvalColumn.miou.name: self.miou, # AutoEvalColumn.num_classes.name: len(self.num_classes) if self.num_classes else 0, } return data_dict def get_raw_eval_results(results_path: str) -> list[EvalResult]: """From the path of the results folder root, extract all needed info for results""" result_filepaths = [] for root, _, files in os.walk(results_path): if len(files) == 0 or all(not f.endswith(".json") for f in files): continue for file in files: if file.endswith(".json"): result_filepaths.append(os.path.join(root, file)) results = [] for result_filepath in result_filepaths: eval_result = EvalResult.init_from_json_file(result_filepath) if eval_result is None: continue try: eval_result.to_dict() # Validate fields results.append(eval_result) except KeyError: continue return results