File size: 3,327 Bytes
68fa4ef fbb6998 06f147b 68fa4ef 06f147b a40eef3 06f147b b6fd6ce 06f147b b6fd6ce 68fa4ef 06f147b a40eef3 06f147b a40eef3 06f147b 68fa4ef a40eef3 06f147b b6fd6ce 68fa4ef 06f147b 68fa4ef 06f147b a40eef3 06f147b b6fd6ce 06f147b b6fd6ce 68fa4ef 06f147b 68fa4ef fbb6998 68fa4ef 06f147b 68fa4ef 06f147b fbb6998 68fa4ef 06f147b fbb6998 06f147b 68fa4ef 06f147b 68fa4ef 06f147b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
import json
import os
from dataclasses import dataclass
from typing import List
from src.display.utils import AutoEvalColumn
@dataclass
class EvalResult:
eval_name: str
result_name: str
date: str
accuracy: float
# precision_score: float
# recall: float
# f1: float
miou: float
# producer_accuracy: List[float]
# user_accuracy: List[float]
# confusion_matrix: List[List[float]]
@classmethod
def init_from_json_file(cls, json_filepath):
try:
with open(json_filepath) as fp:
data = json.load(fp)
except json.decoder.JSONDecodeError:
return None
# Get the directory containing the file
directory = os.path.dirname(json_filepath)
# The username is assumed to be the last folder in the path
username = os.path.basename(directory)
result_name = data.get("result_name", "unknown_result")
result_key = f"{username}_{result_name}"
return cls(
eval_name=result_key,
result_name=result_name,
date=data.get("submitted_time", ""),
accuracy=data.get("accuracy", 0)*100,
# precision_score=data.get("precision")*100,
# recall=data.get("recall")*100,
# f1=data.get("f1")*100,
miou=data.get("miou", 0)*100,
# producer_accuracy=data.get("producer_accuracy", []),
# user_accuracy=data.get("user_accuracy", []),
# confusion_matrix=data.get("confusion_matrix", []),
)
def to_dict(self):
"""Converts the EvalResult to a dict compatible with dataframe or leaderboard display."""
data_dict = {
AutoEvalColumn.eval_name.name: self.eval_name,
AutoEvalColumn.result_name.name: self.result_name,
AutoEvalColumn.date.name: self.date,
AutoEvalColumn.accuracy.name: self.accuracy,
# AutoEvalColumn.precision_score.name: self.precision_score,
# AutoEvalColumn.recall.name: self.recall,
# AutoEvalColumn.f1.name: self.f1,
# AutoEvalColumn.producer_accuracy.name: self.producer_accuracy,
# AutoEvalColumn.user_accuracy.name: self.user_accuracy,
# AutoEvalColumn.confusion_matrix.name: self.confusion_matrix,
AutoEvalColumn.miou.name: self.miou,
# AutoEvalColumn.num_classes.name: len(self.num_classes) if self.num_classes else 0,
}
return data_dict
def get_raw_eval_results(results_path: str) -> list[EvalResult]:
"""From the path of the results folder root, extract all needed info for results"""
result_filepaths = []
for root, _, files in os.walk(results_path):
if len(files) == 0 or all(not f.endswith(".json") for f in files):
continue
for file in files:
if file.endswith(".json"):
result_filepaths.append(os.path.join(root, file))
results = []
for result_filepath in result_filepaths:
eval_result = EvalResult.init_from_json_file(result_filepath)
if eval_result is None:
continue
try:
eval_result.to_dict() # Validate fields
results.append(eval_result)
except KeyError:
continue
return results
|