Spaces:
Running
Running
File size: 1,482 Bytes
2c50826 34046e2 2c50826 34046e2 2c50826 34046e2 2c50826 34046e2 2c50826 34046e2 2c50826 34046e2 2c50826 34046e2 2c50826 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import json
from pathlib import Path
from typing import Any, Dict, Iterator, List, Tuple
import requests
class GenEvalPrompts:
def __init__(self):
super().__init__()
self._download_geneval_file()
metadata_path = Path("downloads/geneval/evaluation_metadata.jsonl")
self.entries: List[Dict[str, Any]] = []
with open(metadata_path, "r") as f:
for line in f:
if line.strip():
self.entries.append(json.loads(line))
def __iter__(self) -> Iterator[Tuple[Dict[str, Any], Path]]:
for i, entry in enumerate(self.entries):
folder_name = f"{i:05d}"
yield entry, folder_name
def _download_geneval_file(self) -> None:
folder_name = Path("downloads/geneval")
folder_name.mkdir(parents=True, exist_ok=True)
metadata_url = "https://raw.githubusercontent.com/djghosh13/geneval/main/prompts/evaluation_metadata.jsonl"
metadata_path = folder_name / "evaluation_metadata.jsonl"
if not metadata_path.exists():
response = requests.get(metadata_url)
with open(metadata_path, "w") as f:
f.write(response.text)
@property
def name(self) -> str:
return "geneval"
@property
def size(self) -> int:
return len(self.entries)
@property
def metrics(self) -> List[str]:
raise NotImplementedError("GenEval requires custom evaluation, see README.md")
|