Spaces:
Sleeping
Sleeping
File size: 1,651 Bytes
6376749 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 |
from concurrent.futures import ProcessPoolExecutor
import queue
import subprocess
import os
def evaluate(dataset, gpu):
print('*******dataset:', dataset)
model_name = "Pythia_2B_143000_SVFT_CR15K"
save_dir= "results/" + model_name
if not os.path.exists(save_dir):
try:
os.makedirs(save_dir)
except:
pass
save_path = os.path.join(save_dir, dataset + ".txt")
command = f"CUDA_VISIBLE_DEVICES={gpu} python commonsense_evaluate_latest.py \
--model LLaMA-7B \
--adapter LoRA \
--dataset {dataset} \
--base_model './{model_name}' \
--batch_size 1| tee -a {save_path}"
result = subprocess.run(command, shell=True, text=True, capture_output=False)
print(f"Evaluation results for dataset {dataset} on GPU {gpu}:\n{result.stdout}")
return gpu
datasets = ["boolq", "social_i_qa", "piqa", "ARC-Easy", "ARC-Challenge", "winogrande", "openbookqa", "hellaswag"]
gpus = [0, 0, 0, 0]
tasks_queue = queue.Queue()
gpu_queue = queue.Queue()
for gpu in gpus:
gpu_queue.put(gpu)
for task in datasets:
tasks_queue.put(task)
num_processes = min(len(datasets), len(gpus)) # number of processes to run in parallel
with ProcessPoolExecutor(max_workers=num_processes) as executor:
futures = [executor.submit(evaluate, tasks_queue.get(), gpu_queue.get()) for i in range(num_processes)]
for future in futures:
gpu_id = future.result()
gpu_queue.put(gpu_id)
if tasks_queue.qsize() > 0:
futures.append(executor.submit(evaluate, tasks_queue.get(), gpu_queue.get()))
|