import argparse import json from tqdm.auto import tqdm import pandas as pd import numpy as np from app.recommendations import RecommenderSystem def precision_at_k(recommended_items, relevant_items, k): recommended_at_k = set(recommended_items[:k]) return len(recommended_at_k & relevant_items) / k def average_precision_at_k(recommended_items, relevant_items, k): relevant_items = set(relevant_items) apk_sum = 0.0 for m, item in enumerate(recommended_items): if item in relevant_items: apk_sum += precision_at_k(recommended_items, relevant_items, m+1) return apk_sum / min(k, len(relevant_items)) def evaluate_recsys( val_ratings_path, faiss_index_path, db_path, n_recommend_items=10, metrics_savepath=None ): recsys = RecommenderSystem( faiss_index_path=faiss_index_path, db_path=db_path) val_ratings = pd.read_csv(val_ratings_path) grouped_items = val_ratings.groupby("user_id")["item_id"].apply(list).reset_index() grouped_items = grouped_items["item_id"].tolist() metric_arrays = { "ap@5": [], } for item_group in tqdm(grouped_items): if len(item_group) == 1: continue ### Metrics are computed for each edge. ### We will first aggregate it over all edges for user ### And after that - aggregate over all users user_metric_arrays = dict() for metric in metric_arrays.keys(): user_metric_arrays[metric] = [] for item in item_group: recommend_items = list(recsys.recommend_items(item, n_recommend_items)) relevant_items = set(item_group) - {item} user_metric_arrays["ap@5"].append( average_precision_at_k(recommend_items, relevant_items, k=5)) for metric in metric_arrays.keys(): user_metric = np.mean(user_metric_arrays[metric]) metric_arrays[metric].append(user_metric) metrics = dict() for metric, array in metric_arrays.items(): metrics[metric] = np.mean(array) if metrics_savepath is not None: with open(metrics_savepath, "w") as f: json.dump(metrics, f) print(f"Saved metrics to {metrics_savepath}") return metrics if __name__ == "__main__": parser = argparse.ArgumentParser(description="Evaluate a recommendation system.") parser.add_argument("--metrics_savepath", required=True, type=str, help="Path to save the evaluation metrics.") parser.add_argument("--val_ratings_path", required=True, type=str, help="Path to the csv file with validation ratings.") parser.add_argument("--faiss_index_path", required=True, type=str, help="Path to the FAISS index.") parser.add_argument("--db_path", required=True, type=str, help="Path to the database file.") parser.add_argument("--n_recommend_items", type=int, default=10, help="Number of items to recommend.") args = parser.parse_args() evaluate_recsys(**vars(args))