Sets default scaffolding method to 'personal' method; adjusts topic selection with new preds_df columns; removes print and log statements

audit_utils.py

@@ -115,8 +115,6 @@ readable_to_internal = {
 }
 internal_to_readable = {v: k for k, v in readable_to_internal.items()}
 
-def get_system_preds_df():
-    return system_preds_df
 
 ########################################
 # Data storage helper functions
@@ -455,7 +453,7 @@ def get_predictions_by_user_and_item(predictions):
 # - model: trained model
 # - user_ids: list of user IDs to compute predictions for
 # - sys_eval_df: dataframe of system eval labels (pre-computed)
-def get_preds_df(model, user_ids, sys_eval_df=sys_eval_df, bins=BINS):
+def get_preds_df(model, user_ids, sys_eval_df=sys_eval_df, bins=BINS, debug=False):
     # Prep dataframe for all predictions we'd like to request
     start = time.time()
     sys_eval_comment_ids = sys_eval_df.item_id.unique().tolist()
@@ -464,7 +462,8 @@ def get_preds_df(model, user_ids, sys_eval_df=sys_eval_df, bins=BINS):
     for user_id in user_ids:
         empty_ratings_rows.extend([[user_id, c_id, 0] for c_id in sys_eval_comment_ids])
     empty_ratings_df = pd.DataFrame(empty_ratings_rows, columns=["user_id", "item_id", "rating"])
-    print("setup", time.time() - start)
+    if debug:
+        print("setup", time.time() - start)
     
     # Evaluate model to get predictions
     start = time.time() 
@@ -472,7 +471,8 @@ def get_preds_df(model, user_ids, sys_eval_df=sys_eval_df, bins=BINS):
     eval_set_data = Dataset.load_from_df(empty_ratings_df, reader)
     _, testset = train_test_split(eval_set_data, test_size=1.)
     predictions = model.test(testset)
-    print("train_test_split", time.time() - start)
+    if debug:
+        print("train_test_split", time.time() - start)
     
     # Update dataframe with predictions
     start = time.time()
@@ -513,7 +513,7 @@ def train_user_model(ratings_df, train_df=train_df, model_eval_df=model_eval_df,
 # - train_df: dataframe of training labels
 # - model_eval_df: dataframe of model eval labels (validation set)
 # - model_type: type of model to train
-def train_model(train_df, model_eval_df, model_type="SVD", sim_type=None, user_based=True):
+def train_model(train_df, model_eval_df, model_type="SVD", sim_type=None, user_based=True, debug=False):
     # Train model
     reader = Reader(rating_scale=(0, 4))
     train_data = Dataset.load_from_df(train_df, reader)
@@ -542,7 +542,8 @@ def train_model(train_df, model_eval_df, model_type="SVD", sim_type=None, user_b
     mae = accuracy.mae(predictions)
     mse = accuracy.mse(predictions)
     
-    print(f"MAE: {mae}, MSE: {mse}, RMSE: {rmse}, FCP: {fcp}")
+    if debug:
+        print(f"MAE: {mae}, MSE: {mse}, RMSE: {rmse}, FCP: {fcp}")
     perf = [mae, mse, rmse, fcp]
     
     return algo, perf
@@ -1038,7 +1039,7 @@ def plot_overall_vis_cluster(cur_user, preds_df, error_type, n_comments=None, bi
 
     return final_plot, df
 
-def get_cluster_comments(df, error_type, threshold=TOXIC_THRESHOLD, sys_col="rating_sys", use_model=True):    
+def get_cluster_comments(df, error_type, threshold=TOXIC_THRESHOLD, sys_col="rating_sys", use_model=True, debug=False):    
     df["user_color"] = [get_user_color(user, threshold) for user in df["pred"].tolist()]  # get cell colors
     df["system_color"] = [get_user_color(sys, threshold) for sys in df[sys_col].tolist()]  # get cell colors
     df["error_color"] = [get_system_color(sys, user, threshold) for sys, user in zip(df[sys_col].tolist(), df["pred"].tolist())]  # get cell colors
@@ -1049,7 +1050,8 @@ def get_cluster_comments(df, error_type, threshold=TOXIC_THRESHOLD, sys_col="rat
     if use_model:
         df = df.sort_values(by=["error_amt"], ascending=False) # surface largest errors first
     else:
-        print("get_cluster_comments; not using model")
+        if debug:
+            print("get_cluster_comments; not using model")
         df = df.sort_values(by=[sys_col], ascending=True)
 
     df["id"] = df["item_id"]

indie_label_svelte/src/ClusterResults.svelte

@@ -55,12 +55,10 @@
     //your code goes here on location change 
         let cur_url = window.location.href;
         let cur_url_elems = cur_url.split("#");
-        // console.log(cur_url_elems)
         if (cur_url_elems.length > 0) {
             let path = cur_url_elems[2];
             if (path == "comment") {
                 let comment_id = cur_url_elems[1].split("/")[0];
-                console.log("comment_id", comment_id)
                 selected_comment_id = parseInt(comment_id);
                 let table_ind = null;
                 for (let i = 0; i < items.length; i++) {
@@ -130,7 +128,6 @@
         items = data["cluster_comments"];
         set_length = items.length;
     }
-    // console.log(set_length);
     
     let cur_open_evidence;
     open_evidence.subscribe(value => {

indie_label_svelte/src/Explore.svelte

@@ -48,7 +48,6 @@
         const text = await response.text();
         const data = JSON.parse(text);
         cur_examples = JSON.parse(data["examples"]);
-        console.log(cur_examples); // TEMP
         return true;
     }
 </script>

indie_label_svelte/src/HypothesisPanel.svelte

@@ -35,14 +35,11 @@
     // Handle routing
     let searchParams = new URLSearchParams(window.location.search);
     let scaffold_method = searchParams.get("scaffold");
+    if (scaffold_method == null) {
+        scaffold_method = "personal"; // Default to personalized model scaffold
+    }
     let topic_vis_method = searchParams.get("topic_vis_method");
 
-    // TODO: connect to selected["error_type"] so changes on main panel affect report panel
-    // let cur_error_type;
-    // error_type.subscribe(value => {
-	// 	cur_error_type = value;
-	// });
-
     // Handle drawer
     let open = false;
     let selected = null;

indie_label_svelte/src/KeywordSearch.svelte

@@ -36,7 +36,6 @@
             keyword: keyword,
             error_type: cur_error_type,
         };
-        console.log("topic_df_ids", topic_df_ids);
         let params = new URLSearchParams(req_params).toString();
         const response = await fetch("./get_cluster_results?" + params);
         const text = await response.text();

indie_label_svelte/src/Labeling.svelte

@@ -93,7 +93,6 @@
         const response = await fetch("./get_group_model?" + params);
         const text = await response.text();
         const data = JSON.parse(text);
-        console.log("getGroupModel", data);
         return data
     }
 

indie_label_svelte/src/TopicTraining.svelte

@@ -75,7 +75,6 @@
             topic: topic,
         };
 
-        console.log("topic training model name", model_name);
         let params = new URLSearchParams(req_params).toString();
         const response = await fetch("./get_personalized_model_topic?" + params); // TODO
         const text = await response.text();
@@ -84,7 +83,6 @@
         model_name = data["new_model_name"];
         model_chosen.update((value) => model_name);
 
-        console.log("topicTraining", data);
         return data;
     }
 </script>

server.py

@@ -203,7 +203,7 @@ def get_group_size():
 ########################################
 # ROUTE: /GET_GROUP_MODEL
 @app.route("/get_group_model")
-def get_group_model():
+def get_group_model(debug=DEBUG):
     # Fetch info for initial labeling component
     model_name = request.args.get("model_name")
     user = request.args.get("user")
@@ -236,7 +236,8 @@ def get_group_model():
     mae, mse, rmse, avg_diff, ratings_prev = utils.train_updated_model(model_name, ratings_grp, user)
 
     duration = time.time() - start
-    print("Time to train/cache:", duration)
+    if debug:
+        print("Time to train/cache:", duration)
 
     context = {
         "group_size": group_size,
@@ -360,13 +361,14 @@ def get_personalized_model(debug=DEBUG):
 ########################################
 # ROUTE: /GET_PERSONALIZED_MODEL_TOPIC
 @app.route("/get_personalized_model_topic")
-def get_personalized_model_topic():
+def get_personalized_model_topic(debug=DEBUG):
     model_name = request.args.get("model_name")
     ratings_json = request.args.get("ratings")
     user = request.args.get("user")
     ratings = json.loads(ratings_json)
     topic = request.args.get("topic")
-    print(ratings)
+    if debug:
+        print(ratings)
     start = time.time()
 
     # Modify model name
@@ -375,14 +377,13 @@ def get_personalized_model_topic():
 
     # Handle existing or new model cases
     # Train model and cache predictions using new labels
-    print("get_personalized_model_topic train")
+    if debug:
+        print("get_personalized_model_topic train")
     mae, mse, rmse, avg_diff, ratings_prev = utils.train_updated_model(model_name, ratings, user, topic=topic)
         
-    duration = time.time() - start
-    print("Time to train/cache:", duration) 
-
-    def round_metric(x):
-        return np.round(abs(x), 3)
+    if debug:
+        duration = time.time() - start
+        print("Time to train/cache:", duration) 
 
     results = {
         "success": "success",
@@ -499,8 +500,8 @@ def get_topic_errors(df, topic_vis_method, threshold=2):
     topic_errors = {}
     for topic in topics:
         t_df = df[df["topic"] == topic]
-        y_true = t_df["pred"].to_numpy()
-        y_pred = t_df["rating"].to_numpy()
+        y_true = t_df["pred"].to_numpy()  # Predicted user rating (treated as ground truth)
+        y_pred = t_df["rating_sys"].to_numpy()  # System rating (which we're auditing)
         if topic_vis_method == "mae":
             t_err = mean_absolute_error(y_true, y_pred)
         elif topic_vis_method == "mse":
@@ -508,8 +509,8 @@ def get_topic_errors(df, topic_vis_method, threshold=2):
         elif topic_vis_method == "avg_diff":
             t_err = np.mean(y_true - y_pred)
         elif topic_vis_method == "fp_proportion":
-            y_true = [0 if rating < threshold else 1 for rating in t_df["pred"].tolist()]
-            y_pred = [0 if rating < threshold else 1 for rating in t_df["rating"].tolist()]
+            y_true = [0 if rating < threshold else 1 for rating in y_true]
+            y_pred = [0 if rating < threshold else 1 for rating in y_pred]
             try:
                 tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
             except:
@@ -517,8 +518,8 @@ def get_topic_errors(df, topic_vis_method, threshold=2):
             total = float(len(y_true))
             t_err = fp / total
         elif topic_vis_method == "fn_proportion":
-            y_true = [0 if rating < threshold else 1 for rating in t_df["pred"].tolist()]
-            y_pred = [0 if rating < threshold else 1 for rating in t_df["rating"].tolist()]
+            y_true = [0 if rating < threshold else 1 for rating in y_true]
+            y_pred = [0 if rating < threshold else 1 for rating in y_pred]
             try:
                 tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
             except:
@@ -529,16 +530,14 @@ def get_topic_errors(df, topic_vis_method, threshold=2):
         
     return topic_errors
 
-def get_personal_scaffold(cur_user, model, topic_vis_method, n_topics=200, n=5):
+def get_personal_scaffold(cur_user, model, topic_vis_method, n_topics=200, n=5, debug=DEBUG):
     threshold = utils.get_toxic_threshold()
 
     # Get topics with greatest amount of error
     preds_file = utils.get_preds_file(cur_user, model)
     with open(preds_file, "rb") as f:
         preds_df = pickle.load(f)
-        system_preds_df = utils.get_system_preds_df()
-        preds_df_mod = preds_df.merge(system_preds_df, on="item_id", how="left", suffixes=('', '_sys'))
-        preds_df_mod = preds_df_mod[preds_df_mod["user_id"] == cur_user].sort_values(by=["item_id"]).reset_index()
+        preds_df_mod = preds_df[preds_df["user_id"] == cur_user].sort_values(by=["item_id"]).reset_index()
         preds_df_mod = preds_df_mod[preds_df_mod["topic_id"] < n_topics]
 
         if topic_vis_method == "median":
@@ -557,11 +556,12 @@ def get_personal_scaffold(cur_user, model, topic_vis_method, n_topics=200, n=5):
             df = preds_df_mod.groupby(["topic", "user_id"]).mean().reset_index()
 
         # Get system error
-        df = df[(df["topic"] != "53_maiareficco_kallystas_dyisisitmanila_tractorsazi") & (df["topic"] != "79_idiot_dumb_stupid_dumber")]
+        junk_topics = ["53_maiareficco_kallystas_dyisisitmanila_tractorsazi", "-1_dude_bullshit_fight_ain"]
+        df = df[~df["topic"].isin(junk_topics)]  # Exclude known "junk topics"
         
         if topic_vis_method == "median" or topic_vis_method == "mean":
-            df["error_magnitude"] = [utils.get_error_magnitude(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
-            df["error_type"] = [utils.get_error_type_radio(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+            df["error_magnitude"] = [utils.get_error_magnitude(sys, user, threshold) for sys, user in zip(df["rating_sys"].tolist(), df["pred"].tolist())]
+            df["error_type"] = [utils.get_error_type_radio(sys, user, threshold) for sys, user in zip(df["rating_sys"].tolist(), df["pred"].tolist())]
 
             df_under = df[df["error_type"] == "System is under-sensitive"]
             df_under = df_under.sort_values(by=["error_magnitude"], ascending=False).head(n) # surface largest errors first
@@ -577,17 +577,21 @@ def get_personal_scaffold(cur_user, model, topic_vis_method, n_topics=200, n=5):
         elif topic_vis_method == "fp_fn":
             df_under = df.sort_values(by=["fn_proportion"], ascending=False).head(n)
             df_under = df_under[df_under["fn_proportion"] > 0]
+            if debug:
+                print(df_under[["topic", "fn_proportion"]])
             report_under = [get_empty_report(row["topic"], "System is under-sensitive") for _, row in df_under.iterrows()]
             
             df_over = df.sort_values(by=["fp_proportion"], ascending=False).head(n)
             df_over = df_over[df_over["fp_proportion"] > 0]
+            if debug:
+                print(df_over[["topic", "fp_proportion"]])
             report_over = [get_empty_report(row["topic"], "System is over-sensitive") for _, row in df_over.iterrows()]
 
             reports = (report_under + report_over)
             random.shuffle(reports)
         else:
             df = df.sort_values(by=[topic_vis_method], ascending=False).head(n * 2)
-            df["error_type"] = [utils.get_error_type_radio(sys, user, threshold) for sys, user in zip(df["rating"].tolist(), df["pred"].tolist())]
+            df["error_type"] = [utils.get_error_type_radio(sys, user, threshold) for sys, user in zip(df["rating_sys"].tolist(), df["pred"].tolist())]
             reports = [get_empty_report(row["topic"], row["error_type"]) for _, row in df.iterrows()]
 
         return reports