sorry this git history is v messy

.gitignore

@@ -11,3 +11,5 @@ eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
+.gradio/
+.evals/

README.md

@@ -1,13 +1,15 @@
 ---
-title: Reward Bench V2
-emoji: 🥇
-colorFrom: green
-colorTo: indigo
+title: Reward Bench Leaderboard
+emoji: 📐
+colorFrom: pink
+colorTo: blue
 sdk: gradio
+sdk_version: 5.30.0
 app_file: app.py
 pinned: true
 license: apache-2.0
-short_description: Internal leaderboard for reward bench v2 (WIP)
+tags:
+  - leaderboard
 ---
 
 # Start the configuration

app.py

@@ -1,14 +1,15 @@
-import gradio as gr
 import os
-from huggingface_hub import HfApi, snapshot_download
-from apscheduler.schedulers.background import BackgroundScheduler
+from pathlib import Path
+
+import gradio as gr
+import numpy as np
 from datasets import load_dataset
-from src.utils import load_all_data
-from src.md import ABOUT_TEXT, TOP_TEXT
-from src.plt import plot_avg_correlation
-from src.constants import subset_mapping, length_categories, example_counts
+from huggingface_hub import HfApi, snapshot_download
+
+from src.constants import example_counts, length_categories, subset_mapping
 from src.css import custom_css
-import numpy as np
+from src.md import *
+from src.utils import load_all_data
 
 api = HfApi()
 
@@ -18,40 +19,44 @@ evals_repo = "allenai/reward-bench-v2-results"
 eval_set_repo = "allenai/reward-bench-v2-v0"
 repo_dir_rewardbench = "./evals/rewardbench/"
 
+
 def restart_space():
     api.restart_space(repo_id="allenai/reward-bench-v2", token=COLLAB_TOKEN)
 
+
 print("Pulling evaluation results")
 repo = snapshot_download(
     local_dir=repo_dir_rewardbench,
     ignore_patterns=["pref-sets-scores/*", "eval-set-scores/*"],
     repo_id=evals_repo,
     use_auth_token=COLLAB_TOKEN,
-    tqdm_class=None, 
+    tqdm_class=None,
     etag_timeout=30,
     repo_type="dataset",
 )
 
+
 def avg_over_rewardbench_v2(dataframe_core):
-    domain_cols = ['chat', 'factuality', 'safety', 'math', 'precise if', 'ties']
-    domain_weights=[0,1,1,1,1,1]
+    domain_cols = ["chat", "factuality", "safety", "math", "precise if", "ties"]
+    domain_weights = [0, 1, 1, 1, 1, 1]
     new_df = dataframe_core.copy()
 
     # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
     # Get the domain data and handle missing values
     domain_data = new_df[domain_cols].values
     masked_data = np.ma.masked_array(domain_data, np.isnan(domain_data))
-    
+
     # Calculate weighted average
     average = np.ma.average(masked_data, axis=1, weights=domain_weights)
     new_df["average"] = average.filled(np.nan)
-    
+
     # Rearrange columns for consistent output
     keep_columns = ["model", "model_type", "average"] + domain_cols
     new_df = new_df[keep_columns]
-    
+
     return new_df
 
+
 def avg_over_rewardbench(dataframe_core, dataframe_prefs):
     """
     Averages over the subsets alpacaeval, mt-bench, llmbar, refusals, hep and returns dataframe with only these columns.
@@ -69,13 +74,19 @@ def avg_over_rewardbench(dataframe_core, dataframe_prefs):
     # for main subsets, keys in subset_mapping, take the weighted avg by example_counts and store for the models
     for subset, sub_subsets in subset_mapping.items():
         subset_cols = [col for col in new_df.columns if col in sub_subsets]
-        sub_data = new_df[subset_cols].values # take the relevant column values
-        sub_counts = [example_counts[s] for s in subset_cols] # take the example counts
-        new_df[subset] = np.average(sub_data, axis=1, weights=sub_counts) # take the weighted average
+        sub_data = new_df[subset_cols].values  # take the relevant column values
+        sub_counts = [example_counts[s] for s in subset_cols]  # take the example counts
+        new_df[subset] = np.average(sub_data, axis=1, weights=sub_counts)  # take the weighted average
         # new_df[subset] = np.round(np.nanmean(new_df[subset_cols].values, axis=1), 2)
 
     data_cols = list(subset_mapping.keys())
-    keep_columns = ["model",] + ["model_type"] + data_cols
+    keep_columns = (
+        [
+            "model",
+        ]
+        + ["model_type"]
+        + data_cols
+    )
     # keep_columns = ["model", "average"] + subsets
     new_df = new_df[keep_columns]
 
@@ -97,7 +108,7 @@ def avg_over_rewardbench(dataframe_core, dataframe_prefs):
             # new_df.at[i, "Prior Sets (0.5 weight)"] = dataframe_prefs[dataframe_prefs["model"] == model]["Prior Sets (0.5 weight)"].values[0]
         else:
             values.append(np.nan)
-    
+
     new_df["Prior Sets (0.5 weight)"] = values
 
     # add total average
@@ -114,6 +125,7 @@ def avg_over_rewardbench(dataframe_core, dataframe_prefs):
     new_df = new_df[keep_columns]
     return new_df
 
+
 def expand_subsets(dataframe):
     # TODO need to modify data/ script to do this
     pass
@@ -125,7 +137,7 @@ def length_bias_check(dataframe):
     Then, take the average of the three buckets as "average"
     """
     new_df = dataframe.copy()
-    existing_subsets = new_df.columns[3:] # model, model_type, average
+    existing_subsets = new_df.columns[3:]  # model, model_type, average
     final_subsets = ["Length Bias", "Neutral", "Terse Bias"]
     # new data is empty list dict for each final subset
     new_data = {s: [] for s in final_subsets}
@@ -154,17 +166,17 @@ def length_bias_check(dataframe):
     return new_df
 
 
-    
-rewardbench_data = load_all_data(repo_dir_rewardbench, subdir="eval-set").sort_values(by='average', ascending=False)
+rewardbench_data = load_all_data(repo_dir_rewardbench, subdir="eval-set").sort_values(by="average", ascending=False)
 # rewardbench_data_length = length_bias_check(rewardbench_data).sort_values(by='Terse Bias', ascending=False)
 # prefs_data = load_all_data(repo_dir_rewardbench, subdir="pref-sets").sort_values(by='average', ascending=False)
 # prefs_data_sub = expand_subsets(prefs_data).sort_values(by='average', ascending=False)
 
-rewardbench_data_avg = avg_over_rewardbench_v2(rewardbench_data).sort_values(by='average', ascending=False)
+rewardbench_data_avg = avg_over_rewardbench_v2(rewardbench_data).sort_values(by="average", ascending=False)
+
 
 def prep_df(df):
     # add column to 0th entry with count (column name itself empty)
-    df.insert(0, '', range(1, 1 + len(df)))
+    df.insert(0, "", range(1, 1 + len(df)))
 
     # replace "model" with "Model" and "model_type" with "Model Type" and "average" with "Average"
     df = df.rename(columns={"model": "Model", "model_type": "Model Type", "average": "Average"})
@@ -173,33 +185,36 @@ def prep_df(df):
     if "Model Type" in df.columns:
         # get model_types that have generative in them
         mask = df["Model Type"].str.contains("generative", case=False, na=False)
-        
+
         # set these values to "Generative"
         df.loc[mask, "Model Type"] = "Generative"
 
     return df
 
+
 # add count column to all dataframes
 rewardbench_data = prep_df(rewardbench_data)
 rewardbench_data_avg = prep_df(rewardbench_data_avg).rename(columns={"Average": "Score"})
 # adjust weight of this average to 50% for Prior Sets (0.5 weight), 1 for others
 
 # rewardbench_data_length = prep_df(rewardbench_data_length)
-#prefs_data = prep_df(prefs_data)
+# prefs_data = prep_df(prefs_data)
 
 col_types_rewardbench = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data.columns) - 1)
-col_types_rewardbench_avg = ["number"] + ["markdown"]+ ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
-#cols_rewardbench_data_length = ["markdown"] + ["number"] * (len(rewardbench_data_length.columns) - 1)
-#col_types_prefs = ["number"] + ["markdown"] + ["number"] * (len(prefs_data.columns) - 1)
+col_types_rewardbench_avg = ["number"] + ["markdown"] + ["str"] + ["number"] * (len(rewardbench_data_avg.columns) - 1)
+# cols_rewardbench_data_length = ["markdown"] + ["number"] * (len(rewardbench_data_length.columns) - 1)
+# col_types_prefs = ["number"] + ["markdown"] + ["number"] * (len(prefs_data.columns) - 1)
 ## col_types_prefs_sub = ["markdown"] + ["number"] * (len(prefs_data_sub.columns) - 1)
 
 # for showing random samples
 eval_set = load_dataset(eval_set_repo, use_auth_token=COLLAB_TOKEN, split="test")
+
+
 def random_sample(r: gr.Request, subset):
     if subset is None or subset == []:
         sample_index = np.random.randint(0, len(eval_set) - 1)
         sample = eval_set[sample_index]
-    else: # filter by subsets (can be list)
+    else:  # filter by subsets (can be list)
         if isinstance(subset, str):
             subset = [subset]
         # filter down dataset to only include the subset(s)
@@ -207,9 +222,10 @@ def random_sample(r: gr.Request, subset):
         sample_index = np.random.randint(0, len(eval_set_filtered) - 1)
         sample = eval_set_filtered[sample_index]
 
-    markdown_text = '\n\n'.join([f"**{key}**:\n\n{value}" for key, value in sample.items()])
+    markdown_text = "\n\n".join([f"**{key}**:\n\n{value}" for key, value in sample.items()])
     return markdown_text
 
+
 subsets = eval_set.unique("subset")
 
 color_map = {
@@ -218,6 +234,8 @@ color_map = {
     "Seq. Classifier": "#ffcd75",
     "DPO": "#75809c",
 }
+
+
 def color_model_type_column(df, color_map):
     """
     Apply color to the 'Model Type' column of the DataFrame based on a given color mapping.
@@ -229,17 +247,19 @@ def color_model_type_column(df, color_map):
     Returns:
     pd.Styler: The styled DataFrame.
     """
+
     # Function to apply color based on the model type
     def apply_color(val):
         color = color_map.get(val, "default")  # Default color if not specified in color_map
-        return f'background-color: {color}'
-    
+        return f"background-color: {color}"
+
     # Format for different columns
-    format_dict = {col: "{:.1f}" for col in df.columns if col not in ['Average', 'Model', 'Model Type']}
-    format_dict['Average'] = "{:.2f}"
-    format_dict[''] = "{:d}"
+    format_dict = {col: "{:.1f}" for col in df.columns if col not in ["Average", "Model", "Model Type"]}
+    format_dict["Average"] = "{:.2f}"
+    format_dict[""] = "{:d}"
+
+    return df.style.applymap(apply_color, subset=["Model Type"]).format(format_dict, na_rep="")
 
-    return df.style.applymap(apply_color, subset=['Model Type']).format(format_dict, na_rep='')
 
 def regex_table(dataframe, regex, filter_button, style=True):
     """
@@ -248,18 +268,18 @@ def regex_table(dataframe, regex, filter_button, style=True):
     # Split regex statement by comma and trim whitespace around regexes
     regex_list = [x.strip() for x in regex.split(",")]
     # Join the list into a single regex pattern with '|' acting as OR
-    combined_regex = '|'.join(regex_list)
+    combined_regex = "|".join(regex_list)
 
     # remove internal ai2 data
     dataframe = dataframe[~dataframe["Model"].str.contains("ai2", case=False, na=False)]
-    
+
     # if filter_button, remove all rows with "ai2" in the model name
     update_scores = False
     if isinstance(filter_button, list) or isinstance(filter_button, str):
-        if "Prior Sets" not in filter_button and 'Prior Sets (0.5 weight)' in dataframe.columns:
+        if "Prior Sets" not in filter_button and "Prior Sets (0.5 weight)" in dataframe.columns:
             update_scores = True
             # remove the column "Prior Sets (0.5 weight)" from the outputted table
-            dataframe = dataframe.drop(columns=['Prior Sets (0.5 weight)'])
+            dataframe = dataframe.drop(columns=["Prior Sets (0.5 weight)"])
         if "Seq. Classifiers" not in filter_button:
             dataframe = dataframe[~dataframe["Model Type"].str.contains("Seq. Classifier", case=False, na=False)]
         if "DPO" not in filter_button:
@@ -277,12 +297,12 @@ def regex_table(dataframe, regex, filter_button, style=True):
         # if "Prior Sets (0.5 weight)" in data.columns:
         # data["Prior Sets (0.5 weight)"] = np.nan
         # sort array by Score column
-        data = data.sort_values(by='Score', ascending=False)
+        data = data.sort_values(by="Score", ascending=False)
 
     data.reset_index(drop=True, inplace=True)
 
     # replace column '' with count/rank
-    data[''] = np.arange(1, 1 + len(data))
+    data[""] = np.arange(1, 1 + len(data))
 
     # if Score exists, round to 2 decimals
     if "Score" in data.columns:
@@ -293,7 +313,7 @@ def regex_table(dataframe, regex, filter_button, style=True):
     for col in data.columns:
         if col not in ["", "Model", "Model Type", "Score", "Average"]:
             # replace any data[col].values == '' with np.nan
-            data[col] = data[col].replace('', np.nan)
+            data[col] = data[col].replace("", np.nan)
             data[col] = np.round(np.array(data[col].values).astype(float), 1)
     if style:
         # apply color
@@ -301,156 +321,145 @@ def regex_table(dataframe, regex, filter_button, style=True):
 
     return data
 
+
 # import ipdb; ipdb.set_trace()
 
-total_models = len(regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False).values)
+total_models = len(
+    regex_table(
+        rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], style=False
+    ).values
+)
+assets = Path("src").resolve()  # absolute dir with the image
+
+# Using a string for a predefined color
+theme = gr.themes.Default(primary_hue="blue")
 
-with gr.Blocks(css=custom_css) as app:
+with gr.Blocks(theme=theme, css=custom_css) as app:
     # create tabs for the app, moving the current table to one titled "rewardbench" and the benchmark_text to a tab called "About"
     with gr.Row():
         with gr.Column(scale=6):
-            gr.Markdown(TOP_TEXT.format(str(total_models)))
-        with gr.Column(scale=4):
-            # search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model")
-            # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True)
-            # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500)
-            gr.Markdown("""
-                        ![](file/src/logo.png)
-                        """)
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏆 RewardBench Leaderboard"):
-            with gr.Row():
-                search_1 = gr.Textbox(label="Model Search (delimit with , )", 
-                                      placeholder="Model Search (delimit with , )",
-                                      show_label=False)
-                model_types_1 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative", "Prior Sets"], 
-                                                 value=["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
-                                                 label="Model Types", 
-                                                 show_label=False,
-                                                #  info="Which model types to include.",
-                                                 )
-            with gr.Row():
-                # reference data
-                rewardbench_table_hidden = gr.Dataframe(
-                    rewardbench_data_avg.values,
-                    datatype=col_types_rewardbench_avg,
-                    headers=rewardbench_data_avg.columns.tolist(),
-                    visible=False,
-                )
-                rewardbench_table = gr.Dataframe(
-                    regex_table(rewardbench_data_avg.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers",  "Generative"]),
-                    datatype=col_types_rewardbench_avg,
-                    headers=rewardbench_data_avg.columns.tolist(),
-                    elem_id="rewardbench_dataframe_avg",
-                    # height=1000,
-                )
-                
-        with gr.TabItem("🔍 RewardBench - Detailed"):
+            gr.Markdown(TOP_TEXT)
+        # with gr.Column(scale=4):
+        #     # search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model")
+        #     # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True)
+        #     # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500)
+        #     gr.Markdown("""
+        #                 ![](/gradio_api/file=src/logo.png)
+        #                 """)
+
+    with gr.Tabs(elem_id="outer-tabs", elem_classes="tabs-big") as tabs_big:
+        with gr.TabItem("🏆 RewardBench 2", scale=1.5):
             with gr.Row():
-                search_2 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
-                model_types_2 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"], 
-                                                 value=["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"], 
-                                                 label="Model Types", 
-                                                 show_label=False,
-                                                #  info="Which model types to include."
-                                                 )
+                with gr.Column(scale=7):
+                    gr.Markdown(CAPTION_V2.format(str(total_models)))
+                with gr.Column(scale=3):
+                    # search = gr.Textbox(label="Model Search (delimit with , )", placeholder="Regex search for a model")
+                    # filter_button = gr.Checkbox(label="Include AI2 training runs (or type ai2 above).", interactive=True)
+                    # img = gr.Image(value="https://private-user-images.githubusercontent.com/10695622/310698241-24ed272a-0844-451f-b414-fde57478703e.png", width=500)
+                    gr.Markdown(
+                        """
+                                ![](/gradio_api/file=src/logo.png)
+                                """
+                    )
+            with gr.Tabs(elem_id="inner-tabs", elem_classes="tabs-small") as tabs:
+                with gr.TabItem("Leaderboard"):
+                    with gr.Row():
+                        search_1 = gr.Textbox(
+                            label="Model Search (delimit with , )",
+                            placeholder="Model Search (delimit with , )",
+                            show_label=False,
+                        )
+                        model_types_1 = gr.CheckboxGroup(
+                            ["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
+                            value=["Seq. Classifiers", "Custom Classifiers", "Generative"],
+                            label="Model Types",
+                            show_label=False,
+                            #  info="Which model types to include.",
+                        )
+                    with gr.Row():
+                        # reference data
+                        rewardbench_table_hidden = gr.Dataframe(
+                            rewardbench_data_avg.values,
+                            datatype=col_types_rewardbench_avg,
+                            headers=rewardbench_data_avg.columns.tolist(),
+                            visible=False,
+                        )
+                        rewardbench_table = gr.Dataframe(
+                            regex_table(
+                                rewardbench_data_avg.copy(),
+                                "",
+                                ["Seq. Classifiers", "Custom Classifiers", "Generative"],
+                            ),
+                            datatype=col_types_rewardbench_avg,
+                            headers=rewardbench_data_avg.columns.tolist(),
+                            elem_id="rewardbench_dataframe_avg",
+                            max_height=800,  # 800 px ≈ ~25 rows on default row-height
+                        )
+
+                # removed because the data does not have sub-domains
+                # with gr.TabItem("Detailed"):
+                #     with gr.Row():
+                #         search_2 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
+                #         model_types_2 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
+                #                                         value=["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"],
+                #                                         label="Model Types",
+                #                                         show_label=False,
+                #                                         #  info="Which model types to include."
+                #                                         )
+                #     with gr.Row():
+                #         # ref data
+                #         rewardbench_table_detailed_hidden = gr.Dataframe(
+                #             rewardbench_data.values,
+                #             datatype=col_types_rewardbench,
+                #             headers=rewardbench_data.columns.tolist(),
+                #             visible=False,
+                #         )
+                #         rewardbench_table_detailed = gr.Dataframe(
+                #             regex_table(rewardbench_data.copy(), "", ["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"]),
+                #             datatype=col_types_rewardbench,
+                #             headers=rewardbench_data.columns.tolist(),
+                #             elem_id="rewardbench_dataframe",
+                #             # height=1000,
+                #         )
+
+                with gr.TabItem("About"):
+                    with gr.Row():
+                        gr.Markdown(ABOUT_TEXT_V2)
+
+                with gr.TabItem("Dataset Viewer"):
+                    with gr.Row():
+                        # loads one sample
+                        gr.Markdown("""## Random Dataset Sample Viewer""")
+                        subset_selector = gr.Dropdown(subsets, label="Subset", value=None, multiselect=True)
+                        button = gr.Button("Show Random Sample")
+
+                    with gr.Row():
+                        sample_display = gr.Markdown("{sampled data loads here}")
+
+                    button.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display])
+        with gr.TabItem("RewardBench", scale=1.5):
             with gr.Row():
-                # ref data
-                rewardbench_table_detailed_hidden = gr.Dataframe(
-                    rewardbench_data.values,
-                    datatype=col_types_rewardbench,
-                    headers=rewardbench_data.columns.tolist(),
-                    visible=False,
-                )
-                rewardbench_table_detailed = gr.Dataframe(
-                    regex_table(rewardbench_data.copy(), "", ["Seq. Classifiers", "DPO",  "Generative", "Custom Classifiers"]),
-                    datatype=col_types_rewardbench,
-                    headers=rewardbench_data.columns.tolist(),
-                    elem_id="rewardbench_dataframe",
-                    # height=1000,
-                )
-        # with gr.TabItem("rewardbench Eval Set - Length Bias"):
-        #     with gr.Row():
-        #         # backup
-        #         rewardbench_table_len_hidden = gr.Dataframe(
-        #             rewardbench_data_length.values,
-        #             datatype=cols_rewardbench_data_length,
-        #             headers=rewardbench_data_length.columns.tolist(),
-        #             visible=False,
-        #         )
-        #         rewardbench_table_len = gr.Dataframe(
-        #             regex_table(rewardbench_data_length.copy(), "", False).values,
-        #             datatype=cols_rewardbench_data_length,
-        #             headers=rewardbench_data_length.columns.tolist(),
-        #             elem_id="rewardbench_dataframe_length",
-        #             height=1000,
-        #         )
-        # with gr.TabItem("Prior Test Sets"):
-        #     with gr.Row():
-        #         search_3 = gr.Textbox(label="Model Search (delimit with , )", show_label=False, placeholder="Model Search (delimit with , )")
-        #         model_types_3 = gr.CheckboxGroup(["Seq. Classifiers", "DPO", "Custom Classifiers", "Generative"],
-        #                                          value=["Seq. Classifiers", "DPO", "Custom Classifiers"], 
-        #                                          label="Model Types",
-        #                                          show_label=False, 
-        #                                         #  info="Which model types to include.",
-        #                                          )
-        #     with gr.Row():
-        #         PREF_SET_TEXT = """
-        #         For more information, see the [dataset](https://huggingface.co/datasets/allenai/pref-test-sets). Only the subsets Anthropic Helpful, Anthropic HHH, Stanford SHP, and OpenAI's Summarize data are used in the leaderboard ranking.
-        #         """
-        #         gr.Markdown(PREF_SET_TEXT)
-        #     with gr.Row():
-        #         # backup
-        #         pref_sets_table_hidden = gr.Dataframe(
-        #             prefs_data.values,
-        #             datatype=col_types_prefs,
-        #             headers=prefs_data.columns.tolist(),
-        #             visible=False,
-        #         )
-        #         pref_sets_table = gr.Dataframe(
-        #             regex_table(prefs_data.copy(), "", ["Seq. Classifiers", "DPO", "Custom Classifiers"]),
-        #             datatype=col_types_prefs,
-        #             headers=prefs_data.columns.tolist(),
-        #             elem_id="prefs_dataframe",
-        #             # height=1000,
-        #         )
-
-
-        with gr.TabItem("About"):
-            with gr.Row():
-                gr.Markdown(ABOUT_TEXT)
-
-        with gr.TabItem("Dataset Viewer"):
-            with gr.Row():
-                # loads one sample
-                gr.Markdown("""## Random Dataset Sample Viewer
-Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
-                subset_selector = gr.Dropdown(subsets, label="Subset", value=None, multiselect=True)
-                button = gr.Button("Show Random Sample")
-
-            with gr.Row():
-                sample_display = gr.Markdown("{sampled data loads here}")
-
-            button.click(fn=random_sample, inputs=[subset_selector], outputs=[sample_display])
-        # removed plot because not pretty enough
-        # with gr.TabItem("Model Correlation"):
-        #     with gr.Row():
-        #         plot = plot_avg_correlation(rewardbench_data_avg, prefs_data)
-        #         gr.Plot(plot)
+                gr.Markdown(CAPTION_V1.format(str(total_models)))
 
     search_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
-    search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
-    # search.change(regex_table, inputs=[rewardbench_table_len_hidden, search, filter_button], outputs=rewardbench_table_len)
-    # search_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)    
-    
-    model_types_1.change(regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table)
-    model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
-    # model_types_3.change(regex_table, inputs=[pref_sets_table_hidden, search_3, model_types_3], outputs=pref_sets_table)
+    # search_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
+
+    model_types_1.change(
+        regex_table, inputs=[rewardbench_table_hidden, search_1, model_types_1], outputs=rewardbench_table
+    )
+    # model_types_2.change(regex_table, inputs=[rewardbench_table_detailed_hidden, search_2, model_types_2], outputs=rewardbench_table_detailed)
 
     with gr.Row():
         with gr.Accordion("📚 Citation", open=False):
             citation_button = gr.Textbox(
-                value=r"""@misc{RewardBench,
+                value=r"""@misc{RewardBench2,
+    title={RewardBench 2: Advancing Reward Model Evaluation},
+    author={Malik, Saumya and Pyatkin, Valentina and Land, Sander and Morrison, Jacob and Smith, Noah A. and Hajishirzi, Hannaneh and Lambert, Nathan},
+    year={2024},
+    howpublished={\url{https://huggingface.co/spaces/allenai/reward-bench-2}},
+}
+
+@misc{RewardBench,
     title={RewardBench: Evaluating Reward Models for Language Modeling},
     author={Lambert, Nathan and Pyatkin, Valentina and Morrison, Jacob and Miranda, LJ and Lin, Bill Yuchen and Chandu, Khyathi and Dziri, Nouha and Kumar, Sachin and Zick, Tom and Choi, Yejin and Smith, Noah A. and Hajishirzi, Hannaneh},
     year={2024},
@@ -461,18 +470,5 @@ Warning, refusals, XSTest, and donotanswer datasets have sensitive content.""")
                 elem_id="citation-button",
                 show_copy_button=True,
             )
-# Load data when app starts, TODO make this used somewhere...
-# def load_data_on_start():
-#     data_rewardbench = load_all_data(repo_dir_rewardbench)
-#     rewardbench_table.update(data_rewardbench)
-
-#     data_rewardbench_avg = avg_over_rewardbench(repo_dir_rewardbench)
-#     rewardbench_table.update(data_rewardbench_avg)
-
-#     data_prefs = load_all_data(repo_dir_prefs)
-#     pref_sets_table.update(data_prefs)
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=10800) # restarted every 3h
-scheduler.start()
-app.launch(allowed_paths=['src/']) # had .queue() before launch before... not sure if that's necessary
+app.launch(allowed_paths=[str(assets)])  # had .queue() before launch before... not sure if that's necessary

src/about.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/constants.py

@@ -1,28 +1,28 @@
 # reference for length bias categories
 length_categories = {
-    'alpacaeval-easy': 'True',
-    'alpacaeval-hard': 'True',
-    'alpacaeval-length': 'Neutral',
-    'donotanswer': 'False',
-    'hep-cpp': 'Neutral',
-    'hep-go': 'Neutral',
-    'hep-java': 'Neutral',
-    'hep-js': 'Neutral',
-    'hep-python': 'Neutral',
-    'hep-rust': 'Neutral',
-    'llmbar-adver-GPTInst': 'False',
-    'llmbar-adver-GPTOut': 'Neutral',
-    'llmbar-adver-manual': 'False',
-    'llmbar-adver-neighbor': 'False',
-    'llmbar-natural': 'Neutral',
-    'math-prm': 'Neutral',
-    'mt-bench-easy': 'False',
-    'mt-bench-hard': 'False',
-    'mt-bench-med': 'Neutral',
-    'refusals-dangerous': 'False',
-    'refusals-offensive': 'False',
-    'xstest-should-refuse': 'False',
-    'xstest-should-respond': 'True'
+    "alpacaeval-easy": "True",
+    "alpacaeval-hard": "True",
+    "alpacaeval-length": "Neutral",
+    "donotanswer": "False",
+    "hep-cpp": "Neutral",
+    "hep-go": "Neutral",
+    "hep-java": "Neutral",
+    "hep-js": "Neutral",
+    "hep-python": "Neutral",
+    "hep-rust": "Neutral",
+    "llmbar-adver-GPTInst": "False",
+    "llmbar-adver-GPTOut": "Neutral",
+    "llmbar-adver-manual": "False",
+    "llmbar-adver-neighbor": "False",
+    "llmbar-natural": "Neutral",
+    "math-prm": "Neutral",
+    "mt-bench-easy": "False",
+    "mt-bench-hard": "False",
+    "mt-bench-med": "Neutral",
+    "refusals-dangerous": "False",
+    "refusals-offensive": "False",
+    "xstest-should-refuse": "False",
+    "xstest-should-respond": "True",
 }
 
 example_counts = {
@@ -32,7 +32,7 @@ example_counts = {
     "mt-bench-easy": 28,
     "mt-bench-med": 40,
     "mt-bench-hard": 37,
-    "math-prm": 984, # actual length 447, upweighting to be equal to code
+    "math-prm": 984,  # actual length 447, upweighting to be equal to code
     "refusals-dangerous": 100,
     "refusals-offensive": 100,
     "llmbar-natural": 100,
@@ -41,20 +41,33 @@ example_counts = {
     "llmbar-adver-GPTOut": 47,
     "llmbar-adver-manual": 46,
     "xstest-should-refuse": 154,
-    "xstest-should-respond": 250, # Note, refuse and respond were accidentally swapped until 9 Sept 2024
+    "xstest-should-respond": 250,  # Note, refuse and respond were accidentally swapped until 9 Sept 2024
     "donotanswer": 136,
     "hep-cpp": 164,
     "hep-go": 164,
     "hep-java": 164,
     "hep-js": 164,
     "hep-python": 164,
-    "hep-rust": 164
+    "hep-rust": 164,
 }
 
 # note, this order should match the dataframe.
 subset_mapping = {
-    "Chat": ['alpacaeval-easy', 'alpacaeval-hard', 'alpacaeval-length', 'mt-bench-easy', 'mt-bench-med'],
-    "Chat Hard": ['llmbar-adver-GPTInst', 'llmbar-adver-GPTOut', 'llmbar-adver-manual', 'llmbar-adver-neighbor', 'llmbar-natural', 'mt-bench-hard'],
-    "Safety": ['donotanswer', 'refusals-dangerous', 'refusals-offensive', 'xstest-should-refuse', 'xstest-should-respond'],
-    "Reasoning": ["hep-cpp", "hep-go", "hep-java", "hep-js", "hep-python", "hep-rust", "math-prm"]
+    "Chat": ["alpacaeval-easy", "alpacaeval-hard", "alpacaeval-length", "mt-bench-easy", "mt-bench-med"],
+    "Chat Hard": [
+        "llmbar-adver-GPTInst",
+        "llmbar-adver-GPTOut",
+        "llmbar-adver-manual",
+        "llmbar-adver-neighbor",
+        "llmbar-natural",
+        "mt-bench-hard",
+    ],
+    "Safety": [
+        "donotanswer",
+        "refusals-dangerous",
+        "refusals-offensive",
+        "xstest-should-refuse",
+        "xstest-should-respond",
+    ],
+    "Reasoning": ["hep-cpp", "hep-go", "hep-java", "hep-js", "hep-python", "hep-rust", "math-prm"],
 }

src/css.py

@@ -1,3 +1,4 @@
+ACCENT = "#245ED4"  # OLMo Blue. Not currently used.
 custom_css = """
 
 /* Full width space */
@@ -11,12 +12,11 @@ custom_css = """
 }
 
 .tab-buttons button {
-  font-size: 20px;
+  font-size: 30px;
 }
 
 h1 {
   font-size: 32px !important;
   margin-top: 0px !important;
 }
-
-"""
+"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-#leaderboard-table td:nth-child(2),
-#leaderboard-table th:nth-child(2) {
-    max-width: 600px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,110 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/md.py

@@ -1,7 +1,10 @@
 from datetime import datetime
+
 import pytz
 
-ABOUT_TEXT = """
+ABOUT_TEXT_V2 = """TODO"""
+
+ABOUT_TEXT_V1 = """
 We compute the win percentage for a reward model on hand curated chosen-rejected pairs for each prompt.
 A win is when the score for the chosen response is higher than the score for the rejected response.
 
@@ -96,11 +99,20 @@ For more details, see the [dataset](https://huggingface.co/datasets/allenai/rewa
 """
 
 # Get Pacific time zone (handles PST/PDT automatically)
-pacific_tz = pytz.timezone('America/Los_Angeles')
+pacific_tz = pytz.timezone("America/Los_Angeles")
 current_time = datetime.now(pacific_tz).strftime("%H:%M %Z, %d %b %Y")
 
-TOP_TEXT = f"""# RewardBench: Evaluating Reward Models
+TOP_TEXT = """# RewardBench: Evaluating Reward Models
 ### Evaluating the capabilities, safety, and pitfalls of reward models
-[Code](https://github.com/allenai/reward-bench) | [Eval. Dataset](https://huggingface.co/datasets/allenai/reward-bench-v2-v0) | [Prior Test Sets](https://huggingface.co/datasets/allenai/pref-test-sets) | [Results](https://huggingface.co/datasets/allenai/reward-bench-v2-results) | [Paper](https://arxiv.org/abs/2403.13787) | Total models: {{}} | * Unverified models | ⚠️ Dataset Contamination | Last restart (PST): {current_time}
+"""
+
+CAPTION_V2 = f"""The *new version* of RewardBench that is based on unseen human data and designed to be substantially more difficult!
 
-⚠️ Many of the top models were trained on unintentionally contaminated, AI-generated data, for more information, see this [gist](https://gist.github.com/natolambert/1aed306000c13e0e8c5bc17c1a5dd300)."""
+[Code](https://github.com/allenai/reward-bench) | [Eval. Dataset](https://huggingface.co/datasets/allenai/reward-bench-v2-v0) | [Prior Test Sets](https://huggingface.co/datasets/allenai/pref-test-sets) | [Results](https://huggingface.co/datasets/allenai/reward-bench-v2-results) | [Paper (TODO)](TODO) | Total models: {{}} | Last restart (PST): {current_time}"""
+
+CAPTION_V1 = """The original RewardBench -- the first reward model evaluation.
+
+**Note**: This leaderboard is frozen and will not be updated. The final version of the evaluation results are available [here](TODO).
+
+⚠️ Many of the top models were trained on unintentionally contaminated, AI-generated data, for more information, see this [gist](https://gist.github.com/natolambert/1aed306000c13e0e8c5bc17c1a5dd300).
+"""

src/plt.py

@@ -1,53 +1,55 @@
 import matplotlib.pyplot as plt
 import pandas as pd
+
 from .utils import undo_hyperlink
 
+
 def plot_avg_correlation(df1, df2):
     """
     Plots the "average" column for each unique model that appears in both dataframes.
-    
+
     Parameters:
     - df1: pandas DataFrame containing columns "model" and "average".
     - df2: pandas DataFrame containing columns "model" and "average".
     """
     # Identify the unique models that appear in both DataFrames
-    common_models = pd.Series(list(set(df1['model']) & set(df2['model'])))
-    
+    common_models = pd.Series(list(set(df1["model"]) & set(df2["model"])))
+
     # Set up the plot
     plt.figure(figsize=(13, 6), constrained_layout=True)
 
-    # axes from 0 to 1 for x and y 
+    # axes from 0 to 1 for x and y
     plt.xlim(0.475, 0.8)
     plt.ylim(0.475, 0.8)
 
     # larger font (16)
-    plt.rcParams.update({'font.size': 12, 'axes.labelsize': 14,'axes.titlesize': 14})
+    plt.rcParams.update({"font.size": 12, "axes.labelsize": 14, "axes.titlesize": 14})
     # plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.1)
     # plt.tight_layout()
     # plt.margins(0,0)
 
     for model in common_models:
         # Filter data for the current model
-        df1_model_data = df1[df1['model'] == model]['average'].values
-        df2_model_data = df2[df2['model'] == model]['average'].values
-        
+        df1_model_data = df1[df1["model"] == model]["average"].values
+        df2_model_data = df2[df2["model"] == model]["average"].values
+
         # Plotting
         plt.scatter(df1_model_data, df2_model_data, label=model)
         m_name = undo_hyperlink(model)
         if m_name == "No text found":
             m_name = "Random"
-        # Add text above each point like 
+        # Add text above each point like
         # plt.text(x[i] + 0.1, y[i] + 0.1, label, ha='left', va='bottom')
-        plt.text(df1_model_data - .005, df2_model_data, m_name, horizontalalignment='right', verticalalignment='center')
+        plt.text(
+            df1_model_data - 0.005, df2_model_data, m_name, horizontalalignment="right", verticalalignment="center"
+        )
 
     # add correlation line to scatter plot
     # first, compute correlation
-    corr = df1['average'].corr(df2['average'])
+    corr = df1["average"].corr(df2["average"])
     # add correlation line based on corr
-    
-
 
-    plt.xlabel('HERM Eval. Set Avg.', fontsize=16)
-    plt.ylabel('Pref. Test Sets Avg.', fontsize=16)
+    plt.xlabel("HERM Eval. Set Avg.", fontsize=16)
+    plt.ylabel("Pref. Test Sets Avg.", fontsize=16)
     # plt.legend(title='Model', bbox_to_anchor=(1.05, 1), loc='upper left')
-    return plt
+    return plt

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

src/utils.py

@@ -1,9 +1,10 @@
-import pandas as pd
-from pathlib import Path
-from datasets import load_dataset
-import numpy as np
 import os
 import re
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+from datasets import load_dataset
 
 UNVERIFIED_MODELS = [
     "nvidia/Nemotron-4-340B-Reward",
@@ -35,9 +36,10 @@ CONTAMINATED_MODELS = [
     "SF-Foundation/TextEval-Llama3.1-70B",
     "ZiyiYe/Con-J-Qwen2-7B",
     "Ray2333/Gemma-2B-rewardmodel-ft",
-    "Ray2333/GRM-Gemma-2B-rewardmodel-ft"
+    "Ray2333/GRM-Gemma-2B-rewardmodel-ft",
 ]
 
+
 # From Open LLM Leaderboard
 def model_hyperlink(link, model_name):
     # if model_name is above 50 characters, return first 47 characters and "..."
@@ -63,9 +65,10 @@ def model_hyperlink(link, model_name):
         output += " ⚠️"
     return output
 
+
 def undo_hyperlink(html_string):
     # Regex pattern to match content inside > and <
-    pattern = r'>[^<]+<'
+    pattern = r">[^<]+<"
     match = re.search(pattern, html_string)
     if match:
         # Extract the matched text and remove leading '>' and trailing '<'
@@ -75,7 +78,7 @@ def undo_hyperlink(html_string):
 
 
 # Define a function to fetch and process data
-def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to pull the git repo
+def load_all_data(data_repo, subdir: str, subsubsets=False):  # use HF api to pull the git repo
     dir = Path(data_repo)
     data_dir = dir / subdir
     orgs = [d for d in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, d))]
@@ -93,21 +96,20 @@ def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to p
 
     # load all json data in the list models_results one by one to avoid not having the same entries
     for model in models_results:
-        model_data = load_dataset("json", data_files=data_repo + subdir+ "/" + model, split="train")
+        model_data = load_dataset("json", data_files=data_repo + subdir + "/" + model, split="train")
         df2 = pd.DataFrame(model_data)
         # add to df
         df = pd.concat([df2, df])
 
-
     # remove chat_template comlumn
     df = df.drop(columns=["chat_template"])
 
     # sort columns alphabetically
     df = df.reindex(sorted(df.columns), axis=1)
-    
+
     # move column "model" to the front
     cols = list(df.columns)
-    cols.insert(0, cols.pop(cols.index('model')))
+    cols.insert(0, cols.pop(cols.index("model")))
     df = df.loc[:, cols]
 
     # select all columns except "model"
@@ -123,7 +125,7 @@ def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to p
     if "model_beaker" in cols:
         cols.remove("model_beaker")
         df = df.drop(columns=["model_beaker"])
-    
+
     # remove column xstest (outdated data)
     # if xstest is a column
     if "xstest" in cols:
@@ -148,24 +150,24 @@ def load_all_data(data_repo, subdir:str, subsubsets=False):    # use HF api to p
         df = df.drop(columns=["pku_safer"])
         cols.remove("pku_safer")
 
-    # convert to score 
-    df[cols] = (df[cols]*100)
-    avg = np.nanmean(df[cols].values,axis=1)
+    # convert to score
+    df[cols] = df[cols] * 100
+    avg = np.nanmean(df[cols].values, axis=1)
     # add average column
     df["average"] = avg
-    
+
     # apply model_hyperlink function to column "model"
     df["model"] = df["model"].apply(lambda x: model_hyperlink(f"https://huggingface.co/{x}", x))
 
     # move average column to the second
     cols = list(df.columns)
-    cols.insert(1, cols.pop(cols.index('average')))
+    cols.insert(1, cols.pop(cols.index("average")))
     df = df.loc[:, cols]
 
     # move model_type column to first
     if "model_type" in cols:
         cols = list(df.columns)
-        cols.insert(1, cols.pop(cols.index('model_type')))
+        cols.insert(1, cols.pop(cols.index("model_type")))
         df = df.loc[:, cols]
 
     # remove models with DPO Ref. Free as type (future work)