Actual latest data refresh and clickable link

app.py

@@ -60,10 +60,10 @@ def get_description_text():
         "NVIDIA runs on A10",
     ]
     msg = ["**" + x + "**" for x in msg] + [""]
-    if Ci_results.last_update_time:
-        msg.append(f"*Result overview by model and hardware (last updated: {Ci_results.last_update_time})*")
+    if Ci_results.latest_update_msg:
+        msg.append(f"*Result overview by hardware for important models ({Ci_results.latest_update_msg})*")
     else:
-        msg.append("*Result overview by model and hardware (loading...)*")
+        msg.append("*Result overview by hardware for important models (loading...)*")
     return "<br>".join(msg)
 
 # Load CSS from external file

data.py

@@ -5,6 +5,7 @@ from datetime import datetime
 import threading
 import traceback
 import json
+import re
 
 # NOTE: if caching is an issue, try adding `use_listings_cache=False`
 fs = HfFileSystem()
@@ -49,27 +50,69 @@ KEYS_TO_KEEP = [
     "job_link_nvidia",
 ]
 
-def read_one_dataframe(json_path: str, device_label: str) -> pd.DataFrame:
-    logger.info(f"Reading df located at {json_path}")
+
+def log_dataframe_link(link: str) -> str:
+    """
+    Adds the link to the dataset in the logs, modifies it to get a clockable link and then returns the date of the 
+    report.
+    """
+    logger.info(f"Reading df located at {link}")
+    # Make sure the links starts with an http adress
+    if link.startswith("hf://"):
+        link = "https://huggingface.co/" + link.removeprefix("hf://")
+    # Pattern to match transformers_daily_ci followed by any path, then a date (YYYY-MM-DD format)
+    pattern = r'transformers_daily_ci(.*?)/(\d{4}-\d{2}-\d{2})'
+    match = re.search(pattern, link)
+    # Failure case: 
+    if not match:
+        logger.error("Could not find transformers_daily_ci and.or date in the link")
+        return "9999-99-99"
+    # Replace the path between with blob/main
+    path_between = match.group(1)
+    link = link.replace("transformers_daily_ci" + path_between, "transformers_daily_ci/blob/main")
+    logger.info(f"Link to data source: {link}")
+    # Return the date
+    return match.group(2)
+
+def infer_latest_update_msg(date_df_amd: str, date_df_nvidia: str) -> str:
+    # Early return if one of the dates is invalid
+    if date_df_amd.startswith("9999") and date_df_nvidia.startswith("9999"):
+        return "could not find last update time"
+    # Warn if dates are not the same
+    if date_df_amd != date_df_nvidia:
+        logger.warning(f"Different dates found: {date_df_amd} (AMD) vs {date_df_nvidia} (NVIDIA)")
+    # Take the latest date and format it
+    try:
+        latest_date = max(date_df_amd, date_df_nvidia)
+        yyyy, mm, dd = latest_date.split("-")
+        return f"last updated {mm}/{dd}/{yyyy}"
+    except Exception as e:
+        logger.error(f"When trying to infer latest date, got error {e}")
+        return "could not find last update time"
+
+def read_one_dataframe(json_path: str, device_label: str) -> tuple[pd.DataFrame, str]:
+    df_upload_date = log_dataframe_link(json_path)
     df = pd.read_json(json_path, orient="index")
     df.index.name = "model_name"
     df[f"failed_multi_no_{device_label}"] = df["failures"].apply(lambda x: len(x["multi"]) if "multi" in x else 0)
     df[f"failed_single_no_{device_label}"] = df["failures"].apply(lambda x: len(x["single"]) if "single" in x else 0)
-    return df
+    return df, df_upload_date
 
-def get_distant_data() -> pd.DataFrame:
+def get_distant_data() -> tuple[pd.DataFrame, str]:
     # Retrieve AMD dataframe
     amd_src = "hf://datasets/optimum-amd/transformers_daily_ci/**/runs/**/ci_results_run_models_gpu/model_results.json"
     files_amd = sorted(fs.glob(amd_src, refresh=True), reverse=True)
-    df_amd = read_one_dataframe(f"hf://{files_amd[0]}", "amd")
+    df_amd, date_df_amd = read_one_dataframe(f"hf://{files_amd[0]}", "amd")
     # Retrieve NVIDIA dataframe, which pattern should be:
-    # hf://datasets/hf-internal-testing/transformers_daily_ci/raw/main/YYYY-MM-DD/ci_results_run_models_gpu/model_results.json 
+    # hf://datasets/hf-internal-testing`/transformers_daily_ci/raw/main/YYYY-MM-DD/ci_results_run_models_gpu/model_results.json 
     nvidia_src = "hf://datasets/hf-internal-testing/transformers_daily_ci/*/ci_results_run_models_gpu/model_results.json"
     files_nvidia = sorted(fs.glob(nvidia_src, refresh=True), reverse=True)
     # NOTE: should this be removeprefix instead of lstrip?
     nvidia_path = files_nvidia[0].lstrip('datasets/hf-internal-testing/transformers_daily_ci/')
     nvidia_path = "https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/main/" + nvidia_path
-    df_nvidia = read_one_dataframe(nvidia_path, "nvidia")
+    df_nvidia, date_df_nvidia = read_one_dataframe(nvidia_path, "nvidia")
+    # Infer and format the latest df date
+    latest_update_msg = infer_latest_update_msg(date_df_amd, date_df_nvidia)
     # Join both dataframes
     joined = df_amd.join(df_nvidia, rsuffix="_nvidia", lsuffix="_amd", how="outer")
     joined = joined[KEYS_TO_KEEP]
@@ -81,7 +124,7 @@ def get_distant_data() -> pd.DataFrame:
     for model in IMPORTANT_MODELS:
         if model not in filtered_joined.index:
             print(f"[WARNING] Model {model} was missing from index.")            
-    return filtered_joined
+    return filtered_joined, latest_update_msg
 
 
 def get_sample_data() -> pd.DataFrame:
@@ -140,14 +183,15 @@ class CIResults:
     def __init__(self):
         self.df = pd.DataFrame()
         self.available_models = []
-        self.last_update_time = ""
+        self.latest_update_msg = ""
 
     def load_data(self) -> None:
         """Load data from the data source."""
         # Try loading the distant data, and fall back on sample data for local tinkering
         try:
             logger.info("Loading distant data...")
-            new_df = get_distant_data()
+            new_df, latest_update_msg = get_distant_data()
+            self.latest_update_msg = latest_update_msg
         except Exception as e:
             error_msg = [
                 "Loading data failed:",
@@ -161,11 +205,10 @@ class CIResults:
         # Update attributes
         self.df = new_df
         self.available_models = new_df.index.tolist()
-        self.last_update_time = datetime.now().strftime('%H:%M')
         # Log and return distant load status
         logger.info(f"Data loaded successfully: {len(self.available_models)} models")
         logger.info(f"Models: {self.available_models[:5]}{'...' if len(self.available_models) > 5 else ''}")
-        logger.info(f"Last update: {self.last_update_time}")
+        logger.info(f"Latest update message: {self.latest_update_msg}")
         # Log a preview of the df
         msg = {}
         for model in self.available_models[:3]: