Test count display, important models, typo fix

README.md

@@ -12,11 +12,11 @@ short_description: A dashboard
 
 # TCID
 
-This space displays the state of the `transformers` CI on two hardwares, for a subset of models. The CI is run daily, on both AMD MI325 and Nvidia A10. The CI runs a different number of tests for each model. When a test finishes, it is assigned a status depending on its outcome:
+This space displays the state of the `transformers` CI on two hardwares, for a **subset of models**. The CI is run daily, on both AMD MI325 and Nvidia A10. The CI runs a different number of tests for each model. When a test finishes, it is assigned a status depending on its outcome:
 
 - passed: the test finsihed and the expected output (or outputs) were retrieved;
 - failed: the test either did not finish or the output was different from the expected output;
-- skipped: the test was not run, which usually happens when a test is incompatible with a model. For instance, some models skip `flash-attention`-related tests because they are incompatible with `flash-attention`;
+- skipped: the test was not run, but it not expected to. More details on this at the end of the README;
 - error: the test did not finish and python crashed;
 
 The dashboard is divided in two main parts:
@@ -26,8 +26,18 @@ The dashboard is divided in two main parts:
 On the summary page, you can see a snapshot of the mix of test passed, failed and skipped for each model. The summary page also features an "Overall failures rate" for AMD and NVIDIA, which is computed this way:
 ```overall_failure_rate = (failed + error) / (passed + failed + error)``` 
 
-We do not account for the test skipped in this overall failure rate, because skipped test have no chance to neither pass nor fail.
+We do not account for the test skipped in this overall failure rate, because skipped test have no chance to neither pass nor fail.  
+We only consider the tests for a **subset of models** out of all the models supported in `transformers`. This subset is named important models, and is mainly defined by model usage.
 
 ## Models page
 
 From the sidebar, you can access a detailled view of each model. In it, you will find the breakdown of test statuses and the names of the test that failed for single and multi-gpu runs.
+
+## Skipped test
+
+You can probably see many skipped tests in the `transformers` CI, which be perplexing. When a test is skipped, it's usually one of three reasons:
+- the test requires a package that is not included in the default transformers docker that the CI uses, like flash attention 3 or deepspeed;
+- the hardware is not the correct one, for instance there are a bunch of MPS (apple hardware) tests that are of course not run on AMD or Nvidia CI;
+- the model is incompatible with what the test is for, say torch.fx or flash-attention, which are incompatible with some models architecture;
+
+Skipping tests rather than not collecting them offers the advantage of having similar test counts across CIs that do not run on the same hardware. Thus, it total test count differs between two CIs, one can immediately know that one of the two only ran partialy. This would not be the case if some skipped tests were not collected at all.

app.py

@@ -61,7 +61,7 @@ def get_description_text():
     ]
     msg = ["**" + x + "**" for x in msg] + [""]
     if Ci_results.latest_update_msg:
-        msg.append(f"*({Ci_results.latest_update_msg})*")
+        msg.append(f"*This dashboard only tracks important models*<br>*({Ci_results.latest_update_msg})*")
     else:
         msg.append("*(loading...)*")
     return "<br>".join(msg)

data.py

@@ -19,7 +19,7 @@ IMPORTANT_MODELS = [
     "vit",  # old (vision) - fixed comma
     "clip",  # old but dominant (vision)
     "detr",  # objection detection, segmentation (vision)
-    "table-transformer",  # objection detection (visioin) - maybe just detr?
+    "table_transformer",  # objection detection (visioin) - maybe just detr?
     "got_ocr2",  # ocr (vision)
     "whisper",  # old but dominant (audio)
     "wav2vec2",  # old (audio)

summary_page.py

@@ -38,15 +38,17 @@ LABEL_OFFSET = 1  # Distance of label from bar
 FAILURE_RATE_FONT_SIZE = 28
 
 
-def calculate_overall_failure_rates(df: pd.DataFrame, available_models: list[str]) -> tuple[float, float]:
+def get_overall_stats(df: pd.DataFrame, available_models: list[str]) -> tuple[list[int], list[int]]:
     """Calculate overall failure rates for AMD and NVIDIA across all models."""
     if df.empty or not available_models:
         return 0.0, 0.0
     
-    total_amd_tests = 0
-    total_amd_failures = 0
-    total_nvidia_tests = 0
-    total_nvidia_failures = 0
+    total_amd_passed = 0
+    total_amd_failed = 0
+    total_amd_skipped = 0
+    total_nvidia_passed = 0
+    total_nvidia_failed = 0
+    total_nvidia_skipped = 0
     
     for model_name in available_models:
         if model_name not in df.index:
@@ -56,21 +58,16 @@ def calculate_overall_failure_rates(df: pd.DataFrame, available_models: list[str
         amd_stats, nvidia_stats = extract_model_data(row)[:2]
         
         # AMD totals
-        amd_total = amd_stats['passed'] + amd_stats['failed'] + amd_stats['error']
-        if amd_total > 0:
-            total_amd_tests += amd_total
-            total_amd_failures += amd_stats['failed'] + amd_stats['error']
-        
+        total_amd_passed += amd_stats['passed']
+        total_amd_failed += amd_stats['failed'] + amd_stats['error']
+        total_amd_skipped += amd_stats['skipped']
+
         # NVIDIA totals
-        nvidia_total = nvidia_stats['passed'] + nvidia_stats['failed'] + nvidia_stats['error']
-        if nvidia_total > 0:
-            total_nvidia_tests += nvidia_total
-            total_nvidia_failures += nvidia_stats['failed'] + nvidia_stats['error']
-    
-    amd_failure_rate = (total_amd_failures / total_amd_tests * 100) if total_amd_tests > 0 else 0.0
-    nvidia_failure_rate = (total_nvidia_failures / total_nvidia_tests * 100) if total_nvidia_tests > 0 else 0.0
+        total_nvidia_passed += nvidia_stats['passed']
+        total_nvidia_failed += nvidia_stats['failed'] + nvidia_stats['error']
+        total_nvidia_skipped += nvidia_stats['skipped']
     
-    return amd_failure_rate, nvidia_failure_rate
+    return [total_amd_passed, total_amd_failed, total_amd_skipped], [total_nvidia_passed, total_nvidia_failed, total_nvidia_skipped]
 
 
 def draw_text_and_bar(
@@ -118,7 +115,12 @@ def create_summary_page(df: pd.DataFrame, available_models: list[str]) -> plt.Fi
         return fig
     
     # Calculate overall failure rates
-    amd_failure_rate, nvidia_failure_rate = calculate_overall_failure_rates(df, available_models)
+    amd_counts, nvidia_counts = get_overall_stats(df, available_models)
+
+    amd_failure_rate = (amd_counts[1] / sum(amd_counts)) if sum(amd_counts) > 0 else 0.0
+    amd_failure_rate *= 100
+    nvidia_failure_rate = (nvidia_counts[1] / sum(nvidia_counts)) if sum(nvidia_counts) > 0 else 0.0
+    nvidia_failure_rate *= 100
 
     # Calculate dimensions for N-column layout
     model_count = len(available_models)
@@ -181,6 +183,26 @@ def create_summary_page(df: pd.DataFrame, available_models: list[str]) -> plt.Fi
         visible_model_count += 1
 
 
+    # Add AMD and NVIDIA test totals in the bottom left
+    # Calculate line spacing to align middle with legend
+    line_height = 0.4  # Height between lines
+    legend_y = max_y + 1
+
+    # Position the two lines so their middle aligns with legend_y
+    amd_y = legend_y - line_height / 2
+    nvidia_y = legend_y + line_height / 2
+
+    amd_totals_text =    f"AMD Tests    - Passed: {amd_counts[0]}, Failed: {amd_counts[1]}, Skipped: {amd_counts[2]}"
+    nvidia_totals_text = f"NVIDIA Tests - Passed: {nvidia_counts[0]}, Failed: {nvidia_counts[1]}, Skipped: {nvidia_counts[2]}"
+
+    ax.text(0, amd_y, amd_totals_text,
+           ha='left', va='bottom', color='#CCCCCC',
+           fontsize=14, fontfamily='monospace')
+
+    ax.text(0, nvidia_y, nvidia_totals_text,
+           ha='left', va='bottom', color='#CCCCCC',
+           fontsize=14, fontfamily='monospace')
+
     # Add legend horizontally in bottom right corner
     patch_height = 0.3
     patch_width = 3
@@ -190,12 +212,6 @@ def create_summary_page(df: pd.DataFrame, available_models: list[str]) -> plt.Fi
     legend_spacing = 10
     legend_font_size = 15
 
-    # Add failure rate explanation text on the left
-    # explanation_text = "Failure rate = failed / (passed + failed)"
-    # ax.text(0, legend_y, explanation_text, 
-    #        ha='left', va='bottom', color='#CCCCCC', 
-    #        fontsize=legend_font_size, fontfamily='monospace', style='italic')
-    
     # Legend entries
     legend_items = [
         ('passed', 'Passed'),