app.py

import gradio as gr
import pandas as pd
import plotly.express as px
import time
from datasets import load_dataset
# Using the stable, community-built RangeSlider component
from gradio_rangeslider import RangeSlider

# --- Constants ---
PARAM_CHOICES = ['< 1B', '1B', '5B', '12B', '32B', '64B', '128B', '256B', '> 500B']
PARAM_CHOICES_DEFAULT_INDICES = (0, len(PARAM_CHOICES) - 1)

TOP_K_CHOICES = list(range(5, 51, 5))
HF_DATASET_ID = "evijit/modelverse_daily_data"
TAG_FILTER_CHOICES = [ "Audio & Speech", "Time series", "Robotics", "Music", "Video", "Images", "Text", "Biomedical", "Sciences" ]
PIPELINE_TAGS = [ 'text-generation', 'text-to-image', 'text-classification', 'text2text-generation', 'audio-to-audio', 'feature-extraction', 'image-classification', 'translation', 'reinforcement-learning', 'fill-mask', 'text-to-speech', 'automatic-speech-recognition', 'image-text-to-text', 'token-classification', 'sentence-similarity', 'question-answering', 'image-feature-extraction', 'summarization', 'zero-shot-image-classification', 'object-detection', 'image-segmentation', 'image-to-image', 'image-to-text', 'audio-classification', 'visual-question-answering', 'text-to-video', 'zero-shot-classification', 'depth-estimation', 'text-ranking', 'image-to-video', 'multiple-choice', 'unconditional-image-generation', 'video-classification', 'text-to-audio', 'time-series-forecasting', 'any-to-any', 'video-text-to-text', 'table-question-answering' ]


def load_models_data():
    overall_start_time = time.time()
    print(f"Attempting to load dataset from Hugging Face Hub: {HF_DATASET_ID}")
    try:
        dataset_dict = load_dataset(HF_DATASET_ID)
        df = dataset_dict[list(dataset_dict.keys())[0]].to_pandas()
        if 'params' in df.columns:
            # IMPORTANT CHANGE: Fill NaN/coerce errors with -1 to signify unknown size
            # This aligns with the utility function's return of -1.0 for unknown sizes.
            df['params'] = pd.to_numeric(df['params'], errors='coerce').fillna(-1)
        else:
            # If 'params' column doesn't exist, assume all are unknown
            df['params'] = -1
        msg = f"Successfully loaded dataset in {time.time() - overall_start_time:.2f}s."
        print(msg)
        return df, True, msg
    except Exception as e:
        err_msg = f"Failed to load dataset. Error: {e}"
        print(err_msg)
        return pd.DataFrame(), False, err_msg

def get_param_range_values(param_range_labels):
    min_label, max_label = param_range_labels
    min_val = 0.0 if '<' in min_label else float(min_label.replace('B', ''))
    max_val = float('inf') if '>' in max_label else float(max_label.replace('B', ''))
    return min_val, max_val

def make_treemap_data(df, count_by, top_k=25, tag_filter=None, pipeline_filter=None, param_range=None, skip_orgs=None, include_unknown_param_size=True):
    if df is None or df.empty: return pd.DataFrame()
    filtered_df = df.copy()

    # New: Filter based on unknown parameter size
    # If include_unknown_param_size is False, exclude models where params is -1 (unknown)
    if not include_unknown_param_size and 'params' in filtered_df.columns:
        filtered_df = filtered_df[filtered_df['params'] != -1]

    col_map = { "Audio & Speech": "is_audio_speech", "Music": "has_music", "Robotics": "has_robot", "Biomedical": "is_biomed", "Time series": "has_series", "Sciences": "has_science", "Video": "has_video", "Images": "has_image", "Text": "has_text" }
    if tag_filter and tag_filter in col_map and col_map[tag_filter] in filtered_df.columns:
        filtered_df = filtered_df[filtered_df[col_map[tag_filter]]]
    if pipeline_filter and "pipeline_tag" in filtered_df.columns:
        filtered_df = filtered_df[filtered_df["pipeline_tag"].astype(str) == pipeline_filter]
    if param_range:
        min_params, max_params = get_param_range_values(param_range)
        is_default_range = (param_range[0] == PARAM_CHOICES[0] and param_range[1] == PARAM_CHOICES[-1])
        # Apply parameter range filter only if it's not the default (all range) AND params column exists
        # This filter will naturally exclude -1 if the min_params is >= 0, as it should.
        if not is_default_range and 'params' in filtered_df.columns:
            if min_params is not None: filtered_df = filtered_df[filtered_df['params'] >= min_params]
            if max_params is not None and max_params != float('inf'): filtered_df = filtered_df[filtered_df['params'] < max_params]

    if skip_orgs and len(skip_orgs) > 0 and "organization" in filtered_df.columns:
        filtered_df = filtered_df[~filtered_df["organization"].isin(skip_orgs)]
    if filtered_df.empty: return pd.DataFrame()
    if count_by not in filtered_df.columns: filtered_df[count_by] = 0.0
    filtered_df[count_by] = pd.to_numeric(filtered_df[count_by], errors='coerce').fillna(0.0)
    org_totals = filtered_df.groupby("organization")[count_by].sum().nlargest(top_k, keep='first')
    top_orgs_list = org_totals.index.tolist()
    treemap_data = filtered_df[filtered_df["organization"].isin(top_orgs_list)][["id", "organization", count_by]].copy()
    treemap_data["root"] = "models"
    return treemap_data

def create_treemap(treemap_data, count_by, title=None):
    if treemap_data.empty:
        fig = px.treemap(names=["No data matches filters"], parents=[""], values=[1])
        fig.update_layout(title="No data matches the selected filters", margin=dict(t=50, l=25, r=25, b=25))
        return fig
    fig = px.treemap(treemap_data, path=["root", "organization", "id"], values=count_by, title=title, color_discrete_sequence=px.colors.qualitative.Plotly)
    fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))
    fig.update_traces(textinfo="label+value+percent root", hovertemplate="<b>%{label}</b><br>%{value:,} " + count_by + "<br>%{percentRoot:.2%} of total<extra></extra>")
    return fig

# --- FINAL, CORRECTED CSS ---
custom_css = """
/* Hide the extra UI elements from the RangeSlider component */
#param-slider-wrapper .head,
#param-slider-wrapper div[data-testid="range-slider"] > span {
    display: none !important;
}
/* 
  THIS IS THE KEY FIX:
  We target all the individual component containers (divs with class .block)
  that are *direct children* of our custom-classed group.
  
  This removes the "box-in-a-box" effect by making the inner component
  containers transparent. The parent gr.Group now acts as the single card,
  which is exactly what we want.
*/
.model-parameters-group > .block {
    background: none !important;
    border: none !important;
    box-shadow: none !important;
}
"""

with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css) as demo:
    models_data_state = gr.State(pd.DataFrame())
    loading_complete_state = gr.State(False)
    
    with gr.Row():
        gr.Markdown("# 🤗 ModelVerse Explorer")

    with gr.Row():
        with gr.Column(scale=1):
            
            # This section remains un-grouped for a consistent flat look
            count_by_dropdown = gr.Dropdown(label="Metric", choices=[("Downloads (last 30 days)", "downloads"), ("Downloads (All Time)", "downloadsAllTime"), ("Likes", "likes")], value="downloads")
            filter_choice_radio = gr.Radio(label="Filter Type", choices=["None", "Tag Filter", "Pipeline Filter"], value="None")
            
            tag_filter_dropdown = gr.Dropdown(label="Select Tag", choices=TAG_FILTER_CHOICES, value=None, visible=False)
            pipeline_filter_dropdown = gr.Dropdown(label="Select Pipeline Tag", choices=PIPELINE_TAGS, value=None, visible=False)
            
            # This group's styling will be modified by the custom CSS
            with gr.Group(elem_classes="model-parameters-group"):
                gr.Markdown("<div style='font-weight: 500;'>Model Parameters</div>")
                param_range_slider = RangeSlider(
                    minimum=0,
                    maximum=len(PARAM_CHOICES) - 1,
                    value=PARAM_CHOICES_DEFAULT_INDICES,
                    step=1,
                    label=None,
                    show_label=False,
                    elem_id="param-slider-wrapper"
                )
                param_range_display = gr.Markdown(f"Range: `{PARAM_CHOICES[0]}` to `{PARAM_CHOICES[-1]}`")
                # New: Checkbox for including unknown parameter sizes
                include_unknown_params_checkbox = gr.Checkbox(label="Include models with unknown parameter size", value=True)
            
            # This section remains un-grouped
            top_k_dropdown = gr.Dropdown(label="Number of Top Organizations", choices=TOP_K_CHOICES, value=25)
            skip_orgs_textbox = gr.Textbox(label="Organizations to Skip (comma-separated)", value="TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski")
            
            generate_plot_button = gr.Button(value="Generate Plot", variant="primary", interactive=False)

        with gr.Column(scale=3):
            plot_output = gr.Plot()
            status_message_md = gr.Markdown("Initializing...")
            data_info_md = gr.Markdown("")
    
    def update_param_display(value: tuple):
        min_idx, max_idx = int(value[0]), int(value[1])
        return f"Range: `{PARAM_CHOICES[min_idx]}` to `{PARAM_CHOICES[max_idx]}`"
    
    # New function to toggle the unknown params checkbox interactivity
    def _toggle_unknown_params_checkbox(param_range_indices):
        min_idx, max_idx = int(param_range_indices[0]), int(param_range_indices[1])
        is_default_range = (min_idx == PARAM_CHOICES_DEFAULT_INDICES[0] and 
                            max_idx == PARAM_CHOICES_DEFAULT_INDICES[1])
        
        # If a specific range is selected (not the default all-inclusive range), disable the checkbox
        # and uncheck it to ensure consistency.
        if not is_default_range:
            return gr.update(interactive=False, value=False) # Disable and uncheck
        else:
            return gr.update(interactive=True) # Enable
            
    param_range_slider.change(update_param_display, param_range_slider, param_range_display)
    # Connect the new toggle function to the param_range_slider's change event
    param_range_slider.change(
        fn=_toggle_unknown_params_checkbox, 
        inputs=[param_range_slider], 
        outputs=[include_unknown_params_checkbox]
    )

    def _update_button_interactivity(is_loaded_flag): return gr.update(interactive=is_loaded_flag)
    loading_complete_state.change(fn=_update_button_interactivity, inputs=loading_complete_state, outputs=generate_plot_button)

    def _toggle_filters_visibility(choice): 
        return gr.update(visible=choice == "Tag Filter"), gr.update(visible=choice == "Pipeline Filter")
    filter_choice_radio.change(fn=_toggle_filters_visibility, inputs=filter_choice_radio, outputs=[tag_filter_dropdown, pipeline_filter_dropdown])

    ## CHANGE: Renamed and modified ui_load_data_controller to also generate the initial plot
    def load_and_generate_initial_plot(progress=gr.Progress()):
        progress(0, desc=f"Loading dataset '{HF_DATASET_ID}'...")
        # --- Part 1: Data Loading ---
        try:
            current_df, load_success_flag, status_msg_from_load = load_models_data()
            if load_success_flag:
                progress(0.5, desc="Processing data...")
                date_display = "Pre-processed (date unavailable)"
                if 'data_download_timestamp' in current_df.columns and pd.notna(current_df['data_download_timestamp'].iloc[0]):
                    ts = pd.to_datetime(current_df['data_download_timestamp'].iloc[0], utc=True)
                    date_display = ts.strftime('%B %d, %Y, %H:%M:%S %Z')
                # Count models where params is not -1 (known size)
                param_count = (current_df['params'] != -1).sum() if 'params' in current_df.columns else 0
                unknown_param_count = (current_df['params'] == -1).sum() if 'params' in current_df.columns else 0

                data_info_text = f"### Data Information\n- Source: `{HF_DATASET_ID}`\n- Status: {status_msg_from_load}\n- Total models loaded: {len(current_df):,}\n- Models with known parameter counts: {param_count:,}\n- Models with unknown parameter counts: {unknown_param_count:,}\n- Data as of: {date_display}\n"
            else:
                data_info_text = f"### Data Load Failed\n- {status_msg_from_load}"
        except Exception as e:
            status_msg_from_load = f"An unexpected error occurred: {str(e)}"
            data_info_text = f"### Critical Error\n- {status_msg_from_load}"
            load_success_flag = False
            current_df = pd.DataFrame()
            print(f"Critical error in load_and_generate_initial_plot: {e}")

        # --- Part 2: Generate Initial Plot ---
        progress(0.6, desc="Generating initial plot...")
        # Get default values directly from the UI component definitions
        default_metric = "downloads"
        default_filter_type = "None"
        default_tag = None
        default_pipeline = None
        default_param_indices = PARAM_CHOICES_DEFAULT_INDICES
        default_k = 25
        default_skip_orgs = "TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski"
        # New default: include unknown params initially (matches checkbox default)
        default_include_unknown_params = True 

        # Reuse the existing controller function for plotting
        initial_plot, initial_status = ui_generate_plot_controller(
            default_metric, default_filter_type, default_tag, default_pipeline, 
            default_param_indices, default_k, default_skip_orgs, default_include_unknown_params, current_df, progress
        )
        
        # Return all the necessary updates for the UI
        return current_df, load_success_flag, data_info_text, initial_status, initial_plot

    def ui_generate_plot_controller(metric_choice, filter_type, tag_choice, pipeline_choice, 
                                   param_range_indices, k_orgs, skip_orgs_input, include_unknown_param_size_flag, df_current_models, progress=gr.Progress()):
        if df_current_models is None or df_current_models.empty:
            return create_treemap(pd.DataFrame(), metric_choice, "Error: Model Data Not Loaded"), "Model data is not loaded. Cannot generate plot."
        
        progress(0.1, desc="Preparing data...")
        tag_to_use = tag_choice if filter_type == "Tag Filter" else None
        pipeline_to_use = pipeline_choice if filter_type == "Pipeline Filter" else None
        orgs_to_skip = [org.strip() for org in skip_orgs_input.split(',') if org.strip()]
        
        min_label = PARAM_CHOICES[int(param_range_indices[0])]
        max_label = PARAM_CHOICES[int(param_range_indices[1])]
        param_labels_for_filtering = [min_label, max_label]
        
        treemap_df = make_treemap_data(
            df_current_models, 
            metric_choice, 
            k_orgs, 
            tag_to_use, 
            pipeline_to_use, 
            param_labels_for_filtering, 
            orgs_to_skip, 
            include_unknown_param_size_flag # Pass the new flag
        )
        
        progress(0.7, desc="Generating plot...")
        title_labels = {"downloads": "Downloads (last 30 days)", "downloadsAllTime": "Downloads (All Time)", "likes": "Likes"}
        chart_title = f"HuggingFace Models - {title_labels.get(metric_choice, metric_choice)} by Organization"
        plotly_fig = create_treemap(treemap_df, metric_choice, chart_title)
        
        if treemap_df.empty:
            plot_stats_md = "No data matches the selected filters. Please try different options."
        else:
            total_items_in_plot = len(treemap_df['id'].unique())
            total_value_in_plot = treemap_df[metric_choice].sum()
            plot_stats_md = f"## Plot Statistics\n- **Models shown**: {total_items_in_plot:,}\n- **Total {metric_choice}**: {int(total_value_in_plot):,}"
        return plotly_fig, plot_stats_md

    ## CHANGE: Updated demo.load to call the new function and to add plot_output to the outputs list
    demo.load(
        fn=load_and_generate_initial_plot, 
        inputs=[], 
        outputs=[models_data_state, loading_complete_state, data_info_md, status_message_md, plot_output]
    )

    generate_plot_button.click(
        fn=ui_generate_plot_controller,
        inputs=[count_by_dropdown, filter_choice_radio, tag_filter_dropdown, pipeline_filter_dropdown,
                param_range_slider, top_k_dropdown, skip_orgs_textbox, include_unknown_params_checkbox, models_data_state], # Add checkbox to inputs
        outputs=[plot_output, status_message_md]
    )

if __name__ == "__main__":
    print(f"Application starting...")
    demo.queue().launch()