evijit HF Staff commited on
Commit
d858aa5
·
verified ·
1 Parent(s): b06975a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +81 -50
app.py CHANGED
@@ -4,43 +4,82 @@ import gradio as gr
4
  import pandas as pd
5
  import plotly.express as px
6
  import time
 
7
  from datasets import load_dataset
8
 
9
  # --- Constants ---
10
  PARAM_CHOICES = ['< 1B', '1B', '5B', '12B', '32B', '64B', '128B', '256B', '> 500B']
11
- PARAM_CHOICES_DEFAULT_INDICES = [0, len(PARAM_CHOICES) - 1]
12
 
13
- # --- NEW: Define choices for the Top-K dropdown ---
14
  TOP_K_CHOICES = list(range(5, 51, 5))
15
-
16
  HF_DATASET_ID = "evijit/orgstats_daily_data"
17
  TAG_FILTER_CHOICES = [ "Audio & Speech", "Time series", "Robotics", "Music", "Video", "Images", "Text", "Biomedical", "Sciences" ]
18
  PIPELINE_TAGS = [ 'text-generation', 'text-to-image', 'text-classification', 'text2text-generation', 'audio-to-audio', 'feature-extraction', 'image-classification', 'translation', 'reinforcement-learning', 'fill-mask', 'text-to-speech', 'automatic-speech-recognition', 'image-text-to-text', 'token-classification', 'sentence-similarity', 'question-answering', 'image-feature-extraction', 'summarization', 'zero-shot-image-classification', 'object-detection', 'image-segmentation', 'image-to-image', 'image-to-text', 'audio-classification', 'visual-question-answering', 'text-to-video', 'zero-shot-classification', 'depth-estimation', 'text-ranking', 'image-to-video', 'multiple-choice', 'unconditional-image-generation', 'video-classification', 'text-to-audio', 'time-series-forecasting', 'any-to-any', 'video-text-to-text', 'table-question-answering' ]
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  def load_models_data():
21
  overall_start_time = time.time()
22
  print(f"Attempting to load dataset from Hugging Face Hub: {HF_DATASET_ID}")
23
  try:
24
  dataset_dict = load_dataset(HF_DATASET_ID)
25
- if not dataset_dict: raise ValueError(f"Dataset '{HF_DATASET_ID}' loaded but appears empty.")
26
- split_name = list(dataset_dict.keys())[0]
27
- df = dataset_dict[split_name].to_pandas()
28
- elapsed = time.time() - overall_start_time
29
  if 'params' in df.columns:
30
  df['params'] = pd.to_numeric(df['params'], errors='coerce').fillna(0)
31
  else:
32
  df['params'] = 0
33
- print("CRITICAL WARNING: 'params' column not found in data. Parameter filtering will not work.")
34
- msg = f"Successfully loaded dataset '{HF_DATASET_ID}' (split: {split_name}) from HF Hub in {elapsed:.2f}s. Shape: {df.shape}"
35
  print(msg)
36
  return df, True, msg
37
  except Exception as e:
38
- err_msg = f"Failed to load dataset '{HF_DATASET_ID}' from Hugging Face Hub. Error: {e}"
39
  print(err_msg)
40
  return pd.DataFrame(), False, err_msg
41
 
42
  def get_param_range_values(param_range_labels):
43
- if not param_range_labels or len(param_range_labels) != 2: return None, None
44
  min_label, max_label = param_range_labels
45
  min_val = 0.0 if '<' in min_label else float(min_label.replace('B', ''))
46
  max_val = float('inf') if '>' in max_label else float(max_label.replace('B', ''))
@@ -81,11 +120,18 @@ def create_treemap(treemap_data, count_by, title=None):
81
  fig.update_traces(textinfo="label+value+percent root", hovertemplate="<b>%{label}</b><br>%{value:,} " + count_by + "<br>%{percentRoot:.2%} of total<extra></extra>")
82
  return fig
83
 
84
- with gr.Blocks(title="ModelVerse Explorer", fill_width=True) as demo:
 
 
 
 
 
 
85
  models_data_state = gr.State(pd.DataFrame())
86
  loading_complete_state = gr.State(False)
87
 
88
- with gr.Row(): gr.Markdown("# 🤗 The Hub Org-Model Atlas")
 
89
  with gr.Row():
90
  with gr.Column(scale=1):
91
  count_by_dropdown = gr.Dropdown(label="Metric", choices=[("Downloads (last 30 days)", "downloads"), ("Downloads (All Time)", "downloadsAllTime"), ("Likes", "likes")], value="downloads")
@@ -94,22 +140,20 @@ with gr.Blocks(title="ModelVerse Explorer", fill_width=True) as demo:
94
  pipeline_filter_dropdown = gr.Dropdown(label="Select Pipeline Tag", choices=PIPELINE_TAGS, value=None, visible=False)
95
 
96
  with gr.Group():
97
- with gr.Row():
98
- param_label_display = gr.Markdown("<div style='font-weight: 500;'>Parameters</div>")
99
- reset_params_button = gr.Button("🔄 Reset", visible=False, size="sm", min_width=80)
100
- param_slider = gr.Slider(
101
- minimum=0, maximum=len(PARAM_CHOICES) - 1, step=1,
102
- value=PARAM_CHOICES_DEFAULT_INDICES,
103
- label="Parameter Range", show_label=False
104
- )
 
 
 
 
105
 
106
- # --- MODIFIED: Replaced Slider with Dropdown for Top-K selection ---
107
- top_k_dropdown = gr.Dropdown(
108
- label="Number of Top Organizations",
109
- choices=TOP_K_CHOICES,
110
- value=25
111
- )
112
-
113
  skip_orgs_textbox = gr.Textbox(label="Organizations to Skip (comma-separated)", value="TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski")
114
  generate_plot_button = gr.Button(value="Generate Plot", variant="primary", interactive=False)
115
 
@@ -118,21 +162,6 @@ with gr.Blocks(title="ModelVerse Explorer", fill_width=True) as demo:
118
  status_message_md = gr.Markdown("Initializing...")
119
  data_info_md = gr.Markdown("")
120
 
121
- def _update_slider_ui_elements(current_range_indices):
122
- if not isinstance(current_range_indices, list) or len(current_range_indices) != 2: return gr.update(), gr.update()
123
- min_idx, max_idx = int(current_range_indices[0]), int(current_range_indices[1])
124
- min_label, max_label = PARAM_CHOICES[min_idx], PARAM_CHOICES[max_idx]
125
- label_md = f"<div style='font-weight: 500;'>Parameters <span style='float: right; font-weight: normal; color: #555;'>{min_label} to {max_label}</span></div>"
126
- is_default = (min_idx == 0 and max_idx == len(PARAM_CHOICES) - 1)
127
- return label_md, gr.update(visible=not is_default)
128
-
129
- def _reset_param_slider_and_ui():
130
- default_label = "<div style='font-weight: 500;'>Parameters</div>"
131
- return gr.update(value=PARAM_CHOICES_DEFAULT_INDICES), default_label, gr.update(visible=False)
132
-
133
- param_slider.release(fn=_update_slider_ui_elements, inputs=param_slider, outputs=[param_label_display, reset_params_button])
134
- reset_params_button.click(fn=_reset_param_slider_and_ui, outputs=[param_slider, param_label_display, reset_params_button])
135
-
136
  def _update_button_interactivity(is_loaded_flag): return gr.update(interactive=is_loaded_flag)
137
  loading_complete_state.change(fn=_update_button_interactivity, inputs=loading_complete_state, outputs=generate_plot_button)
138
 
@@ -156,21 +185,23 @@ with gr.Blocks(title="ModelVerse Explorer", fill_width=True) as demo:
156
  data_info_text = f"### Data Load Failed\n- {status_msg_from_load}"
157
  status_msg_ui = status_msg_from_load
158
  except Exception as e:
159
- status_msg_ui = f"An unexpected error occurred during data loading: {str(e)}"
160
  data_info_text = f"### Critical Error\n- {status_msg_ui}"
161
  load_success_flag = False
162
  print(f"Critical error in ui_load_data_controller: {e}")
163
  return current_df, load_success_flag, data_info_text, status_msg_ui
164
 
165
  def ui_generate_plot_controller(metric_choice, filter_type, tag_choice, pipeline_choice,
166
- param_range_indices, k_orgs, skip_orgs_input, df_current_models, progress=gr.Progress()):
167
  if df_current_models is None or df_current_models.empty:
168
  return create_treemap(pd.DataFrame(), metric_choice, "Error: Model Data Not Loaded"), "Model data is not loaded."
 
169
  progress(0.1, desc="Preparing data...")
170
  tag_to_use = tag_choice if filter_type == "Tag Filter" else None
171
  pipeline_to_use = pipeline_choice if filter_type == "Pipeline Filter" else None
172
  orgs_to_skip = [org.strip() for org in skip_orgs_input.split(',') if org.strip()]
173
 
 
174
  min_label = PARAM_CHOICES[int(param_range_indices[0])]
175
  max_label = PARAM_CHOICES[int(param_range_indices[1])]
176
  param_labels_for_filtering = [min_label, max_label]
@@ -186,22 +217,22 @@ with gr.Blocks(title="ModelVerse Explorer", fill_width=True) as demo:
186
  plot_stats_md = "No data matches the selected filters. Please try different options."
187
  else:
188
  total_items_in_plot = len(treemap_df['id'].unique())
189
- total_value_in_plot = treemap_df[count_by].sum()
190
  plot_stats_md = f"## Plot Statistics\n- **Models shown**: {total_items_in_plot:,}\n- **Total {metric_choice}**: {int(total_value_in_plot):,}"
191
  return plotly_fig, plot_stats_md
192
 
193
- demo.load(fn=ui_load_data_controller, inputs=[], outputs=[models_data_state, loading_complete_state, data_info_md, status_message_md])
 
194
 
195
- # --- MODIFIED: The inputs list now uses top_k_dropdown ---
196
  generate_plot_button.click(
197
  fn=ui_generate_plot_controller,
198
  inputs=[count_by_dropdown, filter_choice_radio, tag_filter_dropdown, pipeline_filter_dropdown,
199
- param_slider, top_k_dropdown, skip_orgs_textbox, models_data_state],
200
  outputs=[plot_output, status_message_md]
201
  )
202
 
203
  if __name__ == "__main__":
204
- print(f"Application starting. Data will be loaded from Hugging Face dataset: {HF_DATASET_ID}")
205
  demo.queue().launch()
206
 
207
  # --- END OF FINAL POLISHED FILE app.py ---
 
4
  import pandas as pd
5
  import plotly.express as px
6
  import time
7
+ import json
8
  from datasets import load_dataset
9
 
10
  # --- Constants ---
11
  PARAM_CHOICES = ['< 1B', '1B', '5B', '12B', '32B', '64B', '128B', '256B', '> 500B']
12
+ PARAM_CHOICES_DEFAULT_INDICES_JSON = json.dumps([0, len(PARAM_CHOICES) - 1])
13
 
 
14
  TOP_K_CHOICES = list(range(5, 51, 5))
 
15
  HF_DATASET_ID = "evijit/orgstats_daily_data"
16
  TAG_FILTER_CHOICES = [ "Audio & Speech", "Time series", "Robotics", "Music", "Video", "Images", "Text", "Biomedical", "Sciences" ]
17
  PIPELINE_TAGS = [ 'text-generation', 'text-to-image', 'text-classification', 'text2text-generation', 'audio-to-audio', 'feature-extraction', 'image-classification', 'translation', 'reinforcement-learning', 'fill-mask', 'text-to-speech', 'automatic-speech-recognition', 'image-text-to-text', 'token-classification', 'sentence-similarity', 'question-answering', 'image-feature-extraction', 'summarization', 'zero-shot-image-classification', 'object-detection', 'image-segmentation', 'image-to-image', 'image-to-text', 'audio-classification', 'visual-question-answering', 'text-to-video', 'zero-shot-classification', 'depth-estimation', 'text-ranking', 'image-to-video', 'multiple-choice', 'unconditional-image-generation', 'video-classification', 'text-to-audio', 'time-series-forecasting', 'any-to-any', 'video-text-to-text', 'table-question-answering' ]
18
 
19
+ # --- Custom HTML, CSS, and JavaScript for the Slider ---
20
+ custom_slider_js = """
21
+ function createCustomSlider() {
22
+ const paramChoices = [<1B>, <1B>, <5B>, <12B>, <32B>, <64B>, <128B>, <256B>, <500B>];
23
+ const slider = document.getElementById('noui-slider-container');
24
+ if (slider.noUiSlider) {
25
+ slider.noUiSlider.destroy();
26
+ }
27
+ noUiSlider.create(slider, {
28
+ start: [0, paramChoices.length - 1],
29
+ connect: true,
30
+ step: 1,
31
+ range: { 'min': 0, 'max': paramChoices.length - 1 },
32
+ pips: {
33
+ mode: 'values',
34
+ values: Array.from(Array(paramChoices.length).keys()),
35
+ density: 100 / (paramChoices.length - 1),
36
+ format: { to: function(value) { return paramChoices[value]; } }
37
+ }
38
+ });
39
+
40
+ const paramRangeStateInput = document.querySelector('#param-range-state-js textarea');
41
+ slider.noUiSlider.on('update', function (values) {
42
+ const intValues = values.map(v => parseInt(v, 10));
43
+ const newValue = JSON.stringify(intValues);
44
+ if (paramRangeStateInput.value !== newValue) {
45
+ paramRangeStateInput.value = newValue;
46
+ const event = new Event('input', { bubbles: true });
47
+ paramRangeStateInput.dispatchEvent(event);
48
+ }
49
+ });
50
+
51
+ function highlightPips(values) {
52
+ const intValues = values.map(v => parseInt(v, 10));
53
+ document.querySelectorAll('.noUi-value').forEach((pip, index) => {
54
+ const pipIsSelected = index >= intValues[0] && index <= intValues[1];
55
+ pip.style.fontWeight = pipIsSelected ? 'bold' : 'normal';
56
+ pip.style.color = pipIsSelected ? '#000' : '#777';
57
+ });
58
+ }
59
+ slider.noUiSlider.on('update', highlightPips);
60
+ highlightPips([0, paramChoices.length - 1]);
61
+ }
62
+ """
63
+
64
  def load_models_data():
65
  overall_start_time = time.time()
66
  print(f"Attempting to load dataset from Hugging Face Hub: {HF_DATASET_ID}")
67
  try:
68
  dataset_dict = load_dataset(HF_DATASET_ID)
69
+ df = dataset_dict[list(dataset_dict.keys())[0]].to_pandas()
 
 
 
70
  if 'params' in df.columns:
71
  df['params'] = pd.to_numeric(df['params'], errors='coerce').fillna(0)
72
  else:
73
  df['params'] = 0
74
+ msg = f"Successfully loaded dataset in {time.time() - overall_start_time:.2f}s."
 
75
  print(msg)
76
  return df, True, msg
77
  except Exception as e:
78
+ err_msg = f"Failed to load dataset. Error: {e}"
79
  print(err_msg)
80
  return pd.DataFrame(), False, err_msg
81
 
82
  def get_param_range_values(param_range_labels):
 
83
  min_label, max_label = param_range_labels
84
  min_val = 0.0 if '<' in min_label else float(min_label.replace('B', ''))
85
  max_val = float('inf') if '>' in max_label else float(max_label.replace('B', ''))
 
120
  fig.update_traces(textinfo="label+value+percent root", hovertemplate="<b>%{label}</b><br>%{value:,} " + count_by + "<br>%{percentRoot:.2%} of total<extra></extra>")
121
  return fig
122
 
123
+ custom_head = """
124
+ <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/15.7.1/nouislider.min.css">
125
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/noUiSlider/15.7.1/nouislider.min.js"></script>
126
+ """
127
+
128
+ # --- MODIFIED: Added emoji to the browser tab title ---
129
+ with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, head=custom_head) as demo:
130
  models_data_state = gr.State(pd.DataFrame())
131
  loading_complete_state = gr.State(False)
132
 
133
+ # --- MODIFIED: Removed the main title from the page body for a cleaner look ---
134
+
135
  with gr.Row():
136
  with gr.Column(scale=1):
137
  count_by_dropdown = gr.Dropdown(label="Metric", choices=[("Downloads (last 30 days)", "downloads"), ("Downloads (All Time)", "downloadsAllTime"), ("Likes", "likes")], value="downloads")
 
140
  pipeline_filter_dropdown = gr.Dropdown(label="Select Pipeline Tag", choices=PIPELINE_TAGS, value=None, visible=False)
141
 
142
  with gr.Group():
143
+ gr.Markdown("<div style='font-weight: 500;'>Parameters</div>")
144
+ gr.HTML("""
145
+ <div id="noui-slider-container" style="margin: 2rem 1rem;"></div>
146
+ <style>
147
+ .noUi-value { font-size: 12px; }
148
+ .noUi-pips-horizontal { padding: 10px 0; height: 50px; }
149
+ .noUi-connect { background: #333; }
150
+ .noUi-handle { border-radius: 50%; width: 20px; height: 20px; right: -10px; top: -7px; box-shadow: none; border: 2px solid #333; background: #FFF; cursor: pointer; }
151
+ .noUi-handle:focus { outline: none; }
152
+ </style>
153
+ """)
154
+ param_range_state_js = gr.Textbox(value=PARAM_CHOICES_DEFAULT_INDICES_JSON, visible=False, elem_id="param-range-state-js")
155
 
156
+ top_k_dropdown = gr.Dropdown(label="Number of Top Organizations", choices=TOP_K_CHOICES, value=25)
 
 
 
 
 
 
157
  skip_orgs_textbox = gr.Textbox(label="Organizations to Skip (comma-separated)", value="TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski")
158
  generate_plot_button = gr.Button(value="Generate Plot", variant="primary", interactive=False)
159
 
 
162
  status_message_md = gr.Markdown("Initializing...")
163
  data_info_md = gr.Markdown("")
164
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  def _update_button_interactivity(is_loaded_flag): return gr.update(interactive=is_loaded_flag)
166
  loading_complete_state.change(fn=_update_button_interactivity, inputs=loading_complete_state, outputs=generate_plot_button)
167
 
 
185
  data_info_text = f"### Data Load Failed\n- {status_msg_from_load}"
186
  status_msg_ui = status_msg_from_load
187
  except Exception as e:
188
+ status_msg_ui = f"An unexpected error occurred: {str(e)}"
189
  data_info_text = f"### Critical Error\n- {status_msg_ui}"
190
  load_success_flag = False
191
  print(f"Critical error in ui_load_data_controller: {e}")
192
  return current_df, load_success_flag, data_info_text, status_msg_ui
193
 
194
  def ui_generate_plot_controller(metric_choice, filter_type, tag_choice, pipeline_choice,
195
+ param_range_json, k_orgs, skip_orgs_input, df_current_models, progress=gr.Progress()):
196
  if df_current_models is None or df_current_models.empty:
197
  return create_treemap(pd.DataFrame(), metric_choice, "Error: Model Data Not Loaded"), "Model data is not loaded."
198
+
199
  progress(0.1, desc="Preparing data...")
200
  tag_to_use = tag_choice if filter_type == "Tag Filter" else None
201
  pipeline_to_use = pipeline_choice if filter_type == "Pipeline Filter" else None
202
  orgs_to_skip = [org.strip() for org in skip_orgs_input.split(',') if org.strip()]
203
 
204
+ param_range_indices = json.loads(param_range_json)
205
  min_label = PARAM_CHOICES[int(param_range_indices[0])]
206
  max_label = PARAM_CHOICES[int(param_range_indices[1])]
207
  param_labels_for_filtering = [min_label, max_label]
 
217
  plot_stats_md = "No data matches the selected filters. Please try different options."
218
  else:
219
  total_items_in_plot = len(treemap_df['id'].unique())
220
+ total_value_in_plot = treemap_df[metric_choice].sum()
221
  plot_stats_md = f"## Plot Statistics\n- **Models shown**: {total_items_in_plot:,}\n- **Total {metric_choice}**: {int(total_value_in_plot):,}"
222
  return plotly_fig, plot_stats_md
223
 
224
+ demo.load(fn=ui_load_data_controller, inputs=[], outputs=[models_data_state, loading_complete_state, data_info_md, status_message_md]) \
225
+ .then(fn=None, _js=custom_slider_js.replace("<", "'<").replace(">", "'"))
226
 
 
227
  generate_plot_button.click(
228
  fn=ui_generate_plot_controller,
229
  inputs=[count_by_dropdown, filter_choice_radio, tag_filter_dropdown, pipeline_filter_dropdown,
230
+ param_range_state_js, top_k_dropdown, skip_orgs_textbox, models_data_state],
231
  outputs=[plot_output, status_message_md]
232
  )
233
 
234
  if __name__ == "__main__":
235
+ print(f"Application starting...")
236
  demo.queue().launch()
237
 
238
  # --- END OF FINAL POLISHED FILE app.py ---