File size: 4,062 Bytes
a938b8a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
import os
import pickle
from datetime import datetime

import matplotlib.pyplot as plt
import pandas as pd
from huggingface_hub import HfApi

# Define colors for each language
LANGUAGE_COLORS = {
    "english": "orange",
    "spanish": "blue",
}


def fetch_models(cache_file="models_cache.pkl"):
    """Fetch and filter models from HuggingFace Hub with caching"""
    # Check if cached data exists and is less than 24 hours old
    if os.path.exists(cache_file):
        cache_age = datetime.now().timestamp() - os.path.getmtime(cache_file)
        if cache_age < 24 * 3600:  # 24 hours in seconds
            print("Loading models from cache...")
            with open(cache_file, "rb") as f:
                return pickle.load(f)
        else:
            print("Cache is older than 24 hours, fetching fresh data...")
    else:
        print("No cache found, fetching models from Hugging Face Hub...")

    hf_api = HfApi()
    all_models = list(hf_api.list_models(full=True))

    # Filter models by language
    english_filter = filter(
        lambda m: any(tag == "language:en" for tag in m.tags)
        and not any(
            tag.startswith("language:") and tag != "language:en" for tag in m.tags
        ),
        all_models,
    )
    spanish_filter = filter(
        lambda m: any(tag == "language:es" for tag in m.tags)
        and not any(
            tag.startswith("language:") and tag != "language:es" for tag in m.tags
        ),
        all_models,
    )

    filtered_models = {
        "english": list(english_filter),
        "spanish": list(spanish_filter),
    }

    # Cache the filtered models
    print("Saving models to cache...")
    with open(cache_file, "wb") as f:
        pickle.dump(filtered_models, f)

    return filtered_models


def create_stack_area_plot(models, output_dir):
    """Create stacked area plot for English and Spanish models"""
    # Prepare data for all languages
    all_dates = []
    languages = ["english", "spanish"]
    for lang in languages:
        all_dates.extend([d.created_at.date() for d in models[lang]])

    if not all_dates:
        print("No models found for any language. Skipping plot creation.")
        return

    # Create a common date range for all languages
    min_date = min(all_dates)
    max_date = max(all_dates)
    date_range = pd.date_range(start=min_date, end=max_date, freq="MS")

    # Create separate DataFrames for each language
    dfs = {}
    for lang in languages:
        dates = [d.created_at.date() for d in models[lang]]
        df = pd.DataFrame({"Date": dates})
        df["Count"] = 1
        df["Date"] = pd.to_datetime(df["Date"])
        # Reindex to common date range and fill missing values with 0
        df_grouped = df.groupby(pd.Grouper(key="Date", freq="MS")).sum()
        df_grouped = df_grouped.reindex(date_range, fill_value=0)
        dfs[lang] = df_grouped.cumsum()

    # Plot stacked area for English and Spanish
    plt.figure(figsize=(10, 6))
    plt.stackplot(
        date_range,
        [dfs[lang]["Count"].values for lang in languages],
        labels=["English", "Spanish"],
        colors=[LANGUAGE_COLORS[lang] for lang in languages],
    )

    plt.xlabel("Date", fontsize=10)
    plt.ylabel("Cumulative Number of Models", fontsize=10)
    plt.xticks(rotation=45, fontsize=10)
    plt.legend(loc="upper left")
    plt.tight_layout()
    plt.savefig(f"{output_dir}/models_stack_area_en_es.png")
    plt.close()


def main():
    # Create output directory if it doesn't exist
    output_dir = "plots"
    os.makedirs(output_dir, exist_ok=True)

    # Fetch models
    print("Fetching models from Hugging Face Hub...")
    models = fetch_models()

    # Print model counts
    print("\nModel counts:")
    for lang, models_list in models.items():
        print(f"{lang.capitalize()}: {len(models_list)}")

    # Create visualization
    print("\nCreating stack area plot...")
    create_stack_area_plot(models, output_dir)

    print(f"Plot has been saved to the '{output_dir}' directory")


if __name__ == "__main__":
    main()