Spaces:
Runtime error
Runtime error
File size: 1,419 Bytes
aba8087 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 |
#!/usr/bin/env python3
"""
Download script for DeepCoder model
Downloads and caches the model for faster container startup
"""
import os
import logging
from transformers import AutoTokenizer, AutoModelForCausalLM
from huggingface_hub import snapshot_download
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
MODEL_NAME = os.getenv("MODEL_NAME", "ai/deepcoder-preview")
CACHE_DIR = os.getenv("HUGGINGFACE_HUB_CACHE", "/app/cache")
def download_model():
"""Download the model and tokenizer"""
try:
logger.info(f"Downloading model: {MODEL_NAME}")
# Download model files
snapshot_download(
repo_id=MODEL_NAME,
cache_dir=CACHE_DIR,
resume_download=True
)
# Verify by loading tokenizer
tokenizer = AutoTokenizer.from_pretrained(
MODEL_NAME,
cache_dir=CACHE_DIR,
trust_remote_code=True
)
logger.info("Model downloaded successfully")
logger.info(f"Vocab size: {tokenizer.vocab_size}")
logger.info(f"Cache directory: {CACHE_DIR}")
return True
except Exception as e:
logger.error(f"Error downloading model: {str(e)}")
return False
if __name__ == "__main__":
success = download_model()
if not success:
exit(1)
logger.info("Download complete!")
|