debug_hf_spaces.py

#!/usr/bin/env python3
"""
Debug script for Hugging Face Spaces deployment
This script helps diagnose common issues during deployment
"""

import os
import sys
import subprocess
from pathlib import Path

def check_environment():
    """Check environment variables and system info"""
    print("🔍 Checking environment...")
    
    # Check Python version
    print(f"Python version: {sys.version}")
    
    # Check environment variables
    env_vars = [
        "HF_HOME", "TRANSFORMERS_CACHE", "HF_HUB_CACHE", 
        "GEMINI_API_KEY", "PINECONE_API_KEY", "PINECONE_ENVIRONMENT"
    ]
    
    for var in env_vars:
        value = os.getenv(var, "NOT SET")
        if "KEY" in var and value != "NOT SET":
            value = f"{value[:8]}..." if len(value) > 8 else "***"
        print(f"  {var}: {value}")
    
    # Check working directory
    print(f"Working directory: {os.getcwd()}")
    
    # Check if we're in a container
    if os.path.exists("/.dockerenv"):
        print("Running in Docker container: ✅")
    else:
        print("Running in Docker container: ❌")

def check_cache_directories():
    """Check cache directory setup"""
    print("\n🔍 Checking cache directories...")
    
    cache_dir = "/code/.cache/huggingface"
    
    # Check if directory exists
    if os.path.exists(cache_dir):
        print(f"Cache directory exists: ✅ ({cache_dir})")
        
        # Check permissions
        try:
            stat = os.stat(cache_dir)
            print(f"Cache directory permissions: {oct(stat.st_mode)[-3:]}")
            
            # Check if writable
            if os.access(cache_dir, os.W_OK):
                print("Cache directory writable: ✅")
            else:
                print("Cache directory writable: ❌")
                
        except Exception as e:
            print(f"Error checking cache directory: {e}")
    else:
        print(f"Cache directory exists: ❌ ({cache_dir})")
        
        # Try to create it
        try:
            os.makedirs(cache_dir, exist_ok=True)
            os.chmod(cache_dir, 0o777)
            print("Created cache directory: ✅")
        except Exception as e:
            print(f"Failed to create cache directory: {e}")

def check_dependencies():
    """Check if required packages are installed"""
    print("\n🔍 Checking dependencies...")
    
    required_packages = [
        "fastapi", "uvicorn", "sentence_transformers", 
        "transformers", "torch", "numpy", "faiss"
    ]
    
    for package in required_packages:
        try:
            __import__(package)
            print(f"  {package}: ✅")
        except ImportError:
            print(f"  {package}: ❌")

def check_dependency_conflicts():
    """Check for potential dependency conflicts"""
    print("\n🔍 Checking for dependency conflicts...")
    
    try:
        import transformers
        import tokenizers
        
        print(f"  transformers version: {transformers.__version__}")
        print(f"  tokenizers version: {tokenizers.__version__}")
        
        # Check if versions are compatible
        transformers_version = transformers.__version__
        tokenizers_version = tokenizers.__version__
        
        if transformers_version.startswith("4.35"):
            if tokenizers_version.startswith("0.15"):
                print("  ⚠️  Potential conflict: transformers 4.35.x with tokenizers 0.15.x")
                print("     transformers 4.35.x requires tokenizers < 0.15")
            else:
                print("  ✅ transformers and tokenizers versions are compatible")
        else:
            print("  ℹ️  transformers version not in expected range")
            
    except ImportError as e:
        print(f"  ❌ Could not check dependency conflicts: {e}")

def test_model_loading():
    """Test sentence-transformers model loading"""
    print("\n🔍 Testing model loading...")
    
    try:
        from sentence_transformers import SentenceTransformer
        
        # Set cache directory
        cache_dir = "/code/.cache/huggingface"
        os.makedirs(cache_dir, exist_ok=True)
        
        # Try different model names
        models_to_try = [
            "all-MiniLM-L6-v2",
            "sentence-transformers/all-MiniLM-L6-v2",
            "sentence-transformers/paraphrase-MiniLM-L6-v2"
        ]
        
        for model_name in models_to_try:
            try:
                print(f"  Trying {model_name}...")
                model = SentenceTransformer(model_name, cache_folder=cache_dir)
                
                # Test encoding
                test_text = ["This is a test sentence."]
                embeddings = model.encode(test_text)
                print(f"  ✅ {model_name} loaded successfully - shape: {embeddings.shape}")
                return True
                
            except Exception as e:
                print(f"  ❌ {model_name} failed: {e}")
                continue
        
        print("  ❌ All models failed to load")
        return False
        
    except Exception as e:
        print(f"  ❌ Error testing model loading: {e}")
        return False

def check_network_connectivity():
    """Check network connectivity to Hugging Face"""
    print("\n🔍 Checking network connectivity...")
    
    try:
        import requests
        
        # Test basic connectivity
        response = requests.get("https://huggingface.co", timeout=10)
        if response.status_code == 200:
            print("Hugging Face connectivity: ✅")
        else:
            print(f"Hugging Face connectivity: ❌ (Status: {response.status_code})")
            
    except Exception as e:
        print(f"Hugging Face connectivity: ❌ ({e})")

def main():
    """Run all diagnostic checks"""
    print("🚀 HackRx 6.0 - Hugging Face Spaces Debug Script")
    print("=" * 50)
    
    check_environment()
    check_cache_directories()
    check_dependencies()
    check_dependency_conflicts()
    check_network_connectivity()
    model_success = test_model_loading()
    
    print("\n" + "=" * 50)
    print("📊 Summary:")
    
    if model_success:
        print("✅ Model loading successful - deployment should work")
    else:
        print("❌ Model loading failed - check the issues above")
        print("\n💡 Troubleshooting tips:")
        print("1. Check if all environment variables are set")
        print("2. Verify cache directory permissions")
        print("3. Ensure network connectivity to Hugging Face")
        print("4. Check if all dependencies are installed")
        print("5. Review the Docker build logs")
        print("6. Check for dependency conflicts (especially transformers/tokenizers)")

if __name__ == "__main__":
    main()