update

Dockerfile

@@ -24,12 +24,17 @@ ENV HOME=/home/user \
 # Set the working directory to the user's home directory
 WORKDIR $HOME/app
 
-# Copy project files with proper ownership
+# Copy dependency files first for better caching
 COPY --chown=user pyproject.toml uv.lock ./
+
+# Install dependencies (this layer will be cached when only code changes)
+RUN uv sync --frozen
+
+# Copy source code
 COPY --chown=user src/ ./src/
 
-# Install dependencies and the package
-RUN uv sync --frozen && uv pip install -e .
+# Install the local package (this layer will rebuild when code changes)
+RUN uv pip install -e .
 
 # Expose port (default 8501, can be overridden)
 EXPOSE 8501

src/legisqa_local/config/settings.py

@@ -39,13 +39,24 @@ def get_chroma_config():
 
 def setup_chromadb():
     """Setup ChromaDB - use persistent storage (/data) or download from HF Dataset if needed"""
-    # Use session state to prevent duplicate runs in Streamlit
-    import streamlit as st
-    if hasattr(st, 'session_state') and getattr(st.session_state, 'chromadb_setup_complete', False):
-        logger.info("ChromaDB setup already completed in this session")
+    # Use a file-based lock to prevent duplicate runs (more reliable than session state in HF Spaces)
+    lock_file = "/tmp/chromadb_setup.lock"
+    
+    # Check if setup is already in progress or complete
+    if os.path.exists(lock_file):
+        logger.info("ChromaDB setup already in progress or complete (lock file exists)")
         chroma_config = get_chroma_config()
         return chroma_config["persist_directory"]
     
+    # Create lock file
+    try:
+        with open(lock_file, 'w') as f:
+            f.write("ChromaDB setup in progress")
+        logger.info("Created ChromaDB setup lock file")
+    except Exception as e:
+        logger.warning(f"Could not create lock file: {e}")
+        # Continue anyway
+    
     logger.info("=== ChromaDB Setup Starting ===")
     
     chroma_config = get_chroma_config()
@@ -64,29 +75,34 @@ def setup_chromadb():
     if os.path.exists("/data"):
         logger.info("🚀 HF Spaces persistent storage detected at /data")
         
-        # Handle force refresh - clear persistent storage
-        if force_refresh and os.path.exists(persistent_chroma_path):
-            logger.info(f"🗑️  Clearing existing ChromaDB due to CHROMA_FORCE_REFRESH: {persistent_chroma_path}")
-            import shutil
-            try:
-                shutil.rmtree(persistent_chroma_path)
-                logger.info("✅ Existing ChromaDB cleared successfully")
-            except Exception as e:
-                logger.error(f"❌ Error clearing ChromaDB: {e}")
-                logger.info("Continuing with download anyway...")
+        # Always clear /data completely to free up maximum space
+        logger.info("🧹 Clearing entire /data directory to free up space...")
+        import shutil
+        try:
+            # Check initial disk space
+            statvfs = os.statvfs("/data")
+            free_space_gb = (statvfs.f_bavail * statvfs.f_frsize) / (1024 * 1024 * 1024)
+            logger.info(f"💾 Initial free space: {free_space_gb:.2f} GB")
+            
+            for item in os.listdir("/data"):
+                item_path = os.path.join("/data", item)
+                if os.path.isdir(item_path):
+                    shutil.rmtree(item_path)
+                    logger.info(f"   Removed directory: {item}")
+                else:
+                    os.remove(item_path)
+                    logger.info(f"   Removed file: {item}")
+            
+            # Check free space after cleanup
+            statvfs = os.statvfs("/data")
+            free_space_gb = (statvfs.f_bavail * statvfs.f_frsize) / (1024 * 1024 * 1024)
+            logger.info(f"✅ /data directory cleared successfully")
+            logger.info(f"💾 Free space after cleanup: {free_space_gb:.2f} GB")
+        except Exception as e:
+            logger.error(f"❌ Error clearing /data directory: {e}")
+            logger.info("Continuing with download anyway...")
         
-        # Check if ChromaDB exists in persistent storage (after potential clearing)
-        if os.path.exists(persistent_chroma_path) and os.listdir(persistent_chroma_path):
-            logger.info(f"✅ ChromaDB found in persistent storage: {persistent_chroma_path}")
-            # Update environment variable to point to persistent storage
-            os.environ["CHROMA_PERSIST_DIRECTORY"] = persistent_chroma_path
-            logger.info(f"Updated CHROMA_PERSIST_DIRECTORY to: {persistent_chroma_path}")
-            # Inspect the existing ChromaDB
-            inspect_chromadb(persistent_chroma_path)
-            # Mark setup as complete
-            if hasattr(st, 'session_state'):
-                st.session_state.chromadb_setup_complete = True
-            return persistent_chroma_path
+        # ChromaDB will always need to be downloaded fresh now
         
         # ChromaDB not found in persistent storage, try to download from HF Dataset
         logger.info("ChromaDB not found in persistent storage, checking HF Dataset configuration...")
@@ -106,9 +122,6 @@ def setup_chromadb():
                 logger.info(f"✅ ChromaDB download successful! Updated path to: {persistent_chroma_path}")
                 # Inspect the downloaded ChromaDB
                 inspect_chromadb(persistent_chroma_path)
-                # Mark setup as complete
-                if hasattr(st, 'session_state'):
-                    st.session_state.chromadb_setup_complete = True
                 return persistent_chroma_path
             else:
                 logger.error("❌ ChromaDB download from HF Dataset failed!")
@@ -128,9 +141,6 @@ def setup_chromadb():
             logger.info(f"✅ ChromaDB found at {chroma_path}")
             # Inspect the fallback ChromaDB
             inspect_chromadb(chroma_path)
-            # Mark setup as complete
-            if hasattr(st, 'session_state'):
-                st.session_state.chromadb_setup_complete = True
             return chroma_path
         else:
             logger.warning(f"ChromaDB directory exists but is empty: {chroma_path}")
@@ -139,9 +149,6 @@ def setup_chromadb():
     
     logger.warning(f"⚠️  Using default ChromaDB path: {chroma_path}")
     logger.info("=== ChromaDB Setup Complete ===")
-    # Mark setup as complete even if no ChromaDB found (prevents infinite retries)
-    if hasattr(st, 'session_state'):
-        st.session_state.chromadb_setup_complete = True
     return chroma_path
 
 def download_chromadb_from_hf_dataset(dataset_repo: str, local_path: str) -> bool:
@@ -164,11 +171,14 @@ def download_chromadb_from_hf_dataset(dataset_repo: str, local_path: str) -> boo
         import uuid
         temp_download_path = f"{local_path}_temp_{uuid.uuid4().hex[:8]}"
         
+        # Use /tmp for cache (we cleared /data completely)
+        cache_dir = "/tmp/hf_chromadb_cache"
+        
         downloaded_path = snapshot_download(
             repo_id=dataset_repo,
             repo_type="dataset",
             local_dir=temp_download_path,
-            cache_dir="/tmp/hf_chromadb_cache"
+            cache_dir=cache_dir
             # Note: resume_download and local_dir_use_symlinks are now handled automatically
         )
         
@@ -195,6 +205,15 @@ def download_chromadb_from_hf_dataset(dataset_repo: str, local_path: str) -> boo
                 logger.info(f"Cleaning up temporary directory: {temp_download_path}")
                 shutil.rmtree(temp_download_path)
                 
+                # Clean up HF cache to save disk space
+                if os.path.exists(cache_dir):
+                    logger.info(f"Cleaning up HF cache directory: {cache_dir}")
+                    try:
+                        shutil.rmtree(cache_dir)
+                        logger.info("✅ HF cache cleaned up successfully")
+                    except Exception as e:
+                        logger.warning(f"Could not clean up HF cache: {e}")
+                
                 logger.info(f"✅ ChromaDB files moved to: {local_path}")
             except Exception as e:
                 logger.error(f"❌ Error moving ChromaDB files: {e}")