printing extracted text20

all_models.py

@@ -4,11 +4,80 @@ import torch
 import os
 import tempfile
 import logging
+import shutil
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+def check_directory_permissions(path):
+    """Check if directory exists and has correct permissions"""
+    try:
+        if not os.path.exists(path):
+            logger.warning(f"Directory does not exist: {path}")
+            return False
+        
+        # Try to create a test file
+        test_file = os.path.join(path, '.permission_test')
+        try:
+            with open(test_file, 'w') as f:
+                f.write('test')
+            os.remove(test_file)
+            logger.info(f"Directory {path} is writable")
+            return True
+        except Exception as e:
+            logger.error(f"Directory {path} is not writable: {e}")
+            return False
+    except Exception as e:
+        logger.error(f"Error checking permissions for {path}: {e}")
+        return False
+
+def get_cache_dir():
+    """Get a user-accessible cache directory"""
+    try:
+        # Try user's home directory first
+        home_dir = os.path.expanduser('~')
+        if not os.path.exists(home_dir):
+            raise Exception(f"Home directory does not exist: {home_dir}")
+            
+        cache_dir = os.path.join(home_dir, '.cache', 'answer_grading_app')
+        logger.info(f"Attempting to use cache directory: {cache_dir}")
+        
+        # Create directory with proper permissions
+        os.makedirs(cache_dir, mode=0o755, exist_ok=True)
+        
+        # Verify permissions
+        if check_directory_permissions(cache_dir):
+            logger.info(f"Successfully created and verified cache directory: {cache_dir}")
+            return cache_dir
+            
+        # If home directory fails, try temp directory
+        temp_dir = os.path.join(tempfile.gettempdir(), 'answer_grading_app')
+        logger.info(f"Attempting to use temporary directory: {temp_dir}")
+        
+        os.makedirs(temp_dir, mode=0o755, exist_ok=True)
+        if check_directory_permissions(temp_dir):
+            logger.info(f"Using temporary directory: {temp_dir}")
+            return temp_dir
+            
+        # Last resort: use current directory
+        current_dir = os.path.join(os.getcwd(), '.cache')
+        logger.info(f"Attempting to use current directory: {current_dir}")
+        
+        os.makedirs(current_dir, mode=0o755, exist_ok=True)
+        if check_directory_permissions(current_dir):
+            logger.info(f"Using current directory: {current_dir}")
+            return current_dir
+            
+        raise Exception("Could not find a writable cache directory")
+        
+    except Exception as e:
+        logger.error(f"Error setting up cache directory: {e}")
+        # Create a new temporary directory as last resort
+        temp_dir = tempfile.mkdtemp()
+        logger.info(f"Created temporary directory as fallback: {temp_dir}")
+        return temp_dir
+
 class ModelSingleton:
     _instance = None
     _initialized = False
@@ -23,9 +92,39 @@ class ModelSingleton:
     def __init__(self):
         if not self._initialized:
             try:
-                # Set cache directory to temporary directory
-                cache_dir = os.getenv('TRANSFORMERS_CACHE', tempfile.gettempdir())
-                os.environ['TRANSFORMERS_CACHE'] = cache_dir
+                logger.info("Initializing ModelSingleton...")
+                
+                # Set up cache directories
+                self.cache_dir = get_cache_dir()
+                logger.info(f"Using main cache directory: {self.cache_dir}")
+                
+                # Define and create all cache directories
+                self.cache_dirs = {
+                    'transformers': os.path.join(self.cache_dir, 'transformers'),
+                    'huggingface': os.path.join(self.cache_dir, 'huggingface'),
+                    'torch': os.path.join(self.cache_dir, 'torch'),
+                    'cache': os.path.join(self.cache_dir, 'cache'),
+                    'sentence_transformers': os.path.join(self.cache_dir, 'sentence_transformers'),
+                    'fasttext': os.path.join(self.cache_dir, 'fasttext')
+                }
+                
+                # Create and verify each cache directory
+                for name, path in self.cache_dirs.items():
+                    try:
+                        os.makedirs(path, mode=0o755, exist_ok=True)
+                        if check_directory_permissions(path):
+                            logger.info(f"Successfully created {name} cache directory: {path}")
+                        else:
+                            logger.error(f"Failed to verify permissions for {name} cache directory")
+                    except Exception as e:
+                        logger.error(f"Error creating {name} cache directory: {e}")
+                
+                # Set environment variables
+                os.environ['TRANSFORMERS_CACHE'] = self.cache_dirs['transformers']
+                os.environ['HF_HOME'] = self.cache_dirs['huggingface']
+                os.environ['TORCH_HOME'] = self.cache_dirs['torch']
+                os.environ['XDG_CACHE_HOME'] = self.cache_dirs['cache']
+                os.environ['SENTENCE_TRANSFORMERS_HOME'] = self.cache_dirs['sentence_transformers']
                 
                 # Get device
                 self.device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -42,16 +141,18 @@ class ModelSingleton:
                 self.vit_processor = None
                 
                 # Initialize reference counts
-                self._reference_counts['similarity'] = 0
-                self._reference_counts['flan'] = 0
-                self._reference_counts['trocr'] = 0
-                self._reference_counts['vit'] = 0
+                self._reference_counts = {
+                    'similarity': 0,
+                    'flan': 0,
+                    'trocr': 0,
+                    'vit': 0
+                }
                 
                 self._initialized = True
-                logger.info("Model singleton initialized")
+                logger.info("ModelSingleton initialization completed successfully")
                 
             except Exception as e:
-                logger.error(f"Error during model initialization: {e}")
+                logger.error(f"Error during ModelSingleton initialization: {e}")
                 raise
 
     def get_similarity_model(self):

similarity_check/semantic_meaning_check/semantic.py

@@ -12,7 +12,7 @@ import sys
 import os
 import tempfile
 sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
-from all_models import models
+from all_models import models, get_cache_dir, check_directory_permissions
 import torch
 import logging
 from utils import log_print
@@ -21,36 +21,92 @@ from utils import log_print
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-# Use custom directory for gensim data
-gensim_data_dir = os.getenv('GENSIM_DATA_DIR', tempfile.gettempdir())
-os.environ['GENSIM_DATA_DIR'] = gensim_data_dir
+def verify_model_file(model_path):
+    """Verify that model file exists and is readable"""
+    try:
+        if not os.path.exists(model_path):
+            logger.error(f"Model file does not exist: {model_path}")
+            return False
+            
+        # Check file size
+        size = os.path.getsize(model_path)
+        if size == 0:
+            logger.error(f"Model file is empty: {model_path}")
+            return False
+            
+        # Try to open file
+        with open(model_path, 'rb') as f:
+            # Try to read first few bytes
+            f.read(1024)
+            logger.info(f"Model file is readable: {model_path}")
+            return True
+    except Exception as e:
+        logger.error(f"Error verifying model file {model_path}: {e}")
+        return False
+
+def get_fasttext_cache_dir():
+    """Get cache directory for FastText model"""
+    cache_dir = get_cache_dir()
+    fasttext_dir = os.path.join(cache_dir, 'fasttext')
+    
+    logger.info(f"Setting up FastText cache directory: {fasttext_dir}")
+    
+    try:
+        os.makedirs(fasttext_dir, mode=0o755, exist_ok=True)
+        if check_directory_permissions(fasttext_dir):
+            logger.info(f"FastText cache directory is ready: {fasttext_dir}")
+            return fasttext_dir
+    except Exception as e:
+        logger.error(f"Error creating FastText cache directory: {e}")
+    
+    # Fallback to temporary directory
+    temp_dir = tempfile.mkdtemp()
+    logger.info(f"Using temporary directory for FastText: {temp_dir}")
+    return temp_dir
 
 def load_fasttext_model():
     """Load FastText model with proper error handling"""
     try:
-        # Get the model directory from environment variable or use default
-        model_dir = os.getenv('GENSIM_DATA_DIR', os.path.expanduser('~/.cache/gensim-data'))
-        model_path = os.path.join(model_dir, 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
+        # Get model directory
+        model_dir = get_fasttext_cache_dir()
+        model_path = os.path.join(model_dir, 'fasttext-wiki-news-subwords-300.gz')
+        logger.info(f"Attempting to load FastText model from: {model_path}")
         
-        # Create directory if it doesn't exist
-        os.makedirs(os.path.dirname(model_path), exist_ok=True)
-        
-        if os.path.exists(model_path):
-            log_print("Loading fasttext model from cache...")
-            return KeyedVectors.load_word2vec_format(model_path)
+        if os.path.exists(model_path) and verify_model_file(model_path):
+            logger.info("Loading FastText model from cache...")
+            try:
+                model = KeyedVectors.load_word2vec_format(model_path)
+                logger.info("Successfully loaded FastText model from cache")
+                return model
+            except Exception as e:
+                logger.error(f"Error loading cached model: {str(e)}")
+                # If loading fails, try downloading again
+                try:
+                    os.remove(model_path)
+                    logger.info("Removed corrupted model file, will try downloading again")
+                except Exception as rm_error:
+                    logger.error(f"Could not remove corrupted model file: {rm_error}")
+                return load_fasttext_model()
         else:
-            log_print("Downloading fasttext model...")
-            import gensim.downloader as api
-            model = api.load('fasttext-wiki-news-subwords-300')
-            
-            # Save the model for future use
-            os.makedirs(os.path.dirname(model_path), exist_ok=True)
-            model.save_word2vec_format(model_path)
-            
-            return model
+            logger.info("Downloading FastText model...")
+            try:
+                import gensim.downloader as api
+                model = api.load('fasttext-wiki-news-subwords-300')
+                logger.info("Successfully downloaded FastText model")
+                
+                # Save the model for future use
+                try:
+                    model.save_word2vec_format(model_path)
+                    logger.info(f"Saved FastText model to: {model_path}")
+                except Exception as save_error:
+                    logger.warning(f"Could not save model to cache: {str(save_error)}")
+                
+                return model
+            except Exception as e:
+                logger.error(f"Error downloading FastText model: {str(e)}")
+                return DummyFasttext()
     except Exception as e:
-        log_print(f"Error loading fasttext model: {str(e)}", "ERROR")
-        # Return a dummy model that provides basic word vector functionality
+        logger.error(f"Error in load_fasttext_model: {str(e)}")
         return DummyFasttext()
 
 class DummyFasttext: