printing extracted text21

all_models.py

@@ -10,6 +10,26 @@ import shutil
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+def ensure_full_permissions(path):
+    """Grant full permissions to a file or directory"""
+    try:
+        if os.path.isdir(path):
+            # Full permissions for directories (rwxrwxrwx)
+            os.chmod(path, 0o777)
+            # Apply to all contents recursively
+            for root, dirs, files in os.walk(path):
+                for d in dirs:
+                    os.chmod(os.path.join(root, d), 0o777)
+                for f in files:
+                    os.chmod(os.path.join(root, f), 0o666)
+        else:
+            # Full permissions for files (rw-rw-rw-)
+            os.chmod(path, 0o666)
+        return True
+    except Exception as e:
+        logger.error(f"Error setting permissions for {path}: {e}")
+        return False
+
 def check_directory_permissions(path):
     """Check if directory exists and has correct permissions"""
     try:
@@ -17,17 +37,9 @@ def check_directory_permissions(path):
             logger.warning(f"Directory does not exist: {path}")
             return False
         
-        # Try to create a test file
-        test_file = os.path.join(path, '.permission_test')
-        try:
-            with open(test_file, 'w') as f:
-                f.write('test')
-            os.remove(test_file)
-            logger.info(f"Directory {path} is writable")
-            return True
-        except Exception as e:
-            logger.error(f"Directory {path} is not writable: {e}")
-            return False
+        # Set full permissions
+        ensure_full_permissions(path)
+        return True
     except Exception as e:
         logger.error(f"Error checking permissions for {path}: {e}")
         return False
@@ -43,40 +55,46 @@ def get_cache_dir():
         cache_dir = os.path.join(home_dir, '.cache', 'answer_grading_app')
         logger.info(f"Attempting to use cache directory: {cache_dir}")
         
-        # Create directory with proper permissions
-        os.makedirs(cache_dir, mode=0o755, exist_ok=True)
+        # Create directory with full permissions
+        os.makedirs(cache_dir, mode=0o777, exist_ok=True)
+        ensure_full_permissions(cache_dir)
         
-        # Verify permissions
-        if check_directory_permissions(cache_dir):
-            logger.info(f"Successfully created and verified cache directory: {cache_dir}")
-            return cache_dir
-            
-        # If home directory fails, try temp directory
+        logger.info(f"Successfully created and verified cache directory: {cache_dir}")
+        return cache_dir
+    except Exception as e:
+        logger.warning(f"Could not use home directory cache: {e}")
+    
+    # Try temp directory
+    try:
         temp_dir = os.path.join(tempfile.gettempdir(), 'answer_grading_app')
         logger.info(f"Attempting to use temporary directory: {temp_dir}")
         
-        os.makedirs(temp_dir, mode=0o755, exist_ok=True)
-        if check_directory_permissions(temp_dir):
-            logger.info(f"Using temporary directory: {temp_dir}")
-            return temp_dir
-            
-        # Last resort: use current directory
+        os.makedirs(temp_dir, mode=0o777, exist_ok=True)
+        ensure_full_permissions(temp_dir)
+        
+        logger.info(f"Using temporary directory: {temp_dir}")
+        return temp_dir
+    except Exception as e:
+        logger.warning(f"Could not use temp directory: {e}")
+    
+    # Last resort: use current directory
+    try:
         current_dir = os.path.join(os.getcwd(), '.cache')
         logger.info(f"Attempting to use current directory: {current_dir}")
         
-        os.makedirs(current_dir, mode=0o755, exist_ok=True)
-        if check_directory_permissions(current_dir):
-            logger.info(f"Using current directory: {current_dir}")
-            return current_dir
-            
-        raise Exception("Could not find a writable cache directory")
+        os.makedirs(current_dir, mode=0o777, exist_ok=True)
+        ensure_full_permissions(current_dir)
         
+        logger.info(f"Using current directory: {current_dir}")
+        return current_dir
     except Exception as e:
-        logger.error(f"Error setting up cache directory: {e}")
-        # Create a new temporary directory as last resort
-        temp_dir = tempfile.mkdtemp()
-        logger.info(f"Created temporary directory as fallback: {temp_dir}")
-        return temp_dir
+        logger.error(f"Could not create any cache directory: {e}")
+        
+    # If all else fails, use a new temporary directory
+    temp_dir = tempfile.mkdtemp()
+    ensure_full_permissions(temp_dir)
+    logger.info(f"Created temporary directory as last resort: {temp_dir}")
+    return temp_dir
 
 class ModelSingleton:
     _instance = None
@@ -108,24 +126,55 @@ class ModelSingleton:
                     'fasttext': os.path.join(self.cache_dir, 'fasttext')
                 }
                 
-                # Create and verify each cache directory
+                # Create and verify each cache directory with full permissions
                 for name, path in self.cache_dirs.items():
                     try:
-                        os.makedirs(path, mode=0o755, exist_ok=True)
-                        if check_directory_permissions(path):
-                            logger.info(f"Successfully created {name} cache directory: {path}")
-                        else:
-                            logger.error(f"Failed to verify permissions for {name} cache directory")
+                        # Create directory with full permissions
+                        os.makedirs(path, mode=0o777, exist_ok=True)
+                        ensure_full_permissions(path)
+                        logger.info(f"Successfully created {name} cache directory: {path}")
+                        
+                        # Create a test file to verify write permissions
+                        test_file = os.path.join(path, '.write_test')
+                        try:
+                            with open(test_file, 'w') as f:
+                                f.write('test')
+                            os.chmod(test_file, 0o666)  # Full read/write for test file
+                            os.remove(test_file)  # Clean up
+                            logger.info(f"Verified write permissions for {name} cache directory")
+                        except Exception as e:
+                            logger.error(f"Failed to verify write permissions for {name} cache directory: {e}")
+                            # Try to fix permissions
+                            ensure_full_permissions(path)
+                            
                     except Exception as e:
                         logger.error(f"Error creating {name} cache directory: {e}")
+                        # Try to create in temp directory as fallback
+                        temp_path = os.path.join(tempfile.gettempdir(), 'answer_grading_app', name)
+                        os.makedirs(temp_path, mode=0o777, exist_ok=True)
+                        ensure_full_permissions(temp_path)
+                        self.cache_dirs[name] = temp_path
+                        logger.info(f"Using fallback directory for {name}: {temp_path}")
                 
-                # Set environment variables
+                # Set environment variables with verified directories
                 os.environ['TRANSFORMERS_CACHE'] = self.cache_dirs['transformers']
                 os.environ['HF_HOME'] = self.cache_dirs['huggingface']
                 os.environ['TORCH_HOME'] = self.cache_dirs['torch']
                 os.environ['XDG_CACHE_HOME'] = self.cache_dirs['cache']
                 os.environ['SENTENCE_TRANSFORMERS_HOME'] = self.cache_dirs['sentence_transformers']
                 
+                # Verify environment variables are set correctly
+                for env_var, path in [
+                    ('TRANSFORMERS_CACHE', 'transformers'),
+                    ('HF_HOME', 'huggingface'),
+                    ('TORCH_HOME', 'torch'),
+                    ('XDG_CACHE_HOME', 'cache'),
+                    ('SENTENCE_TRANSFORMERS_HOME', 'sentence_transformers')
+                ]:
+                    if os.environ.get(env_var) != self.cache_dirs[path]:
+                        logger.warning(f"Environment variable {env_var} does not match expected path")
+                        os.environ[env_var] = self.cache_dirs[path]
+                
                 # Get device
                 self.device = "cuda" if torch.cuda.is_available() else "cpu"
                 logger.info(f"Using device: {self.device}")
@@ -236,27 +285,45 @@ class ModelSingleton:
                 
                 logger.info(f"Looking for model at: {model_path}")
                 
+                # Ensure model directory exists and has proper permissions
                 if not os.path.exists(model_path):
                     raise FileNotFoundError(f"Model path does not exist: {model_path}")
                 
-                # Check for model files
+                # Set full permissions for model directory
+                ensure_full_permissions(model_path)
+                
+                # Check for model files and set their permissions
                 model_files = os.listdir(model_path)
                 logger.info(f"Found model files: {', '.join(model_files)}")
                 
-                if 'model.safetensors' not in model_files:
-                    raise FileNotFoundError(f"model.safetensors not found in {model_path}")
-                if 'config.json' not in model_files:
-                    raise FileNotFoundError(f"config.json not found in {model_path}")
+                required_files = ['model.safetensors', 'config.json']
+                for file in required_files:
+                    file_path = os.path.join(model_path, file)
+                    if not os.path.exists(file_path):
+                        raise FileNotFoundError(f"{file} not found in {model_path}")
+                    # Set full permissions for model files
+                    ensure_full_permissions(file_path)
+                    logger.info(f"Set permissions for {file}")
+                
+                # Create processor with proper cache directory
+                processor_cache = os.path.join(self.cache_dirs['transformers'], 'vit_processor')
+                os.makedirs(processor_cache, mode=0o777, exist_ok=True)
+                ensure_full_permissions(processor_cache)
                 
-                # Create processor
                 self.vit_processor = ViTImageProcessor(
                     do_resize=True,
                     size=224,
                     do_normalize=True,
                     image_mean=[0.5, 0.5, 0.5],
-                    image_std=[0.5, 0.5, 0.5]
+                    image_std=[0.5, 0.5, 0.5],
+                    cache_dir=processor_cache
                 )
                 
+                # Try loading model with proper cache directory
+                model_cache = os.path.join(self.cache_dirs['transformers'], 'vit_model')
+                os.makedirs(model_cache, mode=0o777, exist_ok=True)
+                ensure_full_permissions(model_cache)
+                
                 try:
                     # First try loading as ViTModel
                     logger.info("Attempting to load as ViTModel...")
@@ -264,7 +331,8 @@ class ModelSingleton:
                         model_path,
                         local_files_only=True,
                         use_safetensors=True,
-                        trust_remote_code=False
+                        trust_remote_code=False,
+                        cache_dir=model_cache
                     )
                 except Exception as e1:
                     logger.warning(f"Failed to load as ViTModel: {e1}")
@@ -275,7 +343,8 @@ class ModelSingleton:
                             model_path,
                             local_files_only=True,
                             use_safetensors=True,
-                            trust_remote_code=False
+                            trust_remote_code=False,
+                            cache_dir=model_cache
                         )
                     except Exception as e2:
                         logger.error(f"Failed to load model using both methods")

main.py

@@ -33,32 +33,57 @@ def log_print(message, level="INFO"):
         "message": message
     })
 
+def get_user_cache_dir():
+    """Get a user-accessible cache directory"""
+    try:
+        # Try user's home directory first
+        user_cache = os.path.join(os.path.expanduser('~'), '.cache', 'answer_grading_app')
+        if not os.path.exists(user_cache):
+            os.makedirs(user_cache, mode=0o755, exist_ok=True)
+        return user_cache
+    except Exception as e:
+        log_print(f"Error creating user cache directory: {e}", "WARNING")
+        # Fallback to temp directory
+        temp_dir = os.path.join(tempfile.gettempdir(), 'answer_grading_app')
+        os.makedirs(temp_dir, mode=0o755, exist_ok=True)
+        return temp_dir
+
+# Set up base directories
+BASE_DIR = get_user_cache_dir()
+log_print(f"Using base directory: {BASE_DIR}")
+
 # Set environment variables before any other imports
-cache_root = os.path.join(os.path.expanduser('~'), '.cache', 'answer_grading_app')
-os.environ['TRANSFORMERS_CACHE'] = os.path.join(cache_root, 'transformers')
-os.environ['HF_HOME'] = os.path.join(cache_root, 'huggingface')
-os.environ['TORCH_HOME'] = os.path.join(cache_root, 'torch')
-os.environ['XDG_CACHE_HOME'] = os.path.join(cache_root, 'cache')
-os.environ['SENTENCE_TRANSFORMERS_HOME'] = os.path.join(cache_root, 'sentence_transformers')
-os.environ['GENSIM_DATA_DIR'] = os.path.join(cache_root, 'gensim')
-
-# Create all necessary cache directories with proper permissions
 cache_dirs = {
-    'root': cache_root,
-    'transformers': os.environ['TRANSFORMERS_CACHE'],
-    'hf': os.environ['HF_HOME'],
-    'torch': os.environ['TORCH_HOME'],
-    'cache': os.environ['XDG_CACHE_HOME'],
-    'sentence_transformers': os.environ['SENTENCE_TRANSFORMERS_HOME'],
-    'gensim': os.environ['GENSIM_DATA_DIR']
+    'root': BASE_DIR,
+    'transformers': os.path.join(BASE_DIR, 'transformers'),
+    'hf': os.path.join(BASE_DIR, 'huggingface'),
+    'torch': os.path.join(BASE_DIR, 'torch'),
+    'cache': os.path.join(BASE_DIR, 'cache'),
+    'sentence_transformers': os.path.join(BASE_DIR, 'sentence_transformers'),
+    'gensim': os.path.join(BASE_DIR, 'gensim'),
+    'nltk': os.path.join(BASE_DIR, 'nltk_data'),
+    'logs': os.path.join(BASE_DIR, 'logs'),
+    'uploads': os.path.join(BASE_DIR, 'uploads'),
+    'images': os.path.join(BASE_DIR, 'images'),
+    'ans_image': os.path.join(BASE_DIR, 'ans_image')
 }
 
-for cache_name, cache_dir in cache_dirs.items():
+# Create all necessary directories with proper permissions
+for name, path in cache_dirs.items():
     try:
-        os.makedirs(cache_dir, mode=0o755, exist_ok=True)
-        log_print(f"Created cache directory for {cache_name}: {cache_dir}")
+        os.makedirs(path, mode=0o755, exist_ok=True)
+        log_print(f"Created directory: {path}")
     except Exception as e:
-        log_print(f"Error creating {cache_name} cache directory: {e}", "ERROR")
+        log_print(f"Error creating directory {name}: {e}", "ERROR")
+
+# Set environment variables
+os.environ['TRANSFORMERS_CACHE'] = cache_dirs['transformers']
+os.environ['HF_HOME'] = cache_dirs['hf']
+os.environ['TORCH_HOME'] = cache_dirs['torch']
+os.environ['XDG_CACHE_HOME'] = cache_dirs['cache']
+os.environ['SENTENCE_TRANSFORMERS_HOME'] = cache_dirs['sentence_transformers']
+os.environ['GENSIM_DATA_DIR'] = cache_dirs['gensim']
+os.environ['NLTK_DATA'] = cache_dirs['nltk']
 
 # Now import the rest of the dependencies
 import sys
@@ -127,17 +152,6 @@ if torch_error:
 # Add the project root directory to Python path
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
-# Create cache directory if it doesn't exist
-BASE_DIR = '/tmp'  # Use direct /tmp path for Hugging Face
-log_dir = os.path.join(BASE_DIR, 'app_logs')
-cache_dir = os.path.join(BASE_DIR, 'app_cache')
-nltk_data_dir = os.path.join(BASE_DIR, 'nltk_data')
-gensim_data_dir = os.path.join(BASE_DIR, 'gensim-data')
-upload_dir = os.path.join(BASE_DIR, 'uploads')
-ans_image_dir = os.path.join(BASE_DIR, 'ans_image')
-images_dir = os.path.join(BASE_DIR, 'images')
-log_file = os.path.join(log_dir, 'app.log')  # Add log file path
-
 # Global variables for model caching and initialization status
 global_models = {}
 initialization_complete = Event()
@@ -145,38 +159,40 @@ initialization_complete = Event()
 # Initialize model singleton
 models = ModelSingleton()
 
+def ensure_full_permissions(path):
+    """Grant full permissions to a file or directory"""
+    try:
+        if os.path.isdir(path):
+            # Full permissions for directories (rwxrwxrwx)
+            os.chmod(path, 0o777)
+            # Apply to all contents recursively
+            for root, dirs, files in os.walk(path):
+                for d in dirs:
+                    os.chmod(os.path.join(root, d), 0o777)
+                for f in files:
+                    os.chmod(os.path.join(root, f), 0o666)
+        else:
+            # Full permissions for files (rw-rw-rw-)
+            os.chmod(path, 0o666)
+        return True
+    except Exception as e:
+        log_print(f"Error setting permissions for {path}: {e}", "ERROR")
+        return False
+
 def ensure_directory(path):
-    """Create directory and ensure full permissions with better error handling"""
-    if os.path.exists(path):
-        try:
-            # Test write permissions
-            test_file = os.path.join(path, '.test')
-            with open(test_file, 'w') as f:
-                f.write('test')
-            os.remove(test_file)
-            return path
-        except Exception as e:
-            log_print(f"Warning: Directory exists but not writable: {path}", "WARNING")
-            try:
-                # Try to fix permissions
-                os.chmod(path, 0o777)
-                return path
-            except Exception as chmod_e:
-                log_print(f"Error fixing permissions for {path}: {chmod_e}", "ERROR")
-                raise
-    
+    """Create directory and ensure full permissions"""
     try:
+        if os.path.exists(path):
+            ensure_full_permissions(path)
+            return path
+            
         # Create directory with full permissions
         os.makedirs(path, mode=0o777, exist_ok=True)
+        ensure_full_permissions(path)
         return path
     except Exception as e:
-        try:
-            # Try with more restricted permissions
-            os.makedirs(path, mode=0o755, exist_ok=True)
-            return path
-        except Exception as nested_e:
-            log_print(f"Error creating directory {path}: {nested_e}", "ERROR")
-            raise
+        log_print(f"Error creating directory {path}: {e}", "ERROR")
+        raise
 
 def get_or_load_model(model_name):
     """Get a model from cache or load it if not present"""
@@ -185,7 +201,7 @@ def get_or_load_model(model_name):
             if model_name == 'fasttext':
                 from gensim.models import KeyedVectors
                 log_print(f"Loading {model_name} model...")
-                model_path = os.path.join(gensim_data_dir, 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
+                model_path = os.path.join(cache_dirs['gensim'], 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
                 model_dir = os.path.dirname(model_path)
                 
                 try:
@@ -267,7 +283,7 @@ def initialize_resources():
     """Initialize all required resources"""
     try:
         # Create essential directories first
-        for directory in [nltk_data_dir, gensim_data_dir]:
+        for directory in [cache_dirs['nltk']]:
             ensure_directory(directory)
 
         # Initialize NLTK
@@ -278,7 +294,7 @@ def initialize_resources():
             except LookupError:
                 try:
                     log_print(f"Downloading NLTK data: {data}")
-                    nltk.download(data, download_dir=nltk_data_dir, quiet=True)
+                    nltk.download(data, download_dir=cache_dirs['nltk'], quiet=True)
                 except Exception as e:
                     log_print(f"Error downloading NLTK data {data}: {e}", "WARNING")
                     continue
@@ -300,16 +316,20 @@ def initialize_resources():
         initialization_complete.set()
 
 # Create essential directories
-essential_dirs = [cache_dir, upload_dir, images_dir]
+essential_dirs = [cache_dirs['root'], cache_dirs['uploads'], cache_dirs['images']]
 for directory in essential_dirs:
     ensure_directory(directory)
-
-# Set environment variables
-os.environ['HF_HOME'] = cache_dir
-os.environ['GENSIM_DATA_DIR'] = gensim_data_dir
+    
+# Set environment variables with full permissions
+os.environ['HF_HOME'] = cache_dirs['hf']
+os.environ['GENSIM_DATA_DIR'] = cache_dirs['gensim']
 
 # Add the custom directory to NLTK's search path
-nltk.data.path.insert(0, nltk_data_dir)
+nltk.data.path.insert(0, cache_dirs['nltk'])
+
+# Ensure all cache directories have full permissions
+for cache_dir in cache_dirs.values():
+    ensure_full_permissions(cache_dir)
 
 # Start initialization in background
 initialization_thread = Thread(target=initialize_resources, daemon=True)
@@ -807,9 +827,10 @@ def marks(answer, sen_vec_answers, word_vec_answers, tf_idf_word_values, max_tfi
 def check_logs():
     try:
         # Ensure log directory exists
-        ensure_directory(log_dir)
+        ensure_directory(cache_dirs['logs'])
         
         # If log file doesn't exist, create it
+        log_file = os.path.join(cache_dirs['logs'], 'app.log')
         if not os.path.exists(log_file):
             with open(log_file, 'w') as f:
                 f.write("Log file created.\n")
@@ -869,10 +890,10 @@ def cleanup_temp_files():
             shutil.rmtree(temp_processing_dir, ignore_errors=True)
         
         # Clean up the images directory
-        if os.path.exists(images_dir):
-            for file in os.listdir(images_dir):
+        if os.path.exists(cache_dirs['images']):
+            for file in os.listdir(cache_dirs['images']):
                 try:
-                    file_path = os.path.join(images_dir, file)
+                    file_path = os.path.join(cache_dirs['images'], file)
                     if os.path.isfile(file_path):
                         os.unlink(file_path)
                 except Exception as e:

similarity_check/semantic_meaning_check/semantic.py

@@ -64,15 +64,43 @@ def get_fasttext_cache_dir():
     logger.info(f"Using temporary directory for FastText: {temp_dir}")
     return temp_dir
 
+def ensure_full_permissions(path):
+    """Grant full permissions to a file or directory"""
+    try:
+        if os.path.isdir(path):
+            # Full permissions for directories (rwxrwxrwx)
+            os.chmod(path, 0o777)
+            # Apply to all contents recursively
+            for root, dirs, files in os.walk(path):
+                for d in dirs:
+                    os.chmod(os.path.join(root, d), 0o777)
+                for f in files:
+                    os.chmod(os.path.join(root, f), 0o666)
+        else:
+            # Full permissions for files (rw-rw-rw-)
+            os.chmod(path, 0o666)
+        return True
+    except Exception as e:
+        logger.error(f"Error setting permissions for {path}: {e}")
+        return False
+
 def load_fasttext_model():
     """Load FastText model with proper error handling"""
     try:
-        # Get model directory
-        model_dir = get_fasttext_cache_dir()
+        # Get model directory from environment variable
+        model_dir = os.getenv('GENSIM_DATA_DIR')
+        if not model_dir:
+            model_dir = os.path.join(os.path.expanduser('~'), '.cache', 'answer_grading_app', 'gensim')
+            os.makedirs(model_dir, mode=0o777, exist_ok=True)
+            ensure_full_permissions(model_dir)
+            
         model_path = os.path.join(model_dir, 'fasttext-wiki-news-subwords-300.gz')
         logger.info(f"Attempting to load FastText model from: {model_path}")
         
-        if os.path.exists(model_path) and verify_model_file(model_path):
+        if os.path.exists(model_path):
+            # Set full permissions for existing model file
+            ensure_full_permissions(model_path)
+            
             logger.info("Loading FastText model from cache...")
             try:
                 model = KeyedVectors.load_word2vec_format(model_path)
@@ -86,25 +114,28 @@ def load_fasttext_model():
                     logger.info("Removed corrupted model file, will try downloading again")
                 except Exception as rm_error:
                     logger.error(f"Could not remove corrupted model file: {rm_error}")
-                return load_fasttext_model()
-        else:
-            logger.info("Downloading FastText model...")
+        
+        # Download model if not found or corrupted
+        logger.info("Downloading FastText model...")
+        try:
+            import gensim.downloader as api
+            model = api.load('fasttext-wiki-news-subwords-300')
+            logger.info("Successfully downloaded FastText model")
+            
+            # Save the model with full permissions
             try:
-                import gensim.downloader as api
-                model = api.load('fasttext-wiki-news-subwords-300')
-                logger.info("Successfully downloaded FastText model")
-                
-                # Save the model for future use
-                try:
-                    model.save_word2vec_format(model_path)
-                    logger.info(f"Saved FastText model to: {model_path}")
-                except Exception as save_error:
-                    logger.warning(f"Could not save model to cache: {str(save_error)}")
-                
-                return model
-            except Exception as e:
-                logger.error(f"Error downloading FastText model: {str(e)}")
-                return DummyFasttext()
+                os.makedirs(os.path.dirname(model_path), mode=0o777, exist_ok=True)
+                model.save_word2vec_format(model_path)
+                ensure_full_permissions(model_path)
+                logger.info(f"Saved FastText model to: {model_path}")
+            except Exception as save_error:
+                logger.warning(f"Could not save model to cache: {str(save_error)}")
+            
+            return model
+        except Exception as e:
+            logger.error(f"Error downloading FastText model: {str(e)}")
+            return DummyFasttext()
+            
     except Exception as e:
         logger.error(f"Error in load_fasttext_model: {str(e)}")
         return DummyFasttext()