fix memory overlimit issue

all_models.py

@@ -12,6 +12,8 @@ logger = logging.getLogger(__name__)
 class ModelSingleton:
     _instance = None
     _initialized = False
+    _models = {}
+    _reference_counts = {}
 
     def __new__(cls):
         if cls._instance is None:
@@ -29,63 +31,113 @@ class ModelSingleton:
                 self.device = "cuda" if torch.cuda.is_available() else "cpu"
                 logger.info(f"Using device: {self.device}")
 
-                # Sentence transformer model
-                try:
-                    logger.info("Loading sentence transformer model...")
-                    SENTENCE_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
-                    self.similarity_tokenizer = AutoTokenizer.from_pretrained(
-                        SENTENCE_MODEL,
-                        cache_dir=cache_dir
-                    )
-                    self.similarity_model = SentenceTransformer(
-                        SENTENCE_MODEL,
-                        cache_folder=cache_dir
-                    )
-                    self.similarity_model.to(self.device)
-                    logger.info("Sentence transformer model loaded successfully")
-                except Exception as e:
-                    logger.error(f"Error loading sentence transformer model: {e}")
-                    raise
-
-                # Flan-T5-xl model
-                try:
-                    logger.info("Loading Flan-T5 model...")
-                    FLAN_MODEL = "google/flan-t5-xl"
-                    self.flan_tokenizer = AutoTokenizer.from_pretrained(
-                        FLAN_MODEL,
-                        cache_dir=cache_dir
-                    )
-                    self.flan_model = AutoModelForSeq2SeqLM.from_pretrained(
-                        FLAN_MODEL,
-                        cache_dir=cache_dir,
-                        torch_dtype=torch.float16 if self.device == "cuda" else torch.float32
-                    )
-                    self.flan_model.to(self.device)
-                    logger.info("Flan-T5 model loaded successfully")
-                except Exception as e:
-                    logger.error(f"Error loading Flan-T5 model: {e}")
-                    raise
+                # Initialize with None values
+                self.similarity_tokenizer = None
+                self.similarity_model = None
+                self.flan_tokenizer = None
+                self.flan_model = None
+                
+                # Initialize reference counts
+                self._reference_counts['similarity'] = 0
+                self._reference_counts['flan'] = 0
                 
                 self._initialized = True
-                logger.info("All models initialized successfully")
+                logger.info("Model singleton initialized")
                 
             except Exception as e:
                 logger.error(f"Error during model initialization: {e}")
                 raise
 
-    def cleanup(self):
-        """Clean up model resources"""
+    def get_similarity_model(self):
+        """Get sentence transformer model with reference counting"""
+        try:
+            if self.similarity_model is None:
+                logger.info("Loading sentence transformer model...")
+                SENTENCE_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+                self.similarity_tokenizer = AutoTokenizer.from_pretrained(
+                    SENTENCE_MODEL,
+                    cache_dir=os.getenv('TRANSFORMERS_CACHE')
+                )
+                self.similarity_model = SentenceTransformer(
+                    SENTENCE_MODEL,
+                    cache_folder=os.getenv('TRANSFORMERS_CACHE')
+                )
+                self.similarity_model.to(self.device)
+                logger.info("Sentence transformer model loaded successfully")
+            
+            self._reference_counts['similarity'] += 1
+            return self.similarity_model
+        except Exception as e:
+            logger.error(f"Error loading sentence transformer model: {e}")
+            raise
+
+    def get_flan_model(self):
+        """Get Flan-T5 model with reference counting"""
         try:
-            if hasattr(self, 'similarity_model'):
-                del self.similarity_model
-            if hasattr(self, 'flan_model'):
-                del self.flan_model
+            if self.flan_model is None:
+                logger.info("Loading Flan-T5 model...")
+                FLAN_MODEL = "google/flan-t5-xl"
+                self.flan_tokenizer = AutoTokenizer.from_pretrained(
+                    FLAN_MODEL,
+                    cache_dir=os.getenv('TRANSFORMERS_CACHE')
+                )
+                self.flan_model = AutoModelForSeq2SeqLM.from_pretrained(
+                    FLAN_MODEL,
+                    cache_dir=os.getenv('TRANSFORMERS_CACHE'),
+                    torch_dtype=torch.float16 if self.device == "cuda" else torch.float32,
+                    low_cpu_mem_usage=True
+                )
+                self.flan_model.to(self.device)
+                logger.info("Flan-T5 model loaded successfully")
+            
+            self._reference_counts['flan'] += 1
+            return self.flan_model
+        except Exception as e:
+            logger.error(f"Error loading Flan-T5 model: {e}")
+            raise
+
+    def release_similarity_model(self):
+        """Release reference to similarity model"""
+        self._reference_counts['similarity'] -= 1
+        if self._reference_counts['similarity'] <= 0:
+            self._cleanup_similarity_model()
+
+    def release_flan_model(self):
+        """Release reference to Flan-T5 model"""
+        self._reference_counts['flan'] -= 1
+        if self._reference_counts['flan'] <= 0:
+            self._cleanup_flan_model()
+
+    def _cleanup_similarity_model(self):
+        """Clean up similarity model resources"""
+        if self.similarity_model is not None:
+            del self.similarity_model
+            self.similarity_model = None
+            self.similarity_tokenizer = None
+            torch.cuda.empty_cache()
+            logger.info("Similarity model resources cleaned up")
+
+    def _cleanup_flan_model(self):
+        """Clean up Flan-T5 model resources"""
+        if self.flan_model is not None:
+            del self.flan_model
+            self.flan_model = None
+            self.flan_tokenizer = None
             torch.cuda.empty_cache()
-            logger.info("Model resources cleaned up successfully")
+            logger.info("Flan-T5 model resources cleaned up")
+
+    def cleanup(self):
+        """Clean up all model resources"""
+        try:
+            self._cleanup_similarity_model()
+            self._cleanup_flan_model()
+            self._reference_counts['similarity'] = 0
+            self._reference_counts['flan'] = 0
+            logger.info("All model resources cleaned up successfully")
         except Exception as e:
             logger.error(f"Error during cleanup: {e}")
 
-# Create a global instance
+# Create global instance
 models = ModelSingleton()
 
 # Add cleanup function to the global instance

main.py

@@ -1,5 +1,7 @@
 import os
 import tempfile
+import gc
+import psutil
 
 # Set up Hugging Face cache directory
 os.environ['TRANSFORMERS_CACHE'] = os.path.join(tempfile.gettempdir(), 'huggingface_cache')
@@ -420,6 +422,32 @@ def notifications():
 
     return Response(generate(), mimetype='text/event-stream')
 
+def get_memory_usage():
+    """Get current memory usage"""
+    process = psutil.Process(os.getpid())
+    return process.memory_info().rss / 1024 / 1024  # Convert to MB
+
+def cleanup_memory():
+    """Clean up memory by clearing caches and garbage collection"""
+    try:
+        # Clear PyTorch cache
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        
+        # Clear Python garbage collection
+        gc.collect()
+        
+        # Clear model caches
+        if hasattr(models, 'cleanup'):
+            models.cleanup()
+            
+        # Log memory usage
+        memory_usage = get_memory_usage()
+        log_print(f"Memory usage after cleanup: {memory_usage:.2f} MB")
+        
+    except Exception as e:
+        log_print(f"Error during memory cleanup: {e}", "ERROR")
+
 @app.route('/compute_marks', methods=['POST'])
 def compute_marks():
     try:
@@ -534,6 +562,9 @@ def compute_marks():
                     })
                     count += 1
                     
+                    # Clean up memory after each student
+                    cleanup_memory()
+                    
                 except Exception as e:
                     logger.error(f"Error processing {image_path}: {str(e)}")
                     results.append({
@@ -552,6 +583,9 @@ def compute_marks():
         except Exception as e:
             logger.warning(f"Could not clean up temporary files: {e}")
         
+        # Final memory cleanup
+        cleanup_memory()
+        
         return jsonify({"results": results}), 200
         
     except Exception as e:
@@ -561,6 +595,9 @@ def compute_marks():
             "message": error_msg
         })
         return jsonify({"error": error_msg}), 500
+    finally:
+        # Ensure memory is cleaned up even if there's an error
+        cleanup_memory()
 
 def marks(answer, sen_vec_answers, word_vec_answers, tf_idf_word_values, max_tfidf, correct_answers):
     try:

similarity_check/llm_based_scoring/llm.py

@@ -23,69 +23,53 @@ def llm_score(correct_answers, answer):
             correct_answers = [correct_answers]
             
         score = []
-
+        
+        # Get model instance
+        model = models.get_flan_model()
+        tokenizer = models.flan_tokenizer
+        
+        # Process each correct answer
         for correct_answer in correct_answers:
             try:
-                prompt = (
-                    "You are an expert evaluator of answers. Your response must be a *single numeric score (0-10), not a range.*\n\n"
-                    
-                    "The user's answer has been converted from handwriting using OCR, so minor spelling, punctuation, or small word variations may exist. "
-                    "Focus on meaning rather than transcription errors.\n\n"
-                    
-                    "### Evaluation Criteria:\n"
-                    "- *Correctness (90% weight):* Does the answer accurately convey the meaning of the correct answer?\n"
-                    "- *Completeness (10% weight):* Does it cover all key points?\n\n"
-                    
-                    "### Handling OCR Errors:\n"
-                    "- Ignore minor spelling/punctuation mistakes that don't affect meaning.\n"
-                    "- Penalize only if word substitutions change the meaning.\n\n"
-                    
-                    "### Scoring Guidelines:\n"
-                    "- *10:* Fully correct and complete (90-100% accurate).\n"
-                    "- *From 9 to 8:* Mostly correct, minor missing details (80-90% accurate).\n"
-                    "- *From 7 to 6:* Good but missing some key points (60-80% accurate).\n"
-                    "- *From 5 to 4:* Average, with several omissions/errors (40-60% accurate).\n"
-                    "- *From 3 to 2:* Poor, major meaning errors (20-40% accurate).\n"
-                    "- *From 1 to 0:* Incorrect or irrelevant (less than 20% accurate).\n\n"
-                    
-                    "Compare the answers and assign a *single numeric score (0-10)* based on correctness and completeness.\n\n"
-                    
-                    "Correct answer:\n"
-                    f"{correct_answer}\n\n"
-                    "User's answer:\n"
-                    f"{answer}\n\n"
-                    "Final Score (numeric only, strictly between 0 and 10):")
-
-                # Tokenize input prompt
-                inputs = models.flan_tokenizer(prompt, return_tensors="pt").to(device)
+                # Prepare input
+                input_text = f"Compare these answers and give a similarity score between 0 and 1:\nCorrect: {correct_answer}\nStudent: {answer}"
+                inputs = tokenizer(input_text, return_tensors="pt", max_length=512, truncation=True)
+                inputs = {k: v.to(models.device) for k, v in inputs.items()}
                 
-                # Generate response
-                with torch.no_grad():
-                    outputs = models.flan_model.generate(
+                # Generate score
+                with torch.no_grad():  # Disable gradient calculation
+                    outputs = model.generate(
                         **inputs,
-                        max_length=2048,
-                        do_sample=True,
+                        max_length=50,
                         num_return_sequences=1,
-                        num_beams=5,
-                        temperature=0.6,
-                        top_p=0.9,
-                        early_stopping=True,
-                        pad_token_id=models.flan_tokenizer.pad_token_id,
-                        eos_token_id=models.flan_tokenizer.eos_token_id,
-                        bos_token_id=models.flan_tokenizer.bos_token_id,
+                        temperature=0.7,
+                        do_sample=True
                     )
-
-                # Decode and print response
-                response = models.flan_tokenizer.decode(outputs[0], skip_special_tokens=True)
-                print(response)
-                score.append(response)
+                
+                # Decode and extract score
+                score_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+                try:
+                    # Try to extract numeric score
+                    score_value = float(score_text.split()[-1])
+                    score.append(min(max(score_value, 0.0), 1.0))  # Clamp between 0 and 1
+                except (ValueError, IndexError):
+                    # If no numeric score found, use default
+                    score.append(0.5)
                     
             except Exception as e:
-                print(f"Error processing individual answer: {str(e)}")
-                score.append("0")
+                logger.error(f"Error processing answer: {str(e)}")
+                score.append(0.5)  # Use default score on error
                 
+        # Clean up tensors
+        del inputs
+        del outputs
+        torch.cuda.empty_cache()
+        
         return score
         
     except Exception as e:
-        print(f"Error in llm_score: {str(e)}")
-        return ["0"]
+        logger.error(f"Error in llm_score: {str(e)}")
+        return [0.5]  # Return default score on error
+    finally:
+        # Release model reference
+        models.release_flan_model()

similarity_check/semantic_meaning_check/semantic.py

@@ -41,22 +41,44 @@ except Exception as e:
 def question_vector_sentence(correct_answer):
     """Get sentence embedding using shared model"""
     try:
-        return models.similarity_model.encode(correct_answer, convert_to_tensor=True)
+        # Get model instance
+        model = models.get_similarity_model()
+        # Convert to tensor and move to correct device
+        embedding = model.encode(correct_answer, convert_to_tensor=True, device=models.device)
+        return embedding
     except Exception as e:
         logger.error(f"Error in question_vector_sentence: {str(e)}")
         return None
+    finally:
+        # Release model reference
+        models.release_similarity_model()
 
-def similarity_model_score(correct_answer_vector, answer):
-    """Calculate similarity score using shared model"""
+def similarity_model_score(sentence_vectors, answer):
+    """Calculate similarity score using sentence transformer"""
     try:
-        answer_embedding = models.similarity_model.encode(answer, convert_to_tensor=True)
-        cosine_score = float('-inf')
-        for i in correct_answer_vector:
-            cosine_score = max(cosine_score, util.pytorch_cos_sim(i, answer_embedding))
-        return cosine_score
+        # Get model instance
+        model = models.get_similarity_model()
+        
+        # Get answer embedding
+        answer_embedding = model.encode(answer, convert_to_tensor=True, device=models.device)
+        
+        # Calculate similarities
+        similarities = []
+        for vec in sentence_vectors:
+            if vec is not None:
+                similarity = util.pytorch_cos_sim(answer_embedding, vec).item()
+                similarities.append(similarity)
+        
+        if not similarities:
+            return 0.0
+            
+        return max(similarities)
     except Exception as e:
         logger.error(f"Error in similarity_model_score: {str(e)}")
         return 0.0
+    finally:
+        # Release model reference
+        models.release_similarity_model()
 
 def preprocess(sentence):
     """Preprocess text by tokenizing and removing stopwords"""
@@ -106,23 +128,44 @@ def compute_scm(tokens1, tokens2, model):
         return 0.5  # Return default similarity score
 
 def question_vector_word(correct_answer):
-    """Get preprocessed word tokens"""
+    """Get word embeddings using FastText"""
     try:
-        return preprocess(correct_answer)
+        # Tokenize and remove stopwords
+        stop_words = set(stopwords.words('english'))
+        words = word_tokenize(correct_answer.lower())
+        words = [w for w in words if w not in stop_words]
+        
+        # Get word embeddings
+        embeddings = []
+        for word in words:
+            if word in fasttext:
+                embeddings.append(fasttext[word])
+        
+        if not embeddings:
+            return np.zeros(300)  # Return zero vector if no valid words
+            
+        return np.mean(embeddings, axis=0)
     except Exception as e:
         logger.error(f"Error in question_vector_word: {str(e)}")
-        return []
+        return np.zeros(300)
 
-def fasttext_similarity(correct_answer_vector, answer):
-    """Compute fasttext-based similarity between answers"""
+def fasttext_similarity(word_vectors, answer):
+    """Calculate similarity score using FastText word embeddings"""
     try:
-        preprocess_answer = preprocess(answer)
-        soft_cosine = float('-inf')
+        # Get answer word embedding
+        answer_embedding = question_vector_word(answer)
         
-        for i in correct_answer_vector:
-            soft_cosine = max(compute_scm(i, preprocess_answer, fasttext), soft_cosine)
+        # Calculate similarities
+        similarities = []
+        for vec in word_vectors:
+            if vec is not None:
+                similarity = np.dot(answer_embedding, vec) / (np.linalg.norm(answer_embedding) * np.linalg.norm(vec))
+                similarities.append(similarity)
         
-        return soft_cosine
+        if not similarities:
+            return 0.0
+            
+        return max(similarities)
     except Exception as e:
         logger.error(f"Error in fasttext_similarity: {str(e)}")
         return 0.0