main.py

import os
import tempfile
import gc
import psutil
import time
import logging
import queue

# Set up logging first
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[logging.StreamHandler()]
)
logger = logging.getLogger(__name__)

# Create notification queue for real-time updates
notification_queue = queue.Queue()

def log_print(message, level="INFO"):
    """Unified logging function"""
    if level == "ERROR":
        logger.error(message)
    elif level == "WARNING":
        logger.warning(message)
    else:
        logger.info(message)
    # Also put the message in notification queue for frontend
    notification_queue.put({
        "type": level.lower(),
        "message": message
    })

# Set environment variables before any other imports
os.environ['TRANSFORMERS_CACHE'] = os.path.join(tempfile.gettempdir(), 'huggingface_cache')
os.environ['HF_HOME'] = os.path.join(tempfile.gettempdir(), 'huggingface')
os.environ['TORCH_HOME'] = os.path.join(tempfile.gettempdir(), 'torch')
os.environ['XDG_CACHE_HOME'] = os.path.join(tempfile.gettempdir(), 'cache')
os.environ['SENTENCE_TRANSFORMERS_HOME'] = os.path.join(tempfile.gettempdir(), 'sentence_transformers')

# Create all necessary cache directories
cache_dirs = {
    'transformers': os.environ['TRANSFORMERS_CACHE'],
    'hf': os.environ['HF_HOME'],
    'torch': os.environ['TORCH_HOME'],
    'cache': os.environ['XDG_CACHE_HOME'],
    'sentence_transformers': os.environ['SENTENCE_TRANSFORMERS_HOME']
}

for cache_name, cache_dir in cache_dirs.items():
    try:
        os.makedirs(cache_dir, exist_ok=True)
        log_print(f"Created cache directory for {cache_name}: {cache_dir}")
    except Exception as e:
        log_print(f"Error creating {cache_name} cache directory: {e}", "ERROR")

# Now import the rest of the dependencies
import sys
from pathlib import Path
from flask import Flask, request, jsonify, render_template, send_file, Response
from werkzeug.utils import secure_filename
import cv2
import numpy as np
from PIL import Image
import io
import base64
from datetime import datetime
import json
import threading
from threading import Thread, Event
import warnings
from flask_cors import CORS
from dotenv import load_dotenv
warnings.filterwarnings('ignore')

# Import ML libraries
import torch
import nltk
import gensim
from gensim.models import FastText
from sentence_transformers import SentenceTransformer
from transformers import pipeline

# Import ML libraries with timeout protection
def import_with_timeout(import_statement, timeout=30):
    """Import a module with a timeout to prevent hanging"""
    result = {'success': False, 'module': None, 'error': None}
    
    def _import():
        try:
            if isinstance(import_statement, str):
                result['module'] = __import__(import_statement)
            else:
                exec(import_statement)
            result['success'] = True
        except Exception as e:
            result['error'] = str(e)
    
    thread = Thread(target=_import)
    thread.daemon = True
    thread.start()
    thread.join(timeout=timeout)
    
    if thread.is_alive():
        return None, f"Import timed out after {timeout} seconds"
    
    return result['module'], result['error']

# Import ML libraries safely
nltk, nltk_error = import_with_timeout('nltk')
if nltk_error:
    log_print(f"Warning: NLTK import failed: {nltk_error}", "WARNING")

gensim, gensim_error = import_with_timeout('gensim')
if gensim_error:
    log_print(f"Warning: Gensim import failed: {gensim_error}", "WARNING")

torch, torch_error = import_with_timeout('torch')
if torch_error:
    log_print(f"Warning: PyTorch import failed: {torch_error}", "WARNING")

# Add the project root directory to Python path
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

# Create cache directory if it doesn't exist
BASE_DIR = '/tmp'  # Use direct /tmp path for Hugging Face
log_dir = os.path.join(BASE_DIR, 'app_logs')
cache_dir = os.path.join(BASE_DIR, 'app_cache')
nltk_data_dir = os.path.join(BASE_DIR, 'nltk_data')
gensim_data_dir = os.path.join(BASE_DIR, 'gensim-data')
upload_dir = os.path.join(BASE_DIR, 'uploads')
ans_image_dir = os.path.join(BASE_DIR, 'ans_image')
images_dir = os.path.join(BASE_DIR, 'images')
log_file = os.path.join(log_dir, 'app.log')  # Add log file path

# Global variables for model caching and initialization status
global_models = {}
initialization_complete = Event()

def ensure_directory(path):
    """Create directory and ensure full permissions with better error handling"""
    if os.path.exists(path):
        try:
            # Test write permissions
            test_file = os.path.join(path, '.test')
            with open(test_file, 'w') as f:
                f.write('test')
            os.remove(test_file)
            return path
        except Exception as e:
            log_print(f"Warning: Directory exists but not writable: {path}", "WARNING")
            try:
                # Try to fix permissions
                os.chmod(path, 0o777)
                return path
            except Exception as chmod_e:
                log_print(f"Error fixing permissions for {path}: {chmod_e}", "ERROR")
                raise
    
    try:
        # Create directory with full permissions
        os.makedirs(path, mode=0o777, exist_ok=True)
        return path
    except Exception as e:
        try:
            # Try with more restricted permissions
            os.makedirs(path, mode=0o755, exist_ok=True)
            return path
        except Exception as nested_e:
            log_print(f"Error creating directory {path}: {nested_e}", "ERROR")
            raise

def get_or_load_model(model_name):
    """Get a model from cache or load it if not present"""
    if model_name not in global_models:
        try:
            if model_name == 'fasttext':
                from gensim.models import KeyedVectors
                log_print(f"Loading {model_name} model...")
                model_path = os.path.join(gensim_data_dir, 'fasttext-wiki-news-subwords-300', 'fasttext-wiki-news-subwords-300.gz')
                if not os.path.exists(model_path):
                    from gensim.downloader import load
                    log_print("Downloading fasttext model...")
                    model = load('fasttext-wiki-news-subwords-300')
                    # Move model to CPU explicitly
                    if hasattr(model, 'to'):
                        model = model.to('cpu')
                    global_models[model_name] = model
                else:
                    model = KeyedVectors.load_word2vec_format(model_path)
                    # Move model to CPU explicitly
                    if hasattr(model, 'to'):
                        model = model.to('cpu')
                    global_models[model_name] = model
                log_print(f"Successfully loaded {model_name} model")
            elif model_name == 'vit':
                try:
                    from transformers import ViTImageProcessor, ViTModel
                    log_print("Loading ViT model...")
                    
                    # Use a more common ViT model that's guaranteed to exist
                    model_name = "google/vit-base-patch16-224-in21k"
                    cache_dir = os.path.join(os.environ['TRANSFORMERS_CACHE'], 'vit-models')
                    os.makedirs(cache_dir, exist_ok=True)
                    
                    try:
                        # Try to load the processor first
                        log_print("Loading ViT image processor...")
                        processor = ViTImageProcessor.from_pretrained(model_name, 
                                                                    cache_dir=cache_dir,
                                                                    local_files_only=True)
                        log_print("Loading ViT model from cache...")
                        model = ViTModel.from_pretrained(model_name, 
                                                       cache_dir=cache_dir,
                                                       local_files_only=True)
                        
                        # Move model to CPU explicitly
                        model = model.to('cpu')
                        
                        global_models['vit_processor'] = processor
                        global_models['vit_model'] = model
                        log_print("Successfully loaded ViT model from cache")
                        
                    except Exception as cache_error:
                        log_print(f"Cache load failed ({str(cache_error)}), downloading model...")
                        processor = ViTImageProcessor.from_pretrained(model_name, 
                                                                    cache_dir=cache_dir,
                                                                    local_files_only=False)
                        model = ViTModel.from_pretrained(model_name, 
                                                       cache_dir=cache_dir,
                                                       local_files_only=False)
                        
                        # Move model to CPU explicitly
                        model = model.to('cpu')
                        
                        global_models['vit_processor'] = processor
                        global_models['vit_model'] = model
                        log_print("Successfully downloaded and loaded ViT model")
                    
                except Exception as e:
                    log_print(f"Error loading ViT model: {str(e)}", "ERROR")
                    try:
                        log_print("Trying alternative ViT model...")
                        model_name = "google/vit-base-patch16-224"
                        processor = ViTImageProcessor.from_pretrained(model_name)
                        model = ViTModel.from_pretrained(model_name)
                        
                        # Move model to CPU explicitly
                        model = model.to('cpu')
                        
                        global_models['vit_processor'] = processor
                        global_models['vit_model'] = model
                        log_print("Successfully loaded alternative ViT model")
                    except Exception as alt_error:
                        log_print(f"Error loading alternative ViT model: {str(alt_error)}", "ERROR")
                        return None
            elif model_name == 'llm':
                log_print("LLM model loading not implemented", "WARNING")
                return None
        except Exception as e:
            log_print(f"Error loading {model_name} model: {str(e)}", "ERROR")
            return None
    return global_models.get(model_name)

def initialize_resources():
    """Initialize all required resources"""
    try:
        # Create essential directories first
        for directory in [nltk_data_dir, gensim_data_dir]:
            ensure_directory(directory)

        # Initialize NLTK
        required_nltk_data = ['stopwords', 'punkt', 'wordnet']
        for data in required_nltk_data:
            try:
                nltk.data.find(os.path.join('tokenizers', data))
            except LookupError:
                try:
                    log_print(f"Downloading NLTK data: {data}")
                    nltk.download(data, download_dir=nltk_data_dir, quiet=True)
                except Exception as e:
                    log_print(f"Error downloading NLTK data {data}: {e}", "WARNING")
                    continue

        # Initialize models
        try:
            # Load FastText first
            get_or_load_model('fasttext')
            
            # Then load ViT model
            get_or_load_model('vit')
        except Exception as e:
            log_print(f"Warning: Could not preload models: {e}", "WARNING")

    except Exception as e:
        log_print(f"Error during initialization: {e}", "ERROR")
    finally:
        # Signal that initialization is complete
        initialization_complete.set()

# Create essential directories
essential_dirs = [cache_dir, upload_dir, images_dir]
for directory in essential_dirs:
    ensure_directory(directory)

# Set environment variables
os.environ['HF_HOME'] = cache_dir
os.environ['GENSIM_DATA_DIR'] = gensim_data_dir

# Add the custom directory to NLTK's search path
nltk.data.path.insert(0, nltk_data_dir)

# Start initialization in background
initialization_thread = Thread(target=initialize_resources, daemon=True)
initialization_thread.start()

from flask import Flask, request, jsonify, render_template

from HTR.app import extract_text_from_image
from correct_answer_generation.answer_generation_database_creation import database_creation, answer_generation
from similarity_check.tf_idf.tf_idf_score import create_tfidf_values, tfidf_answer_score
from similarity_check.semantic_meaning_check.semantic import similarity_model_score, fasttext_similarity, question_vector_sentence, question_vector_word
from similarity_check.llm_based_scoring.llm import llm_score

app = Flask(__name__)
app.config['JSON_SORT_KEYS'] = False
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024  # 16MB max file size

# Create temporary directories for Hugging Face Spaces
UPLOAD_FOLDER = tempfile.mkdtemp()
ANS_IMAGE_FOLDER = tempfile.mkdtemp()
os.makedirs(UPLOAD_FOLDER, exist_ok=True)
os.makedirs(ANS_IMAGE_FOLDER, exist_ok=True)

app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['ANS_IMAGE_FOLDER'] = ANS_IMAGE_FOLDER

# Configure CORS for all origins
CORS(app, resources={
    r"/*": {
        "origins": "*",
        "methods": ["GET", "POST", "OPTIONS"],
        "allow_headers": ["Content-Type", "Authorization", "Accept"],
        "expose_headers": ["Content-Type"]
    }
})

# Global error handler for all exceptions
@app.errorhandler(Exception)
def handle_exception(e):
    # Log the error for debugging
    app.logger.error(f"Unhandled exception: {str(e)}")
    error_details = {
        "status": "error",
        "error": "Internal server error",
        "message": str(e),
        "type": type(e).__name__,
        "timestamp": datetime.now().isoformat()
    }
    notification_queue.put({
        "type": "error",
        "message": error_details
    })
    return jsonify(error_details), 500

# Handle 404 errors
@app.errorhandler(404)
def not_found_error(error):
    return jsonify({
        "status": "error",
        "error": "Not found",
        "message": "The requested resource was not found"
    }), 404

# Handle 400 Bad Request
@app.errorhandler(400)
def bad_request_error(error):
    return jsonify({
        "status": "error",
        "error": "Bad request",
        "message": str(error)
    }), 400

@app.route('/')
def index():
    return render_template('2.html')

def new_value(value, old_min, old_max, new_min, new_max):
    """Calculate new value with proper error handling"""
    try:
        if old_max == old_min:
            return new_min  # Return minimum value if range is zero
        return new_min + ((value - old_min) * (new_max - new_min)) / (old_max - old_min)
    except Exception as e:
        log_print(f"Error in new_value calculation: {e}", "ERROR")
        return new_min  # Return minimum value on error

@app.route('/compute_answers', methods=['POST'])
def compute_answers():
    try:
        file_type = request.form.get('file_type')
        log_print(f"Processing file type: {file_type}")
        
        # For CSV files, process directly without any model initialization
        if file_type == "csv":
            ans_csv_file = request.files.get('ans_csv_file')
            if not ans_csv_file:
                return jsonify({"error": "Missing answer CSV file"}), 400
            
            try:
                # Read CSV content directly without unnecessary processing
                content = ans_csv_file.read().decode('utf-8')
                if not content.strip():
                    return jsonify({"error": "CSV file is empty"}), 400
                
                # Process answers more efficiently
                c_answers = []
                for line in content.splitlines():
                    if line.strip():
                        answers = [ans.strip() for ans in line.split('\\n') if ans.strip()]
                        if answers:  # Only add if there are valid answers
                            c_answers.append(answers)
                
                if not c_answers:
                    return jsonify({"error": "No valid answers found in CSV file"}), 400
                
                log_print(f"Successfully processed {len(c_answers)} answers from CSV")
                return jsonify({"answers": c_answers}), 200
                
            except Exception as e:
                log_print(f"Error processing CSV file: {str(e)}", "ERROR")
                return jsonify({"error": f"Error processing CSV file: {str(e)}"}), 400
        
        # For PDF files, continue with existing processing
        elif file_type == 'pdf':
            # Wait for initialization only for PDF files
            wait_for_initialization()
            
            # Create a temporary directory for PDF files
            pdf_dir = tempfile.mkdtemp()
            os.makedirs(pdf_dir, exist_ok=True)
            
            # Save uploaded PDF files
            pdf_files = []
            for file in request.files.getlist('pdf_files[]'):
                if file.filename.endswith('.pdf'):
                    filename = secure_filename(file.filename)
                    filepath = os.path.join(pdf_dir, filename)
                    file.save(filepath)
                    pdf_files.append(filepath)
            
            if not pdf_files:
                return jsonify({"error": "No PDF files uploaded"}), 400
            
            logger.info(f"Processing {len(pdf_files)} PDF files")
            
            # Process PDFs
            for pdf_file in pdf_files:
                database_creation(pdf_file)
            
            # Generate answers
            queries = request.files.get('query_file').read().decode('utf-8').splitlines()
            queries = [q.strip() for q in queries if q.strip()]
            if not queries:
                error_msg = "Query file is empty"
                log_print(error_msg, "ERROR")
                return jsonify({"error": error_msg}), 400
            
            c_answers = []
            for query in queries:
                ans = []
                for pdf_file in pdf_files:
                    ans.append(answer_generation(pdf_file, query))
                c_answers.append(ans)
            
            # Clean up PDF directory
            try:
                shutil.rmtree(pdf_dir)
            except Exception as e:
                logger.warning(f"Could not clean up PDF directory: {e}")
            
            logger.info(f"Generated answers: {c_answers}")
            return jsonify({"answers": c_answers}), 200
        
        else:
            error_msg = "Unsupported file type"
            log_print(error_msg, "ERROR")
            return jsonify({"error": error_msg}), 400
            
    except Exception as e:
        error_msg = f"Error in compute_answers: {str(e)}"
        log_print(error_msg, "ERROR")
        return jsonify({"error": error_msg}), 500

def validate_folder_structure(files):
    """Validate the folder structure of uploaded files"""
    try:
        # Get unique student folders
        student_folders = set()
        for file in files:
            if not file or not file.filename:
                continue
            path_parts = file.filename.split('/')
            if len(path_parts) >= 2:
                student_folders.add(path_parts[-2])
        
        if not student_folders:
            return False, "No valid student folders found. Please create folders with student names."
            
        # Check if each student folder has the same number of files
        file_counts = {}
        for file in files:
            if not file or not file.filename:
                continue
            path_parts = file.filename.split('/')
            if len(path_parts) >= 2:
                student = path_parts[-2]
                file_counts[student] = file_counts.get(student, 0) + 1
                
        if not file_counts:
            return False, "No valid files found in student folders. Please add image files."
            
        # Check if all students have the same number of files
        counts = list(file_counts.values())
        if len(set(counts)) > 1:
            return False, "Inconsistent number of files across student folders. Each student must have the same number of images."
            
        # Validate file extensions
        for file in files:
            if not file or not file.filename:
                continue
            path_parts = file.filename.split('/')
            if len(path_parts) >= 2:
                filename = path_parts[-1]
                ext = os.path.splitext(filename)[1].lower()
                if ext not in ['.jpg', '.jpeg', '.png']:
                    return False, f"Invalid file extension: {ext}. Only .jpg, .jpeg, and .png files are allowed."
            
        return True, f"Valid folder structure with {len(student_folders)} students and {counts[0]} files each"
        
    except Exception as e:
        return False, f"Error validating folder structure: {str(e)}"

@app.route('/notifications')
def notifications():
    def generate():
        error_count = 0
        max_errors = 3
        
        while True:
            try:
                # Get notification from queue (non-blocking)
                try:
                    notification = notification_queue.get_nowait()
                    if notification:
                        yield "data: " + json.dumps(notification) + "\n\n"
                        error_count = 0  # Reset error count on successful notification
                except queue.Empty:
                    # If no notification, yield empty to keep connection alive
                    yield "data: " + json.dumps({"type": "ping"}) + "\n\n"
                    time.sleep(0.5)  # Keep the connection alive
                    
            except Exception as e:
                error_count += 1
                error_msg = str(e).encode('ascii', 'ignore').decode('ascii')
                log_print(f"Error in notification stream: {error_msg}", "ERROR")
                
                yield "data: " + json.dumps({
                    "type": "error",
                    "message": f"Server error: {error_msg}"
                }) + "\n\n"
                
                if error_count >= max_errors:
                    break

    return Response(generate(), mimetype='text/event-stream')

def get_memory_usage():
    """Get current memory usage"""
    process = psutil.Process(os.getpid())
    return process.memory_info().rss / 1024 / 1024  # Convert to MB

def cleanup_memory():
    """Clean up memory by clearing caches and garbage collection"""
    try:
        # Clear PyTorch cache
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
        
        # Clear Python garbage collection
        gc.collect()
        
        # Clear model caches
        if hasattr(models, 'cleanup'):
            models.cleanup()
            
        # Log memory usage
        memory_usage = get_memory_usage()
        log_print(f"Memory usage after cleanup: {memory_usage:.2f} MB")
        
    except Exception as e:
        log_print(f"Error during memory cleanup: {e}", "ERROR")

@app.route('/compute_marks', methods=['POST'])
def compute_marks():
    try:
        # Get answers from request
        a = request.form.get('answers')
        if not a:
            error_msg = "Missing answers in the request"
            notification_queue.put({
                "type": "error",
                "message": error_msg
            })
            return jsonify({"error": error_msg}), 400
            
        try:
            answers = json.loads(a)
            # Validate answers format
            if not isinstance(answers, list):
                raise ValueError("Answers must be a list")
            if not all(isinstance(ans, list) for ans in answers):
                raise ValueError("Each answer must be a list of strings")
            if not all(isinstance(text, str) for ans in answers for text in ans):
                raise ValueError("All answer texts must be strings")
            
            logger.info(f"Received {len(answers)} sets of answers")
            logger.info(f"Answer format: {[len(ans) for ans in answers]} answers per set")
            
        except json.JSONDecodeError as e:
            error_msg = f"Invalid JSON format for answers: {str(e)}"
            notification_queue.put({
                "type": "error",
                "message": error_msg
            })
            return jsonify({"error": error_msg}), 400
        except ValueError as e:
            error_msg = f"Invalid answer format: {str(e)}"
            notification_queue.put({
                "type": "error",
                "message": error_msg
            })
            return jsonify({"error": error_msg}), 400
            
        # Process uploaded files
        files = request.files.getlist('file')
        if not files:
            error_msg = "No files uploaded. Please upload student folders containing images."
            notification_queue.put({
                "type": "error",
                "message": error_msg
            })
            return jsonify({"error": error_msg}), 400
            
        # Validate folder structure and file count
        is_valid, message = validate_folder_structure(files)
        if not is_valid:
            notification_queue.put({
                "type": "error",
                "message": message
            })
            return jsonify({"error": message}), 400
            
        # Create student folders structure
        data = {}
        parent_folder = app.config['ANS_IMAGE_FOLDER']
        
        # Create student folders and save files
        for file in files:
            if file.filename.endswith(('.jpg', '.jpeg', '.png')):
                # Extract student folder from filename
                path_parts = file.filename.split('/')
                if len(path_parts) >= 2:
                    student_folder = secure_filename(path_parts[-2])
                    student_path = os.path.join(parent_folder, student_folder)
                    os.makedirs(student_path, exist_ok=True)
                    
                    # Save the file
                    filename = secure_filename(path_parts[-1])
                    filepath = os.path.join(student_path, filename)
                    file.save(filepath)
                    
                    if student_folder in data:
                        data[student_folder].append((filename, filepath))
                    else:
                        data[student_folder] = [(filename, filepath)]
        
        logger.info(f"Processed files structure: {data}")
        
        # Validate that each student has the correct number of files
        expected_files = len(answers)
        for student, files in data.items():
            if len(files) != expected_files:
                error_msg = f"Student {student} has {len(files)} files but {expected_files} answers were provided"
                notification_queue.put({
                    "type": "error",
                    "message": error_msg
                })
                return jsonify({"error": error_msg}), 400
        
        # Calculate marks
        results = []
        sen_vec_answers = []
        word_vec_answers = []
        
        # Process correct answers
        for i in answers:
            temp_v = []
            temp_w = []
            for j in i:
                temp_v.append(question_vector_sentence(j))
                temp_w.append(question_vector_word(j))
            sen_vec_answers.append(temp_v)
            word_vec_answers.append(temp_w)
        
        # Calculate marks for each student
        for student in data:
            # Sort the image paths by filename
            sorted_images = sorted(data[student], key=lambda x: x[0])
            count = 0
            for filename, image_path in sorted_images:
                try:
                    # Extract text from image
                    s_answer = extract_text_from_image(image_path)
                    logger.info(f"Processing student: {student}, image: {filename}")
                    logger.info(f"Extracted text: {s_answer}")
                    
                    # Handle case where text extraction fails
                    if s_answer is None or s_answer.strip() == '':
                        logger.warning(f"No text extracted from {image_path}")
                        results.append({
                            "subfolder": student,
                            "image": filename,
                            "marks": 0,
                            "extracted_text": "",
                            "correct_answer": answers[count],
                            "error": "No text could be extracted from image. Please check image quality."
                        })
                        count += 1
                        continue
                    
                    # Calculate TF-IDF scores
                    tf_idf_word_values, max_tfidf = create_tfidf_values(answers[count])
                    logger.info(f"TF-IDF max value: {max_tfidf}")
                    
                    # Calculate marks
                    m = marks(s_answer, sen_vec_answers[count], word_vec_answers[count], 
                             tf_idf_word_values, max_tfidf, answers[count])
                    
                    if isinstance(m, torch.Tensor):
                        m = m.item()
                        
                    # Add result with extracted text
                    results.append({
                        "subfolder": student,
                        "image": filename,
                        "marks": round(m, 2),
                        "extracted_text": s_answer,
                        "correct_answer": answers[count]
                    })
                    count += 1
                    
                    # Clean up memory after each student
                    cleanup_memory()
                    
                except Exception as e:
                    logger.error(f"Error processing {image_path}: {str(e)}")
                    results.append({
                        "subfolder": student,
                        "image": filename,
                        "marks": 0,
                        "extracted_text": "",
                        "correct_answer": answers[count] if count < len(answers) else [],
                        "error": f"Error processing image: {str(e)}"
                    })
                    count += 1
                    continue
        
        logger.info(f"Calculated results: {results}")
        
        # Clean up temporary files
        try:
            shutil.rmtree(parent_folder)
        except Exception as e:
            logger.warning(f"Could not clean up temporary files: {e}")
        
        # Final memory cleanup
        cleanup_memory()
        
        return jsonify({
            "results": results,
            "debug_info": {
                "total_students": len(data),
                "total_answers": len(answers),
                "answers_processed": count,
                "successful_extractions": len([r for r in results if r.get('extracted_text')])
            }
        }), 200
        
    except Exception as e:
        error_msg = str(e)
        logger.error(f"Error in compute_marks: {error_msg}")
        notification_queue.put({
            "type": "error",
            "message": error_msg
        })
        return jsonify({"error": error_msg}), 500
    finally:
        # Ensure memory is cleaned up even if there's an error
        cleanup_memory()

def marks(answer, sen_vec_answers, word_vec_answers, tf_idf_word_values, max_tfidf, correct_answers):
    try:
        marks = 0
        logger.info(f"Starting marks calculation for answer: {answer}")
        logger.info(f"Correct answers: {correct_answers}")
        
        # Calculate TF-IDF score
        marks1 = tfidf_answer_score(answer, tf_idf_word_values, max_tfidf, marks=10)
        logger.info(f"Initial TF-IDF score: {marks1}")
        
        if marks1 > 3:
            tfidf_contribution = new_value(marks1, old_min=3, old_max=10, new_min=0, new_max=5)
            marks += tfidf_contribution
            logger.info(f"TF-IDF contribution (>3): {tfidf_contribution}")
            
        if marks1 > 2:
            # Calculate sentence transformer score
            marks2 = similarity_model_score(sen_vec_answers, answer)
            logger.info(f"Sentence transformer raw score: {marks2}")
            
            a = 0
            if marks2 > 0.95:
                marks += 3
                a = 3
                logger.info("High sentence similarity (>0.95): +3 marks")
            elif marks2 > 0.5:
                sentence_contribution = new_value(marks2, old_min=0.5, old_max=0.95, new_min=0, new_max=3)
                marks += sentence_contribution
                a = sentence_contribution
                logger.info(f"Medium sentence similarity (>0.5): +{sentence_contribution} marks")

            # Calculate FastText similarity
            marks3 = fasttext_similarity(word_vec_answers, answer)
            logger.info(f"FastText similarity raw score: {marks3}")
            
            b = 0
            if marks2 > 0.9:
                marks += 2
                b = 2
                logger.info("High word similarity (>0.9): +2 marks")
            elif marks3 > 0.4:
                word_contribution = new_value(marks3, old_min=0.4, old_max=0.9, new_min=0, new_max=2)
                marks += word_contribution
                b = word_contribution
                logger.info(f"Medium word similarity (>0.4): +{word_contribution} marks")

            # Calculate LLM score
            marks4 = llm_score(correct_answers, answer)
            logger.info(f"Raw LLM scores: {marks4}")
            
            for i in range(len(marks4)):
                marks4[i] = float(marks4[i])

            m = max(marks4)
            logger.info(f"Max LLM score: {m}")
            
            # Final score calculation
            final_score = marks/2 + m/2
            logger.info(f"Final score calculation: (marks={marks}/2 + llm={m}/2) = {final_score}")
            marks = final_score
            
        logger.info(f"Final marks awarded: {marks}")
        return marks
        
    except Exception as e:
        logger.error(f"Error in marks calculation: {str(e)}")
        return 0

@app.route('/check_logs')
def check_logs():
    try:
        # Ensure log directory exists
        ensure_directory(log_dir)
        
        # If log file doesn't exist, create it
        if not os.path.exists(log_file):
            with open(log_file, 'w') as f:
                f.write("Log file created.\n")
        
        # Read last 1000 lines of logs
        with open(log_file, 'r') as f:
            logs = f.readlines()[-1000:]
            return jsonify({
                "status": "success",
                "logs": "".join(logs)
            })
    except Exception as e:
        log_print(f"Error reading logs: {str(e)}", "ERROR")
        return jsonify({
            "status": "error",
            "error": str(e)
        }), 500

def is_valid_image_file(filename):
    """Validate image file extensions and basic format"""
    try:
        # Check file extension
        valid_extensions = {'.jpg', '.jpeg', '.png'}
        ext = os.path.splitext(filename)[1].lower()
        if ext not in valid_extensions:
            return False
            
        return True
    except Exception:
        return False

def allowed_file(filename, allowed_extensions):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in allowed_extensions

def wait_for_initialization():
    """Wait for initialization to complete"""
    initialization_complete.wait()
    return True

@app.before_request
def ensure_initialization():
    """Ensure all resources are initialized before processing requests"""
    if request.endpoint == 'compute_marks':
        wait_for_initialization()
    elif request.endpoint == 'compute_answers':
        # Only wait for initialization if processing PDF files
        if request.form.get('file_type') == 'pdf':
            wait_for_initialization()

def cleanup_temp_files():
    """Clean up temporary files with proper error handling"""
    try:
        # Clean up the temporary processing directory
        temp_processing_dir = os.path.join(BASE_DIR, 'temp_processing')
        if os.path.exists(temp_processing_dir):
            shutil.rmtree(temp_processing_dir, ignore_errors=True)
        
        # Clean up the images directory
        if os.path.exists(images_dir):
            for file in os.listdir(images_dir):
                try:
                    file_path = os.path.join(images_dir, file)
                    if os.path.isfile(file_path):
                        os.unlink(file_path)
                except Exception as e:
                    log_print(f"Warning: Could not delete file {file_path}: {e}", "WARNING")
                    
        # Clean up the upload folder
        if os.path.exists(UPLOAD_FOLDER):
            try:
                shutil.rmtree(UPLOAD_FOLDER, ignore_errors=True)
            except Exception as e:
                log_print(f"Warning: Could not clean up upload folder: {e}", "WARNING")
    except Exception as e:
        log_print(f"Error cleaning up temporary files: {e}", "ERROR")

@app.before_first_request
def setup_temp_directories():
    """Set up temporary directories before first request"""
    try:
        # Create temporary directories with proper permissions
        global UPLOAD_FOLDER, ANS_IMAGE_FOLDER
        
        UPLOAD_FOLDER = tempfile.mkdtemp()
        ANS_IMAGE_FOLDER = tempfile.mkdtemp()
        
        # Ensure directories have proper permissions
        ensure_directory(UPLOAD_FOLDER)
        ensure_directory(ANS_IMAGE_FOLDER)
        
        app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
        app.config['ANS_IMAGE_FOLDER'] = ANS_IMAGE_FOLDER
        
        log_print(f"Created temporary directories: {UPLOAD_FOLDER}, {ANS_IMAGE_FOLDER}")
    except Exception as e:
        log_print(f"Error setting up temporary directories: {e}", "ERROR")
        raise

if __name__ == '__main__':
    try:
        # Create essential directories
        for directory in essential_dirs:
            ensure_directory(directory)
        
        # Configure server
        app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
        
        # Start the Flask app
        port = int(os.environ.get('PORT', 7860))
        
        log_print(f"Starting server on port {port}")
        log_print("Server configuration:")
        log_print(f"- Threaded: True")
        log_print(f"- Debug mode: False")
        
        # Run the server with proper configuration
        app.run(
            host='0.0.0.0',
            port=port,
            debug=False,
            use_reloader=False,
            threaded=True
        )
    except Exception as e:
        log_print(f"Fatal error starting server: {str(e)}", "ERROR")
        raise
    finally:
        log_print("Cleaning up temporary files...")
        cleanup_temp_files()
        log_print("Server shutdown complete")