Commit
·
8882e41
1
Parent(s):
3f6ca1b
fix main file handiling
Browse files
main.py
CHANGED
@@ -10,6 +10,7 @@ from flask import Flask, request, jsonify, render_template, make_response
|
|
10 |
from werkzeug.utils import secure_filename
|
11 |
from dotenv import load_dotenv
|
12 |
import json
|
|
|
13 |
|
14 |
# Load environment variables
|
15 |
load_dotenv()
|
@@ -20,12 +21,39 @@ log_dir = os.path.join(BASE_DIR, 'app_logs')
|
|
20 |
cache_dir = os.path.join(BASE_DIR, 'app_cache')
|
21 |
|
22 |
def ensure_directory(path):
|
23 |
-
"""Create directory and ensure full permissions"""
|
24 |
try:
|
|
|
25 |
os.makedirs(path, mode=0o777, exist_ok=True)
|
26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
27 |
except Exception as e:
|
28 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
29 |
|
30 |
# Create necessary directories with full permissions
|
31 |
ensure_directory(log_dir)
|
@@ -262,7 +290,6 @@ def compute_answers():
|
|
262 |
|
263 |
# Clean up PDF directory
|
264 |
try:
|
265 |
-
import shutil
|
266 |
shutil.rmtree(pdf_dir)
|
267 |
except Exception as e:
|
268 |
log_print(f"Warning: Could not clean up PDF directory: {e}", "WARNING")
|
@@ -364,9 +391,17 @@ def compute_marks():
|
|
364 |
"message": message
|
365 |
}), 400
|
366 |
|
367 |
-
# Create a temporary directory
|
368 |
-
|
369 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
370 |
|
371 |
# Dictionary to store results by student folder and image name
|
372 |
results = {}
|
@@ -407,74 +442,94 @@ def compute_marks():
|
|
407 |
results[student_folder] = {}
|
408 |
log_print(f"Initialized results for student: {student_folder}")
|
409 |
|
410 |
-
# Save and process file
|
411 |
-
|
412 |
-
|
413 |
-
|
414 |
-
|
415 |
-
|
416 |
-
|
417 |
-
|
418 |
-
|
419 |
-
|
420 |
-
|
421 |
-
|
422 |
-
|
423 |
-
|
424 |
-
|
425 |
-
|
426 |
-
|
427 |
-
|
428 |
-
continue
|
429 |
-
|
430 |
-
log_print(f"Extracted text: {extracted_text[:100]}...")
|
431 |
-
|
432 |
-
# Clean the extracted text for JSON
|
433 |
-
extracted_text = extracted_text.encode('ascii', 'ignore').decode('ascii')
|
434 |
-
|
435 |
-
# Find best matching answer
|
436 |
-
best_score = 0
|
437 |
-
best_answer_index = 0
|
438 |
-
|
439 |
-
for i, correct_answer in enumerate(correct_answers):
|
440 |
try:
|
441 |
-
|
442 |
-
|
443 |
-
|
444 |
-
# Calculate similarity scores
|
445 |
-
semantic_score = question_vector_sentence(extracted_text, clean_correct_answer)
|
446 |
-
word_score = question_vector_word(extracted_text, clean_correct_answer)
|
447 |
-
tfidf_score = tfidf_answer_score(extracted_text, clean_correct_answer, max_tfidf)
|
448 |
-
ft_score = fasttext_similarity(extracted_text, clean_correct_answer)
|
449 |
-
llm_marks = llm_score(extracted_text, clean_correct_answer)
|
450 |
-
|
451 |
-
combined_score = (
|
452 |
-
semantic_score * 0.3 +
|
453 |
-
word_score * 0.2 +
|
454 |
-
tfidf_score * 0.2 +
|
455 |
-
ft_score * 0.2 +
|
456 |
-
llm_marks * 0.1
|
457 |
-
)
|
458 |
-
|
459 |
-
log_print(f"Scores for answer {i+1}: semantic={semantic_score}, word={word_score}, tfidf={tfidf_score}, ft={ft_score}, llm={llm_marks}, combined={combined_score}")
|
460 |
|
461 |
-
|
462 |
-
|
463 |
-
best_answer_index = i
|
464 |
-
|
465 |
-
except Exception as score_error:
|
466 |
-
error_msg = str(score_error).encode('ascii', 'ignore').decode('ascii')
|
467 |
-
log_print(f"Error calculating scores for {filepath}: {error_msg}", "ERROR")
|
468 |
failed_files.append({
|
469 |
"file": file.filename,
|
470 |
-
"error":
|
471 |
})
|
472 |
continue
|
473 |
-
|
474 |
-
|
475 |
-
|
476 |
-
|
477 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
478 |
except Exception as e:
|
479 |
error_msg = str(e).encode('ascii', 'ignore').decode('ascii')
|
480 |
log_print(f"Error processing file {file.filename}: {error_msg}", "ERROR")
|
|
|
10 |
from werkzeug.utils import secure_filename
|
11 |
from dotenv import load_dotenv
|
12 |
import json
|
13 |
+
import shutil
|
14 |
|
15 |
# Load environment variables
|
16 |
load_dotenv()
|
|
|
21 |
cache_dir = os.path.join(BASE_DIR, 'app_cache')
|
22 |
|
23 |
def ensure_directory(path):
|
24 |
+
"""Create directory and ensure full permissions with better error handling"""
|
25 |
try:
|
26 |
+
# First try to create the directory with full permissions
|
27 |
os.makedirs(path, mode=0o777, exist_ok=True)
|
28 |
+
|
29 |
+
# Double-check the permissions and try to set them again if needed
|
30 |
+
current_mode = os.stat(path).st_mode & 0o777
|
31 |
+
if current_mode != 0o777:
|
32 |
+
os.chmod(path, 0o777)
|
33 |
+
|
34 |
+
# Verify the directory is writable
|
35 |
+
test_file = os.path.join(path, '.test_write')
|
36 |
+
try:
|
37 |
+
with open(test_file, 'w') as f:
|
38 |
+
f.write('test')
|
39 |
+
os.remove(test_file)
|
40 |
+
except Exception as e:
|
41 |
+
raise Exception(f"Directory {path} is not writable: {str(e)}")
|
42 |
+
|
43 |
except Exception as e:
|
44 |
+
# If we can't set full permissions, try with more restrictive ones
|
45 |
+
try:
|
46 |
+
os.makedirs(path, mode=0o755, exist_ok=True)
|
47 |
+
os.chmod(path, 0o755)
|
48 |
+
|
49 |
+
# Verify the directory is at least readable
|
50 |
+
if not os.access(path, os.R_OK | os.W_OK):
|
51 |
+
raise Exception(f"Cannot read/write to directory: {path}")
|
52 |
+
|
53 |
+
except Exception as nested_e:
|
54 |
+
raise Exception(f"Could not create or set permissions for {path}: {str(nested_e)}")
|
55 |
+
|
56 |
+
return path
|
57 |
|
58 |
# Create necessary directories with full permissions
|
59 |
ensure_directory(log_dir)
|
|
|
290 |
|
291 |
# Clean up PDF directory
|
292 |
try:
|
|
|
293 |
shutil.rmtree(pdf_dir)
|
294 |
except Exception as e:
|
295 |
log_print(f"Warning: Could not clean up PDF directory: {e}", "WARNING")
|
|
|
391 |
"message": message
|
392 |
}), 400
|
393 |
|
394 |
+
# Create a temporary directory with proper permissions
|
395 |
+
try:
|
396 |
+
base_temp_dir = os.path.join(BASE_DIR, 'temp_processing')
|
397 |
+
ensure_directory(base_temp_dir)
|
398 |
+
log_print(f"Created temporary directory: {base_temp_dir}")
|
399 |
+
except Exception as e:
|
400 |
+
log_print(f"Error creating temp directory: {str(e)}", "ERROR")
|
401 |
+
return jsonify({
|
402 |
+
"error": "Server error",
|
403 |
+
"message": "Could not create temporary directory"
|
404 |
+
}), 500
|
405 |
|
406 |
# Dictionary to store results by student folder and image name
|
407 |
results = {}
|
|
|
442 |
results[student_folder] = {}
|
443 |
log_print(f"Initialized results for student: {student_folder}")
|
444 |
|
445 |
+
# Save and process file with proper permissions
|
446 |
+
try:
|
447 |
+
student_dir = os.path.join(base_temp_dir, student_folder)
|
448 |
+
ensure_directory(student_dir)
|
449 |
+
|
450 |
+
# Use a more reliable filename
|
451 |
+
safe_filename = secure_filename(f"{student_folder}_{filename}")
|
452 |
+
filepath = os.path.join(student_dir, safe_filename)
|
453 |
+
|
454 |
+
# Save file with proper permissions
|
455 |
+
file.save(filepath)
|
456 |
+
os.chmod(filepath, 0o666) # Make file readable/writable
|
457 |
+
log_print(f"Saved file to: {filepath}")
|
458 |
+
|
459 |
+
# Extract text
|
460 |
+
extracted_text = extract_text_from_image(filepath)
|
461 |
+
|
462 |
+
# Clean up the file immediately after processing
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
463 |
try:
|
464 |
+
os.remove(filepath)
|
465 |
+
except Exception as e:
|
466 |
+
log_print(f"Warning: Could not remove temp file {filepath}: {e}", "WARNING")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
467 |
|
468 |
+
if not extracted_text:
|
469 |
+
log_print(f"No text extracted from: {filepath}", "WARNING")
|
|
|
|
|
|
|
|
|
|
|
470 |
failed_files.append({
|
471 |
"file": file.filename,
|
472 |
+
"error": "No text could be extracted from the image"
|
473 |
})
|
474 |
continue
|
475 |
+
|
476 |
+
log_print(f"Extracted text: {extracted_text[:100]}...")
|
477 |
+
|
478 |
+
# Clean the extracted text for JSON
|
479 |
+
extracted_text = extracted_text.encode('ascii', 'ignore').decode('ascii')
|
480 |
+
|
481 |
+
# Find best matching answer
|
482 |
+
best_score = 0
|
483 |
+
best_answer_index = 0
|
484 |
+
|
485 |
+
for i, correct_answer in enumerate(correct_answers):
|
486 |
+
try:
|
487 |
+
# Clean the correct answer for comparison
|
488 |
+
clean_correct_answer = correct_answer.encode('ascii', 'ignore').decode('ascii')
|
489 |
+
|
490 |
+
# Calculate similarity scores
|
491 |
+
semantic_score = question_vector_sentence(extracted_text, clean_correct_answer)
|
492 |
+
word_score = question_vector_word(extracted_text, clean_correct_answer)
|
493 |
+
tfidf_score = tfidf_answer_score(extracted_text, clean_correct_answer, max_tfidf)
|
494 |
+
ft_score = fasttext_similarity(extracted_text, clean_correct_answer)
|
495 |
+
llm_marks = llm_score(extracted_text, clean_correct_answer)
|
496 |
+
|
497 |
+
combined_score = (
|
498 |
+
semantic_score * 0.3 +
|
499 |
+
word_score * 0.2 +
|
500 |
+
tfidf_score * 0.2 +
|
501 |
+
ft_score * 0.2 +
|
502 |
+
llm_marks * 0.1
|
503 |
+
)
|
504 |
+
|
505 |
+
log_print(f"Scores for answer {i+1}: semantic={semantic_score}, word={word_score}, tfidf={tfidf_score}, ft={ft_score}, llm={llm_marks}, combined={combined_score}")
|
506 |
+
|
507 |
+
if combined_score > best_score:
|
508 |
+
best_score = combined_score
|
509 |
+
best_answer_index = i
|
510 |
+
|
511 |
+
except Exception as score_error:
|
512 |
+
error_msg = str(score_error).encode('ascii', 'ignore').decode('ascii')
|
513 |
+
log_print(f"Error calculating scores for {filepath}: {error_msg}", "ERROR")
|
514 |
+
failed_files.append({
|
515 |
+
"file": file.filename,
|
516 |
+
"error": f"Error calculating scores: {error_msg}"
|
517 |
+
})
|
518 |
+
continue
|
519 |
+
|
520 |
+
marks = new_value(best_score, 0, 1, 0, 5)
|
521 |
+
results[student_folder][filename] = round(marks, 2)
|
522 |
+
log_print(f"Assigned marks for {filename} in {student_folder}: {marks}")
|
523 |
+
|
524 |
+
except Exception as e:
|
525 |
+
error_msg = str(e).encode('ascii', 'ignore').decode('ascii')
|
526 |
+
log_print(f"Error processing file {file.filename}: {error_msg}", "ERROR")
|
527 |
+
failed_files.append({
|
528 |
+
"file": file.filename,
|
529 |
+
"error": error_msg
|
530 |
+
})
|
531 |
+
continue
|
532 |
+
|
533 |
except Exception as e:
|
534 |
error_msg = str(e).encode('ascii', 'ignore').decode('ascii')
|
535 |
log_print(f"Error processing file {file.filename}: {error_msg}", "ERROR")
|