# language_checker.py import re import traceback from typing import List, Dict, Any import language_tool_python from text_utils import convert_markdown_to_plain_text # config.py (setting JAVA_HOME) should be imported early in app.py def perform_language_checks(markdown_text_from_filtered_pdf: str) -> Dict[str, Any]: """ Performs LanguageTool checks on plain text derived from font-filtered Markdown. Filters issues to only include those between "abstract" and "references/bibliography" found within this specific text. """ if not markdown_text_from_filtered_pdf or not markdown_text_from_filtered_pdf.strip(): print("LT_Checker: Input Markdown text is empty.") return {"total_issues": 0, "issues_list": [], "text_used_for_analysis": ""} plain_text_from_markdown = convert_markdown_to_plain_text(markdown_text_from_filtered_pdf) text_for_lt_analysis = plain_text_from_markdown.replace('\n', ' ') text_for_lt_analysis = re.sub(r'\s+', ' ', text_for_lt_analysis).strip() if not text_for_lt_analysis: print("LT_Checker: Plain text derived from Markdown is empty after cleaning.") return {"total_issues": 0, "issues_list": [], "text_used_for_analysis": ""} text_for_lt_analysis_lower = text_for_lt_analysis.lower() abstract_match = re.search(r'\babstract\b', text_for_lt_analysis_lower) content_start_index = abstract_match.start() if abstract_match else 0 if abstract_match: print(f"LT_Checker: Found 'abstract' at index {content_start_index} in its text.") else: print(f"LT_Checker: Did not find 'abstract', LT analysis from index 0 of its text.") # Determine end boundary (references or bibliography) references_match = re.search(r'\breferences\b', text_for_lt_analysis_lower) bibliography_match = re.search(r'\bbibliography\b', text_for_lt_analysis_lower) content_end_index = len(text_for_lt_analysis) if references_match and bibliography_match: content_end_index = min(references_match.start(), bibliography_match.start()) print(f"LT_Checker: Found 'references' at {references_match.start()} and 'bibliography' at {bibliography_match.start()}. Using {content_end_index} as end boundary.") elif references_match: content_end_index = references_match.start() print(f"LT_Checker: Found 'references' at {content_end_index}. Using it as end boundary.") elif bibliography_match: content_end_index = bibliography_match.start() print(f"LT_Checker: Found 'bibliography' at {content_end_index}. Using it as end boundary.") else: print(f"LT_Checker: Did not find 'references' or 'bibliography'. LT analysis up to end of its text (index {content_end_index}).") if content_start_index >= content_end_index: print(f"LT_Checker: Warning: Content start index ({content_start_index}) is not before end index ({content_end_index}) in its text. No LT issues will be reported from this range.") tool = None processed_lt_issues: List[Dict[str, Any]] = [] try: tool = language_tool_python.LanguageTool('en-US') raw_lt_matches = tool.check(text_for_lt_analysis) lt_issues_in_range = 0 for idx, match in enumerate(raw_lt_matches): if match.ruleId == "EN_SPLIT_WORDS_HYPHEN": continue # Common rule to ignore if not (content_start_index <= match.offset < content_end_index): continue lt_issues_in_range += 1 # Text of the error itself error_text_verbatim = match.matchedText # The actual text that LanguageTool flagged # New context extraction for ~10 words: words_around = 1 # Number of words to try and get on each side # Text before the error pre_error_text = text_for_lt_analysis[:match.offset] words_before = pre_error_text.split()[-words_around:] # Text after the error post_error_text = text_for_lt_analysis[match.offset + match.errorLength:] words_after = post_error_text.split()[:words_around] # Combine to form the new wider context context_parts = [] if words_before: context_parts.append(" ".join(words_before)) context_parts.append(error_text_verbatim) # The actual error phrase if words_after: context_parts.append(" ".join(words_after)) wider_context_str = " ".join(context_parts) processed_lt_issues.append({ '_internal_id': f"lt_{idx}", 'ruleId': match.ruleId, 'message': match.message, 'context_text': wider_context_str, 'error_text_verbatim': error_text_verbatim, # Store the verbatim error text 'offset_in_text': match.offset, 'error_length': match.errorLength, 'replacements_suggestion': match.replacements[:3] if match.replacements else [], 'category_name': match.category, 'source_check_type': 'LanguageTool', 'is_mapped_to_pdf': False, 'pdf_coordinates_list': [], 'mapped_page_number': -1 }) print(f"LT_Checker: LanguageTool found {len(raw_lt_matches)} raw issues, {lt_issues_in_range} issues within defined content range of its text.") return { "total_issues": len(processed_lt_issues), "issues_list": processed_lt_issues, "text_used_for_analysis": text_for_lt_analysis } except Exception as e: print(f"Error in perform_language_checks: {e}\n{traceback.format_exc()}") return {"error": str(e), "total_issues": 0, "issues_list": [], "text_used_for_analysis": text_for_lt_analysis} finally: if tool: tool.close()