pdf_processing.py

# pdf_processing.py
import fitz  # PyMuPDF
import pymupdf4llm
import os
import traceback
from typing import Any, Dict, List, Optional # Use standard List, Dict, Optional
from collections import Counter

def convert_rect_to_dict(rect: fitz.Rect) -> Dict[str, float] | None:
    """Converts a fitz.Rect object to a dictionary."""
    if not rect or not isinstance(rect, fitz.Rect):
        # print(f"Warning: Invalid rect object received: {rect}") # Can be verbose
        return None
    return {
        "x0": rect.x0, "y0": rect.y0, "x1": rect.x1, "y1": rect.y1,
        "width": rect.width, "height": rect.height
    }

def _get_specific_error_rect_in_context(
    page: fitz.Page,
    context_rect: fitz.Rect,
    error_text_verbatim: str
) -> Optional[fitz.Rect]:
    """
    Tries to find the precise bounding box of error_text_verbatim within
    the larger context_rect on the given page.
    """
    if not error_text_verbatim or error_text_verbatim.isspace():
        print(f"Debug: _get_specific_error_rect_in_context: error_text_verbatim is empty or whitespace.")
        return None

    # Extract words sorted by position within the given context_rect
    # Each word_data is (x0, y0, x1, y1, "text", block_no, line_no, word_no)
    words_on_page_in_clip = page.get_text("words", clip=context_rect, sort=True)
    
    # print(f"Debug: _get_specific_error_rect_in_context: Searching for '{error_text_verbatim}' in {len(words_on_page_in_clip)} words within clip {context_rect}")

    error_tokens = error_text_verbatim.strip().split()
    if not error_tokens:
        print(f"Debug: _get_specific_error_rect_in_context: No tokens from error_text_verbatim '{error_text_verbatim}'.")
        return None

    found_rects_for_error_sequence = []

    for i in range(len(words_on_page_in_clip) - len(error_tokens) + 1):
        match = True
        current_sequence_rects = []
        # print(f"Debug: _get_specific_error_rect_in_context: Trying match starting at PDF word '{words_on_page_in_clip[i][4]}'")
        for j in range(len(error_tokens)):
            pdf_word_text = words_on_page_in_clip[i+j][4]
            error_token_to_match = error_tokens[j]
            
            # Basic normalization for comparison
            pdf_word_normalized = pdf_word_text.strip().lower()
            error_token_normalized = error_token_to_match.strip().lower()

            # A more robust comparison might involve removing common punctuation
            # or handling hyphenation if LanguageTool splits differently than PyMuPDF.
            if error_token_normalized != pdf_word_normalized:
                # print(f"Debug: _get_specific_error_rect_in_context: Mismatch: '{error_token_normalized}' (expected) vs '{pdf_word_normalized}' (pdf word)")
                match = False
                break
            current_sequence_rects.append(fitz.Rect(words_on_page_in_clip[i+j][:4]))
        
        if match:
            # print(f"Debug: _get_specific_error_rect_in_context: Found match for '{error_text_verbatim}'")
            found_rects_for_error_sequence = current_sequence_rects
            break # Found the first full match of the error_text_verbatim

    if found_rects_for_error_sequence:
        final_error_bbox = fitz.Rect() # Start with an empty rect
        for r_part in found_rects_for_error_sequence:
            final_error_bbox.include_rect(r_part) # Expand to include this part
        
        if not final_error_bbox.is_empty:
            # print(f"Debug: _get_specific_error_rect_in_context: Combined bbox: {final_error_bbox}")
            return final_error_bbox
        else:
            # print(f"Debug: _get_specific_error_rect_in_context: Combined bbox was empty.")
            pass
    else:
        # print(f"Debug: _get_specific_error_rect_in_context: No match found for '{error_text_verbatim}'.")
        pass
    return None


def try_map_issues_to_page_rects(
    issues_to_map_for_context: List[Dict[str, Any]],
    pdf_rects_from_search: List[fitz.Rect], # Rects for occurrences of the wider context string
    page_number_for_mapping: int,
    page: fitz.Page # The current PyMuPDF page object
) -> int:
    mapped_count = 0
    # We assume that the number of issues for a given context string on a page
    # should not exceed the number of times that context string appears.
    # If it does, we only map up to the number of found context occurrences.
    limit = min(len(issues_to_map_for_context), len(pdf_rects_from_search))

    for i in range(limit):
        issue_to_update = issues_to_map_for_context[i]
        if issue_to_update['is_mapped_to_pdf']: 
            continue

        # This is the rectangle for the i-th occurrence of the wider context string
        context_occurrence_rect = pdf_rects_from_search[i]
        
        final_rect_for_issue = context_occurrence_rect # Default to the whole context rect

        # For LanguageTool issues, try to refine the rect to the specific error text
        if issue_to_update.get('source_check_type') == 'LanguageTool':
            error_text_verbatim = issue_to_update.get('error_text_verbatim')
            if error_text_verbatim:
                # print(f"Debug: Refining LT issue: '{error_text_verbatim}' within context rect {context_occurrence_rect}")
                specific_error_rect = _get_specific_error_rect_in_context(
                    page, context_occurrence_rect, error_text_verbatim
                )
                if specific_error_rect:
                    final_rect_for_issue = specific_error_rect
                    # print(f"Debug: Refined rect to: {final_rect_for_issue}")
                else:
                    # print(f"Debug: Could not refine rect, using context rect: {context_occurrence_rect}")
                    pass # Stick with the wider context_occurrence_rect if specific not found
        
        coord_dict = convert_rect_to_dict(final_rect_for_issue)
        if coord_dict:
            issue_to_update['pdf_coordinates_list'] = [coord_dict]
            issue_to_update['is_mapped_to_pdf'] = True
            issue_to_update['mapped_page_number'] = page_number_for_mapping
            mapped_count += 1
    return mapped_count


# ... (rest of pdf_processing.py, including extract_majority_font_text_directly and extract_plain_text_from_original_pdf) ...

def extract_majority_font_text_directly(pdf_path: str) -> str:
    """
    Extracts text from PDF, identifies the majority font and size,
    and then directly assembles a plain text string containing only the text
    that matches this majority font, attempting to preserve basic structure.
    This method does NOT create an intermediate PDF document.
    """
    original_doc = None
    try:
        # 1. Open PDF and Perform Font Analysis (similar to before)
        original_doc = fitz.open(pdf_path)
        if not original_doc.page_count:
            print("FontFilter (Direct): PDF has no pages.")
            return ""

        font_char_counts: Counter = Counter()
        pdf_basename = os.path.basename(pdf_path)
        print(f"FontFilter (Direct): Analyzing fonts in '{pdf_basename}' ({original_doc.page_count} pages)...")

        # First pass: Analyze fonts to find the majority
        for page_num_analysis in range(original_doc.page_count):
            page_analysis = original_doc[page_num_analysis]
            # Using TEXTFLAGS_TEXT for potentially cleaner text from spans
            text_dict_analysis = page_analysis.get_text("dict", flags=fitz.TEXTFLAGS_TEXT)
            for block_analysis in text_dict_analysis.get("blocks", []):
                if block_analysis.get("type") == 0:  # type 0 is a text block
                    for line_analysis in block_analysis.get("lines", []):
                        for span_analysis in line_analysis.get("spans", []):
                            font_name = span_analysis["font"]
                            font_size = span_analysis.get("size")
                            if font_size is None: continue  # Skip if size is not available

                            font_size_rounded = int(round(font_size))
                            text = span_analysis["text"]
                            if not text.strip(): continue  # Skip purely whitespace spans

                            font_char_counts[(font_name, font_size_rounded)] += len(text)

        if not font_char_counts:
            print("FontFilter (Direct): No text with font information found in PDF.")
            return ""

        majority_font_tuple_info = font_char_counts.most_common(1)[0]
        (majority_font_name, majority_font_size_rounded) = majority_font_tuple_info[0]
        char_count_for_majority = majority_font_tuple_info[1]
        print(
            f"FontFilter (Direct): Majority font identified: Name='{majority_font_name}', RoundedSize={majority_font_size_rounded}pt ({char_count_for_majority} chars).")

        # 2. Second Pass: Extract and Assemble Text Based on Majority Font
        print(
            f"FontFilter (Direct): Extracting text matching majority font (Name='{majority_font_name}', RoundedSize={majority_font_size_rounded}pt)...")
        all_pages_collected_text = []  # List to hold text from each page (as a list of block texts)

        for page_num_extraction in range(original_doc.page_count):
            page = original_doc[page_num_extraction]
            # Using flags for potentially better whitespace and ligature handling in extracted text
            text_page_dict = page.get_text("dict",
                                           flags=fitz.TEXTFLAGS_TEXT | fitz.TEXT_PRESERVE_LIGATURES | fitz.TEXT_PRESERVE_WHITESPACE)

            page_blocks_text_parts = []  # Collect text from blocks on this page

            for block in text_page_dict.get("blocks", []):
                if block.get("type") == 0:  # Text block
                    current_block_lines_text_parts = []
                    for line in block.get("lines", []):
                        current_line_spans_text_parts = []
                        for span in line.get("spans", []):
                            # Check if this span matches the majority font
                            current_span_font_name = span["font"]
                            current_span_font_size = span.get("size")

                            if current_span_font_size is not None and \
                                    current_span_font_name == majority_font_name and \
                                    int(round(current_span_font_size)) == majority_font_size_rounded:
                                current_line_spans_text_parts.append(span["text"])

                        if current_line_spans_text_parts:
                            # Join text from selected spans within a line with a single space
                            line_text = " ".join(current_line_spans_text_parts)
                            current_block_lines_text_parts.append(line_text)

                    if current_block_lines_text_parts:
                        # Join lines within a block with a single newline
                        block_text = "\n".join(current_block_lines_text_parts)
                        page_blocks_text_parts.append(block_text)

            if page_blocks_text_parts:
                # Join blocks on a page with a double newline (simulating paragraph breaks)
                all_pages_collected_text.append("\n\n".join(page_blocks_text_parts))

        if not all_pages_collected_text:
            print("FontFilter (Direct): No text matching the majority font was found to extract.")
            return ""

        # Join text from all pages.
        # A page break is already handled by the \n\n between blocks of different pages.
        # If more distinct page separation is needed, a custom separator could be added here.
        final_text = "\n\n".join(all_pages_collected_text)
        print(f"FontFilter (Direct): Successfully extracted text. Total length: {len(final_text)} characters.")
        return final_text

    except Exception as e:
        print(f"Error in extract_majority_font_text_directly for '{pdf_path}': {e}\n{traceback.format_exc()}")
        return ""
    finally:
        if original_doc: original_doc.close()


def extract_plain_text_from_original_pdf(pdf_path: str) -> str:
    """
    Extracts raw plain text from the PDF at pdf_path without any filtering.
    Expects pdf_path to be a valid path to a PDF file.
    """
    doc_orig_text = None
    try:
        doc_orig_text = fitz.open(pdf_path)
        full_text_parts = [page.get_text("text") for page in doc_orig_text]
        # print(full_text_parts) # This was the user's debug print, can be noisy
        return "".join(full_text_parts)
    except Exception as e:
        print(f"Error extracting plain text from original PDF '{pdf_path}': {e}\n{traceback.format_exc()}")
        return ""
    finally:
        if doc_orig_text: doc_orig_text.close()