parser/txt_parser.py

from typing import Iterator
from langchain_core.documents import Document
from langchain_community.document_loaders.base import BaseBlobParser
from langchain_community.document_loaders.blob_loaders import Blob


class TxtParser(BaseBlobParser):
    """Parse plain text files from a blob."""

    # type: ignore[valid-type]
    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
        """Parse a plain text file into the Document iterator.

        Args:
            blob: The blob to parse.

        Returns: An iterator of Documents.
        """
        supported_mime_types = [
            "text/plain",  # .txt
        ]

        # Debugging: Print MIME type
        print(f"Blob MIME type: {blob.mimetype}")

        # type: ignore[attr-defined]
        if blob.mimetype not in supported_mime_types:
            raise ValueError(
                f"This blob type is not supported for this parser. Supported types are: {supported_mime_types}"
            )

        # Read text file content
        with blob.as_bytes_io() as txt_file:  # type: ignore[attr-defined]
            text = txt_file.read().decode("utf-8")
            metadata = {"source": blob.source}  # type: ignore[attr-defined]
            yield Document(page_content=text, metadata=metadata)