test_processors.py

"""
Unit tests for processor helpers — no I/O to raw data directories needed.
"""
import pytest
from langchain_core.documents import Document

from processors.document_processor import clean_text, process_documents
from processors.source_metadata import build_source_metadata, ensure_source_metadata


class TestCleanText:
    def test_removes_null_bytes(self):
        assert "\x00" not in clean_text("hello\x00world")

    def test_collapses_whitespace(self):
        assert clean_text("  a   b  ") == "a b"

    def test_empty_returns_empty(self):
        assert clean_text("") == ""

    def test_none_returns_empty(self):
        assert clean_text(None) == ""


class TestProcessDocuments:
    def _make_doc(self, content, source="test.pdf", doc_type=None):
        meta = {"source": source, "source_type": "pdf"}
        if doc_type:
            meta["type"] = doc_type
        return Document(page_content=content, metadata=meta)

    def test_removes_empty_docs(self):
        docs = [self._make_doc(""), self._make_doc("valid content")]
        chunks = process_documents(docs)
        assert all(d.page_content for d in chunks)

    def test_splits_long_doc(self):
        long_text = "word " * 500
        docs = [self._make_doc(long_text)]
        chunks = process_documents(docs)
        assert len(chunks) > 1

    def test_summary_not_split(self):
        """Summary documents must not be fragmented across chunk boundaries."""
        long_summary = "summary content " * 500
        doc = self._make_doc(long_summary, doc_type="summary")
        chunks = process_documents([doc])
        summary_chunks = [c for c in chunks if c.metadata.get("type") == "summary"]
        assert len(summary_chunks) == 1

    def test_metadata_preserved(self):
        doc = self._make_doc("content", source="manual.pdf")
        chunks = process_documents([doc])
        for chunk in chunks:
            assert chunk.metadata.get("source") == "manual.pdf"


class TestBuildSourceMetadata:
    def test_includes_required_fields(self):
        meta = build_source_metadata("/path/to/file.pdf", "pdf")
        assert "source" in meta
        assert "source_name" in meta
        assert "source_path" in meta
        assert "source_type" in meta

    def test_source_is_basename(self):
        meta = build_source_metadata("/some/path/manual.pdf", "pdf")
        assert meta["source"] == "manual.pdf"

    def test_source_type_set(self):
        meta = build_source_metadata("/some/path/data.xlsx", "excel")
        assert meta["source_type"] == "excel"

    def test_extra_kwargs_included(self):
        meta = build_source_metadata("/path/img.jpg", "image", image_path="/path/img.jpg")
        assert "image_path" in meta


class TestEnsureSourceMetadata:
    def test_extracts_basename(self):
        meta = ensure_source_metadata({"source": "data/raw/pdfs/manual.pdf"})
        assert meta["source"] == "manual.pdf"

    def test_sets_default_type(self):
        meta = ensure_source_metadata({})
        assert meta["source_type"] == "unknown"

    def test_overrides_type(self):
        meta = ensure_source_metadata({}, source_type="pdf")
        assert meta["source_type"] == "pdf"

    def test_handles_none(self):
        meta = ensure_source_metadata(None)
        assert meta["source"] == "unknown"