text_processor.py

"""
Plain-text and Markdown ingestion.
"""
import os
import logging

from langchain_community.document_loaders import TextLoader

from config.settings import RAW_TEXT_PATH
from processors.source_metadata import ensure_source_metadata

logger = logging.getLogger(__name__)


def process_texts():
    """Load .txt and .md files as Document objects (not yet chunked)."""
    documents = []
    if not os.path.exists(RAW_TEXT_PATH):
        print("Text folder not found")
        return documents

    for filename in os.listdir(RAW_TEXT_PATH):
        if not filename.lower().endswith((".txt", ".md")):
            continue
        filepath = os.path.join(RAW_TEXT_PATH, filename)
        print(f"Processing: {filename}")
        try:
            loader = TextLoader(filepath, encoding="utf-8")
            docs = loader.load()
            for doc in docs:
                doc.metadata = ensure_source_metadata(doc.metadata, source_type="text")
            documents.extend(docs)
        except Exception as e:
            logger.warning("Skipping unreadable text file: %s (%s)", filename, e)
            print(f"[WARN] Skipping unreadable text file: {filename} ({e})")
            continue

    print(f"Text loading complete: {len(documents)} files")
    return documents


if __name__ == "__main__":
    docs = process_texts()