evaluate.py

"""用 RAGAS 評估 RAG 系統四大指標（教材第 14 章）：

- Faithfulness：答案是否忠於文件（抓幻覺）
- Answer Relevancy：答案是否相關
- Context Precision：取出的 chunks 相關性
- Context Recall：是否取齊回答所需的所有資訊（需 ground truth）

使用方式：
    pip install ragas datasets
    python -m src.evaluate
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

from src.config import ROOT_DIR, config
from src.rag import build_chain

DEFAULT_TESTSET = ROOT_DIR / "tests" / "testset.json"


def load_testset(path: Path) -> list[dict]:
    if not path.exists():
        print(f"[錯誤] 找不到測試集 {path}")
        print("請建立一個 JSON 檔，格式為：")
        print(
            json.dumps(
                [
                    {
                        "question": "公司年假規定是什麼？",
                        "ground_truth": "員工每年有 14 天年假，需提前 3 個工作日申請。",
                    }
                ],
                ensure_ascii=False,
                indent=2,
            )
        )
        sys.exit(1)
    return json.loads(path.read_text(encoding="utf-8"))


def run_rag_on_testset(testset: list[dict]) -> dict:
    """跑 RAG 取得每題的答案與 contexts，組成 RAGAS 需要的格式。"""
    chain, retriever = build_chain()

    questions, answers, contexts, ground_truths = [], [], [], []

    for i, item in enumerate(testset, 1):
        q = item["question"]
        gt = item["ground_truth"]
        print(f"  [{i}/{len(testset)}] {q}")

        docs = retriever.invoke(q)
        ctxs = [d.page_content for d in docs]
        answer = chain.invoke(q)

        questions.append(q)
        answers.append(answer)
        contexts.append(ctxs)
        ground_truths.append(gt)

    return {
        "question": questions,
        "answer": answers,
        "contexts": contexts,
        "ground_truth": ground_truths,
    }


def evaluate(testset_path: Path = DEFAULT_TESTSET) -> None:
    try:
        from datasets import Dataset
        from ragas import evaluate as ragas_evaluate
        from ragas.metrics import (
            answer_relevancy,
            context_precision,
            context_recall,
            faithfulness,
        )
    except ImportError:
        print("[錯誤] 請先安裝：pip install ragas datasets")
        sys.exit(1)

    print(f"\n=== 載入測試集：{testset_path} ===")
    testset = load_testset(testset_path)
    print(f"共 {len(testset)} 題")

    print("\n=== 跑 RAG 取得答案 ===")
    data = run_rag_on_testset(testset)

    print("\n=== 跑 RAGAS 評估（這一步會打多次 OpenAI API）===")
    dataset = Dataset.from_dict(data)
    result = ragas_evaluate(
        dataset,
        metrics=[
            faithfulness,
            answer_relevancy,
            context_precision,
            context_recall,
        ],
    )

    print("\n=== 評估結果 ===")
    print(result)

    # 也存個 csv 方便分析
    df = result.to_pandas()
    out_path = ROOT_DIR / "evaluation_result.csv"
    df.to_csv(out_path, index=False, encoding="utf-8-sig")
    print(f"\n詳細結果已存到：{out_path}")


if __name__ == "__main__":
    evaluate()