pablan/backend/tests/evals/test_self_knowledge_eval.py

"""Does Pablan find its own help pages when asked about itself? (`make eval`)

Runs against the CONFIGURED embedding endpoint — it must be live.

The help pages under `help/` are imported into `documents` on every start and
are searchable like anything else, which is what lets Pablan answer "how do I
share a document?" from its own retrieval path instead of from a hardcoded
FAQ. That only works if the question actually retrieves the right page, and
the pages are written in one voice about one product, so they compete with
each other far more than the corpus documents do. This measures exactly that:
a question a user would type, against the help pages as shipped.

The company corpus is indexed alongside them on purpose — an instance is
never only help pages, and "how do I report a fault?" must not pull the help
page about writing documents.
"""

from pathlib import Path

import pytest
from sqlalchemy.ext.asyncio import AsyncSession

from app.auth.passwords import hash_password
from app.config import get_settings
from app.help_import import parse_help_document
from app.models import (
    Department,
    Document,
    DocumentStatus,
    DocumentVisibility,
    User,
    UserRole,
)
from app.rag.indexing import reindex_document
from app.rag.retrieval import results_are_low_confidence, search
from tests.fixtures.loader import load_corpus

pytestmark = pytest.mark.eval

RECALL_FLOOR = 0.8  # baseline 2026-08: 10/10 = 1.00

# Questions in the words a user would use, and the help page (`key` in the
# file's frontmatter) that has to be in the top 5.
QUESTIONS: list[tuple[str, set[str]]] = [
    ("Wie funktioniert Pablan?", {"pablan-ueberblick"}),
    ("Was ist der Unterschied zwischen Fragen und Festhalten?", {"pablan-ueberblick"}),
    (
        "Wie teile ich ein Dokument mit einer anderen Abteilung?",
        {"dokumente-und-sichtbarkeit"},
    ),
    ("Was bedeutet der Status Entwurf?", {"dokumente-und-sichtbarkeit"}),
    (
        "Wie bitte ich eine Kollegin, ein Dokument zu prüfen?",
        {"dokumente-und-sichtbarkeit", "wissen-festhalten"},
    ),
    ("Warum sehe ich unter einer Antwort Quellen?", {"fragen-und-antworten"}),
    ("Wie schreibe ich ein Dokument mit der Assistenz?", {"wissen-festhalten"}),
    ("Wo stelle ich die Sprachmodell-Endpunkte ein?", {"administration"}),
    ("Wie lege ich eine neue Abteilung an?", {"administration"}),
    ("Wo finde ich das Profil einer Kollegin?", {"kolleginnen-und-profil"}),
]

# A question about the company, asked in an instance that also has help pages:
# the help must stay out of the way.
COMPANY_QUESTIONS = [
    "Welcher Solldruck gilt für die Hydraulikpresse?",
    "Wie beantrage ich Urlaub?",
    "Was bedeutet Fehlercode E-203?",
]


async def test_help_pages_answer_questions_about_the_product(
    db: AsyncSession,
) -> None:
    department = Department(name="Eval")
    db.add(department)
    await db.flush()
    user = User(
        email="eval@test.dev",
        name="Eval User",
        role=UserRole.member,
        password_hash=hash_password("eval-only"),
        department_id=department.id,
    )
    db.add(user)
    await db.flush()

    help_key_by_document_id: dict = {}
    for path in sorted(Path(get_settings().help_dir).glob("*.md")):
        key, title, body = parse_help_document(path.read_text())
        document = Document(
            title=title,
            status=DocumentStatus.published,
            visibility=DocumentVisibility.public,
            content_md=body,
            meta={"help": key},
            is_builtin=True,
        )
        db.add(document)
        await db.flush()
        help_key_by_document_id[document.id] = key
        await reindex_document(db, document)

    # The company corpus, so the help pages have real competition.
    for doc in load_corpus():
        document = Document(
            title=doc.title,
            status=DocumentStatus.published,
            visibility=DocumentVisibility.public,
            content_md=doc.content_md,
            meta={"slug": doc.slug},
            author_id=user.id,
            department_id=department.id,
        )
        db.add(document)
        await db.flush()
        await reindex_document(db, document)
    await db.commit()

    hits = 0
    report: list[str] = []
    for question, expected in QUESTIONS:
        results = await search(db, question, user=user, top_k=5)
        found = [help_key_by_document_id.get(result.document_id) for result in results]
        hit = any(key in expected for key in found if key)
        hits += int(hit)
        report.append(
            f"{'HIT ' if hit else 'MISS'} {question[:52]!r} -> {[f for f in found][:3]}"
        )

    # A product question must also be ANSWERABLE, not just retrieved: a
    # low-confidence result set is presented as "nothing documented", which
    # for a question about Pablan itself is simply wrong.
    unanswered = []
    for question, _ in QUESTIONS:
        results = await search(db, question, user=user, top_k=5)
        if results_are_low_confidence(results):
            unanswered.append(question)

    leaked = []
    for question in COMPANY_QUESTIONS:
        results = await search(db, question, user=user, top_k=5)
        top = results[0] if results else None
        if top is not None and top.document_id in help_key_by_document_id:
            leaked.append(f"{question!r} -> {help_key_by_document_id[top.document_id]}")

    recall = hits / len(QUESTIONS)
    print("\n" + "\n".join(report))
    print(f"\nself-knowledge recall@5: {hits}/{len(QUESTIONS)} = {recall:.2f}")

    assert not unanswered, f"asked about itself and had no answer: {unanswered}"
    assert not leaked, f"a help page outranked the company documents: {leaked}"
    assert recall >= RECALL_FLOOR, f"self-knowledge recall {recall:.2f}"