Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/tests/evals/test_self_knowledge_eval.py

@@ -0,0 +1,153 @@
+"""Does Pablan find its own help pages when asked about itself? (`make eval`)
+
+Runs against the CONFIGURED embedding endpoint — it must be live.
+
+The help pages under `help/` are imported into `documents` on every start and
+are searchable like anything else, which is what lets Pablan answer "how do I
+share a document?" from its own retrieval path instead of from a hardcoded
+FAQ. That only works if the question actually retrieves the right page, and
+the pages are written in one voice about one product, so they compete with
+each other far more than the corpus documents do. This measures exactly that:
+a question a user would type, against the help pages as shipped.
+
+The company corpus is indexed alongside them on purpose — an instance is
+never only help pages, and "how do I report a fault?" must not pull the help
+page about writing documents.
+"""
+
+from pathlib import Path
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.passwords import hash_password
+from app.config import get_settings
+from app.help_import import parse_help_document
+from app.models import (
+    Department,
+    Document,
+    DocumentStatus,
+    DocumentVisibility,
+    User,
+    UserRole,
+)
+from app.rag.indexing import reindex_document
+from app.rag.retrieval import results_are_low_confidence, search
+from tests.fixtures.loader import load_corpus
+
+pytestmark = pytest.mark.eval
+
+RECALL_FLOOR = 0.8  # baseline 2026-08: 10/10 = 1.00
+
+# Questions in the words a user would use, and the help page (`key` in the
+# file's frontmatter) that has to be in the top 5.
+QUESTIONS: list[tuple[str, set[str]]] = [
+    ("Wie funktioniert Pablan?", {"pablan-ueberblick"}),
+    ("Was ist der Unterschied zwischen Fragen und Festhalten?", {"pablan-ueberblick"}),
+    (
+        "Wie teile ich ein Dokument mit einer anderen Abteilung?",
+        {"dokumente-und-sichtbarkeit"},
+    ),
+    ("Was bedeutet der Status Entwurf?", {"dokumente-und-sichtbarkeit"}),
+    (
+        "Wie bitte ich eine Kollegin, ein Dokument zu prüfen?",
+        {"dokumente-und-sichtbarkeit", "wissen-festhalten"},
+    ),
+    ("Warum sehe ich unter einer Antwort Quellen?", {"fragen-und-antworten"}),
+    ("Wie schreibe ich ein Dokument mit der Assistenz?", {"wissen-festhalten"}),
+    ("Wo stelle ich die Sprachmodell-Endpunkte ein?", {"administration"}),
+    ("Wie lege ich eine neue Abteilung an?", {"administration"}),
+    ("Wo finde ich das Profil einer Kollegin?", {"kolleginnen-und-profil"}),
+]
+
+# A question about the company, asked in an instance that also has help pages:
+# the help must stay out of the way.
+COMPANY_QUESTIONS = [
+    "Welcher Solldruck gilt für die Hydraulikpresse?",
+    "Wie beantrage ich Urlaub?",
+    "Was bedeutet Fehlercode E-203?",
+]
+
+
+async def test_help_pages_answer_questions_about_the_product(
+    db: AsyncSession,
+) -> None:
+    department = Department(name="Eval")
+    db.add(department)
+    await db.flush()
+    user = User(
+        email="eval@test.dev",
+        name="Eval User",
+        role=UserRole.member,
+        password_hash=hash_password("eval-only"),
+        department_id=department.id,
+    )
+    db.add(user)
+    await db.flush()
+
+    help_key_by_document_id: dict = {}
+    for path in sorted(Path(get_settings().help_dir).glob("*.md")):
+        key, title, body = parse_help_document(path.read_text())
+        document = Document(
+            title=title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=body,
+            meta={"help": key},
+            is_builtin=True,
+        )
+        db.add(document)
+        await db.flush()
+        help_key_by_document_id[document.id] = key
+        await reindex_document(db, document)
+
+    # The company corpus, so the help pages have real competition.
+    for doc in load_corpus():
+        document = Document(
+            title=doc.title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=doc.content_md,
+            meta={"slug": doc.slug},
+            author_id=user.id,
+            department_id=department.id,
+        )
+        db.add(document)
+        await db.flush()
+        await reindex_document(db, document)
+    await db.commit()
+
+    hits = 0
+    report: list[str] = []
+    for question, expected in QUESTIONS:
+        results = await search(db, question, user=user, top_k=5)
+        found = [help_key_by_document_id.get(result.document_id) for result in results]
+        hit = any(key in expected for key in found if key)
+        hits += int(hit)
+        report.append(
+            f"{'HIT ' if hit else 'MISS'} {question[:52]!r} -> {[f for f in found][:3]}"
+        )
+
+    # A product question must also be ANSWERABLE, not just retrieved: a
+    # low-confidence result set is presented as "nothing documented", which
+    # for a question about Pablan itself is simply wrong.
+    unanswered = []
+    for question, _ in QUESTIONS:
+        results = await search(db, question, user=user, top_k=5)
+        if results_are_low_confidence(results):
+            unanswered.append(question)
+
+    leaked = []
+    for question in COMPANY_QUESTIONS:
+        results = await search(db, question, user=user, top_k=5)
+        top = results[0] if results else None
+        if top is not None and top.document_id in help_key_by_document_id:
+            leaked.append(f"{question!r} -> {help_key_by_document_id[top.document_id]}")
+
+    recall = hits / len(QUESTIONS)
+    print("\n" + "\n".join(report))
+    print(f"\nself-knowledge recall@5: {hits}/{len(QUESTIONS)} = {recall:.2f}")
+
+    assert not unanswered, f"asked about itself and had no answer: {unanswered}"
+    assert not leaked, f"a help page outranked the company documents: {leaked}"
+    assert recall >= RECALL_FLOOR, f"self-knowledge recall {recall:.2f}"