Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/tests/evals/test_refine_eval.py

@@ -0,0 +1,88 @@
+"""Section-refinement eval (`make eval`).
+
+Runs against the CONFIGURED chat endpoint — it must be live. Proves the core
+of writing-first capture on a real model: a rough section becomes mature
+prose, its heading is kept, ONLY that section comes back (FIM), the language
+is preserved, and no load-bearing fact is dropped.
+"""
+
+import pytest
+
+from app.authoring.prompts import render_refine_prompt
+from app.llm.client import chat_stream
+
+pytestmark = pytest.mark.eval
+
+# Same flag the /refine endpoint uses: turn off the reasoning channel so the
+# call is fast and the eval measures the answer, not the thinking.
+_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
+
+PREFIX = "## Zweck\nDieser Ablauf beschreibt die monatliche Rechnungsstellung."
+SECTION = (
+    "## Ablauf\n"
+    "also man macht das am monatsanfang. erst die stunden exportieren, dann in "
+    "die vorlage kopieren und per mail raus. bei einer PO muss die nummer drauf."
+)
+SUFFIX = "## Fallstricke"
+
+
+async def _refine(
+    section: str,
+    prefix: str,
+    suffix: str,
+    knowledge: list[str] | None = None,
+) -> str:
+    messages = render_refine_prompt(
+        section,
+        prefix=prefix,
+        suffix=suffix,
+        persona="Du bist ein präziser Fachredakteur, der Abläufe dokumentiert.",
+        hint="Die Schritte in Reihenfolge, als Liste.",
+        knowledge=knowledge,
+    )
+    parts: list[str] = []
+    async for token in chat_stream(
+        messages, role="chat", temperature=0.4, extra_body=_NO_THINKING
+    ):
+        parts.append(token)
+    return "".join(parts).strip()
+
+
+async def test_refinement_matures_only_the_active_section() -> None:
+    out = await _refine(SECTION, PREFIX, SUFFIX)
+
+    assert out, "refinement returned nothing"
+    # Kept the section's own heading.
+    assert out.lstrip().startswith("## Ablauf")
+    # ONLY this section: the surrounding headings must not be re-emitted.
+    assert "## Zweck" not in out
+    assert "## Fallstricke" not in out
+    # Used the input and kept the load-bearing PO fact.
+    assert any(token in out for token in ("PO", "Purchase", "Bestell"))
+    # Language preserved (German): a switch to English would be a regression.
+    lowered = out.lower()
+    assert any(word in lowered for word in (" der ", " die ", " und ", " wird "))
+
+
+# A related document the company already has. It shares the topic but carries a
+# distinctive fact the section itself never mentions.
+GROUNDING = [
+    'From "Zahlungsbedingungen" (Fristen): Rechnungen sind binnen 14 Tagen '
+    "fällig, mit zwei Prozent Skonto bei Zahlung binnen sieben Tagen."
+]
+
+
+async def test_grounding_informs_without_being_copied_in() -> None:
+    out = await _refine(SECTION, PREFIX, SUFFIX, knowledge=GROUNDING)
+
+    assert out, "refinement returned nothing"
+    # Grounding does not break the one-section contract.
+    assert out.lstrip().startswith("## Ablauf")
+    assert "## Zweck" not in out and "## Fallstricke" not in out
+    # The section keeps its own load-bearing fact.
+    assert any(token in out for token in ("PO", "Purchase", "Bestell"))
+    # Grounding is a reference, not a fact source: the related document's own
+    # detail must not be imported into this section, and the reference framing
+    # must not be echoed back.
+    assert "Skonto" not in out
+    assert "Zahlungsbedingungen" not in out

backend/tests/evals/test_retrieval_eval.py

@@ -0,0 +1,101 @@
+"""Retrieval quality eval over the golden query set (`make eval`).
+
+Runs against the CONFIGURED embedding endpoint — it must be live. The
+corpus is indexed as public documents for a single eval user: this measures
+retrieval QUALITY; permission behavior is covered by the unit tests.
+"""
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.passwords import hash_password
+from app.models import (
+    Department,
+    Document,
+    DocumentStatus,
+    DocumentVisibility,
+    User,
+    UserRole,
+)
+from app.rag.indexing import reindex_document
+from app.rag.retrieval import results_are_low_confidence, search
+from tests.fixtures.loader import load_corpus, load_golden_queries
+
+pytestmark = pytest.mark.eval
+
+RECALL_FLOOR = 0.8  # baseline 2026-07: 25/25 = 1.00
+
+
+async def test_retrieval_golden_set(db: AsyncSession) -> None:
+    corpus = load_corpus()
+    queries = load_golden_queries()
+
+    department = Department(name="Eval")
+    db.add(department)
+    await db.flush()
+    user = User(
+        email="eval@test.dev",
+        name="Eval User",
+        role=UserRole.member,
+        password_hash=hash_password("eval-only"),
+        department_id=department.id,
+    )
+    db.add(user)
+    await db.flush()
+
+    slug_by_document_id = {}
+    for doc in corpus:
+        document = Document(
+            title=doc.title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=doc.content_md,
+            meta={"slug": doc.slug},
+            author_id=user.id,
+            department_id=department.id,
+        )
+        db.add(document)
+        await db.flush()
+        slug_by_document_id[document.id] = doc.slug
+        await reindex_document(db, document)  # real embeddings
+    await db.commit()
+
+    hits = 0
+    expected_total = 0
+    misses: list[str] = []
+    no_answer_violations: list[str] = []
+    report: list[str] = []
+
+    for entry in queries:
+        results = await search(db, entry["query"], user=user, top_k=5)
+        top_slugs = [slug_by_document_id[r.document_id] for r in results]
+        if entry["expected"]:
+            expected_total += 1
+            hit = any(slug in entry["expected"] for slug in top_slugs)
+            hits += int(hit)
+            if not hit:
+                misses.append(entry["query"])
+            report.append(
+                f"{'HIT ' if hit else 'MISS'} {entry['query'][:58]!r} -> {top_slugs[:3]}"
+            )
+        else:
+            top = results[0] if results else None
+            fts = top.fts_match if top else False
+            distance = top.vector_distance if top else None
+            report.append(
+                f"NOANS {entry['query'][:58]!r} fts={fts} distance={distance:.3f}"
+                if distance is not None
+                else f"NOANS {entry['query'][:58]!r} fts={fts} distance=None"
+            )
+            confident_nothing = results_are_low_confidence(results)
+            if not confident_nothing:
+                no_answer_violations.append(entry["query"])
+
+    recall = hits / expected_total
+    print("\n".join(report))
+    print(f"\nrecall@5: {hits}/{expected_total} = {recall:.2f}")
+
+    assert recall >= RECALL_FLOOR, f"recall {recall:.2f} below floor; misses: {misses}"
+    assert not no_answer_violations, (
+        f"no-answer queries returned confident results: {no_answer_violations}"
+    )

backend/tests/evals/test_self_knowledge_eval.py

@@ -0,0 +1,153 @@
+"""Does Pablan find its own help pages when asked about itself? (`make eval`)
+
+Runs against the CONFIGURED embedding endpoint — it must be live.
+
+The help pages under `help/` are imported into `documents` on every start and
+are searchable like anything else, which is what lets Pablan answer "how do I
+share a document?" from its own retrieval path instead of from a hardcoded
+FAQ. That only works if the question actually retrieves the right page, and
+the pages are written in one voice about one product, so they compete with
+each other far more than the corpus documents do. This measures exactly that:
+a question a user would type, against the help pages as shipped.
+
+The company corpus is indexed alongside them on purpose — an instance is
+never only help pages, and "how do I report a fault?" must not pull the help
+page about writing documents.
+"""
+
+from pathlib import Path
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.passwords import hash_password
+from app.config import get_settings
+from app.help_import import parse_help_document
+from app.models import (
+    Department,
+    Document,
+    DocumentStatus,
+    DocumentVisibility,
+    User,
+    UserRole,
+)
+from app.rag.indexing import reindex_document
+from app.rag.retrieval import results_are_low_confidence, search
+from tests.fixtures.loader import load_corpus
+
+pytestmark = pytest.mark.eval
+
+RECALL_FLOOR = 0.8  # baseline 2026-08: 10/10 = 1.00
+
+# Questions in the words a user would use, and the help page (`key` in the
+# file's frontmatter) that has to be in the top 5.
+QUESTIONS: list[tuple[str, set[str]]] = [
+    ("Wie funktioniert Pablan?", {"pablan-ueberblick"}),
+    ("Was ist der Unterschied zwischen Fragen und Festhalten?", {"pablan-ueberblick"}),
+    (
+        "Wie teile ich ein Dokument mit einer anderen Abteilung?",
+        {"dokumente-und-sichtbarkeit"},
+    ),
+    ("Was bedeutet der Status Entwurf?", {"dokumente-und-sichtbarkeit"}),
+    (
+        "Wie bitte ich eine Kollegin, ein Dokument zu prüfen?",
+        {"dokumente-und-sichtbarkeit", "wissen-festhalten"},
+    ),
+    ("Warum sehe ich unter einer Antwort Quellen?", {"fragen-und-antworten"}),
+    ("Wie schreibe ich ein Dokument mit der Assistenz?", {"wissen-festhalten"}),
+    ("Wo stelle ich die Sprachmodell-Endpunkte ein?", {"administration"}),
+    ("Wie lege ich eine neue Abteilung an?", {"administration"}),
+    ("Wo finde ich das Profil einer Kollegin?", {"kolleginnen-und-profil"}),
+]
+
+# A question about the company, asked in an instance that also has help pages:
+# the help must stay out of the way.
+COMPANY_QUESTIONS = [
+    "Welcher Solldruck gilt für die Hydraulikpresse?",
+    "Wie beantrage ich Urlaub?",
+    "Was bedeutet Fehlercode E-203?",
+]
+
+
+async def test_help_pages_answer_questions_about_the_product(
+    db: AsyncSession,
+) -> None:
+    department = Department(name="Eval")
+    db.add(department)
+    await db.flush()
+    user = User(
+        email="eval@test.dev",
+        name="Eval User",
+        role=UserRole.member,
+        password_hash=hash_password("eval-only"),
+        department_id=department.id,
+    )
+    db.add(user)
+    await db.flush()
+
+    help_key_by_document_id: dict = {}
+    for path in sorted(Path(get_settings().help_dir).glob("*.md")):
+        key, title, body = parse_help_document(path.read_text())
+        document = Document(
+            title=title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=body,
+            meta={"help": key},
+            is_builtin=True,
+        )
+        db.add(document)
+        await db.flush()
+        help_key_by_document_id[document.id] = key
+        await reindex_document(db, document)
+
+    # The company corpus, so the help pages have real competition.
+    for doc in load_corpus():
+        document = Document(
+            title=doc.title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=doc.content_md,
+            meta={"slug": doc.slug},
+            author_id=user.id,
+            department_id=department.id,
+        )
+        db.add(document)
+        await db.flush()
+        await reindex_document(db, document)
+    await db.commit()
+
+    hits = 0
+    report: list[str] = []
+    for question, expected in QUESTIONS:
+        results = await search(db, question, user=user, top_k=5)
+        found = [help_key_by_document_id.get(result.document_id) for result in results]
+        hit = any(key in expected for key in found if key)
+        hits += int(hit)
+        report.append(
+            f"{'HIT ' if hit else 'MISS'} {question[:52]!r} -> {[f for f in found][:3]}"
+        )
+
+    # A product question must also be ANSWERABLE, not just retrieved: a
+    # low-confidence result set is presented as "nothing documented", which
+    # for a question about Pablan itself is simply wrong.
+    unanswered = []
+    for question, _ in QUESTIONS:
+        results = await search(db, question, user=user, top_k=5)
+        if results_are_low_confidence(results):
+            unanswered.append(question)
+
+    leaked = []
+    for question in COMPANY_QUESTIONS:
+        results = await search(db, question, user=user, top_k=5)
+        top = results[0] if results else None
+        if top is not None and top.document_id in help_key_by_document_id:
+            leaked.append(f"{question!r} -> {help_key_by_document_id[top.document_id]}")
+
+    recall = hits / len(QUESTIONS)
+    print("\n" + "\n".join(report))
+    print(f"\nself-knowledge recall@5: {hits}/{len(QUESTIONS)} = {recall:.2f}")
+
+    assert not unanswered, f"asked about itself and had no answer: {unanswered}"
+    assert not leaked, f"a help page outranked the company documents: {leaked}"
+    assert recall >= RECALL_FLOOR, f"self-knowledge recall {recall:.2f}"

backend/tests/evals/test_topic_retrieval_eval.py

@@ -0,0 +1,111 @@
+"""Topic-summary vs raw-message retrieval (`make eval`).
+
+Query mode retrieves over the last user message today. On a topic-losing
+follow-up ("Hi", "what was my first question") that message finds nothing, even
+when the conversation is clearly about a documented subject. An LLM topic
+summary of the whole conversation should recover it. This eval measures how
+much better — the number that decides whether query mode should adopt it.
+Runs against the CONFIGURED embedding + utility endpoints.
+"""
+
+import pytest
+from pydantic import BaseModel
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.passwords import hash_password
+from app.authoring.prompts import render_topic_summary_prompt
+from app.llm.client import chat_json
+from app.models import (
+    Department,
+    Document,
+    DocumentStatus,
+    DocumentVisibility,
+    User,
+    UserRole,
+)
+from app.rag.indexing import reindex_document
+from app.rag.retrieval import search
+from tests.fixtures.loader import load_conversation_snippets, load_corpus
+
+pytestmark = pytest.mark.eval
+
+_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
+
+
+class _Topic(BaseModel):
+    topic: str
+
+
+def _transcript(messages: list[dict[str, str]]) -> str:
+    return "\n".join(f"{message['role']}: {message['content']}" for message in messages)
+
+
+async def _hit(db: AsyncSession, query: str, user: User, slug_by_id, expected) -> bool:
+    results = await search(db, query, user=user, top_k=5)
+    slugs = {slug_by_id.get(result.document_id) for result in results}
+    return bool(expected & slugs)
+
+
+async def test_topic_summary_beats_the_raw_message(db: AsyncSession) -> None:
+    corpus = load_corpus()
+    snippets = load_conversation_snippets()
+
+    department = Department(name="Eval")
+    db.add(department)
+    await db.flush()
+    user = User(
+        email="eval@test.dev",
+        name="Eval User",
+        role=UserRole.member,
+        password_hash=hash_password("eval-only"),
+        department_id=department.id,
+    )
+    db.add(user)
+    await db.flush()
+
+    slug_by_id: dict = {}
+    for doc in corpus:
+        document = Document(
+            title=doc.title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=doc.content_md,
+            author_id=user.id,
+            department_id=department.id,
+            meta={"slug": doc.slug},
+        )
+        db.add(document)
+        await db.flush()
+        await reindex_document(db, document)
+        slug_by_id[document.id] = doc.slug
+    await db.commit()
+
+    raw_hits = 0
+    topic_hits = 0
+    for snippet in snippets:
+        expected = set(snippet["expected"])
+        last = snippet["messages"][-1]["content"]
+        raw_hit = await _hit(db, last, user, slug_by_id, expected)
+
+        topic = (
+            await chat_json(
+                render_topic_summary_prompt(_transcript(snippet["messages"])),
+                _Topic,
+                extra_body=_NO_THINKING,
+            )
+        ).topic
+        topic_hit = await _hit(db, topic, user, slug_by_id, expected)
+
+        raw_hits += int(raw_hit)
+        topic_hits += int(topic_hit)
+        print(
+            f"[{'HIT ' if topic_hit else 'MISS'}] expected={expected} "
+            f"raw={'hit' if raw_hit else 'miss'} topic={topic!r}"
+        )
+
+    n = len(snippets)
+    print(f"\nraw recall {raw_hits}/{n}  |  topic-summary recall {topic_hits}/{n}")
+    # The whole point: a topic summary must not do worse than the raw message,
+    # and must actually recover the documented subject on these follow-ups.
+    assert topic_hits >= raw_hits
+    assert topic_hits >= max(1, raw_hits + 1)