Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/tests/evals/test_refine_eval.py

@@ -0,0 +1,88 @@
+"""Section-refinement eval (`make eval`).
+
+Runs against the CONFIGURED chat endpoint — it must be live. Proves the core
+of writing-first capture on a real model: a rough section becomes mature
+prose, its heading is kept, ONLY that section comes back (FIM), the language
+is preserved, and no load-bearing fact is dropped.
+"""
+
+import pytest
+
+from app.authoring.prompts import render_refine_prompt
+from app.llm.client import chat_stream
+
+pytestmark = pytest.mark.eval
+
+# Same flag the /refine endpoint uses: turn off the reasoning channel so the
+# call is fast and the eval measures the answer, not the thinking.
+_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
+
+PREFIX = "## Zweck\nDieser Ablauf beschreibt die monatliche Rechnungsstellung."
+SECTION = (
+    "## Ablauf\n"
+    "also man macht das am monatsanfang. erst die stunden exportieren, dann in "
+    "die vorlage kopieren und per mail raus. bei einer PO muss die nummer drauf."
+)
+SUFFIX = "## Fallstricke"
+
+
+async def _refine(
+    section: str,
+    prefix: str,
+    suffix: str,
+    knowledge: list[str] | None = None,
+) -> str:
+    messages = render_refine_prompt(
+        section,
+        prefix=prefix,
+        suffix=suffix,
+        persona="Du bist ein präziser Fachredakteur, der Abläufe dokumentiert.",
+        hint="Die Schritte in Reihenfolge, als Liste.",
+        knowledge=knowledge,
+    )
+    parts: list[str] = []
+    async for token in chat_stream(
+        messages, role="chat", temperature=0.4, extra_body=_NO_THINKING
+    ):
+        parts.append(token)
+    return "".join(parts).strip()
+
+
+async def test_refinement_matures_only_the_active_section() -> None:
+    out = await _refine(SECTION, PREFIX, SUFFIX)
+
+    assert out, "refinement returned nothing"
+    # Kept the section's own heading.
+    assert out.lstrip().startswith("## Ablauf")
+    # ONLY this section: the surrounding headings must not be re-emitted.
+    assert "## Zweck" not in out
+    assert "## Fallstricke" not in out
+    # Used the input and kept the load-bearing PO fact.
+    assert any(token in out for token in ("PO", "Purchase", "Bestell"))
+    # Language preserved (German): a switch to English would be a regression.
+    lowered = out.lower()
+    assert any(word in lowered for word in (" der ", " die ", " und ", " wird "))
+
+
+# A related document the company already has. It shares the topic but carries a
+# distinctive fact the section itself never mentions.
+GROUNDING = [
+    'From "Zahlungsbedingungen" (Fristen): Rechnungen sind binnen 14 Tagen '
+    "fällig, mit zwei Prozent Skonto bei Zahlung binnen sieben Tagen."
+]
+
+
+async def test_grounding_informs_without_being_copied_in() -> None:
+    out = await _refine(SECTION, PREFIX, SUFFIX, knowledge=GROUNDING)
+
+    assert out, "refinement returned nothing"
+    # Grounding does not break the one-section contract.
+    assert out.lstrip().startswith("## Ablauf")
+    assert "## Zweck" not in out and "## Fallstricke" not in out
+    # The section keeps its own load-bearing fact.
+    assert any(token in out for token in ("PO", "Purchase", "Bestell"))
+    # Grounding is a reference, not a fact source: the related document's own
+    # detail must not be imported into this section, and the reference framing
+    # must not be echoed back.
+    assert "Skonto" not in out
+    assert "Zahlungsbedingungen" not in out