Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/tests/evals/test_retrieval_eval.py

@@ -0,0 +1,101 @@
+"""Retrieval quality eval over the golden query set (`make eval`).
+
+Runs against the CONFIGURED embedding endpoint — it must be live. The
+corpus is indexed as public documents for a single eval user: this measures
+retrieval QUALITY; permission behavior is covered by the unit tests.
+"""
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.passwords import hash_password
+from app.models import (
+    Department,
+    Document,
+    DocumentStatus,
+    DocumentVisibility,
+    User,
+    UserRole,
+)
+from app.rag.indexing import reindex_document
+from app.rag.retrieval import results_are_low_confidence, search
+from tests.fixtures.loader import load_corpus, load_golden_queries
+
+pytestmark = pytest.mark.eval
+
+RECALL_FLOOR = 0.8  # baseline 2026-07: 25/25 = 1.00
+
+
+async def test_retrieval_golden_set(db: AsyncSession) -> None:
+    corpus = load_corpus()
+    queries = load_golden_queries()
+
+    department = Department(name="Eval")
+    db.add(department)
+    await db.flush()
+    user = User(
+        email="eval@test.dev",
+        name="Eval User",
+        role=UserRole.member,
+        password_hash=hash_password("eval-only"),
+        department_id=department.id,
+    )
+    db.add(user)
+    await db.flush()
+
+    slug_by_document_id = {}
+    for doc in corpus:
+        document = Document(
+            title=doc.title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=doc.content_md,
+            meta={"slug": doc.slug},
+            author_id=user.id,
+            department_id=department.id,
+        )
+        db.add(document)
+        await db.flush()
+        slug_by_document_id[document.id] = doc.slug
+        await reindex_document(db, document)  # real embeddings
+    await db.commit()
+
+    hits = 0
+    expected_total = 0
+    misses: list[str] = []
+    no_answer_violations: list[str] = []
+    report: list[str] = []
+
+    for entry in queries:
+        results = await search(db, entry["query"], user=user, top_k=5)
+        top_slugs = [slug_by_document_id[r.document_id] for r in results]
+        if entry["expected"]:
+            expected_total += 1
+            hit = any(slug in entry["expected"] for slug in top_slugs)
+            hits += int(hit)
+            if not hit:
+                misses.append(entry["query"])
+            report.append(
+                f"{'HIT ' if hit else 'MISS'} {entry['query'][:58]!r} -> {top_slugs[:3]}"
+            )
+        else:
+            top = results[0] if results else None
+            fts = top.fts_match if top else False
+            distance = top.vector_distance if top else None
+            report.append(
+                f"NOANS {entry['query'][:58]!r} fts={fts} distance={distance:.3f}"
+                if distance is not None
+                else f"NOANS {entry['query'][:58]!r} fts={fts} distance=None"
+            )
+            confident_nothing = results_are_low_confidence(results)
+            if not confident_nothing:
+                no_answer_violations.append(entry["query"])
+
+    recall = hits / expected_total
+    print("\n".join(report))
+    print(f"\nrecall@5: {hits}/{expected_total} = {recall:.2f}")
+
+    assert recall >= RECALL_FLOOR, f"recall {recall:.2f} below floor; misses: {misses}"
+    assert not no_answer_violations, (
+        f"no-answer queries returned confident results: {no_answer_violations}"
+    )