Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/tests/evals/test_topic_retrieval_eval.py

@@ -0,0 +1,111 @@
+"""Topic-summary vs raw-message retrieval (`make eval`).
+
+Query mode retrieves over the last user message today. On a topic-losing
+follow-up ("Hi", "what was my first question") that message finds nothing, even
+when the conversation is clearly about a documented subject. An LLM topic
+summary of the whole conversation should recover it. This eval measures how
+much better — the number that decides whether query mode should adopt it.
+Runs against the CONFIGURED embedding + utility endpoints.
+"""
+
+import pytest
+from pydantic import BaseModel
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.auth.passwords import hash_password
+from app.authoring.prompts import render_topic_summary_prompt
+from app.llm.client import chat_json
+from app.models import (
+    Department,
+    Document,
+    DocumentStatus,
+    DocumentVisibility,
+    User,
+    UserRole,
+)
+from app.rag.indexing import reindex_document
+from app.rag.retrieval import search
+from tests.fixtures.loader import load_conversation_snippets, load_corpus
+
+pytestmark = pytest.mark.eval
+
+_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
+
+
+class _Topic(BaseModel):
+    topic: str
+
+
+def _transcript(messages: list[dict[str, str]]) -> str:
+    return "\n".join(f"{message['role']}: {message['content']}" for message in messages)
+
+
+async def _hit(db: AsyncSession, query: str, user: User, slug_by_id, expected) -> bool:
+    results = await search(db, query, user=user, top_k=5)
+    slugs = {slug_by_id.get(result.document_id) for result in results}
+    return bool(expected & slugs)
+
+
+async def test_topic_summary_beats_the_raw_message(db: AsyncSession) -> None:
+    corpus = load_corpus()
+    snippets = load_conversation_snippets()
+
+    department = Department(name="Eval")
+    db.add(department)
+    await db.flush()
+    user = User(
+        email="eval@test.dev",
+        name="Eval User",
+        role=UserRole.member,
+        password_hash=hash_password("eval-only"),
+        department_id=department.id,
+    )
+    db.add(user)
+    await db.flush()
+
+    slug_by_id: dict = {}
+    for doc in corpus:
+        document = Document(
+            title=doc.title,
+            status=DocumentStatus.published,
+            visibility=DocumentVisibility.public,
+            content_md=doc.content_md,
+            author_id=user.id,
+            department_id=department.id,
+            meta={"slug": doc.slug},
+        )
+        db.add(document)
+        await db.flush()
+        await reindex_document(db, document)
+        slug_by_id[document.id] = doc.slug
+    await db.commit()
+
+    raw_hits = 0
+    topic_hits = 0
+    for snippet in snippets:
+        expected = set(snippet["expected"])
+        last = snippet["messages"][-1]["content"]
+        raw_hit = await _hit(db, last, user, slug_by_id, expected)
+
+        topic = (
+            await chat_json(
+                render_topic_summary_prompt(_transcript(snippet["messages"])),
+                _Topic,
+                extra_body=_NO_THINKING,
+            )
+        ).topic
+        topic_hit = await _hit(db, topic, user, slug_by_id, expected)
+
+        raw_hits += int(raw_hit)
+        topic_hits += int(topic_hit)
+        print(
+            f"[{'HIT ' if topic_hit else 'MISS'}] expected={expected} "
+            f"raw={'hit' if raw_hit else 'miss'} topic={topic!r}"
+        )
+
+    n = len(snippets)
+    print(f"\nraw recall {raw_hits}/{n}  |  topic-summary recall {topic_hits}/{n}")
+    # The whole point: a topic summary must not do worse than the raw message,
+    # and must actually recover the documented subject on these follow-ups.
+    assert topic_hits >= raw_hits
+    assert topic_hits >= max(1, raw_hits + 1)