ProfessorNovaandClaude Opus 5
784b76baf7
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
112 lines
3.6 KiB
Python
112 lines
3.6 KiB
Python
"""Topic-summary vs raw-message retrieval (`make eval`).
|
|
|
|
Query mode retrieves over the last user message today. On a topic-losing
|
|
follow-up ("Hi", "what was my first question") that message finds nothing, even
|
|
when the conversation is clearly about a documented subject. An LLM topic
|
|
summary of the whole conversation should recover it. This eval measures how
|
|
much better — the number that decides whether query mode should adopt it.
|
|
Runs against the CONFIGURED embedding + utility endpoints.
|
|
"""
|
|
|
|
import pytest
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.auth.passwords import hash_password
|
|
from app.authoring.prompts import render_topic_summary_prompt
|
|
from app.llm.client import chat_json
|
|
from app.models import (
|
|
Department,
|
|
Document,
|
|
DocumentStatus,
|
|
DocumentVisibility,
|
|
User,
|
|
UserRole,
|
|
)
|
|
from app.rag.indexing import reindex_document
|
|
from app.rag.retrieval import search
|
|
from tests.fixtures.loader import load_conversation_snippets, load_corpus
|
|
|
|
pytestmark = pytest.mark.eval
|
|
|
|
_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
|
|
|
|
|
|
class _Topic(BaseModel):
|
|
topic: str
|
|
|
|
|
|
def _transcript(messages: list[dict[str, str]]) -> str:
|
|
return "\n".join(f"{message['role']}: {message['content']}" for message in messages)
|
|
|
|
|
|
async def _hit(db: AsyncSession, query: str, user: User, slug_by_id, expected) -> bool:
|
|
results = await search(db, query, user=user, top_k=5)
|
|
slugs = {slug_by_id.get(result.document_id) for result in results}
|
|
return bool(expected & slugs)
|
|
|
|
|
|
async def test_topic_summary_beats_the_raw_message(db: AsyncSession) -> None:
|
|
corpus = load_corpus()
|
|
snippets = load_conversation_snippets()
|
|
|
|
department = Department(name="Eval")
|
|
db.add(department)
|
|
await db.flush()
|
|
user = User(
|
|
email="eval@test.dev",
|
|
name="Eval User",
|
|
role=UserRole.member,
|
|
password_hash=hash_password("eval-only"),
|
|
department_id=department.id,
|
|
)
|
|
db.add(user)
|
|
await db.flush()
|
|
|
|
slug_by_id: dict = {}
|
|
for doc in corpus:
|
|
document = Document(
|
|
title=doc.title,
|
|
status=DocumentStatus.published,
|
|
visibility=DocumentVisibility.public,
|
|
content_md=doc.content_md,
|
|
author_id=user.id,
|
|
department_id=department.id,
|
|
meta={"slug": doc.slug},
|
|
)
|
|
db.add(document)
|
|
await db.flush()
|
|
await reindex_document(db, document)
|
|
slug_by_id[document.id] = doc.slug
|
|
await db.commit()
|
|
|
|
raw_hits = 0
|
|
topic_hits = 0
|
|
for snippet in snippets:
|
|
expected = set(snippet["expected"])
|
|
last = snippet["messages"][-1]["content"]
|
|
raw_hit = await _hit(db, last, user, slug_by_id, expected)
|
|
|
|
topic = (
|
|
await chat_json(
|
|
render_topic_summary_prompt(_transcript(snippet["messages"])),
|
|
_Topic,
|
|
extra_body=_NO_THINKING,
|
|
)
|
|
).topic
|
|
topic_hit = await _hit(db, topic, user, slug_by_id, expected)
|
|
|
|
raw_hits += int(raw_hit)
|
|
topic_hits += int(topic_hit)
|
|
print(
|
|
f"[{'HIT ' if topic_hit else 'MISS'}] expected={expected} "
|
|
f"raw={'hit' if raw_hit else 'miss'} topic={topic!r}"
|
|
)
|
|
|
|
n = len(snippets)
|
|
print(f"\nraw recall {raw_hits}/{n} | topic-summary recall {topic_hits}/{n}")
|
|
# The whole point: a topic summary must not do worse than the raw message,
|
|
# and must actually recover the documented subject on these follow-ups.
|
|
assert topic_hits >= raw_hits
|
|
assert topic_hits >= max(1, raw_hits + 1)
|