ProfessorNovaandClaude Opus 5
784b76baf7
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
102 lines
3.3 KiB
Python
102 lines
3.3 KiB
Python
"""Retrieval quality eval over the golden query set (`make eval`).
|
|
|
|
Runs against the CONFIGURED embedding endpoint — it must be live. The
|
|
corpus is indexed as public documents for a single eval user: this measures
|
|
retrieval QUALITY; permission behavior is covered by the unit tests.
|
|
"""
|
|
|
|
import pytest
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.auth.passwords import hash_password
|
|
from app.models import (
|
|
Department,
|
|
Document,
|
|
DocumentStatus,
|
|
DocumentVisibility,
|
|
User,
|
|
UserRole,
|
|
)
|
|
from app.rag.indexing import reindex_document
|
|
from app.rag.retrieval import results_are_low_confidence, search
|
|
from tests.fixtures.loader import load_corpus, load_golden_queries
|
|
|
|
pytestmark = pytest.mark.eval
|
|
|
|
RECALL_FLOOR = 0.8 # baseline 2026-07: 25/25 = 1.00
|
|
|
|
|
|
async def test_retrieval_golden_set(db: AsyncSession) -> None:
|
|
corpus = load_corpus()
|
|
queries = load_golden_queries()
|
|
|
|
department = Department(name="Eval")
|
|
db.add(department)
|
|
await db.flush()
|
|
user = User(
|
|
email="eval@test.dev",
|
|
name="Eval User",
|
|
role=UserRole.member,
|
|
password_hash=hash_password("eval-only"),
|
|
department_id=department.id,
|
|
)
|
|
db.add(user)
|
|
await db.flush()
|
|
|
|
slug_by_document_id = {}
|
|
for doc in corpus:
|
|
document = Document(
|
|
title=doc.title,
|
|
status=DocumentStatus.published,
|
|
visibility=DocumentVisibility.public,
|
|
content_md=doc.content_md,
|
|
meta={"slug": doc.slug},
|
|
author_id=user.id,
|
|
department_id=department.id,
|
|
)
|
|
db.add(document)
|
|
await db.flush()
|
|
slug_by_document_id[document.id] = doc.slug
|
|
await reindex_document(db, document) # real embeddings
|
|
await db.commit()
|
|
|
|
hits = 0
|
|
expected_total = 0
|
|
misses: list[str] = []
|
|
no_answer_violations: list[str] = []
|
|
report: list[str] = []
|
|
|
|
for entry in queries:
|
|
results = await search(db, entry["query"], user=user, top_k=5)
|
|
top_slugs = [slug_by_document_id[r.document_id] for r in results]
|
|
if entry["expected"]:
|
|
expected_total += 1
|
|
hit = any(slug in entry["expected"] for slug in top_slugs)
|
|
hits += int(hit)
|
|
if not hit:
|
|
misses.append(entry["query"])
|
|
report.append(
|
|
f"{'HIT ' if hit else 'MISS'} {entry['query'][:58]!r} -> {top_slugs[:3]}"
|
|
)
|
|
else:
|
|
top = results[0] if results else None
|
|
fts = top.fts_match if top else False
|
|
distance = top.vector_distance if top else None
|
|
report.append(
|
|
f"NOANS {entry['query'][:58]!r} fts={fts} distance={distance:.3f}"
|
|
if distance is not None
|
|
else f"NOANS {entry['query'][:58]!r} fts={fts} distance=None"
|
|
)
|
|
confident_nothing = results_are_low_confidence(results)
|
|
if not confident_nothing:
|
|
no_answer_violations.append(entry["query"])
|
|
|
|
recall = hits / expected_total
|
|
print("\n".join(report))
|
|
print(f"\nrecall@5: {hits}/{expected_total} = {recall:.2f}")
|
|
|
|
assert recall >= RECALL_FLOOR, f"recall {recall:.2f} below floor; misses: {misses}"
|
|
assert not no_answer_violations, (
|
|
f"no-answer queries returned confident results: {no_answer_violations}"
|
|
)
|