ProfessorNovaandClaude Opus 5
97dbff309c
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
89 lines
3.3 KiB
Python
89 lines
3.3 KiB
Python
"""Section-refinement eval (`make eval`).
|
|
|
|
Runs against the CONFIGURED chat endpoint — it must be live. Proves the core
|
|
of writing-first capture on a real model: a rough section becomes mature
|
|
prose, its heading is kept, ONLY that section comes back (FIM), the language
|
|
is preserved, and no load-bearing fact is dropped.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from app.authoring.prompts import render_refine_prompt
|
|
from app.llm.client import chat_stream
|
|
|
|
pytestmark = pytest.mark.eval
|
|
|
|
# Same flag the /refine endpoint uses: turn off the reasoning channel so the
|
|
# call is fast and the eval measures the answer, not the thinking.
|
|
_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
|
|
|
|
PREFIX = "## Zweck\nDieser Ablauf beschreibt die monatliche Rechnungsstellung."
|
|
SECTION = (
|
|
"## Ablauf\n"
|
|
"also man macht das am monatsanfang. erst die stunden exportieren, dann in "
|
|
"die vorlage kopieren und per mail raus. bei einer PO muss die nummer drauf."
|
|
)
|
|
SUFFIX = "## Fallstricke"
|
|
|
|
|
|
async def _refine(
|
|
section: str,
|
|
prefix: str,
|
|
suffix: str,
|
|
knowledge: list[str] | None = None,
|
|
) -> str:
|
|
messages = render_refine_prompt(
|
|
section,
|
|
prefix=prefix,
|
|
suffix=suffix,
|
|
persona="Du bist ein präziser Fachredakteur, der Abläufe dokumentiert.",
|
|
hint="Die Schritte in Reihenfolge, als Liste.",
|
|
knowledge=knowledge,
|
|
)
|
|
parts: list[str] = []
|
|
async for token in chat_stream(
|
|
messages, role="chat", temperature=0.4, extra_body=_NO_THINKING
|
|
):
|
|
parts.append(token)
|
|
return "".join(parts).strip()
|
|
|
|
|
|
async def test_refinement_matures_only_the_active_section() -> None:
|
|
out = await _refine(SECTION, PREFIX, SUFFIX)
|
|
|
|
assert out, "refinement returned nothing"
|
|
# Kept the section's own heading.
|
|
assert out.lstrip().startswith("## Ablauf")
|
|
# ONLY this section: the surrounding headings must not be re-emitted.
|
|
assert "## Zweck" not in out
|
|
assert "## Fallstricke" not in out
|
|
# Used the input and kept the load-bearing PO fact.
|
|
assert any(token in out for token in ("PO", "Purchase", "Bestell"))
|
|
# Language preserved (German): a switch to English would be a regression.
|
|
lowered = out.lower()
|
|
assert any(word in lowered for word in (" der ", " die ", " und ", " wird "))
|
|
|
|
|
|
# A related document the company already has. It shares the topic but carries a
|
|
# distinctive fact the section itself never mentions.
|
|
GROUNDING = [
|
|
'From "Zahlungsbedingungen" (Fristen): Rechnungen sind binnen 14 Tagen '
|
|
"fällig, mit zwei Prozent Skonto bei Zahlung binnen sieben Tagen."
|
|
]
|
|
|
|
|
|
async def test_grounding_informs_without_being_copied_in() -> None:
|
|
out = await _refine(SECTION, PREFIX, SUFFIX, knowledge=GROUNDING)
|
|
|
|
assert out, "refinement returned nothing"
|
|
# Grounding does not break the one-section contract.
|
|
assert out.lstrip().startswith("## Ablauf")
|
|
assert "## Zweck" not in out and "## Fallstricke" not in out
|
|
# The section keeps its own load-bearing fact.
|
|
assert any(token in out for token in ("PO", "Purchase", "Bestell"))
|
|
# Grounding is a reference, not a fact source: the related document's own
|
|
# detail must not be imported into this section, and the reference framing
|
|
# must not be echoed back.
|
|
assert "Skonto" not in out
|
|
assert "Zahlungsbedingungen" not in out
|