"""Section-refinement eval (`make eval`). Runs against the CONFIGURED chat endpoint — it must be live. Proves the core of writing-first capture on a real model: a rough section becomes mature prose, its heading is kept, ONLY that section comes back (FIM), the language is preserved, and no load-bearing fact is dropped. """ import pytest from app.authoring.prompts import render_refine_prompt from app.llm.client import chat_stream pytestmark = pytest.mark.eval # Same flag the /refine endpoint uses: turn off the reasoning channel so the # call is fast and the eval measures the answer, not the thinking. _NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}} PREFIX = "## Zweck\nDieser Ablauf beschreibt die monatliche Rechnungsstellung." SECTION = ( "## Ablauf\n" "also man macht das am monatsanfang. erst die stunden exportieren, dann in " "die vorlage kopieren und per mail raus. bei einer PO muss die nummer drauf." ) SUFFIX = "## Fallstricke" async def _refine( section: str, prefix: str, suffix: str, knowledge: list[str] | None = None, ) -> str: messages = render_refine_prompt( section, prefix=prefix, suffix=suffix, persona="Du bist ein präziser Fachredakteur, der Abläufe dokumentiert.", hint="Die Schritte in Reihenfolge, als Liste.", knowledge=knowledge, ) parts: list[str] = [] async for token in chat_stream( messages, role="chat", temperature=0.4, extra_body=_NO_THINKING ): parts.append(token) return "".join(parts).strip() async def test_refinement_matures_only_the_active_section() -> None: out = await _refine(SECTION, PREFIX, SUFFIX) assert out, "refinement returned nothing" # Kept the section's own heading. assert out.lstrip().startswith("## Ablauf") # ONLY this section: the surrounding headings must not be re-emitted. assert "## Zweck" not in out assert "## Fallstricke" not in out # Used the input and kept the load-bearing PO fact. assert any(token in out for token in ("PO", "Purchase", "Bestell")) # Language preserved (German): a switch to English would be a regression. lowered = out.lower() assert any(word in lowered for word in (" der ", " die ", " und ", " wird ")) # A related document the company already has. It shares the topic but carries a # distinctive fact the section itself never mentions. GROUNDING = [ 'From "Zahlungsbedingungen" (Fristen): Rechnungen sind binnen 14 Tagen ' "fällig, mit zwei Prozent Skonto bei Zahlung binnen sieben Tagen." ] async def test_grounding_informs_without_being_copied_in() -> None: out = await _refine(SECTION, PREFIX, SUFFIX, knowledge=GROUNDING) assert out, "refinement returned nothing" # Grounding does not break the one-section contract. assert out.lstrip().startswith("## Ablauf") assert "## Zweck" not in out and "## Fallstricke" not in out # The section keeps its own load-bearing fact. assert any(token in out for token in ("PO", "Purchase", "Bestell")) # Grounding is a reference, not a fact source: the related document's own # detail must not be imported into this section, and the reference framing # must not be echoed back. assert "Skonto" not in out assert "Zahlungsbedingungen" not in out