"""Conversations API + query mode, end to end against fakes. fake_llm scripts the chat completions, fake_embed the vectors — the SSE contract, persistence rules and permission scoping are what is under test. """ import json import logging import pytest from httpx import AsyncClient from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload from app.api.conversations import stream_turn from app.log import JsonFormatter, apply_content_log_guard from app.models import ( Conversation, ConversationMode, Document, DocumentStatus, DocumentVisibility, Message, MessageRole, User, ) from app.modes import get_mode from app.rag.indexing import reindex_document from tests.fake_openai import FakeOpenAI pytestmark = pytest.mark.usefixtures("fake_llm", "fake_embed") async def _login(client: AsyncClient, email: str = "pablo@test.dev") -> None: response = await client.post( "/api/auth/login", json={"email": email, "password": "secret123"} ) assert response.status_code == 200 async def _indexed_public_doc(db: AsyncSession, author: User) -> Document: document = Document( title="Kaffeemaschine", status=DocumentStatus.published, visibility=DocumentVisibility.public, content_md="## Pflege\n\nDie Kaffeemaschine wird freitags entkalkt.", author_id=author.id, department_id=author.department_id, ) db.add(document) await db.flush() await reindex_document(db, document) await db.commit() return document async def _create_conversation(client: AsyncClient) -> str: response = await client.post("/api/conversations", json={"mode": "query"}) assert response.status_code == 200 return response.json()["id"] def _parse_sse(text: str) -> list[tuple[str, str]]: events = [] for frame in text.split("\n\n"): if not frame.strip(): continue lines = dict(line.split(": ", 1) for line in frame.splitlines() if ": " in line) events.append((lines["event"], lines["data"])) return events async def test_create_and_list(client: AsyncClient, seeded_user: User) -> None: await _login(client) conversation_id = await _create_conversation(client) listing = (await client.get("/api/conversations")).json() assert [c["id"] for c in listing] == [conversation_id] assert listing[0]["title"] is None # no messages yet async def test_unregistered_mode_rejected( client: AsyncClient, seeded_user: User ) -> None: await _login(client) # insight stays unregistered until the EE module provides it. response = await client.post("/api/conversations", json={"mode": "insight"}) assert response.status_code == 400 assert response.json()["code"] == "unknown_mode" async def test_conversations_are_owner_scoped( client: AsyncClient, seeded_user: User, seeded_admin: User ) -> None: await _login(client, "florian@test.dev") foreign_id = await _create_conversation(client) await client.post("/api/auth/logout") await _login(client) assert (await client.get("/api/conversations")).json() == [] assert (await client.get(f"/api/conversations/{foreign_id}")).status_code == 404 assert (await client.delete(f"/api/conversations/{foreign_id}")).status_code == 404 async def test_turn_streams_sources_tokens_done_and_persists( client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI ) -> None: await _indexed_public_doc(db, seeded_user) fake_llm.chat_responses.append({"chunks": ["Frei", "tags."]}) await _login(client) conversation_id = await _create_conversation(client) # All content words exist in the document — websearch_to_tsquery ANDs # terms, and fake embeddings carry no semantics (only FTS can match). response = await client.post( f"/api/conversations/{conversation_id}/messages", json={"content": "Kaffeemaschine entkalkt?"}, ) assert response.status_code == 200 assert response.headers["content-type"].startswith("text/event-stream") events = _parse_sse(response.text) kinds = [kind for kind, _ in events] # searching → results → sources → answering → tokens → done assert kinds[:4] == ["state", "state", "sources", "state"] assert kinds.count("token") == 2 assert kinds[-1] == "done" states = [json.loads(data) for kind, data in events if kind == "state"] assert [s["phase"] for s in states] == ["searching", "results", "answering"] assert states[1]["count"] == 1 # Progress events carry counts only — never the query or any content. for state in states: assert "Kaffeemaschine" not in json.dumps(state) sources = json.loads(dict(events)["sources"])["chunks"] assert sources[0]["title"] == "Kaffeemaschine" assert "freitags entkalkt" in sources[0]["excerpt"] messages = ( ( await db.execute( select(Message) .where(Message.conversation_id == conversation_id) .order_by(Message.created_at) ) ) .scalars() .all() ) assert [m.role for m in messages] == [MessageRole.user, MessageRole.assistant] assert messages[1].content == "Freitags." assert str(messages[1].id) in events[-1][1] # Cache-friendly structure: this turn's excerpt rides the final user turn, # while the static system prompt stays byte-identical for prompt caching. sent = fake_llm.requests[-1]["messages"] assert "freitags entkalkt" not in sent[0]["content"] assert "freitags entkalkt" in sent[-1]["content"] listing = (await client.get("/api/conversations")).json() assert listing[0]["title"] == "Kaffeemaschine entkalkt?" # Citations are snapshotted on the message, so they survive a reload. detail = (await client.get(f"/api/conversations/{conversation_id}")).json() persisted = detail["messages"][1]["sources"] assert [source["title"] for source in persisted] == ["Kaffeemaschine"] assert "freitags entkalkt" in persisted[0]["excerpt"] assert detail["messages"][0]["sources"] == [] # user turn async def test_low_confidence_marks_no_answer( client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI ) -> None: """The knowledge gap is explicit on the wire — the UI turns it into the 'capture this now?' invitation.""" await _indexed_public_doc(db, seeded_user) await _login(client) conversation_id = await _create_conversation(client) response = await client.post( f"/api/conversations/{conversation_id}/messages", json={"content": "Xylophonstimmung Quartalsbericht?"}, ) events = _parse_sse(response.text) phases = [json.loads(data)["phase"] for kind, data in events if kind == "state"] assert phases == ["searching", "no_answer", "answering"] # No passage is USED to ground the answer, but the retrieved-yet-too-weak # passages are still reported (marked unused) so the "?" inspector can # explain why there was no answer. chunks = json.loads(dict(events)["sources"])["chunks"] assert chunks and all(chunk["used"] is False for chunk in chunks) context_turn = fake_llm.requests[-1]["messages"][-1]["content"] # The model still answers (a refusal on every unmatched question makes # the assistant feel broken) — it just may not invent company facts. assert "nothing relevant" in context_turn assert "Answer anyway" in context_turn async def test_turn_logs_contain_no_content( client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI, caplog: pytest.LogCaptureFixture, ) -> None: """Rule 12 for the whole turn: neither the question, the retrieved document text, nor the reply may reach a log line.""" await _indexed_public_doc(db, seeded_user) fake_llm.chat_responses.append({"chunks": ["GEHEIM-ANTWORT-88"]}) await _login(client) conversation_id = await _create_conversation(client) apply_content_log_guard() with caplog.at_level(logging.DEBUG): await client.post( f"/api/conversations/{conversation_id}/messages", json={"content": "Kaffeemaschine entkalkt GEHEIM-FRAGE-77?"}, ) formatter = JsonFormatter() rendered = "\n".join(formatter.format(record) for record in caplog.records) assert "turn finished" in rendered # the turn really was logged assert "GEHEIM-FRAGE-77" not in rendered assert "GEHEIM-ANTWORT-88" not in rendered assert "freitags entkalkt" not in rendered # retrieved content / excerpt async def test_llm_failure_falls_back_to_a_plain_search( client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI ) -> None: """No model, but still a knowledge base: the turn ends as a full-text hit list the user opens themselves. It is persisted like any other reply, so a reload replays it instead of showing an empty assistant turn.""" # Twice: the LLM client retries once on server errors. fake_llm.chat_responses.extend([{"status": 500}, {"status": 500}]) await _login(client) conversation_id = await _create_conversation(client) response = await client.post( f"/api/conversations/{conversation_id}/messages", json={"content": "Hallo?"} ) frames = _parse_sse(response.text) kinds = [kind for kind, _ in frames] assert "fallback" in kinds assert "error" not in kinds assert kinds[-1] == "done" fallback = next(json.loads(data) for kind, data in frames if kind == "fallback") assert fallback["code"] == "llm_failed" stored = ( ( await db.execute( select(Message) .where(Message.conversation_id == conversation_id) .order_by(Message.created_at) ) ) .scalars() .all() ) assert [message.role for message in stored] == [ MessageRole.user, MessageRole.assistant, ] assert stored[-1].content == "" assert stored[-1].meta["fallback"] == "llm_failed" # And it survives the round trip, so the frontend can phrase it on reload. detail = (await client.get(f"/api/conversations/{conversation_id}")).json() assert detail["messages"][-1]["fallback"] == "llm_failed" async def test_a_dead_embedding_endpoint_still_answers( client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI, monkeypatch: pytest.MonkeyPatch, ) -> None: """The three roles are configured separately: with no embedding endpoint, retrieval drops to the full-text index and the chat model still answers.""" from app.llm.errors import LLMError async def _no_endpoint(texts: list[str], *, role: str = "embedding"): raise LLMError( "down", role="embedding", kind="embed", status="error", cause_type="APIConnectionError", ) monkeypatch.setattr("app.rag.retrieval.embed", _no_endpoint) fake_llm.chat_responses.append({"chunks": ["Klar", "doch."]}) await _login(client) conversation_id = await _create_conversation(client) response = await client.post( f"/api/conversations/{conversation_id}/messages", json={"content": "Hallo?"} ) kinds = [kind for kind, _ in _parse_sse(response.text)] assert "fallback" not in kinds assert "error" not in kinds assert kinds[-1] == "done" async def test_delete_conversation_cascades_messages( client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI ) -> None: await _login(client) conversation_id = await _create_conversation(client) await client.post( f"/api/conversations/{conversation_id}/messages", json={"content": "Hi"} ) assert ( await client.delete(f"/api/conversations/{conversation_id}") ).status_code == 204 remaining = (await db.execute(select(func.count(Message.id)))).scalar_one() assert remaining == 0 async def test_empty_message_rejected(client: AsyncClient, seeded_user: User) -> None: await _login(client) conversation_id = await _create_conversation(client) response = await client.post( f"/api/conversations/{conversation_id}/messages", json={"content": ""} ) assert response.status_code == 422 async def test_client_abort_persists_partial( db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI ) -> None: """Stop button: closing the SSE generator mid-stream keeps the tokens that were already delivered.""" conversation = Conversation(mode=ConversationMode.query, user_id=seeded_user.id) db.add(conversation) await db.commit() conversation = ( await db.execute( select(Conversation) .where(Conversation.id == conversation.id) .options(selectinload(Conversation.messages)) ) ).scalar_one() fake_llm.chat_responses.append({"chunks": ["Teil ", "eins ", "und zwei"]}) mode = get_mode("query") assert mode is not None generator = stream_turn(conversation, "Frage?", mode, db) token_frames = 0 async for frame in generator: if frame.startswith("event: token"): token_frames += 1 if token_frames == 2: break await generator.aclose() partial = ( await db.execute( select(Message.content).where(Message.role == MessageRole.assistant) ) ).scalar_one() assert partial == "Teil eins "