Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
@@ -0,0 +1,368 @@
|
||||
"""Conversations API + query mode, end to end against fakes.
|
||||
|
||||
fake_llm scripts the chat completions, fake_embed the vectors — the SSE
|
||||
contract, persistence rules and permission scoping are what is under test.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.api.conversations import stream_turn
|
||||
from app.log import JsonFormatter, apply_content_log_guard
|
||||
from app.models import (
|
||||
Conversation,
|
||||
ConversationMode,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
Message,
|
||||
MessageRole,
|
||||
User,
|
||||
)
|
||||
from app.modes import get_mode
|
||||
from app.rag.indexing import reindex_document
|
||||
from tests.fake_openai import FakeOpenAI
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("fake_llm", "fake_embed")
|
||||
|
||||
|
||||
async def _login(client: AsyncClient, email: str = "pablo@test.dev") -> None:
|
||||
response = await client.post(
|
||||
"/api/auth/login", json={"email": email, "password": "secret123"}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
async def _indexed_public_doc(db: AsyncSession, author: User) -> Document:
|
||||
document = Document(
|
||||
title="Kaffeemaschine",
|
||||
status=DocumentStatus.published,
|
||||
visibility=DocumentVisibility.public,
|
||||
content_md="## Pflege\n\nDie Kaffeemaschine wird freitags entkalkt.",
|
||||
author_id=author.id,
|
||||
department_id=author.department_id,
|
||||
)
|
||||
db.add(document)
|
||||
await db.flush()
|
||||
await reindex_document(db, document)
|
||||
await db.commit()
|
||||
return document
|
||||
|
||||
|
||||
async def _create_conversation(client: AsyncClient) -> str:
|
||||
response = await client.post("/api/conversations", json={"mode": "query"})
|
||||
assert response.status_code == 200
|
||||
return response.json()["id"]
|
||||
|
||||
|
||||
def _parse_sse(text: str) -> list[tuple[str, str]]:
|
||||
events = []
|
||||
for frame in text.split("\n\n"):
|
||||
if not frame.strip():
|
||||
continue
|
||||
lines = dict(line.split(": ", 1) for line in frame.splitlines() if ": " in line)
|
||||
events.append((lines["event"], lines["data"]))
|
||||
return events
|
||||
|
||||
|
||||
async def test_create_and_list(client: AsyncClient, seeded_user: User) -> None:
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
|
||||
listing = (await client.get("/api/conversations")).json()
|
||||
assert [c["id"] for c in listing] == [conversation_id]
|
||||
assert listing[0]["title"] is None # no messages yet
|
||||
|
||||
|
||||
async def test_unregistered_mode_rejected(
|
||||
client: AsyncClient, seeded_user: User
|
||||
) -> None:
|
||||
await _login(client)
|
||||
# insight stays unregistered until the EE module provides it.
|
||||
response = await client.post("/api/conversations", json={"mode": "insight"})
|
||||
assert response.status_code == 400
|
||||
assert response.json()["code"] == "unknown_mode"
|
||||
|
||||
|
||||
async def test_conversations_are_owner_scoped(
|
||||
client: AsyncClient, seeded_user: User, seeded_admin: User
|
||||
) -> None:
|
||||
await _login(client, "florian@test.dev")
|
||||
foreign_id = await _create_conversation(client)
|
||||
await client.post("/api/auth/logout")
|
||||
|
||||
await _login(client)
|
||||
assert (await client.get("/api/conversations")).json() == []
|
||||
assert (await client.get(f"/api/conversations/{foreign_id}")).status_code == 404
|
||||
assert (await client.delete(f"/api/conversations/{foreign_id}")).status_code == 404
|
||||
|
||||
|
||||
async def test_turn_streams_sources_tokens_done_and_persists(
|
||||
client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI
|
||||
) -> None:
|
||||
await _indexed_public_doc(db, seeded_user)
|
||||
fake_llm.chat_responses.append({"chunks": ["Frei", "tags."]})
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
|
||||
# All content words exist in the document — websearch_to_tsquery ANDs
|
||||
# terms, and fake embeddings carry no semantics (only FTS can match).
|
||||
response = await client.post(
|
||||
f"/api/conversations/{conversation_id}/messages",
|
||||
json={"content": "Kaffeemaschine entkalkt?"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"].startswith("text/event-stream")
|
||||
|
||||
events = _parse_sse(response.text)
|
||||
kinds = [kind for kind, _ in events]
|
||||
# searching → results → sources → answering → tokens → done
|
||||
assert kinds[:4] == ["state", "state", "sources", "state"]
|
||||
assert kinds.count("token") == 2
|
||||
assert kinds[-1] == "done"
|
||||
|
||||
states = [json.loads(data) for kind, data in events if kind == "state"]
|
||||
assert [s["phase"] for s in states] == ["searching", "results", "answering"]
|
||||
assert states[1]["count"] == 1
|
||||
# Progress events carry counts only — never the query or any content.
|
||||
for state in states:
|
||||
assert "Kaffeemaschine" not in json.dumps(state)
|
||||
|
||||
sources = json.loads(dict(events)["sources"])["chunks"]
|
||||
assert sources[0]["title"] == "Kaffeemaschine"
|
||||
assert "freitags entkalkt" in sources[0]["excerpt"]
|
||||
|
||||
messages = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Message)
|
||||
.where(Message.conversation_id == conversation_id)
|
||||
.order_by(Message.created_at)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assert [m.role for m in messages] == [MessageRole.user, MessageRole.assistant]
|
||||
assert messages[1].content == "Freitags."
|
||||
assert str(messages[1].id) in events[-1][1]
|
||||
|
||||
# Cache-friendly structure: this turn's excerpt rides the final user turn,
|
||||
# while the static system prompt stays byte-identical for prompt caching.
|
||||
sent = fake_llm.requests[-1]["messages"]
|
||||
assert "freitags entkalkt" not in sent[0]["content"]
|
||||
assert "freitags entkalkt" in sent[-1]["content"]
|
||||
|
||||
listing = (await client.get("/api/conversations")).json()
|
||||
assert listing[0]["title"] == "Kaffeemaschine entkalkt?"
|
||||
|
||||
# Citations are snapshotted on the message, so they survive a reload.
|
||||
detail = (await client.get(f"/api/conversations/{conversation_id}")).json()
|
||||
persisted = detail["messages"][1]["sources"]
|
||||
assert [source["title"] for source in persisted] == ["Kaffeemaschine"]
|
||||
assert "freitags entkalkt" in persisted[0]["excerpt"]
|
||||
assert detail["messages"][0]["sources"] == [] # user turn
|
||||
|
||||
|
||||
async def test_low_confidence_marks_no_answer(
|
||||
client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI
|
||||
) -> None:
|
||||
"""The knowledge gap is explicit on the wire — the UI turns it into the
|
||||
'capture this now?' invitation."""
|
||||
await _indexed_public_doc(db, seeded_user)
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
|
||||
response = await client.post(
|
||||
f"/api/conversations/{conversation_id}/messages",
|
||||
json={"content": "Xylophonstimmung Quartalsbericht?"},
|
||||
)
|
||||
events = _parse_sse(response.text)
|
||||
phases = [json.loads(data)["phase"] for kind, data in events if kind == "state"]
|
||||
assert phases == ["searching", "no_answer", "answering"]
|
||||
# No passage is USED to ground the answer, but the retrieved-yet-too-weak
|
||||
# passages are still reported (marked unused) so the "?" inspector can
|
||||
# explain why there was no answer.
|
||||
chunks = json.loads(dict(events)["sources"])["chunks"]
|
||||
assert chunks and all(chunk["used"] is False for chunk in chunks)
|
||||
context_turn = fake_llm.requests[-1]["messages"][-1]["content"]
|
||||
# The model still answers (a refusal on every unmatched question makes
|
||||
# the assistant feel broken) — it just may not invent company facts.
|
||||
assert "nothing relevant" in context_turn
|
||||
assert "Answer anyway" in context_turn
|
||||
|
||||
|
||||
async def test_turn_logs_contain_no_content(
|
||||
client: AsyncClient,
|
||||
db: AsyncSession,
|
||||
seeded_user: User,
|
||||
fake_llm: FakeOpenAI,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""Rule 12 for the whole turn: neither the question, the retrieved
|
||||
document text, nor the reply may reach a log line."""
|
||||
await _indexed_public_doc(db, seeded_user)
|
||||
fake_llm.chat_responses.append({"chunks": ["GEHEIM-ANTWORT-88"]})
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
|
||||
apply_content_log_guard()
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
await client.post(
|
||||
f"/api/conversations/{conversation_id}/messages",
|
||||
json={"content": "Kaffeemaschine entkalkt GEHEIM-FRAGE-77?"},
|
||||
)
|
||||
|
||||
formatter = JsonFormatter()
|
||||
rendered = "\n".join(formatter.format(record) for record in caplog.records)
|
||||
assert "turn finished" in rendered # the turn really was logged
|
||||
assert "GEHEIM-FRAGE-77" not in rendered
|
||||
assert "GEHEIM-ANTWORT-88" not in rendered
|
||||
assert "freitags entkalkt" not in rendered # retrieved content / excerpt
|
||||
|
||||
|
||||
async def test_llm_failure_falls_back_to_a_plain_search(
|
||||
client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI
|
||||
) -> None:
|
||||
"""No model, but still a knowledge base: the turn ends as a full-text hit
|
||||
list the user opens themselves. It is persisted like any other reply, so a
|
||||
reload replays it instead of showing an empty assistant turn."""
|
||||
# Twice: the LLM client retries once on server errors.
|
||||
fake_llm.chat_responses.extend([{"status": 500}, {"status": 500}])
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
|
||||
response = await client.post(
|
||||
f"/api/conversations/{conversation_id}/messages", json={"content": "Hallo?"}
|
||||
)
|
||||
frames = _parse_sse(response.text)
|
||||
kinds = [kind for kind, _ in frames]
|
||||
assert "fallback" in kinds
|
||||
assert "error" not in kinds
|
||||
assert kinds[-1] == "done"
|
||||
fallback = next(json.loads(data) for kind, data in frames if kind == "fallback")
|
||||
assert fallback["code"] == "llm_failed"
|
||||
|
||||
stored = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Message)
|
||||
.where(Message.conversation_id == conversation_id)
|
||||
.order_by(Message.created_at)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assert [message.role for message in stored] == [
|
||||
MessageRole.user,
|
||||
MessageRole.assistant,
|
||||
]
|
||||
assert stored[-1].content == ""
|
||||
assert stored[-1].meta["fallback"] == "llm_failed"
|
||||
|
||||
# And it survives the round trip, so the frontend can phrase it on reload.
|
||||
detail = (await client.get(f"/api/conversations/{conversation_id}")).json()
|
||||
assert detail["messages"][-1]["fallback"] == "llm_failed"
|
||||
|
||||
|
||||
async def test_a_dead_embedding_endpoint_still_answers(
|
||||
client: AsyncClient,
|
||||
db: AsyncSession,
|
||||
seeded_user: User,
|
||||
fake_llm: FakeOpenAI,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The three roles are configured separately: with no embedding endpoint,
|
||||
retrieval drops to the full-text index and the chat model still answers."""
|
||||
from app.llm.errors import LLMError
|
||||
|
||||
async def _no_endpoint(texts: list[str], *, role: str = "embedding"):
|
||||
raise LLMError(
|
||||
"down",
|
||||
role="embedding",
|
||||
kind="embed",
|
||||
status="error",
|
||||
cause_type="APIConnectionError",
|
||||
)
|
||||
|
||||
monkeypatch.setattr("app.rag.retrieval.embed", _no_endpoint)
|
||||
fake_llm.chat_responses.append({"chunks": ["Klar", "doch."]})
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
|
||||
response = await client.post(
|
||||
f"/api/conversations/{conversation_id}/messages", json={"content": "Hallo?"}
|
||||
)
|
||||
kinds = [kind for kind, _ in _parse_sse(response.text)]
|
||||
assert "fallback" not in kinds
|
||||
assert "error" not in kinds
|
||||
assert kinds[-1] == "done"
|
||||
|
||||
|
||||
async def test_delete_conversation_cascades_messages(
|
||||
client: AsyncClient, db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI
|
||||
) -> None:
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
await client.post(
|
||||
f"/api/conversations/{conversation_id}/messages", json={"content": "Hi"}
|
||||
)
|
||||
|
||||
assert (
|
||||
await client.delete(f"/api/conversations/{conversation_id}")
|
||||
).status_code == 204
|
||||
remaining = (await db.execute(select(func.count(Message.id)))).scalar_one()
|
||||
assert remaining == 0
|
||||
|
||||
|
||||
async def test_empty_message_rejected(client: AsyncClient, seeded_user: User) -> None:
|
||||
await _login(client)
|
||||
conversation_id = await _create_conversation(client)
|
||||
response = await client.post(
|
||||
f"/api/conversations/{conversation_id}/messages", json={"content": ""}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
async def test_client_abort_persists_partial(
|
||||
db: AsyncSession, seeded_user: User, fake_llm: FakeOpenAI
|
||||
) -> None:
|
||||
"""Stop button: closing the SSE generator mid-stream keeps the tokens
|
||||
that were already delivered."""
|
||||
conversation = Conversation(mode=ConversationMode.query, user_id=seeded_user.id)
|
||||
db.add(conversation)
|
||||
await db.commit()
|
||||
conversation = (
|
||||
await db.execute(
|
||||
select(Conversation)
|
||||
.where(Conversation.id == conversation.id)
|
||||
.options(selectinload(Conversation.messages))
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
fake_llm.chat_responses.append({"chunks": ["Teil ", "eins ", "und zwei"]})
|
||||
mode = get_mode("query")
|
||||
assert mode is not None
|
||||
generator = stream_turn(conversation, "Frage?", mode, db)
|
||||
|
||||
token_frames = 0
|
||||
async for frame in generator:
|
||||
if frame.startswith("event: token"):
|
||||
token_frames += 1
|
||||
if token_frames == 2:
|
||||
break
|
||||
await generator.aclose()
|
||||
|
||||
partial = (
|
||||
await db.execute(
|
||||
select(Message.content).where(Message.role == MessageRole.assistant)
|
||||
)
|
||||
).scalar_one()
|
||||
assert partial == "Teil eins "
|
||||
Reference in New Issue
Block a user