Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/tests/test_llm_client.py

@@ -0,0 +1,182 @@
+import pytest
+from pydantic import BaseModel
+
+from app.llm.client import chat_json, chat_stream, embed
+from app.llm.errors import LLMError
+from app.metrics import metrics
+from tests.fake_openai import FakeOpenAI
+
+PING = [{"role": "user", "content": "hi"}]
+
+
+class Verdict(BaseModel):
+    covered: list[str]
+    done: bool
+
+
+def _counter(name: str, **labels: str) -> float:
+    for entry in metrics.snapshot()["counters"].get(name, []):
+        if entry["labels"] == labels:
+            return entry["value"]
+    return 0.0
+
+
+async def test_chat_stream_yields_deltas_and_records_metrics(
+    fake_llm: FakeOpenAI,
+) -> None:
+    fake_llm.chat_responses.append({"chunks": ["Hel", "lo"]})
+    out = [token async for token in chat_stream(PING)]
+    assert out == ["Hel", "lo"]
+
+    body = fake_llm.requests[-1]
+    assert body["stream"] is True
+    assert body["stream_options"] == {"include_usage": True}
+    assert (
+        _counter("llm_calls_total", role="chat", kind="chat_stream", status="ok") == 1
+    )
+    assert _counter("llm_tokens_total", role="chat", direction="completion") == 3
+
+
+async def test_chat_stream_role_override(fake_llm: FakeOpenAI) -> None:
+    [token async for token in chat_stream(PING, role="utility", max_tokens=1)]
+    assert fake_llm.requests[-1]["max_tokens"] == 1
+    assert (
+        _counter("llm_calls_total", role="utility", kind="chat_stream", status="ok")
+        == 1
+    )
+
+
+async def test_chat_stream_error_is_sanitized(fake_llm: FakeOpenAI) -> None:
+    # Twice: the client retries once on connection/server errors.
+    fake_llm.chat_responses.extend([{"status": 500}, {"status": 500}])
+    with pytest.raises(LLMError) as excinfo:
+        [token async for token in chat_stream(PING)]
+    error = excinfo.value
+    assert "chat_stream failed" in str(error)
+    # The server error body must not leak into the exception message.
+    assert "induced failure" not in str(error)
+    assert error.role == "chat"
+    assert error.kind == "chat_stream"
+    assert error.status == "error"
+    assert error.status_code == 500
+    assert error.cause_type  # original exception class name only
+    assert isinstance(error.duration_ms, int)
+    assert (
+        _counter("llm_calls_total", role="chat", kind="chat_stream", status="error")
+        == 1
+    )
+
+
+async def test_chat_json_happy_path(fake_llm: FakeOpenAI) -> None:
+    fake_llm.chat_responses.append({"content": '{"covered": ["a"], "done": true}'})
+    result = await chat_json(PING, Verdict)
+    assert result == Verdict(covered=["a"], done=True)
+
+    body = fake_llm.requests[-1]
+    assert body["response_format"]["type"] == "json_schema"
+    assert body["response_format"]["json_schema"]["name"] == "Verdict"
+    assert (
+        _counter("llm_calls_total", role="utility", kind="chat_json", status="ok") == 1
+    )
+
+
+async def test_chat_json_retries_once_on_invalid_json(fake_llm: FakeOpenAI) -> None:
+    fake_llm.chat_responses.extend(
+        [
+            {"content": "definitely not json"},
+            {"content": '{"covered": [], "done": false}'},
+        ]
+    )
+    result = await chat_json(PING, Verdict)
+    assert result.done is False
+    assert len(fake_llm.requests) == 2
+    retry_messages = fake_llm.requests[-1]["messages"]
+    assert retry_messages[-1]["role"] == "user"
+    assert "JSON" in retry_messages[-1]["content"]
+    assert retry_messages[-2] == {"role": "assistant", "content": "definitely not json"}
+    assert (
+        _counter("llm_calls_total", role="utility", kind="chat_json", status="invalid")
+        == 1
+    )
+
+
+async def test_chat_json_gives_up_after_retry_without_leaking(
+    fake_llm: FakeOpenAI,
+) -> None:
+    fake_llm.chat_responses.extend(
+        [{"content": "SECRET-A 123"}, {"content": "SECRET-B 456"}]
+    )
+    with pytest.raises(LLMError) as excinfo:
+        await chat_json(PING, Verdict)
+    error = excinfo.value
+    assert "Verdict" in str(error)
+    # Structured debugging metadata is present ...
+    assert error.role == "utility"
+    assert error.kind == "chat_json"
+    assert error.status == "invalid"
+    assert error.cause_type == "ValidationError"
+    assert error.attempt == 2
+    assert error.status_code is None
+    assert isinstance(error.duration_ms, int)
+    # ... and neither message nor ANY metadata field carries content.
+    everything = str(error) + repr(vars(error))
+    assert "SECRET-A" not in everything
+    assert "SECRET-B" not in everything
+
+
+async def test_chat_json_http_error(fake_llm: FakeOpenAI) -> None:
+    # Twice: the client retries once on connection/server errors.
+    fake_llm.chat_responses.extend([{"status": 503}, {"status": 503}])
+    with pytest.raises(LLMError) as excinfo:
+        await chat_json(PING, Verdict)
+    error = excinfo.value
+    assert error.status == "error"
+    assert error.status_code == 503
+    assert error.attempt == 1
+    assert error.cause_type
+    assert "induced failure" not in str(error) + repr(vars(error))
+    assert (
+        _counter("llm_calls_total", role="utility", kind="chat_json", status="error")
+        == 1
+    )
+
+
+async def test_embed_preserves_order(fake_llm: FakeOpenAI) -> None:
+    vectors = await embed(["a", "b", "c"])
+    assert len(vectors) == 3
+    assert vectors[0][0] == 0.0
+    assert vectors[2][0] == 2.0
+    assert len(vectors[0]) == fake_llm.embedding_dim
+    assert _counter("llm_calls_total", role="embedding", kind="embed", status="ok") == 1
+
+
+@pytest.mark.parametrize(
+    ("cause_type", "status_code", "expected"),
+    [
+        ("APIConnectionError", None, "llm_unreachable"),
+        ("ConnectError", None, "llm_unreachable"),
+        # A timeout is the busy case, not the down case: the endpoint took the
+        # request and never came back.
+        ("APITimeoutError", None, "llm_busy"),
+        ("RateLimitError", 429, "llm_busy"),
+        ("APIStatusError", 503, "llm_busy"),
+        ("AuthenticationError", 401, "llm_misconfigured"),
+        ("NotFoundError", 404, "llm_misconfigured"),
+        ("BadRequestError", 400, "llm_failed"),
+        ("ValueError", None, "llm_failed"),
+    ],
+)
+def test_error_code_separates_down_from_busy_from_misconfigured(
+    cause_type: str, status_code: int | None, expected: str
+) -> None:
+    """One classification for every surface: retry-in-a-moment, start the
+    endpoint, and fix the config must not read the same to the user."""
+    error = LLMError(
+        "boom",
+        role="chat",
+        kind="chat_stream",
+        status="error",
+        cause_type=cause_type,
+        status_code=status_code,
+    )
+    assert error.code == expected