ProfessorNovaandClaude Opus 5
784b76baf7
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
84 lines
2.9 KiB
Python
84 lines
2.9 KiB
Python
from httpx import AsyncClient
|
|
|
|
from app.models import User
|
|
from tests.fake_openai import FakeOpenAI
|
|
|
|
|
|
async def _login(client: AsyncClient, email: str) -> None:
|
|
response = await client.post(
|
|
"/api/auth/login", json={"email": email, "password": "secret123"}
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
|
|
async def test_llm_test_requires_authentication(client: AsyncClient) -> None:
|
|
response = await client.post("/api/admin/llm/test")
|
|
assert response.status_code == 401
|
|
|
|
|
|
async def test_llm_test_requires_admin_role(
|
|
client: AsyncClient, seeded_user: User
|
|
) -> None:
|
|
await _login(client, "pablo@test.dev")
|
|
response = await client.post("/api/admin/llm/test")
|
|
assert response.status_code == 403
|
|
assert response.json()["code"] == "forbidden"
|
|
|
|
|
|
async def test_llm_test_reports_all_roles_healthy(
|
|
client: AsyncClient, seeded_admin: User, fake_llm: FakeOpenAI
|
|
) -> None:
|
|
await _login(client, "florian@test.dev")
|
|
response = await client.post("/api/admin/llm/test")
|
|
assert response.status_code == 200
|
|
roles = {entry["role"]: entry for entry in response.json()["roles"]}
|
|
assert set(roles) == {"chat", "utility", "embedding"}
|
|
for entry in roles.values():
|
|
assert entry["ok"] is True
|
|
assert entry["error"] is None
|
|
assert isinstance(entry["latency_ms"], int)
|
|
assert entry["model"]
|
|
assert entry["base_url"]
|
|
|
|
|
|
async def test_llm_test_reports_broken_roles(
|
|
client: AsyncClient, seeded_admin: User, fake_llm: FakeOpenAI
|
|
) -> None:
|
|
# Both chat-completion roles fail; embeddings stay healthy. Four
|
|
# errors: two roles x one SDK retry each.
|
|
fake_llm.chat_responses.extend([{"status": 500}] * 4)
|
|
await _login(client, "florian@test.dev")
|
|
response = await client.post("/api/admin/llm/test")
|
|
roles = {entry["role"]: entry for entry in response.json()["roles"]}
|
|
assert roles["chat"]["ok"] is False
|
|
assert "chat_stream failed" in roles["chat"]["error"]
|
|
assert "induced failure" not in roles["chat"]["error"]
|
|
assert roles["utility"]["ok"] is False
|
|
assert roles["embedding"]["ok"] is True
|
|
|
|
|
|
async def test_metrics_endpoint_reflects_llm_calls(
|
|
client: AsyncClient, seeded_admin: User, fake_llm: FakeOpenAI
|
|
) -> None:
|
|
await _login(client, "florian@test.dev")
|
|
await client.post("/api/admin/llm/test")
|
|
|
|
response = await client.get("/api/admin/metrics")
|
|
assert response.status_code == 200
|
|
snapshot = response.json()
|
|
counted_roles = {
|
|
entry["labels"]["role"]
|
|
for entry in snapshot["counters"]["llm_calls_total"]
|
|
if entry["labels"]["status"] == "ok"
|
|
}
|
|
assert counted_roles == {"chat", "utility", "embedding"}
|
|
assert "llm_call_seconds" in snapshot["histograms"]
|
|
|
|
|
|
async def test_metrics_endpoint_requires_admin(
|
|
client: AsyncClient, seeded_user: User
|
|
) -> None:
|
|
await _login(client, "pablo@test.dev")
|
|
response = await client.get("/api/admin/metrics")
|
|
assert response.status_code == 403
|