Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
co-authored by
Claude Opus 5
Claude Opus 5
parent
68d3a43191
commit
784b76baf7
@@ -0,0 +1,347 @@
|
||||
"""The ONLY code that talks to LLM endpoints.
|
||||
|
||||
Exactly three functions: chat_stream, chat_json, embed. Three model roles
|
||||
(chat / utility / embedding), each base_url + api_key + model from settings.
|
||||
The openai SDK is used purely as a client for OpenAI-compatible endpoints
|
||||
(llama.cpp locally, cloud APIs in production).
|
||||
|
||||
Logging policy: metadata only — prompts and responses are logged ONLY at
|
||||
DEBUG level behind PABLAN_DEBUG_LOG_PROMPTS=true (never in production).
|
||||
LLMError messages are sanitized and never contain content.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from functools import lru_cache
|
||||
from typing import Any, Literal, TypeVar
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from app.config import get_settings
|
||||
from app.llm.errors import LLMError, llm_error
|
||||
from app.llm.gate import slot
|
||||
from app.llm.overrides import env_defaults, get_config
|
||||
from app.metrics import metrics
|
||||
|
||||
logger = logging.getLogger("pablan.llm")
|
||||
|
||||
Role = Literal["chat", "utility", "embedding"]
|
||||
ChatMessage = dict[str, str]
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
# Turn off a reasoning model's hidden thinking. Latency-critical calls (a
|
||||
# refinement fires on a typing pause) want the answer, not the deliberation:
|
||||
# ~1s instead of ~10s with no quality loss on mechanical rewrites. Endpoints
|
||||
# and templates that do not know the parameter ignore it.
|
||||
NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
_RETRY_INSTRUCTION = (
|
||||
"Your previous reply did not match the required JSON schema. "
|
||||
"Reply again with ONLY valid JSON matching the schema — no prose."
|
||||
)
|
||||
|
||||
|
||||
def _http_client_factory() -> httpx.AsyncClient | None:
|
||||
"""Tests override this to inject an ASGI transport."""
|
||||
return None
|
||||
|
||||
|
||||
def role_config(role: Role) -> tuple[str, str, str]:
|
||||
"""Effective endpoint config: the DB row, seeded from `.env` at first
|
||||
start (see app/llm/overrides.py — "bootstrap, then DB").
|
||||
|
||||
The `or env` fallbacks are a safety net, not the model: they cover the
|
||||
window before `load_config()` has run (early startup, tests that never
|
||||
touch the table) and a field an admin blanked. In a bootstrapped
|
||||
instance the stored value always wins.
|
||||
"""
|
||||
stored = get_config(role)
|
||||
env = env_defaults(role)
|
||||
return (
|
||||
stored.base_url or env.base_url or "",
|
||||
stored.api_key or env.api_key or "",
|
||||
stored.model or env.model or "",
|
||||
)
|
||||
|
||||
|
||||
def _build_client(base_url: str, api_key: str) -> AsyncOpenAI:
|
||||
kwargs: dict[str, Any] = {
|
||||
"base_url": base_url,
|
||||
"api_key": api_key,
|
||||
"timeout": get_settings().llm_timeout_seconds,
|
||||
# One SDK retry: llama.cpp closes idle keep-alive connections, and
|
||||
# the first call on a stale connection fails with APIConnectionError.
|
||||
# (SDK-internal retries are not separately metered.)
|
||||
"max_retries": 1,
|
||||
}
|
||||
http_client = _http_client_factory()
|
||||
if http_client is not None:
|
||||
kwargs["http_client"] = http_client
|
||||
return AsyncOpenAI(**kwargs)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _client_for(role: Role) -> AsyncOpenAI:
|
||||
base_url, api_key, _ = role_config(role)
|
||||
return _build_client(base_url, api_key)
|
||||
|
||||
|
||||
def rebuild_clients() -> None:
|
||||
"""Apply changed endpoint config without a restart: the cached clients
|
||||
hold the old base_url and key, so they must go."""
|
||||
_client_for.cache_clear()
|
||||
|
||||
|
||||
async def probe(
|
||||
role: Role,
|
||||
*,
|
||||
base_url: str | None = None,
|
||||
api_key: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> None:
|
||||
"""Smallest possible call against a candidate config, so an admin can
|
||||
test an endpoint before saving it. Raises LLMError on failure."""
|
||||
effective_url, effective_key, effective_model = role_config(role)
|
||||
client = _build_client(base_url or effective_url, api_key or effective_key)
|
||||
target = model or effective_model
|
||||
started = time.monotonic()
|
||||
try:
|
||||
if role == "embedding":
|
||||
await client.embeddings.create(model=target, input=["ping"])
|
||||
else:
|
||||
stream = await client.chat.completions.create(
|
||||
model=target,
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
max_tokens=1,
|
||||
stream=True,
|
||||
)
|
||||
async for _ in stream:
|
||||
break
|
||||
except Exception as exc:
|
||||
raise llm_error("probe", role, exc, started) from None
|
||||
|
||||
|
||||
async def list_models(
|
||||
role: Role,
|
||||
*,
|
||||
base_url: str | None = None,
|
||||
api_key: str | None = None,
|
||||
) -> list[str]:
|
||||
"""Ask an endpoint what it serves (`GET /v1/models`).
|
||||
|
||||
Server-side on purpose: the credentials must never leave the backend,
|
||||
and the browser has no business talking to the model endpoint at all.
|
||||
|
||||
Not every OpenAI-compatible server implements the route, so a failure
|
||||
here is ordinary rather than exceptional — the caller degrades to a
|
||||
free-text model field. Raises LLMError so the caller can distinguish
|
||||
"no such route" from "wrong credentials".
|
||||
"""
|
||||
effective_url, effective_key, _ = role_config(role)
|
||||
client = _build_client(base_url or effective_url, api_key or effective_key)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
page = await client.models.list()
|
||||
except Exception as exc:
|
||||
raise llm_error("list_models", role, exc, started) from None
|
||||
# Ids only, sorted for a stable dropdown. Model ids are configuration,
|
||||
# not content, so they may be returned and logged by count.
|
||||
return sorted({model.id for model in page.data if getattr(model, "id", None)})
|
||||
|
||||
|
||||
def _record(
|
||||
role: Role,
|
||||
kind: str,
|
||||
status: str,
|
||||
started: float,
|
||||
usage: Any = None,
|
||||
**extra_fields: Any,
|
||||
) -> None:
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("llm_calls_total", {"role": role, "kind": kind, "status": status})
|
||||
metrics.observe("llm_call_seconds", duration, {"role": role, "kind": kind})
|
||||
extra: dict[str, Any] = {
|
||||
"event": "llm_call",
|
||||
"role": role,
|
||||
"kind": kind,
|
||||
"status": status,
|
||||
"duration_ms": round(duration * 1000),
|
||||
**extra_fields,
|
||||
}
|
||||
if usage is not None:
|
||||
prompt_tokens = getattr(usage, "prompt_tokens", None)
|
||||
completion_tokens = getattr(usage, "completion_tokens", None)
|
||||
if prompt_tokens:
|
||||
metrics.inc(
|
||||
"llm_tokens_total", {"role": role, "direction": "prompt"}, prompt_tokens
|
||||
)
|
||||
extra["prompt_tokens"] = prompt_tokens
|
||||
if completion_tokens:
|
||||
metrics.inc(
|
||||
"llm_tokens_total",
|
||||
{"role": role, "direction": "completion"},
|
||||
completion_tokens,
|
||||
)
|
||||
extra["completion_tokens"] = completion_tokens
|
||||
logger.info("llm call", extra=extra)
|
||||
|
||||
|
||||
def _debug_log_content(label: str, content: Any) -> None:
|
||||
if get_settings().debug_log_prompts:
|
||||
logger.debug("llm content", extra={"label": label, "content": content})
|
||||
|
||||
|
||||
async def chat_stream(
|
||||
messages: list[ChatMessage],
|
||||
*,
|
||||
role: Role = "chat",
|
||||
temperature: float | None = None,
|
||||
max_tokens: int | None = None,
|
||||
extra_body: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream a chat completion as text deltas.
|
||||
|
||||
`extra_body` is passed through to the endpoint verbatim — used to reach
|
||||
non-standard OpenAI-compatible parameters such as
|
||||
`{"chat_template_kwargs": {"enable_thinking": False}}`, which turns off a
|
||||
reasoning model's hidden thinking for latency-critical calls. Only
|
||||
`delta.content` is ever yielded, so a reasoning channel never leaks into
|
||||
the output regardless.
|
||||
"""
|
||||
base_url, _, model = role_config(role)
|
||||
_debug_log_content("chat_stream.messages", messages)
|
||||
options: dict[str, Any] = {}
|
||||
if temperature is not None:
|
||||
options["temperature"] = temperature
|
||||
if max_tokens is not None:
|
||||
options["max_tokens"] = max_tokens
|
||||
if extra_body is not None:
|
||||
options["extra_body"] = extra_body
|
||||
|
||||
started = time.monotonic()
|
||||
status = "ok"
|
||||
usage = None
|
||||
try:
|
||||
# The slot is held until the last token: a streaming completion
|
||||
# occupies its server slot for its whole life (app/llm/gate.py).
|
||||
async with slot(base_url, role):
|
||||
stream = await _client_for(role).chat.completions.create(
|
||||
model=model,
|
||||
messages=messages, # type: ignore[arg-type]
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
**options,
|
||||
)
|
||||
async for chunk in stream:
|
||||
if chunk.usage is not None:
|
||||
usage = chunk.usage
|
||||
if chunk.choices and chunk.choices[0].delta.content:
|
||||
yield chunk.choices[0].delta.content
|
||||
except GeneratorExit:
|
||||
status = "aborted"
|
||||
raise
|
||||
except LLMError:
|
||||
status = "error"
|
||||
raise
|
||||
except Exception as exc:
|
||||
status = "error"
|
||||
raise llm_error("chat_stream", role, exc, started) from None
|
||||
finally:
|
||||
_record(role, "chat_stream", status, started, usage)
|
||||
|
||||
|
||||
async def chat_json(
|
||||
messages: list[ChatMessage],
|
||||
schema: type[T],
|
||||
*,
|
||||
role: Role = "utility",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int | None = None,
|
||||
extra_body: dict[str, Any] | None = None,
|
||||
) -> T:
|
||||
"""Structured output: response_format JSON schema + validation + one retry.
|
||||
|
||||
`extra_body` is passed through verbatim (e.g.
|
||||
`{"chat_template_kwargs": {"enable_thinking": False}}` to skip a reasoning
|
||||
model's hidden thinking on latency-sensitive utility calls)."""
|
||||
base_url, _, model = role_config(role)
|
||||
response_format = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": schema.__name__,
|
||||
"schema": schema.model_json_schema(),
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
options: dict[str, Any] = {"temperature": temperature}
|
||||
if max_tokens is not None:
|
||||
options["max_tokens"] = max_tokens
|
||||
if extra_body is not None:
|
||||
options["extra_body"] = extra_body
|
||||
|
||||
attempt_messages = list(messages)
|
||||
for attempt in (1, 2):
|
||||
_debug_log_content("chat_json.messages", attempt_messages)
|
||||
started = time.monotonic()
|
||||
usage = None
|
||||
try:
|
||||
async with slot(base_url, role):
|
||||
response = await _client_for(role).chat.completions.create(
|
||||
model=model,
|
||||
messages=attempt_messages, # type: ignore[arg-type]
|
||||
response_format=response_format, # type: ignore[arg-type]
|
||||
**options,
|
||||
)
|
||||
usage = response.usage
|
||||
content = response.choices[0].message.content or ""
|
||||
result = schema.model_validate_json(content)
|
||||
_record(role, "chat_json", "ok", started, usage, attempt=attempt)
|
||||
return result
|
||||
except ValidationError:
|
||||
_record(role, "chat_json", "invalid", started, usage, attempt=attempt)
|
||||
_debug_log_content("chat_json.invalid_response", content)
|
||||
attempt_messages = [
|
||||
*attempt_messages,
|
||||
{"role": "assistant", "content": content},
|
||||
{"role": "user", "content": _RETRY_INSTRUCTION},
|
||||
]
|
||||
except LLMError:
|
||||
_record(role, "chat_json", "error", started, usage, attempt=attempt)
|
||||
raise
|
||||
except Exception as exc:
|
||||
_record(role, "chat_json", "error", started, usage, attempt=attempt)
|
||||
raise llm_error("chat_json", role, exc, started, attempt=attempt) from None
|
||||
|
||||
raise LLMError(
|
||||
f"chat_json failed (role={role}): response did not match schema "
|
||||
f"{schema.__name__} after retry",
|
||||
role=role,
|
||||
kind="chat_json",
|
||||
status="invalid",
|
||||
cause_type="ValidationError",
|
||||
duration_ms=round((time.monotonic() - started) * 1000),
|
||||
attempt=2,
|
||||
)
|
||||
|
||||
|
||||
async def embed(texts: list[str], *, role: Role = "embedding") -> list[list[float]]:
|
||||
"""Embed a batch of texts; order of results matches the input order."""
|
||||
base_url, _, model = role_config(role)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
async with slot(base_url, role):
|
||||
response = await _client_for(role).embeddings.create(
|
||||
model=model, input=texts
|
||||
)
|
||||
except LLMError:
|
||||
_record(role, "embed", "error", started, batch_size=len(texts))
|
||||
raise
|
||||
except Exception as exc:
|
||||
_record(role, "embed", "error", started, batch_size=len(texts))
|
||||
raise llm_error("embed", role, exc, started) from None
|
||||
_record(role, "embed", "ok", started, response.usage, batch_size=len(texts))
|
||||
ordered = sorted(response.data, key=lambda item: item.index)
|
||||
return [item.embedding for item in ordered]
|
||||
@@ -0,0 +1,101 @@
|
||||
"""What it means when an endpoint does not answer.
|
||||
|
||||
Separate from `client.py` because eight modules catch this and none of them
|
||||
talk to an endpoint: routers, modes and the authoring code only need to know
|
||||
what went wrong and how to say it. The client itself stays the one place that
|
||||
CALLS an endpoint.
|
||||
|
||||
Nothing here ever carries content — not the prompt, not the reply, not the
|
||||
original exception's message. A failure is described by class name, status
|
||||
code and duration, which is everything a log line may hold (rule 12).
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
# Exception class names that mean "nothing answered at the other end" versus
|
||||
# "the other end is there but not ready for us". The SDK wraps both, so the
|
||||
# class name is all we have: APITimeoutError subclasses APIConnectionError,
|
||||
# which is why timeouts are matched first.
|
||||
_TIMEOUT_CAUSES = frozenset(
|
||||
{"APITimeoutError", "ReadTimeout", "PoolTimeout", "TimeoutError"}
|
||||
)
|
||||
_CONNECTION_CAUSES = frozenset(
|
||||
{"APIConnectionError", "ConnectError", "ConnectTimeout", "RemoteProtocolError"}
|
||||
)
|
||||
# Server said "come back later" (rate limit, no free slot, model still loading).
|
||||
_BUSY_STATUS = frozenset({408, 429, 503, 504})
|
||||
# Server said "not with these credentials / not this model".
|
||||
_SETUP_STATUS = frozenset({401, 403, 404})
|
||||
|
||||
|
||||
class LLMError(Exception):
|
||||
"""Sanitized LLM failure: structured metadata for debugging —
|
||||
never content, never original exception messages.
|
||||
|
||||
Fields: role, kind, status ("error" | "invalid"), cause_type (original
|
||||
exception CLASS NAME only), status_code (HTTP, if any), duration_ms,
|
||||
attempt (chat_json: 1 or 2).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
role: str,
|
||||
kind: str,
|
||||
status: str,
|
||||
cause_type: str | None = None,
|
||||
status_code: int | None = None,
|
||||
duration_ms: int | None = None,
|
||||
attempt: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.role = role
|
||||
self.kind = kind
|
||||
self.status = status
|
||||
self.cause_type = cause_type
|
||||
self.status_code = status_code
|
||||
self.duration_ms = duration_ms
|
||||
self.attempt = attempt
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
"""The API error code for this failure — the ONE place an endpoint
|
||||
failure is classified, so every caller reports the same reason and the
|
||||
frontend can phrase it (`docs/api-protocol.md`).
|
||||
|
||||
`llm_busy` and `llm_unreachable` are worth telling apart: the first is
|
||||
worth retrying in a moment, the second needs someone to start the
|
||||
endpoint.
|
||||
"""
|
||||
if self.status_code in _BUSY_STATUS:
|
||||
return "llm_busy"
|
||||
if self.status_code in _SETUP_STATUS:
|
||||
return "llm_misconfigured"
|
||||
if self.cause_type in _TIMEOUT_CAUSES:
|
||||
return "llm_busy"
|
||||
if self.cause_type in _CONNECTION_CAUSES:
|
||||
return "llm_unreachable"
|
||||
return "llm_failed"
|
||||
|
||||
|
||||
def llm_error(
|
||||
kind: str,
|
||||
role: str,
|
||||
exc: Exception,
|
||||
started: float,
|
||||
*,
|
||||
attempt: int | None = None,
|
||||
) -> LLMError:
|
||||
"""Wrap whatever the SDK raised, keeping only what may be logged."""
|
||||
status_code = getattr(exc, "status_code", None)
|
||||
return LLMError(
|
||||
f"{kind} failed (role={role}): {type(exc).__name__}",
|
||||
role=role,
|
||||
kind=kind,
|
||||
status="error",
|
||||
cause_type=type(exc).__name__,
|
||||
status_code=status_code if isinstance(status_code, int) else None,
|
||||
duration_ms=round((time.monotonic() - started) * 1000),
|
||||
attempt=attempt,
|
||||
)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""How many requests Pablan lets an endpoint see at once.
|
||||
|
||||
A self-hosted llama.cpp server has a fixed number of parallel slots. Sending
|
||||
more than that does not make it faster: the extra requests sit in the server's
|
||||
own queue where Pablan can neither see nor bound them, and every one of them
|
||||
counts against the HTTP timeout. Two colleagues chatting while a reindex runs
|
||||
is enough to turn a working instance into one where everything times out at
|
||||
once.
|
||||
|
||||
So the waiting happens here instead, in front of the endpoint:
|
||||
|
||||
- **One gate per endpoint, not per role.** chat and utility usually point at
|
||||
the same server (they do in the shipped `.env`), and it is the SERVER that
|
||||
has the slots. Keying by base_url is what makes the limit real.
|
||||
- **A bounded wait.** A caller waits at most `llm_queue_wait_seconds` for a
|
||||
slot and then fails as `llm_busy` — a fast, honest "try again" instead of a
|
||||
two-minute timeout that looks like a broken endpoint.
|
||||
- **A bounded queue.** Past `llm_max_queued` waiters the gate stops admitting:
|
||||
when far more work has arrived than the endpoint can absorb, the useful
|
||||
answer is "busy", given immediately, to everyone beyond the line.
|
||||
|
||||
`llm_busy` is already the vocabulary for this (`llm/errors.py`), and the
|
||||
frontend phrases it as "the model is busy" — so a queue rejection reaches the
|
||||
user as the same, correct sentence as a 429 from a cloud provider.
|
||||
|
||||
Admin diagnostics (`probe`, `list_models`) deliberately do NOT pass through
|
||||
the gate: they are single tiny calls, and an admin has to be able to test an
|
||||
endpoint precisely when it is saturated.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from app.config import get_settings
|
||||
from app.llm.errors import LLMError
|
||||
from app.metrics import metrics
|
||||
|
||||
logger = logging.getLogger("pablan.llm")
|
||||
|
||||
|
||||
class _Endpoint:
|
||||
"""The live picture of one endpoint: who is in it, who is waiting."""
|
||||
|
||||
def __init__(self, limit: int) -> None:
|
||||
self.limit = limit
|
||||
self.semaphore = asyncio.Semaphore(limit)
|
||||
self.in_flight = 0
|
||||
self.waiting = 0
|
||||
|
||||
@property
|
||||
def saturated(self) -> bool:
|
||||
return self.in_flight >= self.limit
|
||||
|
||||
|
||||
_endpoints: dict[str, _Endpoint] = {}
|
||||
|
||||
|
||||
def _busy_error(role: str, reason: str, waited: float) -> LLMError:
|
||||
"""A queue rejection, in the same shape as an endpoint's own 503 — the
|
||||
caller classifies it through `LLMError.code` like any other failure."""
|
||||
metrics.inc("llm_queue_rejected_total", {"role": role, "reason": reason})
|
||||
logger.info(
|
||||
"llm request not admitted",
|
||||
extra={
|
||||
"event": "llm_queue_rejected",
|
||||
"role": role,
|
||||
"reason": reason,
|
||||
"waited_ms": round(waited * 1000),
|
||||
},
|
||||
)
|
||||
return LLMError(
|
||||
f"endpoint busy (role={role}): {reason}",
|
||||
role=role,
|
||||
kind="queue",
|
||||
status="error",
|
||||
# 503 is what a saturated endpoint says itself, and what maps to
|
||||
# `llm_busy`. Keeping the queue's own rejection in that vocabulary
|
||||
# means one reason reaches the user, not two.
|
||||
status_code=503,
|
||||
)
|
||||
|
||||
|
||||
def _endpoint_for(base_url: str) -> _Endpoint:
|
||||
limit = max(1, get_settings().llm_max_parallel)
|
||||
endpoint = _endpoints.get(base_url)
|
||||
if endpoint is None or endpoint.limit != limit:
|
||||
# A changed limit (settings reloaded in a test) rebuilds the gate.
|
||||
# In-flight callers hold the old semaphore and still release it.
|
||||
endpoint = _Endpoint(limit)
|
||||
_endpoints[base_url] = endpoint
|
||||
return endpoint
|
||||
|
||||
|
||||
def endpoint_busy(base_url: str) -> bool:
|
||||
"""Is every slot on this endpoint taken right now?
|
||||
|
||||
Read by the query mode so a waiting turn can SAY it is waiting instead of
|
||||
showing a frozen cursor. Advisory: by the time the caller acquires, a slot
|
||||
may well have freed.
|
||||
"""
|
||||
endpoint = _endpoints.get(base_url)
|
||||
return endpoint is not None and endpoint.saturated
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def slot(base_url: str, role: str) -> AsyncIterator[None]:
|
||||
"""Hold one of the endpoint's slots for the whole call.
|
||||
|
||||
For a stream that means until the last token: a streaming completion
|
||||
occupies its server slot until it ends, and releasing early would let the
|
||||
gate admit work the endpoint has no room for.
|
||||
"""
|
||||
settings = get_settings()
|
||||
endpoint = _endpoint_for(base_url)
|
||||
|
||||
if endpoint.saturated and endpoint.waiting >= max(0, settings.llm_max_queued):
|
||||
raise _busy_error(role, "queue_full", 0.0)
|
||||
|
||||
started = time.monotonic()
|
||||
endpoint.waiting += 1
|
||||
metrics.set_gauge("llm_queue_waiting", float(endpoint.waiting), {"role": role})
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
endpoint.semaphore.acquire(), timeout=settings.llm_queue_wait_seconds
|
||||
)
|
||||
except TimeoutError:
|
||||
raise _busy_error(role, "queue_timeout", time.monotonic() - started) from None
|
||||
finally:
|
||||
endpoint.waiting -= 1
|
||||
metrics.set_gauge("llm_queue_waiting", float(endpoint.waiting), {"role": role})
|
||||
|
||||
waited = time.monotonic() - started
|
||||
if waited > 0.01:
|
||||
metrics.observe("llm_queue_wait_seconds", waited, {"role": role})
|
||||
endpoint.in_flight += 1
|
||||
metrics.set_gauge("llm_in_flight", float(endpoint.in_flight), {"role": role})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
endpoint.in_flight -= 1
|
||||
metrics.set_gauge("llm_in_flight", float(endpoint.in_flight), {"role": role})
|
||||
endpoint.semaphore.release()
|
||||
|
||||
|
||||
def reset() -> None:
|
||||
"""Drop every gate. Tests only — a live gate holds waiters."""
|
||||
_endpoints.clear()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""The LLM endpoint configuration, as the process sees it.
|
||||
|
||||
**Bootstrap, then DB.** On the very first start the `PABLAN_*` environment
|
||||
variables are copied into `llm_settings`, one row per role. From that
|
||||
moment the table is the truth: later `.env` edits are ignored, because a
|
||||
configuration an admin can change in the UI and a configuration the
|
||||
deployment can change underneath them cannot both be authoritative. The
|
||||
environment stays reachable as the value a field can be *reset* to, which
|
||||
is what `env_defaults()` is for.
|
||||
|
||||
The configuration lives in a module-level cache because `_role_config` is a
|
||||
hot, synchronous function on every LLM call — it cannot await a query. The
|
||||
cache is filled at startup and refreshed whenever an admin writes, which is
|
||||
also when the OpenAI clients are rebuilt.
|
||||
|
||||
Single-process by design: the customer stack pins `--workers 1` (see
|
||||
architecture.md), so there is exactly one cache to refresh. A multi-worker
|
||||
deployment would need a notification channel instead.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models import LLMSetting
|
||||
|
||||
logger = logging.getLogger("pablan.llm")
|
||||
|
||||
ROLES = ("chat", "utility", "embedding")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoleConfig:
|
||||
"""One role's stored configuration. A None field means the column is
|
||||
empty, which after bootstrap only happens if an admin blanked it."""
|
||||
|
||||
base_url: str | None = None
|
||||
model: str | None = None
|
||||
api_key: str | None = None
|
||||
|
||||
|
||||
_config: dict[str, RoleConfig] = {}
|
||||
|
||||
|
||||
def env_defaults(role: str) -> RoleConfig:
|
||||
"""What `.env` says for this role — the value "reset to .env" restores.
|
||||
|
||||
Read live rather than remembered from bootstrap: an admin who fixes a
|
||||
typo in `.env` and resets the field should get the corrected value, not
|
||||
the one that was wrong at install time.
|
||||
"""
|
||||
settings = get_settings()
|
||||
base_url, api_key, model = {
|
||||
"chat": (settings.chat_base_url, settings.chat_api_key, settings.chat_model),
|
||||
"utility": (
|
||||
settings.utility_base_url,
|
||||
settings.utility_api_key,
|
||||
settings.utility_model,
|
||||
),
|
||||
"embedding": (
|
||||
settings.embedding_base_url,
|
||||
settings.embedding_api_key,
|
||||
settings.embedding_model,
|
||||
),
|
||||
}[role]
|
||||
return RoleConfig(base_url=base_url, model=model, api_key=api_key)
|
||||
|
||||
|
||||
_FIELDS = ("base_url", "model", "api_key")
|
||||
|
||||
|
||||
async def bootstrap_llm_settings(db: AsyncSession) -> int:
|
||||
"""Copy the environment into any field that still defers to it.
|
||||
|
||||
Runs at every startup, but only ever fills blanks: a field is written
|
||||
exactly when it is flagged `*_from_env` AND currently empty. That is
|
||||
true for a fresh install (no rows yet) and for a field an upgrade
|
||||
marked as still belonging to `.env`, and false for anything an admin
|
||||
has typed, which is never touched.
|
||||
|
||||
Returns the number of fields written.
|
||||
"""
|
||||
rows = {row.role: row for row in (await db.execute(select(LLMSetting))).scalars()}
|
||||
written = 0
|
||||
for role in ROLES:
|
||||
row = rows.get(role)
|
||||
if row is None:
|
||||
# Flags set explicitly rather than left to the column defaults:
|
||||
# those only materialise on flush, and the loop below reads them
|
||||
# before that.
|
||||
row = LLMSetting(
|
||||
role=role,
|
||||
base_url_from_env=True,
|
||||
model_from_env=True,
|
||||
api_key_from_env=True,
|
||||
)
|
||||
db.add(row)
|
||||
defaults = env_defaults(role)
|
||||
for field in _FIELDS:
|
||||
if not getattr(row, f"{field}_from_env") or getattr(row, field):
|
||||
continue
|
||||
setattr(row, field, getattr(defaults, field) or None)
|
||||
written += 1
|
||||
if written:
|
||||
await db.commit()
|
||||
# Counts and roles only — never the values, one of which is a key.
|
||||
# NB: not `created` — logging reserves that name on
|
||||
# LogRecord and raises KeyError when an `extra` key collides with it.
|
||||
logger.info(
|
||||
"llm settings bootstrapped",
|
||||
extra={"event": "llm_bootstrap", "fields_written": written},
|
||||
)
|
||||
return written
|
||||
|
||||
|
||||
async def load_config(db: AsyncSession) -> None:
|
||||
"""Re-read every stored row. Call after any write."""
|
||||
rows = (await db.execute(select(LLMSetting))).scalars().all()
|
||||
_config.clear()
|
||||
_config.update(
|
||||
{
|
||||
row.role: RoleConfig(
|
||||
base_url=row.base_url, model=row.model, api_key=row.api_key
|
||||
)
|
||||
for row in rows
|
||||
}
|
||||
)
|
||||
logger.info(
|
||||
"llm settings loaded",
|
||||
extra={"event": "llm_settings_loaded", "roles": sorted(_config)},
|
||||
)
|
||||
|
||||
|
||||
def get_config(role: str) -> RoleConfig:
|
||||
return _config.get(role, RoleConfig())
|
||||
|
||||
|
||||
def clear() -> None:
|
||||
"""Drop the cache — used by tests between cases."""
|
||||
_config.clear()
|
||||
Reference in New Issue
Block a user