Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
@@ -0,0 +1,29 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api import (
|
||||
account,
|
||||
admin,
|
||||
auth,
|
||||
authoring,
|
||||
conversations,
|
||||
departments,
|
||||
documents,
|
||||
people,
|
||||
templates,
|
||||
)
|
||||
|
||||
api_router = APIRouter(prefix="/api")
|
||||
api_router.include_router(auth.router)
|
||||
api_router.include_router(account.router)
|
||||
api_router.include_router(admin.router)
|
||||
api_router.include_router(conversations.router)
|
||||
api_router.include_router(departments.router)
|
||||
api_router.include_router(documents.router)
|
||||
api_router.include_router(authoring.router)
|
||||
api_router.include_router(people.router)
|
||||
api_router.include_router(templates.router)
|
||||
|
||||
|
||||
@api_router.get("/health")
|
||||
async def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Self-service account actions.
|
||||
|
||||
Separate from `auth.py` (login/logout/me) and from `admin.py`: this is what
|
||||
a user may change about themselves.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.deps import get_current_auth_session, get_current_user
|
||||
from app.auth.passwords import hash_password, verify_password
|
||||
from app.auth.sessions import revoke_user_sessions
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import AuthSession, Document, DocumentStatus, Template, User
|
||||
|
||||
router = APIRouter(prefix="/account", tags=["account"])
|
||||
logger = logging.getLogger("pablan.account")
|
||||
|
||||
|
||||
class LocalePreference(BaseModel):
|
||||
"""The languages the interface ships in — the frontend bundles must
|
||||
cover exactly these."""
|
||||
|
||||
# null = follow the browser's Accept-Language again.
|
||||
locale: Literal["de", "en"] | None = None
|
||||
|
||||
|
||||
class PasswordChange(BaseModel):
|
||||
current_password: str = Field(min_length=1, max_length=200)
|
||||
new_password: str = Field(min_length=8, max_length=200)
|
||||
|
||||
|
||||
@router.post("/password", status_code=204)
|
||||
async def change_password(
|
||||
body: PasswordChange,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
session: Annotated[AuthSession, Depends(get_current_auth_session)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> None:
|
||||
"""Change your own password, proving the current one first.
|
||||
|
||||
Every other session of this user is revoked — a
|
||||
password change is how someone reacts to a suspected compromise, so
|
||||
other devices must lose access. The session doing the change survives,
|
||||
otherwise the user is thrown out of the app they are standing in.
|
||||
"""
|
||||
if not verify_password(user.password_hash, body.current_password):
|
||||
raise ApiError(
|
||||
403, "Current password is incorrect.", "invalid_current_password"
|
||||
)
|
||||
|
||||
user.password_hash = hash_password(body.new_password)
|
||||
await revoke_user_sessions(db, user.id, keep_session_id=session.id)
|
||||
await db.commit()
|
||||
# Metadata only — never the password, not even its length.
|
||||
logger.info("password changed", extra={"event": "password_change"})
|
||||
|
||||
|
||||
@router.put("/locale", status_code=204)
|
||||
async def set_locale(
|
||||
body: LocalePreference,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> None:
|
||||
"""Pin the interface language, or clear it to follow the browser again.
|
||||
|
||||
The backend only stores the choice — it never renders UI-language
|
||||
strings (see docs/architecture.md); the frontend does the translating.
|
||||
"""
|
||||
user.locale = body.locale
|
||||
await db.commit()
|
||||
|
||||
|
||||
# The starter catalog's blueprint about a PERSON (role, specialities, who to
|
||||
# ask) rather than a topic. What "the document about you" is made from, named
|
||||
# once here so the frontend does not have to know a blueprint id.
|
||||
PERSONAL_BLUEPRINT = "person"
|
||||
|
||||
|
||||
class PersonalDocument(BaseModel):
|
||||
"""The caller's own document about themselves.
|
||||
|
||||
Either they wrote one — then it is opened and edited like any other
|
||||
document — or they have not, and `template_id` says what to start it from.
|
||||
Both are null when the blueprint is not in this instance and nothing was
|
||||
written yet; the frontend falls back to the ordinary template picker.
|
||||
"""
|
||||
|
||||
document_id: uuid.UUID | None = None
|
||||
title: str | None = None
|
||||
status: DocumentStatus | None = None
|
||||
template_id: uuid.UUID | None = None
|
||||
|
||||
|
||||
@router.get("/document")
|
||||
async def personal_document(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> PersonalDocument:
|
||||
"""What the profile page needs to show: your document about yourself, or
|
||||
the way to start it. Self-scoped, and authorship is the whole rule — a
|
||||
document someone else wrote about you is not this."""
|
||||
document = (
|
||||
await db.execute(
|
||||
select(Document)
|
||||
.where(
|
||||
Document.author_id == user.id,
|
||||
Document.meta["template"].astext == PERSONAL_BLUEPRINT,
|
||||
)
|
||||
# The newest, if a second one was ever started.
|
||||
.order_by(Document.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
template_id = (
|
||||
await db.execute(
|
||||
select(Template.id).where(
|
||||
Template.config["id"].astext == PERSONAL_BLUEPRINT
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if document is None:
|
||||
return PersonalDocument(template_id=template_id)
|
||||
return PersonalDocument(
|
||||
document_id=document.id,
|
||||
title=document.title,
|
||||
status=document.status,
|
||||
template_id=template_id,
|
||||
)
|
||||
@@ -0,0 +1,18 @@
|
||||
"""The admin API: everything only an administrator may do.
|
||||
|
||||
Split by what is being administered — model endpoints, prompts, users,
|
||||
departments, and the metrics snapshot. The admin gate is not repeated per
|
||||
endpoint: it sits on the shared router in `routing.py`, so a new route in any
|
||||
of these modules is admin-only whether or not its author thought about it.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.admin import departments, llm, observability, prompts, users
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(llm.router)
|
||||
router.include_router(prompts.router)
|
||||
router.include_router(users.router)
|
||||
router.include_router(departments.router)
|
||||
router.include_router(observability.router)
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Departments.
|
||||
|
||||
Deliberately unpaged: an SME has a handful of them. The interesting rule is
|
||||
deletion — a department's read grants CASCADE away with it, and that access
|
||||
loss is invisible, so it has to be confirmed rather than discovered later.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.admin.routing import admin_router
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import Department, DocPermission, Document, User
|
||||
|
||||
router = admin_router()
|
||||
|
||||
NAME_TAKEN = ("Department name already exists.", "name_taken")
|
||||
|
||||
|
||||
class DepartmentCreate(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=200)
|
||||
|
||||
|
||||
class AdminDepartmentOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
|
||||
|
||||
@router.post("/departments")
|
||||
async def create_department(
|
||||
body: DepartmentCreate,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> AdminDepartmentOut:
|
||||
department = Department(name=body.name.strip())
|
||||
db.add(department)
|
||||
try:
|
||||
await db.commit()
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
raise ApiError(409, *NAME_TAKEN) from None
|
||||
return AdminDepartmentOut.model_validate(department)
|
||||
|
||||
|
||||
@router.patch("/departments/{department_id}")
|
||||
async def rename_department(
|
||||
department_id: uuid.UUID,
|
||||
body: DepartmentCreate,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> AdminDepartmentOut:
|
||||
department = await db.get(Department, department_id)
|
||||
if department is None:
|
||||
raise ApiError(404, "Department not found.", "not_found")
|
||||
department.name = body.name.strip()
|
||||
try:
|
||||
await db.commit()
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
raise ApiError(409, *NAME_TAKEN) from None
|
||||
return AdminDepartmentOut.model_validate(department)
|
||||
|
||||
|
||||
async def _still_in_use(db: AsyncSession, department_id: uuid.UUID) -> bool:
|
||||
"""Members, owned documents or read grants — anything whose access changes
|
||||
when this department disappears."""
|
||||
for count in (
|
||||
select(func.count(User.id)).where(User.department_id == department_id),
|
||||
select(func.count(Document.id)).where(Document.department_id == department_id),
|
||||
select(func.count())
|
||||
.select_from(DocPermission)
|
||||
.where(DocPermission.department_id == department_id),
|
||||
):
|
||||
if (await db.execute(count)).scalar_one():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@router.delete("/departments/{department_id}", status_code=204)
|
||||
async def delete_department(
|
||||
department_id: uuid.UUID,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
confirm: bool = False,
|
||||
) -> None:
|
||||
"""Delete a department. Members and owned documents survive with their
|
||||
`department_id` set to NULL, but this department's `doc_permissions` grants
|
||||
CASCADE away — silently dropping the shared read access they gave. Because
|
||||
that access loss is invisible, deleting a department that still has members,
|
||||
owned documents or grants requires `?confirm=true` (409 `department_in_use`
|
||||
otherwise)."""
|
||||
department = await db.get(Department, department_id)
|
||||
if department is None:
|
||||
raise ApiError(404, "Department not found.", "not_found")
|
||||
if not confirm and await _still_in_use(db, department_id):
|
||||
raise ApiError(
|
||||
409,
|
||||
"This department is still in use; deleting it drops that access.",
|
||||
"department_in_use",
|
||||
)
|
||||
await db.delete(department)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Model endpoints: test them, configure them, ask what they serve.
|
||||
|
||||
Configuration is bootstrapped from `.env` at first start and lives in the DB
|
||||
afterwards, so this is where an admin changes an endpoint without a restart.
|
||||
Every call runs server-side: the API key must never reach the browser, and the
|
||||
browser must never reach the model endpoint.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.admin.routing import admin_router
|
||||
from app.db import get_db
|
||||
from app.llm.client import (
|
||||
Role,
|
||||
chat_stream,
|
||||
embed,
|
||||
list_models,
|
||||
probe,
|
||||
rebuild_clients,
|
||||
role_config,
|
||||
)
|
||||
from app.llm.errors import LLMError
|
||||
from app.llm.overrides import env_defaults, load_config
|
||||
from app.models import LLMSetting
|
||||
|
||||
router = admin_router()
|
||||
|
||||
ROLES: tuple[Role, ...] = ("chat", "utility", "embedding")
|
||||
|
||||
|
||||
class LLMSettingUpdate(BaseModel):
|
||||
# None means "leave as is"; the reset_* flags restore the value the
|
||||
# environment currently states (same sentinel pattern as
|
||||
# UserUpdate.clear_department).
|
||||
base_url: str | None = Field(default=None, max_length=500)
|
||||
model: str | None = Field(default=None, max_length=200)
|
||||
api_key: str | None = Field(default=None, max_length=500)
|
||||
reset_base_url: bool | None = None
|
||||
reset_model: bool | None = None
|
||||
reset_api_key: bool | None = None
|
||||
|
||||
|
||||
class LLMRoleStatus(BaseModel):
|
||||
role: Role
|
||||
ok: bool
|
||||
base_url: str
|
||||
model: str
|
||||
latency_ms: int | None
|
||||
# Why it failed: `code` is phrased by the frontend, `error` is the
|
||||
# sanitized technical detail (exception class + role) for the admin.
|
||||
code: str | None
|
||||
error: str | None
|
||||
|
||||
|
||||
class LLMTestResponse(BaseModel):
|
||||
roles: list[LLMRoleStatus]
|
||||
|
||||
|
||||
class LLMTestRequest(BaseModel):
|
||||
"""Optional candidate config: test an endpoint BEFORE saving it."""
|
||||
|
||||
role: Role | None = None
|
||||
base_url: str | None = None
|
||||
model: str | None = None
|
||||
api_key: str | None = None
|
||||
|
||||
|
||||
async def _ping_role(
|
||||
role: Role, candidate: LLMSettingUpdate | None = None
|
||||
) -> LLMRoleStatus:
|
||||
"""Ping a role — either its effective config, or a candidate an admin is
|
||||
about to save."""
|
||||
base_url, _, model = role_config(role)
|
||||
if candidate is not None:
|
||||
base_url = candidate.base_url or base_url
|
||||
model = candidate.model or model
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
if candidate is None:
|
||||
if role == "embedding":
|
||||
await embed(["ping"])
|
||||
else:
|
||||
# Drain the (max_tokens=1) stream so the call records as
|
||||
# "ok", not "aborted".
|
||||
async for _ in chat_stream(
|
||||
[{"role": "user", "content": "ping"}], role=role, max_tokens=1
|
||||
):
|
||||
pass
|
||||
else:
|
||||
await probe(
|
||||
role,
|
||||
base_url=candidate.base_url,
|
||||
api_key=candidate.api_key,
|
||||
model=candidate.model,
|
||||
)
|
||||
return LLMRoleStatus(
|
||||
role=role,
|
||||
ok=True,
|
||||
base_url=base_url,
|
||||
model=model,
|
||||
latency_ms=round((time.monotonic() - started) * 1000),
|
||||
code=None,
|
||||
error=None,
|
||||
)
|
||||
except LLMError as exc:
|
||||
return LLMRoleStatus(
|
||||
role=role,
|
||||
ok=False,
|
||||
base_url=base_url,
|
||||
model=model,
|
||||
latency_ms=None,
|
||||
code=exc.code,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/llm/test")
|
||||
async def llm_test(body: LLMTestRequest | None = None) -> LLMTestResponse:
|
||||
"""First-line support tool: pings all three roles, or one candidate
|
||||
configuration without persisting anything."""
|
||||
if body is not None and body.role is not None:
|
||||
candidate = LLMSettingUpdate(
|
||||
base_url=body.base_url, model=body.model, api_key=body.api_key
|
||||
)
|
||||
return LLMTestResponse(roles=[await _ping_role(body.role, candidate)])
|
||||
results = await asyncio.gather(*(_ping_role(role) for role in ROLES))
|
||||
return LLMTestResponse(roles=list(results))
|
||||
|
||||
|
||||
class LLMSettingOut(BaseModel):
|
||||
"""Stored config for one role.
|
||||
|
||||
The api_key is NEVER returned — only whether one is set, and where each
|
||||
field's value came from. `*_from_env` drives the per-field
|
||||
"taken from .env" / "changed here" label and the reset action; it is
|
||||
provenance, not a fallback.
|
||||
"""
|
||||
|
||||
role: Role
|
||||
base_url: str
|
||||
model: str
|
||||
base_url_from_env: bool
|
||||
model_from_env: bool
|
||||
api_key_set: bool
|
||||
api_key_from_env: bool
|
||||
|
||||
|
||||
async def _row_for(db: AsyncSession, role: Role) -> LLMSetting:
|
||||
row = (
|
||||
await db.execute(select(LLMSetting).where(LLMSetting.role == role))
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
# Only reachable if bootstrap never ran (a test, or a role added
|
||||
# after install). Seed it from the environment, same as bootstrap.
|
||||
defaults = env_defaults(role)
|
||||
row = LLMSetting(
|
||||
role=role,
|
||||
base_url=defaults.base_url or None,
|
||||
model=defaults.model or None,
|
||||
api_key=defaults.api_key or None,
|
||||
)
|
||||
db.add(row)
|
||||
await db.flush()
|
||||
return row
|
||||
|
||||
|
||||
def _setting_out(row: LLMSetting) -> LLMSettingOut:
|
||||
return LLMSettingOut(
|
||||
role=row.role, # type: ignore[arg-type]
|
||||
base_url=row.base_url or "",
|
||||
model=row.model or "",
|
||||
base_url_from_env=row.base_url_from_env,
|
||||
model_from_env=row.model_from_env,
|
||||
api_key_set=bool(row.api_key),
|
||||
api_key_from_env=row.api_key_from_env,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/llm/settings")
|
||||
async def llm_settings(
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[LLMSettingOut]:
|
||||
return [_setting_out(await _row_for(db, role)) for role in ROLES]
|
||||
|
||||
|
||||
@router.put("/llm/settings/{role}")
|
||||
async def update_llm_setting(
|
||||
role: Role,
|
||||
body: LLMSettingUpdate,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> LLMSettingOut:
|
||||
"""Change a role's stored config and apply it without a restart.
|
||||
|
||||
Writing a field marks it as changed here; resetting it writes back what
|
||||
`.env` currently says and marks it as coming from the environment
|
||||
again.
|
||||
"""
|
||||
row = await _row_for(db, role)
|
||||
defaults = env_defaults(role)
|
||||
|
||||
if body.reset_base_url:
|
||||
row.base_url, row.base_url_from_env = defaults.base_url or None, True
|
||||
elif body.base_url is not None:
|
||||
row.base_url, row.base_url_from_env = body.base_url or None, False
|
||||
if body.reset_model:
|
||||
row.model, row.model_from_env = defaults.model or None, True
|
||||
elif body.model is not None:
|
||||
row.model, row.model_from_env = body.model or None, False
|
||||
if body.reset_api_key:
|
||||
row.api_key, row.api_key_from_env = defaults.api_key or None, True
|
||||
elif body.api_key:
|
||||
row.api_key, row.api_key_from_env = body.api_key, False
|
||||
|
||||
await db.commit()
|
||||
await load_config(db)
|
||||
# The cached OpenAI clients hold the old base_url and key.
|
||||
rebuild_clients()
|
||||
return _setting_out(row)
|
||||
|
||||
|
||||
class LLMModelsRequest(BaseModel):
|
||||
"""Optional candidate endpoint, so an admin can list the models of a
|
||||
URL they have typed but not saved."""
|
||||
|
||||
base_url: str | None = Field(default=None, max_length=500)
|
||||
api_key: str | None = Field(default=None, max_length=500)
|
||||
|
||||
|
||||
class LLMModelsResponse(BaseModel):
|
||||
models: list[str]
|
||||
# False when the endpoint does not implement GET /v1/models — the UI
|
||||
# keeps its free-text field instead of showing an error.
|
||||
supported: bool
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@router.post("/llm/models/{role}")
|
||||
async def llm_models(
|
||||
role: Role, body: LLMModelsRequest | None = None
|
||||
) -> LLMModelsResponse:
|
||||
"""List what the endpoint serves, so the model field can be a dropdown."""
|
||||
try:
|
||||
models = await list_models(
|
||||
role,
|
||||
base_url=(body.base_url if body else None) or None,
|
||||
api_key=(body.api_key if body else None) or None,
|
||||
)
|
||||
except LLMError as exc:
|
||||
# A 404 means "this server has no /v1/models", which is common
|
||||
# enough to be a normal outcome rather than a failure to report.
|
||||
supported = exc.status_code != 404
|
||||
return LLMModelsResponse(
|
||||
models=[],
|
||||
supported=supported,
|
||||
error=exc.cause_type if supported else None,
|
||||
)
|
||||
return LLMModelsResponse(models=models, supported=True)
|
||||
@@ -0,0 +1,19 @@
|
||||
"""What the running process has counted so far."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.api.admin.routing import admin_router
|
||||
from app.metrics import metrics
|
||||
|
||||
router = admin_router()
|
||||
|
||||
|
||||
@router.get("/metrics")
|
||||
async def metrics_snapshot() -> dict[str, Any]:
|
||||
"""The in-process metrics registry as JSON.
|
||||
|
||||
Per process by design (the app runs one worker), and admin-only: the
|
||||
counters name models and durations, which is operational detail rather
|
||||
than something to expose publicly.
|
||||
"""
|
||||
return metrics.snapshot()
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Editing the shipped system prompts.
|
||||
|
||||
Every prompt has a code default; a row exists only where an admin changed one,
|
||||
and resetting deletes that row rather than storing a copy of the default. The
|
||||
UI labels the keys from its own messages, so nothing user-facing is worded here.
|
||||
"""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.admin.routing import admin_router
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import PromptSetting
|
||||
from app.prompts.defaults import DEFAULTS, PROMPT_KEYS
|
||||
from app.prompts.overrides import get_prompt, is_overridden
|
||||
from app.prompts.overrides import load_config as load_prompt_config
|
||||
|
||||
router = admin_router()
|
||||
|
||||
|
||||
class PromptSettingOut(BaseModel):
|
||||
key: str
|
||||
# The effective text: an admin override if present, else the code default.
|
||||
content: str
|
||||
# True when no override exists, i.e. the shipped default is in force.
|
||||
is_default: bool
|
||||
|
||||
|
||||
class PromptSettingUpdate(BaseModel):
|
||||
# New override text, or `reset` to drop the override back to the default.
|
||||
content: str | None = None
|
||||
reset: bool | None = None
|
||||
|
||||
|
||||
@router.get("/prompts")
|
||||
async def prompt_settings(
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[PromptSettingOut]:
|
||||
"""Every editable system prompt with its effective text and whether it is
|
||||
still the shipped default."""
|
||||
rows = {row.key: row for row in (await db.execute(select(PromptSetting))).scalars()}
|
||||
return [
|
||||
PromptSettingOut(
|
||||
key=key,
|
||||
content=rows[key].content if key in rows else DEFAULTS[key],
|
||||
is_default=key not in rows,
|
||||
)
|
||||
for key in PROMPT_KEYS
|
||||
]
|
||||
|
||||
|
||||
@router.put("/prompts/{key}")
|
||||
async def update_prompt_setting(
|
||||
key: str,
|
||||
body: PromptSettingUpdate,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> PromptSettingOut:
|
||||
"""Override a system prompt (applied without a restart) or reset it to the
|
||||
shipped default. Resetting deletes the override row."""
|
||||
if key not in DEFAULTS:
|
||||
raise ApiError(404, "Unknown prompt.", "not_found")
|
||||
row = (
|
||||
await db.execute(select(PromptSetting).where(PromptSetting.key == key))
|
||||
).scalar_one_or_none()
|
||||
|
||||
if body.reset:
|
||||
if row is not None:
|
||||
await db.delete(row)
|
||||
elif body.content is not None:
|
||||
content = body.content.strip()
|
||||
if not content:
|
||||
raise ApiError(422, "A prompt cannot be empty.", "empty_prompt")
|
||||
if row is None:
|
||||
db.add(PromptSetting(key=key, content=content))
|
||||
else:
|
||||
row.content = content
|
||||
|
||||
await db.commit()
|
||||
await load_prompt_config(db)
|
||||
return PromptSettingOut(
|
||||
key=key, content=get_prompt(key), is_default=not is_overridden(key)
|
||||
)
|
||||
@@ -0,0 +1,16 @@
|
||||
"""The one router constructor the admin modules share.
|
||||
|
||||
The admin gate lives here rather than on each endpoint: a route added to any
|
||||
of these modules is admin-only by construction, and there is no way to forget
|
||||
the dependency.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.auth.deps import require_admin
|
||||
|
||||
|
||||
def admin_router() -> APIRouter:
|
||||
return APIRouter(
|
||||
prefix="/admin", tags=["admin"], dependencies=[Depends(require_admin)]
|
||||
)
|
||||
@@ -0,0 +1,196 @@
|
||||
"""User accounts.
|
||||
|
||||
An admin creates people, corrects their details, and offboards them. The two
|
||||
guardrails here exist because an admin who locks themselves out has no second
|
||||
admin to call: you cannot change your own role, and you cannot delete yourself.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.admin.routing import admin_router
|
||||
from app.auth.deps import get_current_user
|
||||
from app.auth.passwords import hash_password
|
||||
from app.auth.sessions import revoke_user_sessions
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import Department, User, UserRole
|
||||
|
||||
router = admin_router()
|
||||
|
||||
EMAIL_TAKEN = ("Email address already in use.", "email_taken")
|
||||
|
||||
|
||||
class AdminUserOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: uuid.UUID
|
||||
email: str
|
||||
name: str
|
||||
role: UserRole
|
||||
department_id: uuid.UUID | None
|
||||
|
||||
|
||||
class AdminUserPage(BaseModel):
|
||||
items: list[AdminUserOut]
|
||||
total: int
|
||||
per_page: int
|
||||
|
||||
|
||||
class UserCreate(BaseModel):
|
||||
email: str = Field(min_length=3, max_length=320)
|
||||
name: str = Field(min_length=1, max_length=200)
|
||||
role: UserRole = UserRole.member
|
||||
department_id: uuid.UUID | None = None
|
||||
password: str = Field(min_length=8, max_length=200)
|
||||
|
||||
|
||||
class UserUpdate(BaseModel):
|
||||
name: str | None = Field(default=None, min_length=1, max_length=200)
|
||||
# Correcting a typo in an address should not mean deleting the person
|
||||
# and losing everything hanging off their id. Plain `str` like
|
||||
# UserCreate: `EmailStr` would pull in email-validator, a dependency we
|
||||
# deliberately avoid, and the format is already guarded by the browser's
|
||||
# type="email" and the column's unique constraint.
|
||||
email: str | None = Field(default=None, min_length=3, max_length=320)
|
||||
role: UserRole | None = None
|
||||
department_id: uuid.UUID | None = None
|
||||
clear_department: bool | None = None
|
||||
# Setting a password IS the reset mechanism.
|
||||
password: str | None = Field(default=None, min_length=8, max_length=200)
|
||||
|
||||
|
||||
def _normalize_email(email: str) -> str:
|
||||
"""The same normalisation on create and update, or an edited address would
|
||||
stop matching what login looks up."""
|
||||
return email.strip().lower()
|
||||
|
||||
|
||||
async def _department_must_exist(
|
||||
db: AsyncSession, department_id: uuid.UUID | None
|
||||
) -> None:
|
||||
if department_id is not None and await db.get(Department, department_id) is None:
|
||||
raise ApiError(404, "Department not found.", "not_found")
|
||||
|
||||
|
||||
@router.get("/users")
|
||||
async def list_users(
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
search: str | None = Query(None, max_length=200),
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(25, ge=1, le=100),
|
||||
) -> AdminUserPage:
|
||||
"""Paged, because the admin screen is the one place that scales with
|
||||
headcount: a company with two hundred employees would otherwise get two
|
||||
hundred rows and no way to find anyone.
|
||||
|
||||
Departments deliberately stay unpaged: an SME has a handful, and a
|
||||
pager over five rows is furniture.
|
||||
"""
|
||||
filters = []
|
||||
if search:
|
||||
needle = f"%{search.strip()}%"
|
||||
filters.append(or_(User.email.ilike(needle), User.name.ilike(needle)))
|
||||
|
||||
total = (await db.execute(select(func.count(User.id)).where(*filters))).scalar_one()
|
||||
rows = (
|
||||
(
|
||||
await db.execute(
|
||||
select(User)
|
||||
.where(*filters)
|
||||
.order_by(User.email)
|
||||
.offset((page - 1) * per_page)
|
||||
.limit(per_page)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return AdminUserPage(
|
||||
items=[AdminUserOut.model_validate(row) for row in rows],
|
||||
total=total,
|
||||
per_page=per_page,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/users")
|
||||
async def create_user(
|
||||
body: UserCreate,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> AdminUserOut:
|
||||
await _department_must_exist(db, body.department_id)
|
||||
user = User(
|
||||
email=_normalize_email(body.email),
|
||||
name=body.name,
|
||||
role=body.role,
|
||||
department_id=body.department_id,
|
||||
password_hash=hash_password(body.password),
|
||||
)
|
||||
db.add(user)
|
||||
try:
|
||||
await db.commit()
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
raise ApiError(409, *EMAIL_TAKEN) from None
|
||||
return AdminUserOut.model_validate(user)
|
||||
|
||||
|
||||
@router.patch("/users/{user_id}")
|
||||
async def update_user(
|
||||
user_id: uuid.UUID,
|
||||
body: UserUpdate,
|
||||
admin: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> AdminUserOut:
|
||||
user = await db.get(User, user_id)
|
||||
if user is None:
|
||||
raise ApiError(404, "User not found.", "not_found")
|
||||
if user.id == admin.id and body.role is not None and body.role != user.role:
|
||||
raise ApiError(409, "You cannot change your own role.", "self_modification")
|
||||
|
||||
if body.name is not None:
|
||||
user.name = body.name
|
||||
if body.email is not None:
|
||||
user.email = _normalize_email(body.email)
|
||||
if body.role is not None:
|
||||
user.role = body.role
|
||||
if body.clear_department:
|
||||
user.department_id = None
|
||||
elif body.department_id is not None:
|
||||
await _department_must_exist(db, body.department_id)
|
||||
user.department_id = body.department_id
|
||||
if body.password is not None:
|
||||
user.password_hash = hash_password(body.password)
|
||||
# A password change revokes every session of that user — including
|
||||
# the current one if an admin changes their own password.
|
||||
await revoke_user_sessions(db, user.id)
|
||||
|
||||
try:
|
||||
await db.commit()
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
raise ApiError(409, *EMAIL_TAKEN) from None
|
||||
return AdminUserOut.model_validate(user)
|
||||
|
||||
|
||||
@router.delete("/users/{user_id}", status_code=204)
|
||||
async def delete_user(
|
||||
user_id: uuid.UUID,
|
||||
admin: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> None:
|
||||
"""Offboarding: sessions and conversations cascade, documents survive
|
||||
with author set to NULL."""
|
||||
if user_id == admin.id:
|
||||
raise ApiError(409, "You cannot delete your own account.", "self_modification")
|
||||
user = await db.get(User, user_id)
|
||||
if user is None:
|
||||
raise ApiError(404, "User not found.", "not_found")
|
||||
await db.delete(user)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,87 @@
|
||||
import uuid
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Request, Response
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.deps import get_current_user
|
||||
from app.auth.passwords import burn_verification_time, verify_password
|
||||
from app.auth.sessions import (
|
||||
COOKIE_NAME,
|
||||
clear_session_cookie,
|
||||
create_auth_session,
|
||||
set_session_cookie,
|
||||
)
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import AuthSession, User, UserRole
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
email: str
|
||||
password: str
|
||||
|
||||
|
||||
class UserOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: uuid.UUID
|
||||
email: str
|
||||
name: str
|
||||
role: UserRole
|
||||
department_id: uuid.UUID | None
|
||||
# Pinned interface language, or null to follow the browser. Flows to the
|
||||
# UI via /me, so no page needs its own preference fetch. Typed as the
|
||||
# closed set the API accepts, so the generated client is precise too.
|
||||
locale: Literal["de", "en"] | None = None
|
||||
|
||||
|
||||
@router.post("/login")
|
||||
async def login(
|
||||
body: LoginRequest,
|
||||
response: Response,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> UserOut:
|
||||
email = body.email.strip().lower()
|
||||
user = (
|
||||
await db.execute(select(User).where(User.email == email))
|
||||
).scalar_one_or_none()
|
||||
if user is None:
|
||||
burn_verification_time()
|
||||
raise ApiError(401, "Invalid email or password.", "invalid_credentials")
|
||||
if not verify_password(user.password_hash, body.password):
|
||||
raise ApiError(401, "Invalid email or password.", "invalid_credentials")
|
||||
|
||||
session = await create_auth_session(db, user)
|
||||
await db.commit()
|
||||
set_session_cookie(response, session)
|
||||
return UserOut.model_validate(user)
|
||||
|
||||
|
||||
@router.post("/logout", status_code=204)
|
||||
async def logout(
|
||||
request: Request,
|
||||
response: Response,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> None:
|
||||
raw = request.cookies.get(COOKIE_NAME)
|
||||
if raw is not None:
|
||||
try:
|
||||
session_id = uuid.UUID(raw)
|
||||
except ValueError:
|
||||
session_id = None
|
||||
if session_id is not None:
|
||||
session = await db.get(AuthSession, session_id)
|
||||
if session is not None:
|
||||
await db.delete(session)
|
||||
await db.commit()
|
||||
clear_session_cookie(response)
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def me(user: Annotated[User, Depends(get_current_user)]) -> UserOut:
|
||||
return UserOut.model_validate(user)
|
||||
@@ -0,0 +1,18 @@
|
||||
"""The authoring API: the model's help while someone writes.
|
||||
|
||||
Three endpoints under `/documents`, all owner-scoped: refine the section at the
|
||||
cursor (streamed), suggest a title for a finished draft, and suggest which
|
||||
existing document a capture should extend. What they share is `grounding` — the
|
||||
permission-filtered look at what the company already wrote.
|
||||
|
||||
`suggest-similar` has a static path and is registered before the refine module
|
||||
so it cannot be read as a document id.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.authoring import refine, suggest
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(suggest.router)
|
||||
router.include_router(refine.router)
|
||||
@@ -0,0 +1,74 @@
|
||||
"""What the company already wrote about this.
|
||||
|
||||
Before refining a section, the server looks for related published material the
|
||||
author may read and hands it to the prompt as a reference — so a suggestion
|
||||
stays consistent with the rest of the knowledge base instead of inventing a
|
||||
parallel version of it. The same chunks are surfaced to the editor's "?"
|
||||
inspector, so the author can see where a suggestion drew from.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.llm.errors import LLMError
|
||||
from app.models import User
|
||||
from app.rag.chunking import HEADING_RE
|
||||
from app.rag.similarity import (
|
||||
CAPTURE_CONTEXT_MAX_DISTANCE,
|
||||
SimilarChunk,
|
||||
similar_chunks,
|
||||
)
|
||||
|
||||
# A section needs at least this much of its own text (beyond the heading)
|
||||
# before it is worth searching the knowledge base to ground the refinement:
|
||||
# a bare heading matches nothing useful and only adds noise.
|
||||
MIN_CHARS = 40
|
||||
TOP_K = 3
|
||||
# Cap each grounding excerpt so a few long ones cannot crowd out the section.
|
||||
EXCERPT_CHARS = 600
|
||||
|
||||
|
||||
def leading_heading(section_text: str) -> str | None:
|
||||
"""The heading text a section starts with, to match a template hint."""
|
||||
first = section_text.lstrip().splitlines()[0] if section_text.strip() else ""
|
||||
match = HEADING_RE.match(first)
|
||||
return match.group(2).strip() if match else None
|
||||
|
||||
|
||||
def reference(title: str, heading_path: str, content: str) -> str:
|
||||
"""One retrieved chunk, rendered for the prompt: where it comes from, then
|
||||
a bounded excerpt."""
|
||||
excerpt = content.strip()
|
||||
if len(excerpt) > EXCERPT_CHARS:
|
||||
excerpt = excerpt[:EXCERPT_CHARS].rstrip() + " ..."
|
||||
where = f'"{title}" ({heading_path})' if heading_path else f'"{title}"'
|
||||
return f"From {where}:\n{excerpt}"
|
||||
|
||||
|
||||
async def for_section(
|
||||
db: AsyncSession, section_text: str, user: User, document_id: uuid.UUID
|
||||
) -> list[SimilarChunk]:
|
||||
"""Related knowledge for one section, permission-filtered by construction.
|
||||
|
||||
Returns nothing when the section is still too thin to match on, when the
|
||||
current document is the only match, or when the embedding endpoint is
|
||||
unavailable — the refinement then simply proceeds without grounding.
|
||||
"""
|
||||
query = section_text.strip()
|
||||
_, _, after_heading = query.partition("\n")
|
||||
body = after_heading.strip() if leading_heading(query) is not None else query
|
||||
if len(body) < MIN_CHARS:
|
||||
return []
|
||||
try:
|
||||
return await similar_chunks(
|
||||
db,
|
||||
query,
|
||||
user=user,
|
||||
top_k=TOP_K,
|
||||
max_distance=CAPTURE_CONTEXT_MAX_DISTANCE,
|
||||
exclude_builtin=True,
|
||||
exclude_document_id=document_id,
|
||||
)
|
||||
except LLMError:
|
||||
return []
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Section refinement for the writing editor.
|
||||
|
||||
The user writes Markdown; after a pause the client asks the model to refine the
|
||||
section the cursor is in. The whole document is context, but the model
|
||||
regenerates ONLY that section (FIM-style), streamed back as SSE so the
|
||||
suggestion appears progressively and can be aborted the moment the user resumes
|
||||
typing.
|
||||
|
||||
The request body and the streamed response carry document text. That is fine on
|
||||
this owner-scoped endpoint — the same trust boundary as
|
||||
`GET /api/documents/{id}` — but nothing here logs content: metadata only.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.authoring import grounding
|
||||
from app.api.authoring.routing import authoring_router
|
||||
from app.api.documents import readable_document, require_editor
|
||||
from app.api.sse import sse
|
||||
from app.auth.deps import get_current_user
|
||||
from app.authoring.prompts import render_refine_prompt
|
||||
from app.authoring.schema import AuthoringTemplate
|
||||
from app.authoring.sections import ActiveSection, active_section, slice_lines
|
||||
from app.db import get_db
|
||||
from app.llm.client import NO_THINKING, chat_stream
|
||||
from app.llm.errors import LLMError
|
||||
from app.models import Template, User
|
||||
|
||||
router = authoring_router()
|
||||
logger = logging.getLogger("pablan.authoring")
|
||||
|
||||
|
||||
class RefineRequest(BaseModel):
|
||||
content_md: str = Field(max_length=100_000)
|
||||
cursor_line: int = Field(ge=1)
|
||||
|
||||
|
||||
async def _template_for(
|
||||
db: AsyncSession, template_config_id: str | None
|
||||
) -> AuthoringTemplate | None:
|
||||
"""The blueprint a document was started from, if it still exists and still
|
||||
parses — it carries the persona, the temperature and the per-section hints
|
||||
that shape a refinement."""
|
||||
if not template_config_id:
|
||||
return None
|
||||
row = (
|
||||
await db.execute(
|
||||
select(Template).where(Template.config["id"].astext == template_config_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
try:
|
||||
return AuthoringTemplate.model_validate(row.config)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/{document_id}/refine")
|
||||
async def refine_section(
|
||||
document_id: uuid.UUID,
|
||||
body: RefineRequest,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> StreamingResponse:
|
||||
"""Stream a matured version of the section at the cursor.
|
||||
|
||||
SSE frames: one `section` frame with the exact line range the suggestion
|
||||
replaces, then `token` frames, then `done` (or `error`)."""
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_editor(document, user)
|
||||
|
||||
section = active_section(body.content_md, body.cursor_line)
|
||||
prefix, section_text, suffix = slice_lines(
|
||||
body.content_md, section.start_line, section.end_line
|
||||
)
|
||||
meta = document.meta or {}
|
||||
|
||||
persona: str | None = None
|
||||
hint: str | None = None
|
||||
temperature = 0.4
|
||||
template = await _template_for(db, meta.get("template"))
|
||||
if template is not None:
|
||||
persona = template.persona
|
||||
temperature = template.model.temperature
|
||||
heading = grounding.leading_heading(section_text)
|
||||
if heading:
|
||||
hint = template.hint_for(heading)
|
||||
|
||||
# Related, already-published knowledge the author may read. Rendered into
|
||||
# the prompt as grounding, AND surfaced to the editor's "?" inspector so the
|
||||
# author can see where a suggestion drew from.
|
||||
chunks = await grounding.for_section(db, section_text, user, document.id)
|
||||
messages = render_refine_prompt(
|
||||
section_text,
|
||||
prefix=prefix,
|
||||
suffix=suffix,
|
||||
persona=persona,
|
||||
hint=hint,
|
||||
# Background from the chat this capture came from, if any. Like the
|
||||
# document text, it travels only on this owner-scoped call and is
|
||||
# never logged.
|
||||
context=meta.get("context"),
|
||||
knowledge=[
|
||||
grounding.reference(chunk.title, chunk.heading_path, chunk.content)
|
||||
for chunk in chunks
|
||||
],
|
||||
)
|
||||
references = [
|
||||
{"title": chunk.title, "heading_path": chunk.heading_path} for chunk in chunks
|
||||
]
|
||||
|
||||
return StreamingResponse(
|
||||
_stream_refine(messages, section, temperature, str(document.id), references),
|
||||
media_type="text/event-stream",
|
||||
headers={"cache-control": "no-cache", "x-accel-buffering": "no"},
|
||||
)
|
||||
|
||||
|
||||
async def _stream_refine(
|
||||
messages: list[dict[str, str]],
|
||||
section: ActiveSection,
|
||||
temperature: float,
|
||||
document_id: str,
|
||||
references: list[dict[str, str]],
|
||||
) -> AsyncIterator[str]:
|
||||
started = time.monotonic()
|
||||
outcome = "ok"
|
||||
token_events = 0
|
||||
# First: which lines "Accept" will overwrite, so the client can bind the
|
||||
# suggestion to an exact range even as the model streams.
|
||||
yield sse(
|
||||
"section", {"start_line": section.start_line, "end_line": section.end_line}
|
||||
)
|
||||
# What the suggestion is grounding on (the author's own readable material) —
|
||||
# titles + heading paths only, for the "?" inspector. Content-safe.
|
||||
if references:
|
||||
yield sse("grounding", {"references": references})
|
||||
try:
|
||||
async for token in chat_stream(
|
||||
messages, role="chat", temperature=temperature, extra_body=NO_THINKING
|
||||
):
|
||||
token_events += 1
|
||||
yield sse("token", {"text": token})
|
||||
yield sse("done", {})
|
||||
except LLMError as exc:
|
||||
outcome = "error"
|
||||
yield sse("error", {"code": exc.code})
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
outcome = "aborted"
|
||||
raise
|
||||
finally:
|
||||
logger.info(
|
||||
"refine finished",
|
||||
extra={
|
||||
"event": "refine",
|
||||
"outcome": outcome,
|
||||
"duration_ms": round((time.monotonic() - started) * 1000),
|
||||
"token_events": token_events,
|
||||
"document_id": document_id,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,12 @@
|
||||
"""The one router constructor the authoring modules share.
|
||||
|
||||
The prefix is `/documents`: authoring acts ON a document the caller owns, so
|
||||
its endpoints live under the document they belong to rather than in a
|
||||
namespace of their own.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
|
||||
def authoring_router() -> APIRouter:
|
||||
return APIRouter(prefix="/documents", tags=["authoring"])
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Two small suggestions the editor asks for: a title, and what to extend.
|
||||
|
||||
Both are one-shot calls rather than streams, and both are advisory: the author
|
||||
keeps the generic title or starts a new document if the suggestion does not fit.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.api.authoring.routing import authoring_router
|
||||
from app.api.documents import readable_document, require_editor
|
||||
from app.auth.deps import get_current_user
|
||||
from app.authoring.context import summarize_conversation
|
||||
from app.authoring.prompts import render_title_prompt
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.llm.client import NO_THINKING, chat_json
|
||||
from app.llm.errors import LLMError
|
||||
from app.models import Conversation, User
|
||||
from app.rag.similarity import CAPTURE_CONTEXT_MAX_DISTANCE, similar_documents
|
||||
|
||||
router = authoring_router()
|
||||
|
||||
|
||||
class TitleSuggestion(BaseModel):
|
||||
title: str
|
||||
|
||||
|
||||
@router.post("/{document_id}/suggest-title")
|
||||
async def suggest_title(
|
||||
document_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> TitleSuggestion:
|
||||
"""Suggest a concise title from the document's content (review step for a
|
||||
new document). Owner-scoped; content in, title out, nothing logged."""
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_editor(document, user)
|
||||
if not document.content_md.strip():
|
||||
return TitleSuggestion(title=document.title)
|
||||
try:
|
||||
return await chat_json(
|
||||
render_title_prompt(document.content_md),
|
||||
TitleSuggestion,
|
||||
extra_body=NO_THINKING,
|
||||
)
|
||||
except LLMError as exc:
|
||||
raise ApiError(503, "The model endpoint did not answer.", exc.code) from None
|
||||
|
||||
|
||||
class SuggestSimilarRequest(BaseModel):
|
||||
conversation_id: uuid.UUID
|
||||
|
||||
|
||||
class SimilarDocumentOut(BaseModel):
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
|
||||
|
||||
@router.post("/suggest-similar")
|
||||
async def suggest_similar(
|
||||
body: SuggestSimilarRequest,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[SimilarDocumentOut]:
|
||||
"""Existing documents that match the conversation a capture is starting
|
||||
from — found over an LLM TOPIC SUMMARY of the chat (not the raw last
|
||||
message), permission-filtered, help pages excluded. Empty list when the
|
||||
conversation is unknown, empty, or nothing is close enough."""
|
||||
conversation = (
|
||||
await db.execute(
|
||||
select(Conversation)
|
||||
.where(
|
||||
Conversation.id == body.conversation_id,
|
||||
Conversation.user_id == user.id,
|
||||
)
|
||||
.options(selectinload(Conversation.messages))
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if conversation is None:
|
||||
return []
|
||||
topic = await summarize_conversation(conversation)
|
||||
if not topic:
|
||||
return []
|
||||
matches = await similar_documents(
|
||||
db,
|
||||
topic,
|
||||
user=user,
|
||||
max_distance=CAPTURE_CONTEXT_MAX_DISTANCE,
|
||||
exclude_builtin=True,
|
||||
)
|
||||
return [
|
||||
SimilarDocumentOut(document_id=match.document_id, title=match.title)
|
||||
for match in matches
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
"""The conversations API: the chat itself.
|
||||
|
||||
Two halves. `crud` manages conversations as objects a user owns; `turns` is the
|
||||
one place that speaks SSE, turning a mode's events into frames and persisting
|
||||
what was streamed. The ownership gate sits in `access`, the wire shapes in
|
||||
`schemas`, and the row-to-shape mapping in `view`.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.conversations import crud, turns
|
||||
from app.api.conversations.turns import stream_turn
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(crud.router)
|
||||
router.include_router(turns.router)
|
||||
|
||||
# Exported for the tests that drive a turn without going through HTTP.
|
||||
__all__ = ["router", "stream_turn"]
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Whose conversation this is.
|
||||
|
||||
A conversation is private to the user who started it — there is no sharing and
|
||||
no admin view. One gate, used by every endpoint in the package, so the rule
|
||||
cannot quietly differ between reading and writing.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.errors import ApiError
|
||||
from app.models import Conversation, User
|
||||
|
||||
|
||||
async def own_conversation(
|
||||
db: AsyncSession,
|
||||
conversation_id: uuid.UUID,
|
||||
user: User,
|
||||
*,
|
||||
with_messages: bool = False,
|
||||
) -> Conversation:
|
||||
stmt = select(Conversation).where(
|
||||
Conversation.id == conversation_id, Conversation.user_id == user.id
|
||||
)
|
||||
if with_messages:
|
||||
stmt = stmt.options(selectinload(Conversation.messages))
|
||||
conversation = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if conversation is None:
|
||||
# 404 rather than 403: someone else's conversation must not be
|
||||
# confirmed to exist.
|
||||
raise ApiError(404, "Conversation not found.", "not_found")
|
||||
return conversation
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Starting, listing, reading and deleting conversations.
|
||||
|
||||
Everything here is owner-scoped. Deleting is a GDPR surface, not a convenience:
|
||||
a user must be able to remove their own transcripts, and the messages go with
|
||||
them by cascade.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.conversations.access import own_conversation
|
||||
from app.api.conversations.routing import conversations_router
|
||||
from app.api.conversations.schemas import (
|
||||
ConversationCreate,
|
||||
ConversationDetail,
|
||||
ConversationSummary,
|
||||
)
|
||||
from app.api.conversations.view import message_out, title
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import Conversation, Message, User
|
||||
from app.modes import get_mode
|
||||
|
||||
router = conversations_router()
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def create_conversation(
|
||||
body: ConversationCreate,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> ConversationSummary:
|
||||
if get_mode(body.mode.value) is None:
|
||||
raise ApiError(
|
||||
400, f"Mode '{body.mode.value}' is not available.", "unknown_mode"
|
||||
)
|
||||
conversation = Conversation(mode=body.mode, user_id=user.id)
|
||||
db.add(conversation)
|
||||
await db.commit()
|
||||
return ConversationSummary.model_validate(conversation)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_conversations(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[ConversationSummary]:
|
||||
"""The sidebar list, newest activity first. The title is the first message,
|
||||
fetched as a correlated subquery so one statement answers the whole list."""
|
||||
first_message = (
|
||||
select(Message.content)
|
||||
.where(Message.conversation_id == Conversation.id)
|
||||
.order_by(Message.created_at)
|
||||
.limit(1)
|
||||
.correlate(Conversation)
|
||||
.scalar_subquery()
|
||||
)
|
||||
rows = await db.execute(
|
||||
select(Conversation, first_message)
|
||||
.where(Conversation.user_id == user.id)
|
||||
.order_by(Conversation.updated_at.desc())
|
||||
)
|
||||
return [
|
||||
ConversationSummary.model_validate(conversation).model_copy(
|
||||
update={"title": title(first)}
|
||||
)
|
||||
for conversation, first in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/{conversation_id}")
|
||||
async def get_conversation(
|
||||
conversation_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> ConversationDetail:
|
||||
conversation = await own_conversation(db, conversation_id, user, with_messages=True)
|
||||
first = conversation.messages[0].content if conversation.messages else None
|
||||
return ConversationDetail.model_validate(conversation).model_copy(
|
||||
update={
|
||||
"title": title(first),
|
||||
"messages": [message_out(message) for message in conversation.messages],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{conversation_id}", status_code=204)
|
||||
async def delete_conversation(
|
||||
conversation_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> None:
|
||||
"""GDPR: users delete their own conversations; messages cascade."""
|
||||
conversation = await own_conversation(db, conversation_id, user)
|
||||
await db.delete(conversation)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,7 @@
|
||||
"""The one router constructor the conversations modules share."""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
|
||||
def conversations_router() -> APIRouter:
|
||||
return APIRouter(prefix="/conversations", tags=["conversations"])
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Request and response shapes for conversations and their turns."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.models import ConversationMode, MessageRole
|
||||
|
||||
|
||||
class ConversationCreate(BaseModel):
|
||||
mode: ConversationMode
|
||||
|
||||
|
||||
class ConversationSummary(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: uuid.UUID
|
||||
title: str | None = None
|
||||
|
||||
|
||||
class MessageSource(BaseModel):
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
heading_path: str
|
||||
excerpt: str = ""
|
||||
# True when the passage was passed to the model; False for passages that
|
||||
# were retrieved but dropped as too weak (a no-answer turn). Old messages
|
||||
# predate the flag, so it defaults to True (they were all cited).
|
||||
used: bool = True
|
||||
# The cited document has an unanswered request to check it. Snapshotted
|
||||
# with the citation, so a reload shows what was true when it was answered.
|
||||
review_pending: bool = False
|
||||
|
||||
|
||||
class MessageOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: uuid.UUID
|
||||
role: MessageRole
|
||||
content: str
|
||||
created_at: datetime
|
||||
sources: list[MessageSource] = []
|
||||
# Set when no model answered this turn and `sources` is a plain full-text
|
||||
# result list instead: the `llm_*` code that caused it, which the frontend
|
||||
# phrases. Null on every normal turn.
|
||||
fallback: str | None = None
|
||||
|
||||
|
||||
class ConversationDetail(ConversationSummary):
|
||||
messages: list[MessageOut] = []
|
||||
|
||||
|
||||
class SendMessage(BaseModel):
|
||||
content: str = Field(min_length=1, max_length=8000)
|
||||
@@ -0,0 +1,226 @@
|
||||
"""One turn: a question in, an answer streamed out, both persisted.
|
||||
|
||||
This is the only place that knows about SSE. A mode yields `ModeEvent`s and
|
||||
knows nothing about HTTP; here they become frames on the wire. Persistence is
|
||||
deliberately asymmetric: the user message is committed BEFORE streaming starts
|
||||
so it survives anything the endpoint does, while the assistant message is
|
||||
written at the end — complete, partial after an abort, or source-list-only when
|
||||
no model could answer.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from app.api.conversations.access import own_conversation
|
||||
from app.api.conversations.routing import conversations_router
|
||||
from app.api.conversations.schemas import SendMessage
|
||||
from app.api.sse import sse
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.llm.errors import LLMError
|
||||
from app.log import conversation_id as conversation_id_var
|
||||
from app.models import Conversation, Message, MessageRole, User
|
||||
from app.modes import get_mode
|
||||
from app.modes.base import Degraded, Done, Error, Mode, Sources, StateChanged, Token
|
||||
|
||||
router = conversations_router()
|
||||
logger = logging.getLogger("pablan.conversations")
|
||||
|
||||
|
||||
@router.post("/{conversation_id}/messages")
|
||||
async def send_message(
|
||||
conversation_id: uuid.UUID,
|
||||
body: SendMessage,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> StreamingResponse:
|
||||
conversation = await own_conversation(db, conversation_id, user, with_messages=True)
|
||||
mode = get_mode(conversation.mode.value)
|
||||
if mode is None:
|
||||
raise ApiError(
|
||||
400, f"Mode '{conversation.mode.value}' is not available.", "unknown_mode"
|
||||
)
|
||||
|
||||
# The user message is committed before streaming starts — it survives
|
||||
# whatever happens to the LLM call.
|
||||
db.add(
|
||||
Message(
|
||||
conversation_id=conversation.id,
|
||||
role=MessageRole.user,
|
||||
content=body.content,
|
||||
)
|
||||
)
|
||||
conversation.updated_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
|
||||
return StreamingResponse(
|
||||
stream_turn(conversation, body.content, mode, db),
|
||||
media_type="text/event-stream",
|
||||
headers={"cache-control": "no-cache", "x-accel-buffering": "no"},
|
||||
)
|
||||
|
||||
|
||||
def _source_payload(chunks: Any) -> list[dict[str, Any]]:
|
||||
"""The wire shape of a citation — the same dicts are snapshotted onto the
|
||||
message, so a reload shows exactly what was streamed."""
|
||||
return [
|
||||
{
|
||||
"document_id": str(chunk.document_id),
|
||||
"title": chunk.title,
|
||||
"heading_path": chunk.heading_path,
|
||||
"excerpt": chunk.excerpt,
|
||||
"used": chunk.used,
|
||||
"review_pending": chunk.review_pending,
|
||||
}
|
||||
for chunk in chunks
|
||||
]
|
||||
|
||||
|
||||
async def stream_turn(
|
||||
conversation: Conversation, content: str, mode: Mode, db: AsyncSession
|
||||
) -> AsyncIterator[str]:
|
||||
"""Convert ModeEvents to SSE frames; persist the assistant reply —
|
||||
complete on normal end, partial on client abort or endpoint failure."""
|
||||
context_token = conversation_id_var.set(str(conversation.id))
|
||||
started = time.monotonic()
|
||||
parts: list[str] = []
|
||||
sources: list[dict[str, Any]] = []
|
||||
# Set when the mode gave up on the model: the turn still has a reply (the
|
||||
# retrieved documents), so it is persisted and replayed like any other.
|
||||
fallback: str | None = None
|
||||
outcome = "ok"
|
||||
try:
|
||||
try:
|
||||
async for event in mode.handle_turn(conversation, content, db):
|
||||
match event:
|
||||
case Token(text=text):
|
||||
parts.append(text)
|
||||
yield sse("token", {"text": text})
|
||||
case Sources(chunks=chunks):
|
||||
sources = _source_payload(chunks)
|
||||
yield sse("sources", {"chunks": sources})
|
||||
case StateChanged(phase=phase, count=count):
|
||||
yield sse("state", {"phase": phase, "count": count})
|
||||
case Error(code=code):
|
||||
outcome = "error"
|
||||
yield sse("error", {"code": code})
|
||||
case Degraded(code=code):
|
||||
outcome = "degraded"
|
||||
fallback = code
|
||||
yield sse("fallback", {"code": code})
|
||||
case Done():
|
||||
pass # the router emits the final done after persisting
|
||||
except LLMError as exc:
|
||||
# Every endpoint failure inside a mode ends the turn the same way,
|
||||
# wherever it happened. Retrieval embeds before the model is ever
|
||||
# called, so an escaping error would reach the browser as a
|
||||
# truncated stream ("connection lost") instead of the reason.
|
||||
outcome = "error"
|
||||
logger.warning(
|
||||
"turn failed",
|
||||
extra={
|
||||
"event": "turn_error",
|
||||
"mode": mode.name,
|
||||
"code": exc.code,
|
||||
"cause_type": exc.cause_type,
|
||||
"status_code": exc.status_code,
|
||||
},
|
||||
)
|
||||
yield sse("error", {"code": exc.code})
|
||||
except (asyncio.CancelledError, GeneratorExit):
|
||||
# Client aborted (stop button): keep what was already streamed.
|
||||
outcome = "aborted"
|
||||
if parts:
|
||||
await asyncio.shield(
|
||||
_persist_partial(db.bind, conversation.id, "".join(parts), sources)
|
||||
)
|
||||
raise
|
||||
# Whatever arrived before the end is the reply, complete or not — for a
|
||||
# fallback turn that is the source list alone.
|
||||
if parts or fallback:
|
||||
message_id = await _persist_assistant(
|
||||
db, conversation, "".join(parts), sources, fallback=fallback
|
||||
)
|
||||
yield sse("done", {"message_id": str(message_id)})
|
||||
finally:
|
||||
conversation_id_var.reset(context_token)
|
||||
logger.info(
|
||||
"turn finished",
|
||||
extra={
|
||||
"event": "turn",
|
||||
"mode": mode.name,
|
||||
"outcome": outcome,
|
||||
"duration_ms": round((time.monotonic() - started) * 1000),
|
||||
"token_events": len(parts),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _assistant_meta(
|
||||
sources: list[dict[str, Any]], fallback: str | None
|
||||
) -> dict[str, Any]:
|
||||
meta: dict[str, Any] = {}
|
||||
if sources:
|
||||
meta["sources"] = sources
|
||||
if fallback:
|
||||
# Why there is no generated text, kept so a reload replays the turn as
|
||||
# what it was rather than as an empty reply.
|
||||
meta["fallback"] = fallback
|
||||
return meta
|
||||
|
||||
|
||||
async def _persist_assistant(
|
||||
db: AsyncSession,
|
||||
conversation: Conversation,
|
||||
content: str,
|
||||
sources: list[dict[str, Any]],
|
||||
*,
|
||||
fallback: str | None = None,
|
||||
) -> uuid.UUID:
|
||||
message = Message(
|
||||
conversation_id=conversation.id,
|
||||
role=MessageRole.assistant,
|
||||
content=content,
|
||||
meta=_assistant_meta(sources, fallback),
|
||||
)
|
||||
db.add(message)
|
||||
conversation.updated_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
return message.id
|
||||
|
||||
|
||||
async def _persist_partial(
|
||||
bind: Any,
|
||||
conversation_id: uuid.UUID,
|
||||
content: str,
|
||||
sources: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Write what was streamed before the client hung up.
|
||||
|
||||
On a FRESH session on the same engine as the request session: the request
|
||||
session is being torn down mid-cancel, so it cannot be used to commit, and
|
||||
binding to the same engine keeps this working under the test overrides.
|
||||
"""
|
||||
async with async_sessionmaker(bind, expire_on_commit=False)() as db:
|
||||
db.add(
|
||||
Message(
|
||||
conversation_id=conversation_id,
|
||||
role=MessageRole.assistant,
|
||||
content=content,
|
||||
meta=_assistant_meta(sources, None),
|
||||
)
|
||||
)
|
||||
conversation = await db.get(Conversation, conversation_id)
|
||||
if conversation is not None:
|
||||
conversation.updated_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Rows to API shapes.
|
||||
|
||||
A conversation has no title column: the first message is the title, derived
|
||||
here so the list and the detail can never disagree about what a conversation
|
||||
is called.
|
||||
"""
|
||||
|
||||
from app.api.conversations.schemas import MessageOut, MessageSource
|
||||
from app.models import Message
|
||||
from app.modes.query import excerpt as clean_excerpt
|
||||
|
||||
TITLE_LENGTH = 80
|
||||
|
||||
|
||||
def title(first_message: str | None) -> str | None:
|
||||
if not first_message:
|
||||
return None
|
||||
flattened = " ".join(first_message.split())
|
||||
if len(flattened) <= TITLE_LENGTH:
|
||||
return flattened
|
||||
return flattened[: TITLE_LENGTH - 1] + "…"
|
||||
|
||||
|
||||
def message_out(message: Message) -> MessageOut:
|
||||
"""Message + its citation snapshot from `meta` (assistant turns only).
|
||||
|
||||
The excerpt is re-cleaned on the way out, not just on the way in. It is
|
||||
a presentation detail frozen at answer time, so an improvement to the
|
||||
cleaning would otherwise only reach conversations created afterwards,
|
||||
and every existing citation would keep showing raw Markdown forever.
|
||||
Cleaning is idempotent, so text stored by a newer backend passes
|
||||
through untouched.
|
||||
"""
|
||||
meta = message.meta or {}
|
||||
sources = [
|
||||
MessageSource.model_validate(item).model_copy(
|
||||
update={"excerpt": clean_excerpt(item.get("excerpt", ""))}
|
||||
)
|
||||
for item in meta.get("sources", [])
|
||||
]
|
||||
return MessageOut.model_validate(message).model_copy(
|
||||
update={"sources": sources, "fallback": meta.get("fallback")}
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.models import Department, User
|
||||
|
||||
router = APIRouter(prefix="/departments", tags=["departments"])
|
||||
|
||||
|
||||
class DepartmentOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_departments(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[DepartmentOut]:
|
||||
"""Department names for filters and pickers — not secret, any user."""
|
||||
rows = (
|
||||
(await db.execute(select(Department).order_by(Department.name))).scalars().all()
|
||||
)
|
||||
return [DepartmentOut.model_validate(row) for row in rows]
|
||||
@@ -0,0 +1,27 @@
|
||||
"""The documents API.
|
||||
|
||||
Split by what a caller is doing, not by HTTP verb: browsing, the life of one
|
||||
document, its audit trail, the approval workflow, and department sharing. The
|
||||
shared gates live in `access.py` and the shared response shapes in `view.py`,
|
||||
so a rule like "an author keeps access to their own document" exists once.
|
||||
|
||||
**Route order matters.** FastAPI matches in registration order, so `browse`
|
||||
goes first: after `/{document_id}` exists, a request for `/search` would be
|
||||
parsed as a document id.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.documents import browse, crud, history, sharing, workflow
|
||||
from app.api.documents.access import readable_document, require_editor
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(browse.router)
|
||||
router.include_router(crud.router)
|
||||
router.include_router(history.router)
|
||||
router.include_router(workflow.router)
|
||||
router.include_router(sharing.router)
|
||||
|
||||
# The authoring API works on documents the caller may change, so it shares
|
||||
# this package's gate rather than growing a second one.
|
||||
__all__ = ["readable_document", "require_editor", "router"]
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Who may read, edit and publish a document.
|
||||
|
||||
The read gate itself lives in `rag/permissions` as SQL, because there is one
|
||||
place where "which documents may this user see" is decided. What lives here is
|
||||
everything the HTTP layer needs around it: loading one document through that
|
||||
gate, the write gates on top of it, and the Python mirror that can judge a
|
||||
change BEFORE it is committed.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.errors import ApiError
|
||||
from app.models import (
|
||||
AccessReason,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
User,
|
||||
UserRole,
|
||||
)
|
||||
from app.rag.permissions import readable_documents_filter
|
||||
|
||||
BUILTIN_READONLY = (
|
||||
"Built-in help documents are maintained with the product.",
|
||||
"builtin_readonly",
|
||||
)
|
||||
|
||||
|
||||
async def readable_document(
|
||||
db: AsyncSession, document_id: uuid.UUID, user: User
|
||||
) -> Document:
|
||||
"""One document, through the same filter the search uses."""
|
||||
document = (
|
||||
await db.execute(
|
||||
select(Document).where(
|
||||
Document.id == document_id, readable_documents_filter(user)
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if document is None:
|
||||
# 404 for unreadable docs: existence must not leak.
|
||||
raise ApiError(404, "Document not found.", "not_found")
|
||||
return document
|
||||
|
||||
|
||||
def is_open_reviewer(document: Document, user: User) -> bool:
|
||||
"""Someone asked this user to check the document and has not been answered.
|
||||
|
||||
Being asked is what grants the right to change it: a reviewer who spots a
|
||||
wrong number should fix it, not file a second question about it.
|
||||
"""
|
||||
return any(review.reviewer_id == user.id for review in document.open_reviews)
|
||||
|
||||
|
||||
def can_edit(document: Document, user: User) -> bool:
|
||||
"""Mirror of `require_editor` — the UI must predict the gate, never guess
|
||||
it."""
|
||||
if document.is_builtin:
|
||||
return False
|
||||
return (
|
||||
document.author_id == user.id
|
||||
or user.role == UserRole.admin
|
||||
or is_open_reviewer(document, user)
|
||||
)
|
||||
|
||||
|
||||
def require_editor(document: Document, user: User) -> None:
|
||||
if document.is_builtin:
|
||||
# Help pages ship with the product and are re-imported on start;
|
||||
# an edit here would silently vanish on the next deploy.
|
||||
raise ApiError(409, *BUILTIN_READONLY)
|
||||
if not can_edit(document, user):
|
||||
raise ApiError(403, "Not allowed to modify this document.", "forbidden")
|
||||
|
||||
|
||||
def require_author_or_admin(document: Document, user: User) -> None:
|
||||
"""Stricter than `require_editor`: for the decisions that belong to the
|
||||
document's owner, like deleting it or handing out a review request."""
|
||||
if document.is_builtin:
|
||||
raise ApiError(409, *BUILTIN_READONLY)
|
||||
if document.author_id != user.id and user.role != UserRole.admin:
|
||||
raise ApiError(403, "Not allowed to modify this document.", "forbidden")
|
||||
|
||||
|
||||
def access_reason(document: Document, user: User) -> AccessReason:
|
||||
"""Most specific reason first: being the author explains access better
|
||||
than the visibility level does.
|
||||
|
||||
Mirrors `readable_documents_filter`, where the visibility rules only apply
|
||||
to a PUBLISHED document — an unpublished one is visible to its author and
|
||||
to whoever was asked to check it, and to nobody else. So `review` is the
|
||||
reason whenever nothing more durable carries the access, which is exactly
|
||||
the case where the access ends with the answer.
|
||||
"""
|
||||
if document.author_id == user.id:
|
||||
return AccessReason.author
|
||||
published = document.status == DocumentStatus.published
|
||||
if published and document.visibility == DocumentVisibility.public:
|
||||
return AccessReason.public
|
||||
if (
|
||||
published
|
||||
and document.visibility == DocumentVisibility.department
|
||||
and document.department_id is not None
|
||||
and document.department_id == user.department_id
|
||||
):
|
||||
return AccessReason.department
|
||||
if is_open_reviewer(document, user):
|
||||
return AccessReason.review
|
||||
# Everything else that survived the permission filter came via a grant.
|
||||
return AccessReason.granted
|
||||
|
||||
|
||||
def user_can_read(
|
||||
user: User,
|
||||
*,
|
||||
author_id: uuid.UUID | None,
|
||||
visibility: DocumentVisibility,
|
||||
department_id: uuid.UUID | None,
|
||||
granted_department_ids: set[uuid.UUID],
|
||||
) -> bool:
|
||||
"""The Python mirror of `readable_documents_filter` for one document's
|
||||
proposed state — so a change can be checked BEFORE it is committed. Admins
|
||||
get no read-everything bypass (same as the filter).
|
||||
|
||||
Kept next to its only caller so the two cannot drift apart unnoticed; the
|
||||
SQL it mirrors is one import away.
|
||||
"""
|
||||
if author_id is not None and author_id == user.id:
|
||||
return True
|
||||
if visibility == DocumentVisibility.public:
|
||||
return True
|
||||
if (
|
||||
visibility == DocumentVisibility.department
|
||||
and department_id is not None
|
||||
and department_id == user.department_id
|
||||
):
|
||||
return True
|
||||
return (
|
||||
user.department_id is not None and user.department_id in granted_department_ids
|
||||
)
|
||||
|
||||
|
||||
def guard_self_lockout(
|
||||
user: User,
|
||||
*,
|
||||
author_id: uuid.UUID | None,
|
||||
visibility: DocumentVisibility,
|
||||
department_id: uuid.UUID | None,
|
||||
granted_department_ids: set[uuid.UUID],
|
||||
confirm: bool,
|
||||
) -> None:
|
||||
"""Refuse (or, for a confirming admin, allow) a change that would remove the
|
||||
editing user's own read access. An author keeps access as author, so this
|
||||
only ever bites an admin editing a document they do not own."""
|
||||
if user_can_read(
|
||||
user,
|
||||
author_id=author_id,
|
||||
visibility=visibility,
|
||||
department_id=department_id,
|
||||
granted_department_ids=granted_department_ids,
|
||||
):
|
||||
return
|
||||
if user.role != UserRole.admin:
|
||||
# A non-author non-admin cannot reach this state through the API; a
|
||||
# defensive block rather than a silent lockout.
|
||||
raise ApiError(409, "This change would remove your own access.", "self_lockout")
|
||||
if not confirm:
|
||||
raise ApiError(
|
||||
409,
|
||||
"You will lose access to this document after this change.",
|
||||
"self_lockout_warning",
|
||||
)
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Finding documents: the paged list, ranked search, the ZIP export, and the
|
||||
company-wide counts.
|
||||
|
||||
Every route here has a static path, so this router is included FIRST: after
|
||||
`/{document_id}` is registered, "search" would be parsed as a document id.
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
import uuid
|
||||
import zipfile
|
||||
from typing import Annotated
|
||||
|
||||
import yaml
|
||||
from fastapi import Depends, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import exists, func, or_, select, true
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.documents.routing import documents_router
|
||||
from app.api.documents.schemas import (
|
||||
DocumentPage,
|
||||
DocumentSearchHit,
|
||||
DocumentSort,
|
||||
DocumentStats,
|
||||
)
|
||||
from app.api.documents.view import document_fields, summary
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
Department,
|
||||
DocPermission,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
User,
|
||||
)
|
||||
from app.rag.permissions import open_review_for, readable_documents_filter
|
||||
|
||||
# aliased: `search` is also a query parameter on the list endpoint
|
||||
from app.rag.retrieval import search as hybrid_search
|
||||
|
||||
router = documents_router()
|
||||
|
||||
# Chunks retrieved before grouping, and the most documents a search returns.
|
||||
SEARCH_CANDIDATES = 20
|
||||
SEARCH_LIMIT = 20
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_documents(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
department: uuid.UUID | None = None,
|
||||
status: DocumentStatus | None = None,
|
||||
assigned_to_me: bool = False,
|
||||
search: str | None = Query(None, max_length=200),
|
||||
sort: DocumentSort = DocumentSort.updated,
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(30, ge=1, le=100),
|
||||
) -> DocumentPage:
|
||||
"""Browse readable documents.
|
||||
|
||||
Paginated server-side: the list is the one screen that grows without
|
||||
bound as a knowledge base fills up. Search has its own endpoint and is
|
||||
ranked rather than paged.
|
||||
"""
|
||||
filters = [readable_documents_filter(user)]
|
||||
if department is not None:
|
||||
# A department filter matches the owning department OR a shared grant,
|
||||
# so a document shared with a department shows up under it too.
|
||||
filters.append(
|
||||
or_(
|
||||
Document.department_id == department,
|
||||
exists(
|
||||
select(DocPermission.document_id).where(
|
||||
DocPermission.document_id == Document.id,
|
||||
DocPermission.department_id == department,
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
if status is not None:
|
||||
filters.append(Document.status == status)
|
||||
if assigned_to_me:
|
||||
# "Waiting for me": documents someone asked THIS user to check.
|
||||
filters.append(open_review_for(user))
|
||||
if search:
|
||||
filters.append(Document.title.ilike(f"%{search}%"))
|
||||
|
||||
total = (
|
||||
await db.execute(select(func.count(Document.id)).where(*filters))
|
||||
).scalar_one()
|
||||
|
||||
order = (
|
||||
Document.created_at.desc()
|
||||
if sort is DocumentSort.created
|
||||
else Document.updated_at.desc()
|
||||
)
|
||||
documents = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document)
|
||||
.where(*filters)
|
||||
# Built-in help is reference material and belongs after the
|
||||
# team's own documents — sorted in SQL so it holds across page
|
||||
# boundaries, which a client-side sort could not manage.
|
||||
# `Document.id` breaks ties. Without it the order is only
|
||||
# partial: the corpus is seeded in one transaction, so many
|
||||
# rows share a timestamp to the microsecond, and Postgres is
|
||||
# free to return them in a different order per query. Two pages
|
||||
# then overlap and a document is shown twice while another is
|
||||
# never reachable.
|
||||
.order_by(Document.is_builtin.asc(), order, Document.id)
|
||||
.offset((page - 1) * per_page)
|
||||
.limit(per_page)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return DocumentPage(
|
||||
items=[summary(document, user) for document in documents],
|
||||
total=total,
|
||||
per_page=per_page,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
async def search_documents(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
q: Annotated[str, Query(min_length=1, max_length=200)],
|
||||
) -> list[DocumentSearchHit]:
|
||||
"""Find documents through the same hybrid retrieval the chat uses.
|
||||
|
||||
Permission-safe by construction: `search()` requires a user and applies
|
||||
the shared filter. Drafts and pending documents are readable
|
||||
but never indexed, so a title fallback covers them — the one asymmetry
|
||||
between this endpoint and chat retrieval.
|
||||
"""
|
||||
results = await hybrid_search(db, q, user=user, top_k=SEARCH_CANDIDATES)
|
||||
|
||||
# Group chunks per document, keeping the best-scoring chunk's heading.
|
||||
best_heading: dict[uuid.UUID, str] = {}
|
||||
for result in results:
|
||||
best_heading.setdefault(result.document_id, result.heading_path)
|
||||
|
||||
hits: list[DocumentSearchHit] = []
|
||||
if best_heading:
|
||||
documents = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document).where(
|
||||
Document.id.in_(best_heading),
|
||||
readable_documents_filter(user),
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
by_id = {document.id: document for document in documents}
|
||||
# Preserve retrieval order — relevance, not insertion order.
|
||||
for document_id, heading in best_heading.items():
|
||||
document = by_id.get(document_id)
|
||||
if document is not None:
|
||||
hits.append(
|
||||
DocumentSearchHit(
|
||||
**document_fields(document, user),
|
||||
heading_path=heading,
|
||||
)
|
||||
)
|
||||
|
||||
# Title fallback for everything retrieval cannot see.
|
||||
remaining = SEARCH_LIMIT - len(hits)
|
||||
if remaining > 0:
|
||||
by_title = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document)
|
||||
.where(
|
||||
readable_documents_filter(user),
|
||||
Document.title.ilike(f"%{q}%"),
|
||||
Document.id.notin_(best_heading) if best_heading else true(),
|
||||
)
|
||||
.order_by(Document.updated_at.desc())
|
||||
.limit(remaining)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
hits.extend(
|
||||
DocumentSearchHit(**document_fields(document, user))
|
||||
for document in by_title
|
||||
)
|
||||
return hits[:SEARCH_LIMIT]
|
||||
|
||||
|
||||
def _export_name(document: Document, used: set[str]) -> str:
|
||||
"""A stable, de-duplicated `.md` filename for a document in the export."""
|
||||
slug = (document.meta or {}).get("slug")
|
||||
base = slug or re.sub(r"[^a-z0-9]+", "-", document.title.lower()).strip("-")
|
||||
base = base or str(document.id)
|
||||
name = f"{base}.md"
|
||||
counter = 2
|
||||
while name in used:
|
||||
name = f"{base}-{counter}.md"
|
||||
counter += 1
|
||||
used.add(name)
|
||||
return name
|
||||
|
||||
|
||||
@router.get("/export")
|
||||
async def export_documents(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> StreamingResponse:
|
||||
"""The readable knowledge base as a ZIP of Markdown files with YAML
|
||||
frontmatter. Permission-filtered by construction (`readable_documents_filter`
|
||||
— an admin exports what they can read, anyone else the same); built-in help
|
||||
pages are excluded (product content, not the company's knowledge). stdlib
|
||||
only, streamed, no temp files."""
|
||||
documents = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document)
|
||||
.where(readable_documents_filter(user), Document.is_builtin.is_(False))
|
||||
.order_by(Document.title)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
# Resolve department names once for the frontmatter: the owning department
|
||||
# plus any shared grants, so an export records the full reach of a document.
|
||||
dept_names = dict((await db.execute(select(Department.id, Department.name))).all())
|
||||
shared: dict[uuid.UUID, list[str]] = {}
|
||||
for doc_id, dept_id in (
|
||||
await db.execute(select(DocPermission.document_id, DocPermission.department_id))
|
||||
).all():
|
||||
shared.setdefault(doc_id, []).append(dept_names.get(dept_id, ""))
|
||||
|
||||
buffer = io.BytesIO()
|
||||
used: set[str] = set()
|
||||
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
for document in documents:
|
||||
departments = [
|
||||
*(
|
||||
[dept_names[document.department_id]]
|
||||
if document.department_id in dept_names
|
||||
else []
|
||||
),
|
||||
*sorted(shared.get(document.id, [])),
|
||||
]
|
||||
frontmatter = yaml.safe_dump(
|
||||
{
|
||||
"title": document.title,
|
||||
"status": str(document.status),
|
||||
"visibility": str(document.visibility),
|
||||
"departments": departments,
|
||||
},
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
body = f"---\n{frontmatter}---\n\n{document.content_md.rstrip()}\n"
|
||||
archive.writestr(_export_name(document, used), body)
|
||||
|
||||
buffer.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([buffer.getvalue()]),
|
||||
media_type="application/zip",
|
||||
headers={"content-disposition": 'attachment; filename="pablan-export.zip"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
async def document_stats(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentStats:
|
||||
"""Is this a fresh install or a filled one? Read by the landing page's
|
||||
first-run guide."""
|
||||
published = Document.status == DocumentStatus.published
|
||||
return DocumentStats(
|
||||
documents_total=(
|
||||
await db.execute(select(func.count(Document.id)).where(published))
|
||||
).scalar_one(),
|
||||
departments_total=(
|
||||
await db.execute(select(func.count(Department.id)))
|
||||
).scalar_one(),
|
||||
)
|
||||
@@ -0,0 +1,236 @@
|
||||
"""The life of one document: open it, read it, change it, delete it.
|
||||
|
||||
Publishing does NOT live here — a draft becomes public through
|
||||
`workflow.py`, so a content edit can never make a private draft readable by
|
||||
accident. Archiving does, because it is the mirror of the `status` a PATCH
|
||||
already carries.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import Depends
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.api.documents.access import (
|
||||
guard_self_lockout,
|
||||
readable_document,
|
||||
require_author_or_admin,
|
||||
require_editor,
|
||||
)
|
||||
from app.api.documents.routing import documents_router
|
||||
from app.api.documents.schemas import DocumentCreate, DocumentDetail, DocumentUpdate
|
||||
from app.api.documents.view import detail, full_detail, granted_department_ids
|
||||
from app.auth.deps import get_current_user
|
||||
from app.authoring.context import summarize_conversation
|
||||
from app.authoring.document import render_skeleton, render_title
|
||||
from app.authoring.history import record_event
|
||||
from app.authoring.schema import AuthoringTemplate
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.ingestion.handlers import INDEX_DOCUMENT
|
||||
from app.ingestion.queue import enqueue
|
||||
from app.models import (
|
||||
Conversation,
|
||||
Document,
|
||||
DocumentEventAction,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
Template,
|
||||
User,
|
||||
)
|
||||
|
||||
router = documents_router()
|
||||
|
||||
|
||||
@router.post("", status_code=201)
|
||||
async def create_document(
|
||||
body: DocumentCreate,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentDetail:
|
||||
"""Open a new document to write in.
|
||||
|
||||
A `draft` is the author's private working copy: `readable_documents_filter`
|
||||
shows it to no one else (bar a colleague asked to check it) and only
|
||||
`published` documents are indexed, so a draft never reaches another user
|
||||
or an LLM prompt."""
|
||||
title = body.title
|
||||
content_md = ""
|
||||
visibility = body.visibility or DocumentVisibility.department
|
||||
meta: dict[str, Any] = {}
|
||||
|
||||
if body.template_id is not None:
|
||||
row = await db.get(Template, body.template_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "Template not found.", "not_found")
|
||||
try:
|
||||
template = AuthoringTemplate.model_validate(row.config)
|
||||
except ValidationError:
|
||||
raise ApiError(
|
||||
422, "Template is not a valid authoring template.", "invalid_template"
|
||||
) from None
|
||||
content_md = render_skeleton(template)
|
||||
meta = {"template": template.id}
|
||||
if title is None:
|
||||
title = render_title(template, user)
|
||||
if body.visibility is None:
|
||||
visibility = DocumentVisibility(template.metadata.visibility)
|
||||
|
||||
if not title:
|
||||
raise ApiError(422, "A title or a template is required.", "title_required")
|
||||
|
||||
if body.conversation_id is not None:
|
||||
meta.update(await _conversation_context(db, body.conversation_id, user))
|
||||
|
||||
document = Document(
|
||||
title=title,
|
||||
status=DocumentStatus.draft,
|
||||
visibility=visibility,
|
||||
content_md=content_md,
|
||||
meta=meta,
|
||||
author_id=user.id,
|
||||
department_id=user.department_id,
|
||||
# Marks the collection loaded — a brand-new document has no requests,
|
||||
# and the serializer reads them without a session to lazy-load in.
|
||||
reviews=[],
|
||||
)
|
||||
db.add(document)
|
||||
# Flush so the event can reference document.id (the PK default is applied
|
||||
# at flush, not at construction).
|
||||
await db.flush()
|
||||
record_event(db, document, user, DocumentEventAction.created, snapshot=True)
|
||||
await db.commit()
|
||||
return detail(document, user)
|
||||
|
||||
|
||||
async def _conversation_context(
|
||||
db: AsyncSession, conversation_id: uuid.UUID, user: User
|
||||
) -> dict[str, Any]:
|
||||
"""What the chat this capture started from was about, as background for
|
||||
section refinement. Owner-scoped; a foreign or unknown conversation simply
|
||||
contributes nothing."""
|
||||
conversation = (
|
||||
await db.execute(
|
||||
select(Conversation)
|
||||
.where(
|
||||
Conversation.id == conversation_id,
|
||||
Conversation.user_id == user.id,
|
||||
)
|
||||
.options(selectinload(Conversation.messages))
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if conversation is None:
|
||||
return {}
|
||||
topic = await summarize_conversation(conversation)
|
||||
if not topic:
|
||||
return {}
|
||||
return {"context": topic, "conversation_id": str(conversation.id)}
|
||||
|
||||
|
||||
@router.get("/{document_id}")
|
||||
async def get_document(
|
||||
document_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentDetail:
|
||||
document = await readable_document(db, document_id, user)
|
||||
return await full_detail(db, document, user)
|
||||
|
||||
|
||||
@router.patch("/{document_id}")
|
||||
async def update_document(
|
||||
document_id: uuid.UUID,
|
||||
body: DocumentUpdate,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentDetail:
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_editor(document, user)
|
||||
|
||||
# Track the axes of change separately so the audit trail can name what
|
||||
# happened (an edit vs. a visibility change vs. an archive), even though
|
||||
# all three equally invalidate the denormalized chunk copy.
|
||||
body_changed = False
|
||||
if body.title is not None and body.title != document.title:
|
||||
document.title = body.title
|
||||
body_changed = True
|
||||
if body.content_md is not None and body.content_md != document.content_md:
|
||||
document.content_md = body.content_md
|
||||
body_changed = True
|
||||
|
||||
visibility_changed = False
|
||||
if body.visibility is not None and body.visibility != document.visibility:
|
||||
# Who may READ this is the owner's decision, like sharing and deleting:
|
||||
# a colleague asked to check the text may correct it, not re-address it.
|
||||
require_author_or_admin(document, user)
|
||||
# A visibility change can remove the editing user's own access (only an
|
||||
# admin editing a document they do not own — an author keeps access).
|
||||
guard_self_lockout(
|
||||
user,
|
||||
author_id=document.author_id,
|
||||
visibility=body.visibility,
|
||||
department_id=document.department_id,
|
||||
granted_department_ids=await granted_department_ids(db, document.id),
|
||||
confirm=bool(body.confirm_lockout),
|
||||
)
|
||||
document.visibility = body.visibility
|
||||
visibility_changed = True # chunk meta carries a denormalized copy
|
||||
|
||||
if body.conversation_id is not None:
|
||||
# Extending a document out of a chat: same background as a fresh
|
||||
# capture. Metadata only — no event, and no reindex, because nothing
|
||||
# a chunk carries changed.
|
||||
document.meta = {
|
||||
**document.meta,
|
||||
**await _conversation_context(db, body.conversation_id, user),
|
||||
}
|
||||
|
||||
archived = False
|
||||
status_changed = False
|
||||
if body.status is not None and body.status != document.status:
|
||||
archivable = {DocumentStatus.published, DocumentStatus.archived}
|
||||
if body.status not in archivable or document.status not in archivable:
|
||||
# Publishing is its own endpoint: it indexes the document and is
|
||||
# the author's decision, not a field on a content edit.
|
||||
raise ApiError(
|
||||
409,
|
||||
"Only published documents can be archived (and vice versa).",
|
||||
"invalid_status",
|
||||
)
|
||||
document.status = body.status
|
||||
status_changed = True
|
||||
archived = body.status == DocumentStatus.archived
|
||||
|
||||
# Audit: an edit snapshots the new Markdown so the version can be diffed; a
|
||||
# visibility change or archive is a pure transition (no content snapshot).
|
||||
if body_changed:
|
||||
record_event(db, document, user, DocumentEventAction.edited, snapshot=True)
|
||||
if visibility_changed:
|
||||
record_event(db, document, user, DocumentEventAction.visibility_changed)
|
||||
if archived:
|
||||
record_event(db, document, user, DocumentEventAction.archived)
|
||||
|
||||
# Chunks are derivatives of the Markdown: published edits reindex, and
|
||||
# archive/publish transitions add or remove the chunks.
|
||||
content_changed = body_changed or visibility_changed
|
||||
published = document.status == DocumentStatus.published
|
||||
if (content_changed and published) or status_changed:
|
||||
await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document.id)})
|
||||
await db.commit()
|
||||
return detail(document, user)
|
||||
|
||||
|
||||
@router.delete("/{document_id}", status_code=204)
|
||||
async def delete_document(
|
||||
document_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> None:
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_author_or_admin(document, user)
|
||||
await db.delete(document) # chunks cascade
|
||||
await db.commit()
|
||||
@@ -0,0 +1,112 @@
|
||||
"""The audit trail: who changed a document, when, and what that change was.
|
||||
|
||||
Snapshots are written AFTER their event, so an entry's content is the state it
|
||||
produced. Showing "what did this one do" therefore needs the pair (this
|
||||
snapshot and the one before it), which is why the version endpoint returns both.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy import select, tuple_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.documents.access import readable_document
|
||||
from app.api.documents.routing import documents_router
|
||||
from app.api.documents.schemas import DocumentEventOut, DocumentVersion
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import DocumentEvent, User
|
||||
|
||||
router = documents_router()
|
||||
|
||||
# Newest first, with the id as tiebreaker: events written in one transaction
|
||||
# share a timestamp, and only a total order can be paged or walked backwards.
|
||||
_NEWEST_FIRST = (DocumentEvent.created_at.desc(), DocumentEvent.id.desc())
|
||||
|
||||
|
||||
@router.get("/{document_id}/history")
|
||||
async def document_history(
|
||||
document_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[DocumentEventOut]:
|
||||
"""The document's audit trail, newest first: who changed or reviewed it,
|
||||
when, and whether a content snapshot exists to diff against. Same read gate
|
||||
as the document itself, so history never leaks to a user who cannot read the
|
||||
document."""
|
||||
document = await readable_document(db, document_id, user)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(DocumentEvent, User.name)
|
||||
.join(User, DocumentEvent.actor_id == User.id, isouter=True)
|
||||
.where(DocumentEvent.document_id == document.id)
|
||||
.order_by(*_NEWEST_FIRST)
|
||||
)
|
||||
).all()
|
||||
return [
|
||||
DocumentEventOut(
|
||||
id=event.id,
|
||||
action=event.action,
|
||||
actor_id=event.actor_id,
|
||||
actor_name=name,
|
||||
visibility=event.visibility,
|
||||
created_at=event.created_at,
|
||||
has_snapshot=event.content_md is not None,
|
||||
)
|
||||
for event, name in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/{document_id}/versions/{event_id}")
|
||||
async def document_version(
|
||||
document_id: uuid.UUID,
|
||||
event_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentVersion:
|
||||
"""A single past version's frozen content plus the content it replaced, so
|
||||
the caller can show what this event changed. Same read gate as the
|
||||
document."""
|
||||
document = await readable_document(db, document_id, user)
|
||||
row = (
|
||||
await db.execute(
|
||||
select(DocumentEvent, User.name)
|
||||
.join(User, DocumentEvent.actor_id == User.id, isouter=True)
|
||||
.where(
|
||||
DocumentEvent.id == event_id,
|
||||
DocumentEvent.document_id == document.id,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if row is None:
|
||||
raise ApiError(404, "Version not found.", "not_found")
|
||||
event, name = row
|
||||
# The state this event started from: the closest earlier snapshot, in the
|
||||
# same order the history list uses.
|
||||
previous = (
|
||||
await db.execute(
|
||||
select(DocumentEvent.content_md)
|
||||
.where(
|
||||
DocumentEvent.document_id == document.id,
|
||||
DocumentEvent.content_md.is_not(None),
|
||||
tuple_(DocumentEvent.created_at, DocumentEvent.id)
|
||||
< tuple_(event.created_at, event.id),
|
||||
)
|
||||
.order_by(*_NEWEST_FIRST)
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return DocumentVersion(
|
||||
id=event.id,
|
||||
action=event.action,
|
||||
actor_id=event.actor_id,
|
||||
actor_name=name,
|
||||
created_at=event.created_at,
|
||||
title=event.title,
|
||||
content_md=event.content_md,
|
||||
previous_content_md=previous,
|
||||
visibility=event.visibility,
|
||||
)
|
||||
@@ -0,0 +1,13 @@
|
||||
"""The one router constructor the package's modules share.
|
||||
|
||||
Every module builds its own `APIRouter` and `__init__` mounts them in the
|
||||
order that matters. They cannot be prefix-less sub-routers: FastAPI refuses a
|
||||
route whose path and router prefix are BOTH empty, which the browse list ("")
|
||||
would be, so the prefix lives here rather than five times over.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
|
||||
def documents_router() -> APIRouter:
|
||||
return APIRouter(prefix="/documents", tags=["documents"])
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Request and response shapes for the documents API.
|
||||
|
||||
Kept in one module because the whole package answers with the same handful of
|
||||
document shapes: a summary in lists, a detail on a single document, and the
|
||||
few command bodies that change one.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.models import (
|
||||
AccessReason,
|
||||
DocumentEventAction,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
)
|
||||
|
||||
|
||||
class DocumentSummary(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: uuid.UUID
|
||||
title: str
|
||||
status: DocumentStatus
|
||||
visibility: DocumentVisibility
|
||||
department_id: uuid.UUID | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
# Why this user sees it and what they may do — so the UI can explain
|
||||
# access instead of leaving the rules implicit.
|
||||
access_reason: AccessReason
|
||||
can_edit: bool
|
||||
# Unanswered questions about this document. A published document with an
|
||||
# open question is readable but not settled, and every surface that shows
|
||||
# the document says so — including the sources under a chat answer.
|
||||
open_reviews: int
|
||||
# Shipped with the product: read-only, and never deletable.
|
||||
is_builtin: bool
|
||||
|
||||
|
||||
class DepartmentRef(BaseModel):
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
|
||||
|
||||
class ReviewOut(BaseModel):
|
||||
"""One request to check this document. Open while `resolved_at` is null."""
|
||||
|
||||
id: uuid.UUID
|
||||
question: str | None
|
||||
requester_name: str | None
|
||||
reviewer_id: uuid.UUID | None
|
||||
reviewer_name: str | None
|
||||
created_at: datetime
|
||||
resolved_at: datetime | None
|
||||
resolved_by_name: str | None
|
||||
# Whether the caller is the one being asked, so the UI can offer the
|
||||
# answer rather than just showing the question.
|
||||
is_mine: bool
|
||||
|
||||
|
||||
class DocumentDetail(DocumentSummary):
|
||||
content_md: str
|
||||
# Every request on this document, oldest first, open and answered — the
|
||||
# answered ones are the record of what was already checked.
|
||||
reviews: list[ReviewOut] = []
|
||||
# Additional departments the document is shared with, on top of its owning
|
||||
# `department_id` (the `doc_permissions` grants). Resolved where the endpoint
|
||||
# looks it up (get_document, the departments endpoint).
|
||||
shared_departments: list[DepartmentRef] = []
|
||||
|
||||
|
||||
class DocumentSort(StrEnum):
|
||||
"""How the browse list is ordered. Deliberately two options: "what
|
||||
changed" and "what is new" are the two questions people actually ask of
|
||||
a document list."""
|
||||
|
||||
updated = "updated"
|
||||
created = "created"
|
||||
|
||||
|
||||
class DocumentPage(BaseModel):
|
||||
items: list[DocumentSummary]
|
||||
# Total matching the filters, not the page — the UI needs it to know
|
||||
# whether there is a next page at all.
|
||||
total: int
|
||||
per_page: int
|
||||
|
||||
|
||||
class DocumentSearchHit(DocumentSummary):
|
||||
"""A search result: the document plus the section that matched.
|
||||
|
||||
Empty `heading_path` means the match was on the title, not a section.
|
||||
"""
|
||||
|
||||
heading_path: str = ""
|
||||
|
||||
|
||||
class DocumentStats(BaseModel):
|
||||
"""Company-wide counts, read by the landing page's first-run guide.
|
||||
|
||||
Aggregates only — no titles, no per-user data. Deliberately not
|
||||
permission-filtered: a bare count reveals nothing about content.
|
||||
"""
|
||||
|
||||
documents_total: int
|
||||
departments_total: int
|
||||
|
||||
|
||||
class DocumentCreate(BaseModel):
|
||||
"""Start a new document the user will write in the editor.
|
||||
|
||||
With a `template_id` the draft opens on that template's Markdown skeleton
|
||||
and title; without one it starts blank and `title` is required. The result
|
||||
is a `draft` — author-only and never indexed until it is published."""
|
||||
|
||||
template_id: uuid.UUID | None = None
|
||||
title: str | None = None
|
||||
visibility: DocumentVisibility | None = None
|
||||
# When the capture started from a chat: its subject is summarized and kept
|
||||
# on the draft as background for section refinement.
|
||||
conversation_id: uuid.UUID | None = None
|
||||
|
||||
|
||||
class DocumentUpdate(BaseModel):
|
||||
title: str | None = None
|
||||
content_md: str | None = None
|
||||
visibility: DocumentVisibility | None = None
|
||||
# Only the archive transition is settable here; publishing has its own
|
||||
# endpoint, because it indexes the document.
|
||||
status: DocumentStatus | None = None
|
||||
# An admin may knowingly make a change that removes their own access; an
|
||||
# author never can (they keep access as author). See access.guard_self_lockout.
|
||||
# Nullable (not `bool = False`) so it stays optional in the generated client.
|
||||
confirm_lockout: bool | None = None
|
||||
# Continuing an EXISTING document out of a chat: the same background the
|
||||
# create path attaches, for the document that already covers the topic.
|
||||
conversation_id: uuid.UUID | None = None
|
||||
|
||||
|
||||
class DocumentDepartments(BaseModel):
|
||||
"""The full set of ADDITIONAL departments the document is shared with (on
|
||||
top of the owning department) — replaces the existing grants."""
|
||||
|
||||
department_ids: list[uuid.UUID]
|
||||
confirm_lockout: bool | None = None
|
||||
|
||||
|
||||
class ReviewerCandidate(BaseModel):
|
||||
"""A user the author may ask to check a document — id + name only."""
|
||||
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
|
||||
|
||||
class ReviewRequestBody(BaseModel):
|
||||
"""Ask someone to check this document, optionally about something specific
|
||||
("do the holiday numbers still hold?")."""
|
||||
|
||||
reviewer_id: uuid.UUID
|
||||
question: str | None = Field(default=None, max_length=2000)
|
||||
|
||||
|
||||
class DocumentEventOut(BaseModel):
|
||||
"""One entry in a document's history timeline — metadata only."""
|
||||
|
||||
id: uuid.UUID
|
||||
action: DocumentEventAction
|
||||
actor_id: uuid.UUID | None
|
||||
# Null once the actor's account is deleted (SET NULL on the event).
|
||||
actor_name: str | None
|
||||
visibility: DocumentVisibility | None
|
||||
created_at: datetime
|
||||
# A content snapshot exists for this event and can be fetched for diffing.
|
||||
has_snapshot: bool
|
||||
|
||||
|
||||
class DocumentVersion(BaseModel):
|
||||
"""A past version's frozen content, for viewing or diffing.
|
||||
|
||||
A snapshot is taken *after* its event, so `content_md` is the state this
|
||||
event produced and `previous_content_md` the state it started from — the
|
||||
pair is what "what did this change do?" needs. `previous_content_md` is
|
||||
null for the first snapshot, where everything was added.
|
||||
"""
|
||||
|
||||
id: uuid.UUID
|
||||
action: DocumentEventAction
|
||||
actor_id: uuid.UUID | None
|
||||
actor_name: str | None
|
||||
created_at: datetime
|
||||
title: str | None
|
||||
content_md: str | None
|
||||
previous_content_md: str | None
|
||||
visibility: DocumentVisibility | None
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Sharing a document with departments beyond its own.
|
||||
|
||||
Management, not new permission logic: the read filter's EXISTS branch already
|
||||
unions `doc_permissions` in, so this endpoint only maintains those rows. Grants
|
||||
are evaluated live against the table, which is why nothing is reindexed here.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.documents.access import (
|
||||
guard_self_lockout,
|
||||
readable_document,
|
||||
require_author_or_admin,
|
||||
)
|
||||
from app.api.documents.routing import documents_router
|
||||
from app.api.documents.schemas import DocumentDepartments, DocumentDetail
|
||||
from app.api.documents.view import full_detail, granted_department_ids
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import Department, DocPermission, PermissionLevel, User
|
||||
|
||||
router = documents_router()
|
||||
|
||||
|
||||
@router.put("/{document_id}/departments")
|
||||
async def set_shared_departments(
|
||||
document_id: uuid.UUID,
|
||||
body: DocumentDepartments,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentDetail:
|
||||
"""Replace the full set of ADDITIONAL departments this document is shared
|
||||
with. Author or admin only."""
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_author_or_admin(document, user)
|
||||
|
||||
requested = set(body.department_ids)
|
||||
# A document is never "shared with" its own owning department.
|
||||
requested.discard(document.department_id)
|
||||
if requested:
|
||||
found = set(
|
||||
(
|
||||
await db.execute(
|
||||
select(Department.id).where(Department.id.in_(requested))
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if requested - found:
|
||||
raise ApiError(404, "One or more departments do not exist.", "not_found")
|
||||
|
||||
# Removing a grant can drop the editing admin's own department access.
|
||||
guard_self_lockout(
|
||||
user,
|
||||
author_id=document.author_id,
|
||||
visibility=document.visibility,
|
||||
department_id=document.department_id,
|
||||
granted_department_ids=requested,
|
||||
confirm=bool(body.confirm_lockout),
|
||||
)
|
||||
|
||||
existing = await granted_department_ids(db, document.id)
|
||||
for dept_id in existing - requested:
|
||||
await db.execute(
|
||||
delete(DocPermission).where(
|
||||
DocPermission.document_id == document.id,
|
||||
DocPermission.department_id == dept_id,
|
||||
)
|
||||
)
|
||||
for dept_id in requested - existing:
|
||||
db.add(
|
||||
DocPermission(
|
||||
document_id=document.id,
|
||||
department_id=dept_id,
|
||||
level=PermissionLevel.read,
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
return await full_detail(db, document, user)
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Document rows to API shapes.
|
||||
|
||||
Every endpoint in the package answers with `summary` or `detail`, so the
|
||||
per-request fields (why this user sees it, what they may do, what is still
|
||||
open) are computed in exactly one place.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.documents.access import access_reason, can_edit
|
||||
from app.api.documents.schemas import (
|
||||
DepartmentRef,
|
||||
DocumentDetail,
|
||||
DocumentSummary,
|
||||
ReviewOut,
|
||||
)
|
||||
from app.models import Department, DocPermission, Document, ReviewRequest, User
|
||||
|
||||
|
||||
def document_fields(document: Document, user: User) -> dict[str, Any]:
|
||||
"""Built explicitly rather than via model_validate: access_reason and
|
||||
can_edit are per-request, so there is nothing to read them from."""
|
||||
return {
|
||||
"id": document.id,
|
||||
"title": document.title,
|
||||
"status": document.status,
|
||||
"visibility": document.visibility,
|
||||
"department_id": document.department_id,
|
||||
"created_at": document.created_at,
|
||||
"updated_at": document.updated_at,
|
||||
"access_reason": access_reason(document, user),
|
||||
"can_edit": can_edit(document, user),
|
||||
"open_reviews": len(document.open_reviews),
|
||||
"is_builtin": document.is_builtin,
|
||||
}
|
||||
|
||||
|
||||
def summary(document: Document, user: User) -> DocumentSummary:
|
||||
return DocumentSummary(**document_fields(document, user))
|
||||
|
||||
|
||||
def detail(
|
||||
document: Document,
|
||||
user: User,
|
||||
*,
|
||||
reviews: list[ReviewOut] | None = None,
|
||||
shared_departments: list[DepartmentRef] | None = None,
|
||||
) -> DocumentDetail:
|
||||
return DocumentDetail(
|
||||
**document_fields(document, user),
|
||||
content_md=document.content_md,
|
||||
reviews=reviews or [],
|
||||
shared_departments=shared_departments or [],
|
||||
)
|
||||
|
||||
|
||||
async def resolve_reviews(
|
||||
db: AsyncSession, document: Document, user: User
|
||||
) -> list[ReviewOut]:
|
||||
"""The document's requests with the names filled in.
|
||||
|
||||
One query for every name involved, rather than three relationships loaded
|
||||
with every document: the names are needed on the detail page only, while
|
||||
the requests themselves ride along everywhere (they decide who may edit).
|
||||
"""
|
||||
if not document.reviews:
|
||||
return []
|
||||
wanted = {
|
||||
person_id
|
||||
for review in document.reviews
|
||||
for person_id in (
|
||||
review.requester_id,
|
||||
review.reviewer_id,
|
||||
review.resolved_by_id,
|
||||
)
|
||||
if person_id is not None
|
||||
}
|
||||
names = dict(
|
||||
(await db.execute(select(User.id, User.name).where(User.id.in_(wanted)))).all()
|
||||
)
|
||||
return [
|
||||
ReviewOut(
|
||||
id=review.id,
|
||||
question=review.question,
|
||||
requester_name=names.get(review.requester_id),
|
||||
reviewer_id=review.reviewer_id,
|
||||
reviewer_name=names.get(review.reviewer_id),
|
||||
created_at=review.created_at,
|
||||
resolved_at=review.resolved_at,
|
||||
resolved_by_name=names.get(review.resolved_by_id),
|
||||
is_mine=review.reviewer_id == user.id,
|
||||
)
|
||||
for review in document.reviews
|
||||
]
|
||||
|
||||
|
||||
async def resolve_shared_departments(
|
||||
db: AsyncSession, document: Document
|
||||
) -> list[DepartmentRef]:
|
||||
"""The additional departments this document is shared with (its
|
||||
`doc_permissions` grants), resolved to names for display."""
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(Department.id, Department.name)
|
||||
.join(DocPermission, DocPermission.department_id == Department.id)
|
||||
.where(DocPermission.document_id == document.id)
|
||||
.order_by(Department.name)
|
||||
)
|
||||
).all()
|
||||
return [DepartmentRef(id=row.id, name=row.name) for row in rows]
|
||||
|
||||
|
||||
async def granted_department_ids(
|
||||
db: AsyncSession, document_id: uuid.UUID
|
||||
) -> set[uuid.UUID]:
|
||||
return set(
|
||||
(
|
||||
await db.execute(
|
||||
select(DocPermission.department_id).where(
|
||||
DocPermission.document_id == document_id
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
async def full_detail(
|
||||
db: AsyncSession, document: Document, user: User
|
||||
) -> DocumentDetail:
|
||||
"""The detail with everything resolved — for the endpoints that answer
|
||||
with a document the UI is about to render in full."""
|
||||
return detail(
|
||||
document,
|
||||
user,
|
||||
reviews=await resolve_reviews(db, document, user),
|
||||
shared_departments=await resolve_shared_departments(db, document),
|
||||
)
|
||||
|
||||
|
||||
async def open_review_for(
|
||||
db: AsyncSession, document: Document, reviewer_id: uuid.UUID
|
||||
) -> ReviewRequest | None:
|
||||
"""An unanswered request on this document addressed to `reviewer_id`."""
|
||||
return (
|
||||
await db.execute(
|
||||
select(ReviewRequest).where(
|
||||
ReviewRequest.document_id == document.id,
|
||||
ReviewRequest.reviewer_id == reviewer_id,
|
||||
ReviewRequest.resolved_at.is_(None),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
@@ -0,0 +1,174 @@
|
||||
"""From draft to published, and the questions that hang off a document.
|
||||
|
||||
Two things that used to be one. **Publishing** is the author's own decision: a
|
||||
draft is private until they say it is worth reading, one action, no waiting.
|
||||
**A review request** is "please check this", and it is not a status — it can
|
||||
sit on a draft the author is unsure about OR on a document that has been
|
||||
published for months, and it marks the document wherever it appears until
|
||||
someone answers it.
|
||||
|
||||
Being asked is what grants the right to edit: a reviewer who spots a wrong
|
||||
number should fix it rather than file a second question about it.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.documents.access import (
|
||||
readable_document,
|
||||
require_author_or_admin,
|
||||
require_editor,
|
||||
)
|
||||
from app.api.documents.routing import documents_router
|
||||
from app.api.documents.schemas import (
|
||||
DocumentDetail,
|
||||
ReviewerCandidate,
|
||||
ReviewRequestBody,
|
||||
)
|
||||
from app.api.documents.view import full_detail
|
||||
from app.auth.deps import get_current_user
|
||||
from app.authoring.history import record_event
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.ingestion.handlers import INDEX_DOCUMENT
|
||||
from app.ingestion.queue import enqueue
|
||||
from app.models import DocumentEventAction, DocumentStatus, ReviewRequest, User
|
||||
from app.rag.permissions import document_reader_filter
|
||||
|
||||
router = documents_router()
|
||||
|
||||
|
||||
@router.post("/{document_id}/publish")
|
||||
async def publish_document(
|
||||
document_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentDetail:
|
||||
"""Make a draft readable and searchable for everyone its visibility allows.
|
||||
|
||||
The author's own call — an open question about the content does not block
|
||||
it, it travels with the document instead (`open_reviews`), which is what
|
||||
lets a colleague read it AND know it is not settled.
|
||||
|
||||
Author or admin, deliberately not every editor: a colleague asked to check
|
||||
a draft may fix what is wrong in it, but whether the company gets to read
|
||||
it at all is not their call.
|
||||
"""
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_author_or_admin(document, user)
|
||||
if document.status != DocumentStatus.draft:
|
||||
raise ApiError(409, "Only a draft can be published.", "invalid_status")
|
||||
|
||||
document.status = DocumentStatus.published
|
||||
record_event(db, document, user, DocumentEventAction.published)
|
||||
await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document.id)})
|
||||
await db.commit()
|
||||
return await full_detail(db, document, user)
|
||||
|
||||
|
||||
@router.get("/{document_id}/reviewers")
|
||||
async def list_reviewers(
|
||||
document_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[ReviewerCandidate]:
|
||||
"""Who can be asked: everyone who could read this document once published,
|
||||
minus the author. Permission-safe and non-admin (unlike /admin/users), and
|
||||
only id + name leave the server."""
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_editor(document, user)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(User.id, User.name)
|
||||
.where(document_reader_filter(document), User.id != document.author_id)
|
||||
.order_by(User.name)
|
||||
)
|
||||
).all()
|
||||
return [ReviewerCandidate(id=row.id, name=row.name) for row in rows]
|
||||
|
||||
|
||||
@router.post("/{document_id}/reviews")
|
||||
async def request_review(
|
||||
document_id: uuid.UUID,
|
||||
body: ReviewRequestBody,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentDetail:
|
||||
"""Ask a colleague to check this document, optionally about something
|
||||
specific. The request grants them the right to read and edit it until it
|
||||
is answered."""
|
||||
document = await readable_document(db, document_id, user)
|
||||
require_author_or_admin(document, user)
|
||||
if body.reviewer_id == user.id:
|
||||
raise ApiError(422, "You cannot ask yourself.", "invalid_reviewer")
|
||||
|
||||
allowed = (
|
||||
await db.execute(
|
||||
select(User.id).where(
|
||||
User.id == body.reviewer_id,
|
||||
document_reader_filter(document),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if allowed is None:
|
||||
raise ApiError(
|
||||
422, "That user cannot review this document.", "invalid_reviewer"
|
||||
)
|
||||
if any(review.reviewer_id == body.reviewer_id for review in document.open_reviews):
|
||||
raise ApiError(
|
||||
409, "That colleague has already been asked.", "review_already_open"
|
||||
)
|
||||
|
||||
db.add(
|
||||
ReviewRequest(
|
||||
document_id=document.id,
|
||||
requester_id=user.id,
|
||||
reviewer_id=body.reviewer_id,
|
||||
question=(body.question or "").strip() or None,
|
||||
)
|
||||
)
|
||||
record_event(db, document, user, DocumentEventAction.review_requested)
|
||||
await db.commit()
|
||||
# Reload the collection, not just the columns: the serializer reads the
|
||||
# requests, and a lazy load there would be IO in a sync property.
|
||||
await db.refresh(document, attribute_names=["reviews"])
|
||||
return await full_detail(db, document, user)
|
||||
|
||||
|
||||
@router.post("/{document_id}/reviews/{review_id}/resolve")
|
||||
async def resolve_review(
|
||||
document_id: uuid.UUID,
|
||||
review_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentDetail:
|
||||
"""Answer a request: the content was checked.
|
||||
|
||||
The reviewer answers their own request; the author (or an admin) can close
|
||||
one that has become moot, because a question nobody will answer should not
|
||||
mark a document forever.
|
||||
"""
|
||||
document = await readable_document(db, document_id, user)
|
||||
review = next(
|
||||
(review for review in document.reviews if review.id == review_id), None
|
||||
)
|
||||
if review is None:
|
||||
raise ApiError(404, "Review request not found.", "not_found")
|
||||
if review.resolved_at is not None:
|
||||
raise ApiError(409, "This request is already answered.", "already_resolved")
|
||||
if review.reviewer_id != user.id:
|
||||
require_author_or_admin(document, user)
|
||||
|
||||
review.resolved_at = datetime.now(UTC)
|
||||
review.resolved_by_id = user.id
|
||||
record_event(db, document, user, DocumentEventAction.review_resolved)
|
||||
await db.commit()
|
||||
# Reload the collection, not just the columns: the serializer reads the
|
||||
# requests, and a lazy load there would be IO in a sync property.
|
||||
await db.refresh(document, attribute_names=["reviews"])
|
||||
return await full_detail(db, document, user)
|
||||
@@ -0,0 +1,72 @@
|
||||
"""The colleague directory.
|
||||
|
||||
A member-visible directory any authenticated user may browse: colleagues'
|
||||
`{id, name, role, department}` — no email, no password hash. The same
|
||||
permission-safe, non-admin shape as `ReviewerCandidate` in `api/documents.py`,
|
||||
deliberately separate from the admin-only `/admin/users`.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import Department, User, UserRole
|
||||
|
||||
router = APIRouter(prefix="/people", tags=["people"])
|
||||
|
||||
|
||||
class PersonOut(BaseModel):
|
||||
"""A colleague as the directory shows them — never email or credentials."""
|
||||
|
||||
id: uuid.UUID
|
||||
name: str
|
||||
role: UserRole
|
||||
department: str | None
|
||||
|
||||
|
||||
def _select_people():
|
||||
return select(
|
||||
User.id,
|
||||
User.name,
|
||||
User.role,
|
||||
Department.name.label("department"),
|
||||
).join(Department, User.department_id == Department.id, isouter=True)
|
||||
|
||||
|
||||
def _person(row) -> PersonOut:
|
||||
return PersonOut(
|
||||
id=row.id,
|
||||
name=row.name,
|
||||
role=row.role,
|
||||
department=row.department,
|
||||
)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_people(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[PersonOut]:
|
||||
"""Every colleague, ordered by name. Visible to any authenticated user."""
|
||||
rows = (await db.execute(_select_people().order_by(User.name))).all()
|
||||
return [_person(row) for row in rows]
|
||||
|
||||
|
||||
@router.get("/{person_id}")
|
||||
async def get_person(
|
||||
person_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> PersonOut:
|
||||
"""One colleague's profile."""
|
||||
row = (await db.execute(_select_people().where(User.id == person_id))).first()
|
||||
if row is None:
|
||||
raise ApiError(404, "Person not found.", "not_found")
|
||||
return _person(row)
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Server-sent events, the one way this API streams.
|
||||
|
||||
Two endpoints stream (a chat turn and a section refinement) and they frame the
|
||||
same way, so the wire format lives here rather than in both. `event:` names the frame, `data:` carries a JSON object, and
|
||||
a blank line ends it.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sse(event: str, data: dict[str, Any]) -> str:
|
||||
return f"event: {event}\ndata: {json.dumps(data)}\n\n"
|
||||
@@ -0,0 +1,19 @@
|
||||
"""The templates API: the blueprints a document can be started from.
|
||||
|
||||
Reading is for everyone (the picker needs it), changing is admin-only, and the
|
||||
two live on separate routers so the gate is structural rather than repeated per
|
||||
endpoint. `catalog` is what ships with the product, `edit` what the customer
|
||||
made of it.
|
||||
|
||||
**Route order matters.** The catalog's static paths are registered before
|
||||
`/{template_id}`, or "catalog" would be parsed as a row id.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.templates import browse, catalog, edit
|
||||
|
||||
router = APIRouter()
|
||||
router.include_router(catalog.router)
|
||||
router.include_router(edit.router)
|
||||
router.include_router(browse.router)
|
||||
@@ -0,0 +1,67 @@
|
||||
"""A template's blueprint: parsing it, and keeping its id unique.
|
||||
|
||||
The config id is a slug, not a row id — it is what documents record as their
|
||||
origin and what the catalog matches on, so two rows must never share one.
|
||||
Three endpoints need that rule (add from catalog, save from the builder,
|
||||
duplicate), which is why it is written once here.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.authoring.schema import AuthoringTemplate
|
||||
from app.errors import ApiError
|
||||
from app.models import Template
|
||||
from app.template_import import TemplateImportError, parse_template
|
||||
|
||||
|
||||
def parse_or_422(source: str) -> AuthoringTemplate:
|
||||
"""YAML in, validated blueprint out. The parse error is the message: it
|
||||
already says which field is wrong, and an admin is the one reading it."""
|
||||
try:
|
||||
return parse_template(source)
|
||||
except TemplateImportError as exc:
|
||||
raise ApiError(422, str(exc), "invalid_template") from None
|
||||
|
||||
|
||||
async def taken_config_ids(db: AsyncSession) -> set[str]:
|
||||
return set((await db.execute(select(Template.config["id"].astext))).scalars().all())
|
||||
|
||||
|
||||
def unique_config_id(base: str, taken: set[str], *, suffix: str = "") -> str:
|
||||
"""`base`, or `base-2`, `base-3` … until it is free.
|
||||
|
||||
`suffix` marks derived ids (a duplicate becomes `base-kopie`), so a copy
|
||||
reads as a copy in the one place ids are visible.
|
||||
"""
|
||||
stem = f"{base}{suffix}"
|
||||
candidate = stem
|
||||
counter = 2
|
||||
while candidate in taken:
|
||||
candidate = f"{stem}-{counter}"
|
||||
counter += 1
|
||||
return candidate
|
||||
|
||||
|
||||
async def ensure_config_id_free(
|
||||
db: AsyncSession, config_id: str, *, except_row: uuid.UUID
|
||||
) -> None:
|
||||
"""Refuse an edit that would move a config id onto a DIFFERENT row.
|
||||
|
||||
Editing keeps the id stable, so a clash is never the row's own id — it
|
||||
means someone would silently steal another template's identity.
|
||||
"""
|
||||
clash = (
|
||||
await db.execute(
|
||||
select(Template.id).where(
|
||||
Template.config["id"].astext == config_id,
|
||||
Template.id != except_row,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if clash is not None:
|
||||
raise ApiError(
|
||||
409, f"Another template already uses the id '{config_id}'.", "id_taken"
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Reading templates. Any authenticated user: the picker needs them."""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.templates.routing import reader_router
|
||||
from app.api.templates.schemas import TemplateDetail, TemplateSummary
|
||||
from app.api.templates.view import detail, summary
|
||||
from app.auth.deps import get_current_user
|
||||
from app.config import get_settings
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import Template, User
|
||||
|
||||
router = reader_router()
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_templates(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[TemplateSummary]:
|
||||
"""Templates the picker offers. Templates in the reader's language come
|
||||
first — they are customer content, so a mismatched one is still listed
|
||||
rather than hidden."""
|
||||
rows = (await db.execute(select(Template).order_by(Template.name))).scalars().all()
|
||||
wanted = user.locale or get_settings().default_locale
|
||||
ordered = sorted(rows, key=lambda row: row.config.get("locale") != wanted)
|
||||
return [summary(row) for row in ordered]
|
||||
|
||||
|
||||
@router.get("/{template_id}")
|
||||
async def get_template(
|
||||
template_id: uuid.UUID,
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> TemplateDetail:
|
||||
row = await db.get(Template, template_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "Template not found.", "not_found")
|
||||
return detail(row)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""The blueprints that ship with the product.
|
||||
|
||||
Nothing in the catalog is active. It is a shelf an admin takes from: adding a
|
||||
blueprint copies it into an ordinary template row, which the customer then owns
|
||||
and edits. The catalog never touches that row again.
|
||||
"""
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.templates.blueprints import parse_or_422, taken_config_ids
|
||||
from app.api.templates.routing import editor_router
|
||||
from app.api.templates.schemas import CatalogDetail, CatalogSummary, TemplateDetail
|
||||
from app.api.templates.view import detail
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.template_catalog import catalog_for_locale, get_catalog_entry
|
||||
from app.template_import import upsert_template
|
||||
|
||||
router = editor_router()
|
||||
|
||||
|
||||
@router.get("/catalog")
|
||||
async def list_catalog(
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> list[CatalogSummary]:
|
||||
"""The blueprints shipped with Pablan, each marked with whether this
|
||||
instance has already added it."""
|
||||
added = await taken_config_ids(db)
|
||||
return [
|
||||
CatalogSummary(
|
||||
id=entry.id,
|
||||
name=entry.name,
|
||||
description=entry.description,
|
||||
sections=entry.sections,
|
||||
added=entry.id in added,
|
||||
)
|
||||
for entry in catalog_for_locale()
|
||||
]
|
||||
|
||||
|
||||
@router.get("/catalog/{catalog_id}")
|
||||
async def get_catalog_blueprint(catalog_id: str) -> CatalogDetail:
|
||||
"""Read a blueprint before adding it — the whole point of "view" is that
|
||||
an admin can see its structure before committing to it."""
|
||||
entry = get_catalog_entry(catalog_id)
|
||||
if entry is None:
|
||||
raise ApiError(404, "Blueprint not found.", "not_found")
|
||||
return CatalogDetail(
|
||||
id=entry.id,
|
||||
name=entry.name,
|
||||
description=entry.description,
|
||||
sections=entry.sections,
|
||||
added=False,
|
||||
yaml=entry.source,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/catalog/{catalog_id}")
|
||||
async def add_from_catalog(
|
||||
catalog_id: str,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> TemplateDetail:
|
||||
"""Copy a blueprint into this instance. The result is an ordinary
|
||||
template row: editable, and never touched by the catalog again."""
|
||||
entry = get_catalog_entry(catalog_id)
|
||||
if entry is None:
|
||||
raise ApiError(404, "Blueprint not found.", "not_found")
|
||||
if entry.id in await taken_config_ids(db):
|
||||
raise ApiError(
|
||||
409,
|
||||
"This template has already been added — edit or duplicate it instead.",
|
||||
"already_added",
|
||||
)
|
||||
row, _created = await upsert_template(db, parse_or_422(entry.source))
|
||||
await db.commit()
|
||||
return detail(row)
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Changing what this instance offers. Admin only, by the router it hangs on.
|
||||
|
||||
Two ways in, one guarantee: the form builder sends a structured config and the
|
||||
YAML editor sends text, but both end as the same validated blueprint, so
|
||||
neither path can save something the other would reject.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.templates.blueprints import (
|
||||
ensure_config_id_free,
|
||||
parse_or_422,
|
||||
taken_config_ids,
|
||||
unique_config_id,
|
||||
)
|
||||
from app.api.templates.routing import editor_router
|
||||
from app.api.templates.schemas import (
|
||||
TemplateBuildRequest,
|
||||
TemplateDetail,
|
||||
TemplateImportRequest,
|
||||
)
|
||||
from app.api.templates.view import detail
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import Template
|
||||
|
||||
router = editor_router()
|
||||
|
||||
# What a template's config id falls back to when the form has nothing to slug.
|
||||
FALLBACK_ID = "vorlage"
|
||||
|
||||
|
||||
async def _row_or_404(db: AsyncSession, template_id: uuid.UUID) -> Template:
|
||||
row = await db.get(Template, template_id)
|
||||
if row is None:
|
||||
raise ApiError(404, "Template not found.", "not_found")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/build")
|
||||
async def build_template(
|
||||
body: TemplateBuildRequest,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> TemplateDetail:
|
||||
"""Save a template from the structured form builder. Creates a new row
|
||||
(template_id null) or updates one in place."""
|
||||
template = body.config
|
||||
config = template.model_dump(mode="json")
|
||||
|
||||
if body.template_id is None:
|
||||
# The config id is an internal slug the form derives from the name;
|
||||
# make it unique so a second "Onboarding" never overwrites the first.
|
||||
config["id"] = unique_config_id(
|
||||
template.id or FALLBACK_ID, await taken_config_ids(db)
|
||||
)
|
||||
row = Template(name=template.name, version=template.version, config=config)
|
||||
db.add(row)
|
||||
else:
|
||||
await ensure_config_id_free(db, template.id, except_row=body.template_id)
|
||||
row = await _row_or_404(db, body.template_id)
|
||||
row.name = template.name
|
||||
row.version = template.version
|
||||
row.config = config
|
||||
|
||||
await db.commit()
|
||||
return detail(row)
|
||||
|
||||
|
||||
@router.put("/{template_id}")
|
||||
async def update_template(
|
||||
template_id: uuid.UUID,
|
||||
body: TemplateImportRequest,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> TemplateDetail:
|
||||
"""Replace a template's YAML. Validated against the schema on save."""
|
||||
row = await _row_or_404(db, template_id)
|
||||
template = parse_or_422(body.yaml)
|
||||
await ensure_config_id_free(db, template.id, except_row=row.id)
|
||||
|
||||
row.name = template.name
|
||||
row.version = template.version
|
||||
row.config = template.model_dump(mode="json")
|
||||
await db.commit()
|
||||
return detail(row)
|
||||
|
||||
|
||||
async def _copy_name(db: AsyncSession, name: str) -> str:
|
||||
"""The next free "<name> (2)".
|
||||
|
||||
A number rather than a word, because this name is shown in the interface
|
||||
and the backend never renders UI-language strings (CLAUDE.md) — a German
|
||||
"(Kopie)" would sit untranslated in an English admin panel. It is also
|
||||
what file managers do, so it needs no explaining.
|
||||
"""
|
||||
taken = set((await db.execute(select(Template.name))).scalars().all())
|
||||
counter = 2
|
||||
while f"{name} ({counter})" in taken:
|
||||
counter += 1
|
||||
return f"{name} ({counter})"
|
||||
|
||||
|
||||
@router.post("/{template_id}/duplicate")
|
||||
async def duplicate_template(
|
||||
template_id: uuid.UUID,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> TemplateDetail:
|
||||
"""Fork a template — for trying a variant without losing the original.
|
||||
The copy gets a fresh config id so the two never collide."""
|
||||
row = await _row_or_404(db, template_id)
|
||||
config_id = unique_config_id(
|
||||
row.config.get("id", "template"), await taken_config_ids(db), suffix="-copy"
|
||||
)
|
||||
config = {**row.config, "id": config_id, "name": await _copy_name(db, row.name)}
|
||||
copy = Template(name=config["name"], version=row.version, config=config)
|
||||
db.add(copy)
|
||||
await db.commit()
|
||||
return detail(copy)
|
||||
|
||||
|
||||
@router.delete("/{template_id}", status_code=204)
|
||||
async def delete_template(
|
||||
template_id: uuid.UUID,
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> None:
|
||||
"""Remove a template. Documents created from it are independent and
|
||||
survive (a template is only a starting point). If it came from the
|
||||
catalog it can always be added back."""
|
||||
row = await _row_or_404(db, template_id)
|
||||
await db.delete(row)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Two routers, because templates have two audiences.
|
||||
|
||||
Everyone may READ the templates (the picker needs them); only an admin may
|
||||
change what the instance offers. Expressing that as two constructors means a
|
||||
new endpoint is gated by which router it is added to, not by remembering to
|
||||
repeat a dependency.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.auth.deps import require_admin
|
||||
|
||||
|
||||
def reader_router() -> APIRouter:
|
||||
return APIRouter(prefix="/templates", tags=["templates"])
|
||||
|
||||
|
||||
def editor_router() -> APIRouter:
|
||||
return APIRouter(
|
||||
prefix="/templates", tags=["templates"], dependencies=[Depends(require_admin)]
|
||||
)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Request and response shapes for templates and the shipped catalog."""
|
||||
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.authoring.schema import AuthoringTemplate
|
||||
|
||||
|
||||
class TemplateSummary(BaseModel):
|
||||
id: uuid.UUID
|
||||
# The blueprint id from the config (e.g. "onboarding-basis"): stable across
|
||||
# installs, where the row id is not. Anything that wants to offer ONE known
|
||||
# blueprint (the profile page's "write about yourself") finds it by this.
|
||||
config_id: str
|
||||
name: str
|
||||
version: str
|
||||
description: str = ""
|
||||
|
||||
|
||||
class TemplateDetail(TemplateSummary):
|
||||
config: dict[str, Any]
|
||||
# The editable source. Serialized server-side because the frontend has
|
||||
# no YAML library and must not gain one.
|
||||
yaml: str
|
||||
|
||||
|
||||
class CatalogSummary(BaseModel):
|
||||
"""A blueprint on disk. `id` is the config id, NOT a row id — a catalog
|
||||
entry has no row until someone adds it."""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
description: str
|
||||
# How many skeleton sections the blueprint carries hints for.
|
||||
sections: int
|
||||
# Whether a template with this config id already exists, so the UI can
|
||||
# offer "View" instead of a second "Add".
|
||||
added: bool
|
||||
|
||||
|
||||
class CatalogDetail(CatalogSummary):
|
||||
yaml: str
|
||||
|
||||
|
||||
class TemplateImportRequest(BaseModel):
|
||||
yaml: str
|
||||
|
||||
|
||||
class TemplateBuildRequest(BaseModel):
|
||||
"""A template assembled by the form builder. The config is the same schema
|
||||
a pasted YAML parses into, so both paths get one validation guarantee —
|
||||
the frontend has no YAML library and must not gain one, so it sends the
|
||||
structured config instead of serializing it."""
|
||||
|
||||
# The row to update, or null to create a new template. Kept separate from
|
||||
# the config id (a stable slug) so renaming the display name never forks
|
||||
# the row.
|
||||
template_id: uuid.UUID | None = None
|
||||
config: AuthoringTemplate
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Template rows to API shapes.
|
||||
|
||||
Both are built explicitly rather than validated from the row: the blueprint id
|
||||
and the description live inside `config`, and `yaml` is rendered per request,
|
||||
so there is nothing on the row to read them from.
|
||||
"""
|
||||
|
||||
import yaml
|
||||
|
||||
from app.api.templates.schemas import TemplateDetail, TemplateSummary
|
||||
from app.models import Template
|
||||
|
||||
|
||||
def summary(row: Template) -> TemplateSummary:
|
||||
return TemplateSummary(
|
||||
id=row.id,
|
||||
config_id=row.config.get("id", ""),
|
||||
name=row.name,
|
||||
version=row.version,
|
||||
description=row.config.get("description", ""),
|
||||
)
|
||||
|
||||
|
||||
def detail(row: Template) -> TemplateDetail:
|
||||
return TemplateDetail(
|
||||
id=row.id,
|
||||
config_id=row.config.get("id", ""),
|
||||
name=row.name,
|
||||
version=row.version,
|
||||
description=row.config.get("description", ""),
|
||||
config=row.config,
|
||||
yaml=yaml.safe_dump(row.config, allow_unicode=True, sort_keys=False, width=80),
|
||||
)
|
||||
@@ -0,0 +1,40 @@
|
||||
import uuid
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.sessions import COOKIE_NAME, get_valid_session
|
||||
from app.db import get_db
|
||||
from app.errors import ApiError
|
||||
from app.models import AuthSession, User, UserRole
|
||||
|
||||
|
||||
async def get_current_auth_session(
|
||||
request: Request, db: Annotated[AsyncSession, Depends(get_db)]
|
||||
) -> AuthSession:
|
||||
raw = request.cookies.get(COOKIE_NAME)
|
||||
if raw is None:
|
||||
raise ApiError(401, "Not authenticated.", "not_authenticated")
|
||||
try:
|
||||
session_id = uuid.UUID(raw)
|
||||
except ValueError:
|
||||
raise ApiError(401, "Not authenticated.", "not_authenticated") from None
|
||||
session = await get_valid_session(db, session_id)
|
||||
if session is None:
|
||||
raise ApiError(401, "Not authenticated.", "not_authenticated")
|
||||
return session
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
session: Annotated[AuthSession, Depends(get_current_auth_session)],
|
||||
) -> User:
|
||||
return session.user
|
||||
|
||||
|
||||
async def require_admin(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
) -> User:
|
||||
if user.role != UserRole.admin:
|
||||
raise ApiError(403, "Admin privileges required.", "forbidden")
|
||||
return user
|
||||
@@ -0,0 +1,23 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
# Verified against when the user does not exist, so login duration does not
|
||||
# reveal whether an email address is registered.
|
||||
_DUMMY_HASH = _hasher.hash("pablan-dummy-password")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(password_hash, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def burn_verification_time() -> None:
|
||||
verify_password(_DUMMY_HASH, "wrong-password")
|
||||
@@ -0,0 +1,68 @@
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from fastapi import Response
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models import AuthSession, User
|
||||
|
||||
COOKIE_NAME = "pablan_session"
|
||||
|
||||
|
||||
async def create_auth_session(db: AsyncSession, user: User) -> AuthSession:
|
||||
settings = get_settings()
|
||||
session = AuthSession(
|
||||
user_id=user.id,
|
||||
expires_at=datetime.now(UTC) + timedelta(days=settings.auth_session_ttl_days),
|
||||
)
|
||||
db.add(session)
|
||||
await db.flush()
|
||||
return session
|
||||
|
||||
|
||||
async def get_valid_session(
|
||||
db: AsyncSession, session_id: uuid.UUID
|
||||
) -> AuthSession | None:
|
||||
session = await db.get(AuthSession, session_id)
|
||||
if session is None or session.expires_at <= datetime.now(UTC):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
async def revoke_user_sessions(
|
||||
db: AsyncSession, user_id: uuid.UUID, *, keep_session_id: uuid.UUID | None = None
|
||||
) -> None:
|
||||
"""Log a user out everywhere — the session-revocation primitive.
|
||||
|
||||
`keep_session_id` spares the caller's own session, which is what a
|
||||
self-service password change wants: every other device is logged out,
|
||||
the one you are typing on is not.
|
||||
"""
|
||||
statement = delete(AuthSession).where(AuthSession.user_id == user_id)
|
||||
if keep_session_id is not None:
|
||||
statement = statement.where(AuthSession.id != keep_session_id)
|
||||
await db.execute(statement)
|
||||
|
||||
|
||||
def set_session_cookie(response: Response, session: AuthSession) -> None:
|
||||
settings = get_settings()
|
||||
response.set_cookie(
|
||||
COOKIE_NAME,
|
||||
str(session.id),
|
||||
max_age=settings.auth_session_ttl_days * 24 * 60 * 60,
|
||||
httponly=True,
|
||||
secure=settings.cookie_secure,
|
||||
samesite="lax",
|
||||
)
|
||||
|
||||
|
||||
def clear_session_cookie(response: Response) -> None:
|
||||
settings = get_settings()
|
||||
response.delete_cookie(
|
||||
COOKIE_NAME,
|
||||
httponly=True,
|
||||
secure=settings.cookie_secure,
|
||||
samesite="lax",
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Writing-first knowledge capture.
|
||||
|
||||
Capture is not a conversation Mode: the artifact is a Document the user
|
||||
writes directly (Markdown is the source of truth), and the model
|
||||
refines one section at a time (FIM-style). This package holds the template
|
||||
schema, the skeleton/title rendering, the active-section boundary and the
|
||||
refinement prompt. The HTTP surface lives in `app/api/authoring.py`.
|
||||
"""
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Conversation context for a capture.
|
||||
|
||||
When a document is written out of a chat, that chat's subject is useful twice:
|
||||
to find existing documents the user might extend, and as background for section
|
||||
refinement. Both use a short LLM topic summary of the conversation. All LLM
|
||||
traffic goes through `llm/client.py`; nothing here logs content — a failure
|
||||
degrades quietly rather than breaking the capture.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.authoring.prompts import render_topic_summary_prompt
|
||||
from app.llm.client import chat_json
|
||||
from app.llm.errors import LLMError
|
||||
from app.models import Conversation, MessageRole
|
||||
|
||||
logger = logging.getLogger("pablan.authoring")
|
||||
|
||||
# How many recent turns feed the summary — enough for the subject, bounded so
|
||||
# a long thread cannot blow up the utility prompt.
|
||||
MAX_CONTEXT_MESSAGES = 12
|
||||
|
||||
# Skip the reasoning model's hidden thinking: this is a short, latency-
|
||||
# sensitive utility call.
|
||||
_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
|
||||
class _TopicSummary(BaseModel):
|
||||
topic: str
|
||||
|
||||
|
||||
def conversation_transcript(conversation: Conversation) -> str:
|
||||
"""The recent user/assistant turns as a plain transcript."""
|
||||
turns = [
|
||||
message
|
||||
for message in conversation.messages
|
||||
if message.role in (MessageRole.user, MessageRole.assistant)
|
||||
][-MAX_CONTEXT_MESSAGES:]
|
||||
return "\n".join(
|
||||
f"{'User' if message.role == MessageRole.user else 'Assistant'}: "
|
||||
f"{message.content}"
|
||||
for message in turns
|
||||
)
|
||||
|
||||
|
||||
async def summarize_transcript(transcript: str) -> str:
|
||||
"""A short topic summary of a conversation transcript, or '' if none can
|
||||
be made. Failures degrade quietly."""
|
||||
if not transcript.strip():
|
||||
return ""
|
||||
try:
|
||||
result = await chat_json(
|
||||
render_topic_summary_prompt(transcript),
|
||||
_TopicSummary,
|
||||
extra_body=_NO_THINKING,
|
||||
)
|
||||
except LLMError:
|
||||
logger.info("topic summary failed", extra={"event": "topic_summary_failed"})
|
||||
return ""
|
||||
return result.topic.strip()
|
||||
|
||||
|
||||
async def summarize_conversation(conversation: Conversation) -> str:
|
||||
"""A short topic summary of the conversation, or '' if none can be made."""
|
||||
return await summarize_transcript(conversation_transcript(conversation))
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Render a template's Markdown skeleton and title into a new draft document."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.authoring.schema import AuthoringTemplate
|
||||
from app.models import User
|
||||
|
||||
|
||||
def render_title(template: AuthoringTemplate, user: User) -> str:
|
||||
today = datetime.now(UTC).date().isoformat()
|
||||
return (
|
||||
template.title_template.replace("{{user.name}}", user.name)
|
||||
.replace("{{date}}", today)
|
||||
.strip()
|
||||
)
|
||||
|
||||
|
||||
def render_skeleton(template: AuthoringTemplate) -> str:
|
||||
"""The Markdown the editor opens with: the skeleton verbatim, normalized
|
||||
to a single trailing newline. An empty skeleton yields an empty document
|
||||
the author fills from scratch."""
|
||||
skeleton = template.skeleton.strip()
|
||||
return f"{skeleton}\n" if skeleton else ""
|
||||
@@ -0,0 +1,40 @@
|
||||
"""The document audit trail.
|
||||
|
||||
Every content edit and lifecycle transition is appended to `document_events`
|
||||
as an immutable record of who did what, when. Content-bearing actions snapshot
|
||||
the Markdown source of truth (never the disposable chunks) so a past version
|
||||
can later be viewed or diffed.
|
||||
"""
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Document, DocumentEvent, DocumentEventAction, User
|
||||
|
||||
|
||||
def record_event(
|
||||
db: AsyncSession,
|
||||
document: Document,
|
||||
actor: User,
|
||||
action: DocumentEventAction,
|
||||
*,
|
||||
snapshot: bool = False,
|
||||
) -> None:
|
||||
"""Append an audit record for `document`.
|
||||
|
||||
`snapshot` freezes the current Markdown, title and meta so the version can
|
||||
be reconstructed later — pass it for content-bearing events (created /
|
||||
edited). Visibility is small, so it is always recorded. Leave `snapshot`
|
||||
False for pure transitions that carry no new content. The document must
|
||||
already have an id (flush a freshly created document first).
|
||||
"""
|
||||
db.add(
|
||||
DocumentEvent(
|
||||
document_id=document.id,
|
||||
actor_id=actor.id,
|
||||
action=action,
|
||||
content_md=document.content_md if snapshot else None,
|
||||
title=document.title if snapshot else None,
|
||||
visibility=document.visibility,
|
||||
meta=document.meta if snapshot else None,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Refinement prompt for the writing editor — natural language only.
|
||||
|
||||
The model refines exactly ONE section of a document the user is writing. The
|
||||
rest of the document travels as prefix/suffix context so the section stays
|
||||
coherent with its surroundings, but the model regenerates ONLY the section —
|
||||
a large document is never re-emitted whole (FIM-style).
|
||||
|
||||
The base texts (persona, rules, framings) are admin-editable: they come from
|
||||
`app/prompts/overrides.py::get_prompt`, which returns a DB override when one
|
||||
exists and the code default (`app/prompts/defaults.py`) otherwise.
|
||||
"""
|
||||
|
||||
from app.llm.client import ChatMessage
|
||||
from app.prompts.overrides import get_prompt
|
||||
|
||||
|
||||
def render_refine_prompt(
|
||||
section: str,
|
||||
*,
|
||||
prefix: str,
|
||||
suffix: str,
|
||||
persona: str | None,
|
||||
hint: str | None,
|
||||
context: str | None = None,
|
||||
knowledge: list[str] | None = None,
|
||||
) -> list[ChatMessage]:
|
||||
# A template may carry its own persona; otherwise the admin-editable default.
|
||||
system_parts = [persona.strip() if persona else get_prompt("refine_persona")]
|
||||
if hint:
|
||||
system_parts.append(f"What this section should convey: {hint}")
|
||||
if context:
|
||||
# Background from the chat this capture came from, so the refinement
|
||||
# is on-topic — but only as orientation, never a source of new facts.
|
||||
system_parts.append(
|
||||
f"Background (the conversation this document came from, for "
|
||||
f"orientation only — do not invent facts from it): {context}"
|
||||
)
|
||||
system_parts.append(get_prompt("refine_rules"))
|
||||
|
||||
# The document being edited leads the user turn (prefix/suffix/section);
|
||||
# the retrieved knowledge trails it, because it changes on every call and
|
||||
# keeping it last leaves the stable prompt prefix reusable between calls.
|
||||
user_parts: list[str] = []
|
||||
if prefix.strip():
|
||||
user_parts.append(
|
||||
f"Text before the section (context only, do not repeat it):\n{prefix}"
|
||||
)
|
||||
if suffix.strip():
|
||||
user_parts.append(
|
||||
f"Text after the section (context only, do not repeat it):\n{suffix}"
|
||||
)
|
||||
user_parts.append(f"Refine only this section:\n{section}")
|
||||
if knowledge:
|
||||
# What the company has already documented elsewhere. It is grounding,
|
||||
# not source material: it keeps terminology and facts consistent and
|
||||
# lets the section point at related documents, but it must not be
|
||||
# copied in or become a way to add facts the section's notes do not
|
||||
# support.
|
||||
joined = "\n\n".join(knowledge)
|
||||
user_parts.append(f"{get_prompt('grounding_framing')}\n{joined}")
|
||||
|
||||
return [
|
||||
{"role": "system", "content": "\n\n".join(system_parts)},
|
||||
{"role": "user", "content": "\n\n".join(user_parts)},
|
||||
]
|
||||
|
||||
|
||||
def render_topic_summary_prompt(transcript: str) -> list[ChatMessage]:
|
||||
"""Condense a conversation into a short search topic (a few words)."""
|
||||
return [
|
||||
{"role": "system", "content": get_prompt("topic_summary")},
|
||||
{"role": "user", "content": f"Conversation:\n{transcript}"},
|
||||
]
|
||||
|
||||
|
||||
def render_title_prompt(content_md: str) -> list[ChatMessage]:
|
||||
"""Suggest a concise document title from its written content."""
|
||||
return [
|
||||
{"role": "system", "content": get_prompt("title")},
|
||||
{"role": "user", "content": f"Document:\n{content_md}"},
|
||||
]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Pydantic schema for authoring templates (schema version 1.0).
|
||||
|
||||
A template is a **Markdown skeleton** — a starting document with headings the
|
||||
author fills in — plus a persona and optional per-section hints that steer the
|
||||
section-refinement model. It is declarative configuration, not code (see
|
||||
docs/authoring-templates.md), stored in `templates.config` (JSONB) and
|
||||
validated on load.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, field_validator
|
||||
|
||||
|
||||
class TemplateModelHints(BaseModel):
|
||||
temperature: float = 0.4
|
||||
# UI warning when the configured endpoint is weaker than the template
|
||||
# expects; read by api/templates.py.
|
||||
min_class_hint: str | None = None
|
||||
|
||||
|
||||
class SectionHint(BaseModel):
|
||||
"""Steers what the refinement model should draw out of one section.
|
||||
|
||||
`heading` is matched to a skeleton heading by its exact text, so the hint
|
||||
only reaches the model while the author is writing under that heading.
|
||||
"""
|
||||
|
||||
heading: str
|
||||
hint: str
|
||||
|
||||
|
||||
class TemplateMetadata(BaseModel):
|
||||
visibility: Literal["public", "department", "restricted"] = "department"
|
||||
|
||||
|
||||
class AuthoringTemplate(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
version: str
|
||||
# Names the template's shape, so a differently-shaped config is rejected
|
||||
# rather than silently loaded as an authoring template.
|
||||
kind: Literal["authoring"] = "authoring"
|
||||
# The language this template's CONTENT is written in — persona, skeleton
|
||||
# and hints, not the UI. The picker lists matching templates first.
|
||||
locale: Literal["de", "en"] | None = None
|
||||
description: str = ""
|
||||
model: TemplateModelHints = TemplateModelHints()
|
||||
persona: str
|
||||
# The Markdown the editor opens with: headings the author fills in. This
|
||||
# IS the starting content, not a description of it.
|
||||
skeleton: str
|
||||
sections: list[SectionHint] = []
|
||||
title_template: str
|
||||
metadata: TemplateMetadata = TemplateMetadata()
|
||||
|
||||
@field_validator("version", mode="before")
|
||||
@classmethod
|
||||
def _version_to_string(cls, value: object) -> str:
|
||||
# YAML reads an unquoted 1.0 as a float.
|
||||
return str(value)
|
||||
|
||||
def hint_for(self, heading: str) -> str | None:
|
||||
for section in self.sections:
|
||||
if section.heading == heading:
|
||||
return section.hint
|
||||
return None
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Find the section of a Markdown document the cursor sits in.
|
||||
|
||||
The refinement endpoint refines exactly one section at a time (FIM-style),
|
||||
so this is the AUTHORITATIVE boundary computation — the client mirrors it for
|
||||
a visual highlight, but the server owns it. A section runs from the nearest
|
||||
heading at or above the cursor down to the line before the next heading of
|
||||
the same or higher level; content before the first heading is its own
|
||||
section. A section whose body exceeds the chunk cap narrows to the blank-line
|
||||
paragraph at the cursor, so a large document never refines as one giant block.
|
||||
|
||||
Shares the heading regex, fence-awareness and cap with `rag/chunking.py` so
|
||||
"what is a section" means the same thing to refinement and to indexing.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.rag.chunking import HEADING_RE, TARGET_CHUNK_CHARS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActiveSection:
|
||||
start_line: int # 1-based, inclusive, into content_md
|
||||
end_line: int # 1-based, inclusive
|
||||
|
||||
|
||||
def _heading_lines(lines: list[str]) -> list[tuple[int, int]]:
|
||||
"""(line_index_0based, level) for every heading line, ignoring fences."""
|
||||
headings: list[tuple[int, int]] = []
|
||||
in_fence = False
|
||||
for i, line in enumerate(lines):
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
if in_fence:
|
||||
continue
|
||||
match = HEADING_RE.match(line)
|
||||
if match:
|
||||
headings.append((i, len(match.group(1))))
|
||||
return headings
|
||||
|
||||
|
||||
def _paragraph_at(
|
||||
lines: list[str], start0: int, end0: int, cursor0: int
|
||||
) -> tuple[int, int] | None:
|
||||
"""The blank-line-delimited block (fence-aware) at the cursor, within
|
||||
[start0, end0]. Falls back to the block just before the cursor when it
|
||||
sits on a blank gap, else the first block."""
|
||||
blocks: list[tuple[int, int]] = []
|
||||
block_start: int | None = None
|
||||
in_fence = False
|
||||
for i in range(start0, end0 + 1):
|
||||
line = lines[i]
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
if block_start is None:
|
||||
block_start = i
|
||||
continue
|
||||
if not line.strip() and not in_fence:
|
||||
if block_start is not None:
|
||||
blocks.append((block_start, i - 1))
|
||||
block_start = None
|
||||
elif block_start is None:
|
||||
block_start = i
|
||||
if block_start is not None:
|
||||
blocks.append((block_start, end0))
|
||||
if not blocks:
|
||||
return None
|
||||
for b_start, b_end in blocks:
|
||||
if b_start <= cursor0 <= b_end:
|
||||
return b_start, b_end
|
||||
for b_start, b_end in reversed(blocks):
|
||||
if b_end < cursor0:
|
||||
return b_start, b_end
|
||||
return blocks[0]
|
||||
|
||||
|
||||
def active_section(content_md: str, cursor_line: int) -> ActiveSection:
|
||||
lines = content_md.splitlines()
|
||||
n = len(lines)
|
||||
if n == 0:
|
||||
return ActiveSection(1, 1)
|
||||
cursor0 = max(1, min(cursor_line, n)) - 1
|
||||
|
||||
headings = _heading_lines(lines)
|
||||
owner: tuple[int, int] | None = None
|
||||
for idx, level in headings:
|
||||
if idx <= cursor0:
|
||||
owner = (idx, level)
|
||||
else:
|
||||
break
|
||||
|
||||
if owner is None:
|
||||
# Preamble before the first heading (or a document with no headings).
|
||||
start0 = 0
|
||||
end0 = headings[0][0] - 1 if headings else n - 1
|
||||
else:
|
||||
start0, owner_level = owner
|
||||
end0 = n - 1
|
||||
for idx, level in headings:
|
||||
if idx > start0 and level <= owner_level:
|
||||
end0 = idx - 1
|
||||
break
|
||||
|
||||
# Trailing blank lines belong to the separation before the next section,
|
||||
# not to this one: keeping them in the range would let an accepted
|
||||
# suggestion swallow the blank line above the next heading.
|
||||
while end0 > start0 and not lines[end0].strip():
|
||||
end0 -= 1
|
||||
|
||||
body = "\n".join(lines[start0 : end0 + 1])
|
||||
if len(body) > TARGET_CHUNK_CHARS:
|
||||
narrowed = _paragraph_at(lines, start0, end0, cursor0)
|
||||
if narrowed is not None:
|
||||
start0, end0 = narrowed
|
||||
|
||||
return ActiveSection(start_line=start0 + 1, end_line=end0 + 1)
|
||||
|
||||
|
||||
def slice_lines(
|
||||
content_md: str, start_line: int, end_line: int
|
||||
) -> tuple[str, str, str]:
|
||||
"""(prefix, section, suffix) split at the 1-based inclusive line range.
|
||||
|
||||
The section is the lines the model refines; prefix/suffix are the rest of
|
||||
the document, handed to the model as context it must not re-emit.
|
||||
"""
|
||||
lines = content_md.splitlines()
|
||||
prefix = "\n".join(lines[: start_line - 1])
|
||||
section = "\n".join(lines[start_line - 1 : end_line])
|
||||
suffix = "\n".join(lines[end_line:])
|
||||
return prefix, section, suffix
|
||||
@@ -0,0 +1,64 @@
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Repo-root .env for native dev; inside Docker the file is absent and
|
||||
# configuration comes from real environment variables (which take precedence).
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_ENV_FILE = _REPO_ROOT / ".env"
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="PABLAN_", env_file=_ENV_FILE, extra="ignore"
|
||||
)
|
||||
|
||||
env: Literal["development", "production"] = "development"
|
||||
database_url: str = "postgresql+asyncpg://pablan:change-me@localhost:5432/pablan"
|
||||
|
||||
# Secure=false is needed for dev over plain http on non-localhost
|
||||
# addresses (WireGuard IPs) — see .env.example.
|
||||
cookie_secure: bool = True
|
||||
auth_session_ttl_days: int = 14
|
||||
|
||||
query_retention_days: int = 90
|
||||
|
||||
# The shipped template catalog (repo templates/ in dev; the customer
|
||||
# stack mounts the directory and overrides this path).
|
||||
templates_dir: str = str(_REPO_ROOT / "templates")
|
||||
help_dir: str = str(_REPO_ROOT / "help")
|
||||
|
||||
# The instance's own language: which blueprint variant the first-install
|
||||
# starter set uses, and the fallback when a visitor states no preference.
|
||||
# Per-user choice lives on users.locale and wins over this.
|
||||
default_locale: Literal["de", "en"] = "de"
|
||||
|
||||
log_level: str = "INFO"
|
||||
# Content debug logging (prompts/responses) — NEVER in production.
|
||||
debug_log_prompts: bool = False
|
||||
llm_timeout_seconds: float = 120.0
|
||||
# How many requests Pablan lets one endpoint see at once, and how long a
|
||||
# request waits for a free slot before it is answered with "busy". Match
|
||||
# llm_max_parallel to the server's parallel slots (llama.cpp: --parallel).
|
||||
# See app/llm/gate.py.
|
||||
llm_max_parallel: int = 4
|
||||
llm_queue_wait_seconds: float = 20.0
|
||||
llm_max_queued: int = 24
|
||||
job_poll_seconds: float = 1.0
|
||||
|
||||
chat_base_url: str = "http://localhost:8001/v1"
|
||||
chat_api_key: str = "none"
|
||||
chat_model: str = ""
|
||||
utility_base_url: str = "http://localhost:8001/v1"
|
||||
utility_api_key: str = "none"
|
||||
utility_model: str = ""
|
||||
embedding_base_url: str = "http://localhost:8002/v1"
|
||||
embedding_api_key: str = "none"
|
||||
embedding_model: str = ""
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,17 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
engine = create_async_engine(get_settings().database_url)
|
||||
async_session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
async def get_db() -> AsyncIterator[AsyncSession]:
|
||||
async with async_session_factory() as session:
|
||||
yield session
|
||||
@@ -0,0 +1,20 @@
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""API error rendered as the protocol's {detail, code} problem shape."""
|
||||
|
||||
def __init__(self, status_code: int, detail: str, code: str) -> None:
|
||||
self.status_code = status_code
|
||||
self.detail = detail
|
||||
self.code = code
|
||||
|
||||
|
||||
def register_exception_handlers(app: FastAPI) -> None:
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={"detail": exc.detail, "code": exc.code},
|
||||
)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Built-in help documents: Markdown files → documents table.
|
||||
|
||||
The help pages that describe Pablan itself ship with the product and live in
|
||||
the repo-level help/ directory (product content, not code). They are
|
||||
re-imported on every start, so a release always carries the current
|
||||
documentation, and they are flagged `is_builtin` so the API refuses to edit
|
||||
or delete them.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.ingestion.handlers import INDEX_DOCUMENT
|
||||
from app.ingestion.queue import enqueue
|
||||
from app.models import (
|
||||
Document,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("pablan.help")
|
||||
|
||||
FRONTMATTER_SEPARATOR = "---"
|
||||
META_KEY = "help_key"
|
||||
|
||||
|
||||
class HelpImportError(Exception):
|
||||
"""Malformed help file — a packaging bug, never user input."""
|
||||
|
||||
|
||||
def parse_help_document(source: str) -> tuple[str, str, str]:
|
||||
"""Split the `key`/`title` frontmatter from the Markdown body."""
|
||||
if not source.startswith(FRONTMATTER_SEPARATOR):
|
||||
raise HelpImportError("Help document must start with YAML frontmatter.")
|
||||
_, frontmatter, body = source.split(FRONTMATTER_SEPARATOR, 2)
|
||||
try:
|
||||
meta = yaml.safe_load(frontmatter)
|
||||
except yaml.YAMLError as exc:
|
||||
raise HelpImportError(f"Invalid frontmatter: {type(exc).__name__}") from None
|
||||
if not isinstance(meta, dict) or not meta.get("key") or not meta.get("title"):
|
||||
raise HelpImportError("Help frontmatter needs at least 'key' and 'title'.")
|
||||
return str(meta["key"]), str(meta["title"]), body.strip()
|
||||
|
||||
|
||||
async def import_help_documents(db: AsyncSession) -> int:
|
||||
"""Upsert every help/*.md by its key. Returns the number re-indexed."""
|
||||
directory = Path(get_settings().help_dir)
|
||||
if not directory.is_dir():
|
||||
logger.warning("help directory missing", extra={"event": "help_import_skipped"})
|
||||
return 0
|
||||
|
||||
reindexed = 0
|
||||
for path in sorted(directory.glob("*.md")):
|
||||
key, title, body = parse_help_document(path.read_text())
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(Document).where(
|
||||
Document.is_builtin.is_(True),
|
||||
Document.meta[META_KEY].astext == key,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing is None:
|
||||
document = Document(
|
||||
title=title,
|
||||
status=DocumentStatus.published,
|
||||
# Help is for everyone; it has no author and no department.
|
||||
visibility=DocumentVisibility.public,
|
||||
content_md=body,
|
||||
meta={META_KEY: key},
|
||||
is_builtin=True,
|
||||
)
|
||||
db.add(document)
|
||||
await db.flush()
|
||||
elif existing.content_md == body and existing.title == title:
|
||||
continue # unchanged — no need to re-embed
|
||||
else:
|
||||
existing.title = title
|
||||
existing.content_md = body
|
||||
existing.meta = {**existing.meta, META_KEY: key}
|
||||
document = existing
|
||||
|
||||
await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document.id)})
|
||||
reindexed += 1
|
||||
|
||||
await db.commit()
|
||||
logger.info(
|
||||
"help documents imported",
|
||||
extra={"event": "help_import", "reindexed": reindexed},
|
||||
)
|
||||
return reindexed
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Job handlers. Importing this module registers them with the queue."""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.ingestion.queue import enqueue, job_handler
|
||||
from app.models import (
|
||||
AuthSession,
|
||||
Conversation,
|
||||
ConversationMode,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
Job,
|
||||
JobStatus,
|
||||
)
|
||||
from app.rag.indexing import reindex_document, remove_chunks
|
||||
|
||||
logger = logging.getLogger("pablan.queue")
|
||||
|
||||
RETENTION_CLEANUP = "retention_cleanup"
|
||||
INDEX_DOCUMENT = "index_document"
|
||||
REINDEX_ALL = "reindex_all"
|
||||
|
||||
|
||||
@job_handler(INDEX_DOCUMENT)
|
||||
async def index_document(db: AsyncSession, job: Job) -> None:
|
||||
"""(Re)build the chunks of one document; drop them if it is not published."""
|
||||
document_id = uuid.UUID(job.payload["document_id"])
|
||||
document = await db.get(Document, document_id)
|
||||
if document is None:
|
||||
logger.info(
|
||||
"index skipped, document gone",
|
||||
extra={"event": "index_skipped", "document_id": str(document_id)},
|
||||
)
|
||||
return
|
||||
if document.status == DocumentStatus.published:
|
||||
await reindex_document(db, document)
|
||||
else:
|
||||
await remove_chunks(db, document.id)
|
||||
|
||||
|
||||
@job_handler(REINDEX_ALL)
|
||||
async def reindex_all(db: AsyncSession, job: Job) -> None:
|
||||
"""Fan out one index_document job per published document.
|
||||
|
||||
Never embeds the corpus in this handler itself — the queue holds the
|
||||
claim transaction open for the whole handler run.
|
||||
"""
|
||||
document_ids = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document.id).where(Document.status == DocumentStatus.published)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for document_id in document_ids:
|
||||
await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document_id)})
|
||||
logger.info(
|
||||
"reindex fan-out",
|
||||
extra={"event": "reindex_all", "document_count": len(document_ids)},
|
||||
)
|
||||
|
||||
|
||||
@job_handler(RETENTION_CLEANUP)
|
||||
async def retention_cleanup(db: AsyncSession, job: Job) -> None:
|
||||
"""GDPR retention: drop old query conversations and expired auth sessions.
|
||||
|
||||
Messages go with their conversation via ON DELETE CASCADE. Reschedules
|
||||
itself daily.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(days=get_settings().query_retention_days)
|
||||
|
||||
conversations_deleted = (
|
||||
await db.execute(
|
||||
delete(Conversation).where(
|
||||
Conversation.mode == ConversationMode.query,
|
||||
Conversation.updated_at < cutoff,
|
||||
)
|
||||
)
|
||||
).rowcount
|
||||
sessions_deleted = (
|
||||
await db.execute(delete(AuthSession).where(AuthSession.expires_at < now))
|
||||
).rowcount
|
||||
|
||||
await enqueue(db, RETENTION_CLEANUP, run_after=now + timedelta(days=1))
|
||||
logger.info(
|
||||
"retention cleanup",
|
||||
extra={
|
||||
"event": "retention_cleanup",
|
||||
"conversations_deleted": conversations_deleted,
|
||||
"auth_sessions_deleted": sessions_deleted,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def ensure_retention_scheduled(db: AsyncSession) -> None:
|
||||
"""Idempotent startup bootstrap: exactly one pending retention job."""
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(Job.id).where(
|
||||
Job.type == RETENTION_CLEANUP, Job.status == JobStatus.pending
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if existing is None:
|
||||
await enqueue(db, RETENTION_CLEANUP)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Postgres-backed background queue.
|
||||
|
||||
One asyncio loop in the app lifespan claims jobs via
|
||||
SELECT … FOR UPDATE SKIP LOCKED. The claim transaction stays open while the
|
||||
handler runs: a crash rolls everything back and the job remains pending and
|
||||
claimable after restart — handler writes are atomic with job completion.
|
||||
Failure bookkeeping (attempts, backoff, last_error) happens in a follow-up
|
||||
transaction. Single worker per process; the loop moves into a worker
|
||||
container unchanged when scale demands it (Variant B).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import async_session_factory
|
||||
from app.log import safe_error
|
||||
from app.metrics import metrics
|
||||
from app.models import Job, JobStatus
|
||||
|
||||
logger = logging.getLogger("pablan.queue")
|
||||
|
||||
JobHandler = Callable[[AsyncSession, Job], Awaitable[None]]
|
||||
|
||||
_HANDLERS: dict[str, JobHandler] = {}
|
||||
|
||||
MAX_ATTEMPTS = 5
|
||||
BACKOFF_BASE_SECONDS = 30.0 # 30s, 1m, 2m, 4m between retries
|
||||
|
||||
|
||||
def job_handler(job_type: str) -> Callable[[JobHandler], JobHandler]:
|
||||
def register(fn: JobHandler) -> JobHandler:
|
||||
_HANDLERS[job_type] = fn
|
||||
return fn
|
||||
|
||||
return register
|
||||
|
||||
|
||||
def backoff_delay(attempts: int) -> timedelta:
|
||||
return timedelta(seconds=BACKOFF_BASE_SECONDS * 2 ** (attempts - 1))
|
||||
|
||||
|
||||
async def enqueue(
|
||||
db: AsyncSession,
|
||||
job_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
run_after: datetime | None = None,
|
||||
) -> Job:
|
||||
job = Job(type=job_type, payload=payload or {})
|
||||
if run_after is not None:
|
||||
job.run_after = run_after
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
return job
|
||||
|
||||
|
||||
async def process_one(
|
||||
session_factory: async_sessionmaker[AsyncSession] = async_session_factory,
|
||||
) -> bool:
|
||||
"""Claim and process a single due job. Returns True if one was processed."""
|
||||
started = asyncio.get_running_loop().time()
|
||||
async with session_factory() as db:
|
||||
job = (
|
||||
await db.execute(
|
||||
select(Job)
|
||||
.where(Job.status == JobStatus.pending, Job.run_after <= func.now())
|
||||
.order_by(Job.run_after)
|
||||
.limit(1)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
await db.rollback()
|
||||
return False
|
||||
|
||||
job_id, job_type, attempts_before = job.id, job.type, job.attempts
|
||||
try:
|
||||
handler = _HANDLERS.get(job_type)
|
||||
if handler is None:
|
||||
raise LookupError(f"no handler registered for job type {job_type!r}")
|
||||
await handler(db, job)
|
||||
job.status = JobStatus.done
|
||||
job.attempts = attempts_before + 1
|
||||
await db.commit()
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
await _record_failure(session_factory, job_id, exc)
|
||||
duration = asyncio.get_running_loop().time() - started
|
||||
metrics.inc("jobs_processed_total", {"type": job_type, "status": "failed"})
|
||||
metrics.observe("job_seconds", duration, {"type": job_type})
|
||||
logger.warning(
|
||||
"job failed",
|
||||
extra={
|
||||
"event": "job_failed",
|
||||
"job_id": str(job_id),
|
||||
"job_type": job_type,
|
||||
"attempt": attempts_before + 1,
|
||||
"error": safe_error(exc),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
duration = asyncio.get_running_loop().time() - started
|
||||
metrics.inc("jobs_processed_total", {"type": job_type, "status": "done"})
|
||||
metrics.observe("job_seconds", duration, {"type": job_type})
|
||||
logger.info(
|
||||
"job done",
|
||||
extra={
|
||||
"event": "job_done",
|
||||
"job_id": str(job_id),
|
||||
"job_type": job_type,
|
||||
"attempt": attempts_before + 1,
|
||||
"duration_ms": round(duration * 1000),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _record_failure(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
job_id: Any,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
async with session_factory() as db:
|
||||
job = await db.get(Job, job_id, with_for_update=True)
|
||||
if job is None: # pragma: no cover — job deleted underneath us
|
||||
return
|
||||
job.attempts += 1
|
||||
job.last_error = safe_error(exc, limit=500)
|
||||
if job.attempts >= MAX_ATTEMPTS:
|
||||
job.status = JobStatus.failed
|
||||
metrics.inc("jobs_exhausted_total", {"type": job.type})
|
||||
else:
|
||||
job.status = JobStatus.pending
|
||||
job.run_after = datetime.now(UTC) + backoff_delay(job.attempts)
|
||||
metrics.inc("jobs_retried_total", {"type": job.type})
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def _update_depth_gauge(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
async with session_factory() as db:
|
||||
depth = (
|
||||
await db.execute(
|
||||
select(func.count(Job.id)).where(Job.status == JobStatus.pending)
|
||||
)
|
||||
).scalar_one()
|
||||
metrics.set_gauge("jobs_queue_depth", float(depth))
|
||||
|
||||
|
||||
async def run_queue(
|
||||
stop_event: asyncio.Event,
|
||||
session_factory: async_sessionmaker[AsyncSession] = async_session_factory,
|
||||
) -> None:
|
||||
poll_seconds = get_settings().job_poll_seconds
|
||||
logger.info("job queue started", extra={"event": "queue_started"})
|
||||
while not stop_event.is_set():
|
||||
worked = False
|
||||
try:
|
||||
worked = await process_one(session_factory)
|
||||
await _update_depth_gauge(session_factory)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"queue iteration failed",
|
||||
extra={"event": "queue_error", "error": safe_error(exc)},
|
||||
)
|
||||
if not worked:
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
logger.info("job queue stopped", extra={"event": "queue_stopped"})
|
||||
@@ -0,0 +1,347 @@
|
||||
"""The ONLY code that talks to LLM endpoints.
|
||||
|
||||
Exactly three functions: chat_stream, chat_json, embed. Three model roles
|
||||
(chat / utility / embedding), each base_url + api_key + model from settings.
|
||||
The openai SDK is used purely as a client for OpenAI-compatible endpoints
|
||||
(llama.cpp locally, cloud APIs in production).
|
||||
|
||||
Logging policy: metadata only — prompts and responses are logged ONLY at
|
||||
DEBUG level behind PABLAN_DEBUG_LOG_PROMPTS=true (never in production).
|
||||
LLMError messages are sanitized and never contain content.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from functools import lru_cache
|
||||
from typing import Any, Literal, TypeVar
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from app.config import get_settings
|
||||
from app.llm.errors import LLMError, llm_error
|
||||
from app.llm.gate import slot
|
||||
from app.llm.overrides import env_defaults, get_config
|
||||
from app.metrics import metrics
|
||||
|
||||
logger = logging.getLogger("pablan.llm")
|
||||
|
||||
Role = Literal["chat", "utility", "embedding"]
|
||||
ChatMessage = dict[str, str]
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
# Turn off a reasoning model's hidden thinking. Latency-critical calls (a
|
||||
# refinement fires on a typing pause) want the answer, not the deliberation:
|
||||
# ~1s instead of ~10s with no quality loss on mechanical rewrites. Endpoints
|
||||
# and templates that do not know the parameter ignore it.
|
||||
NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
_RETRY_INSTRUCTION = (
|
||||
"Your previous reply did not match the required JSON schema. "
|
||||
"Reply again with ONLY valid JSON matching the schema — no prose."
|
||||
)
|
||||
|
||||
|
||||
def _http_client_factory() -> httpx.AsyncClient | None:
|
||||
"""Tests override this to inject an ASGI transport."""
|
||||
return None
|
||||
|
||||
|
||||
def role_config(role: Role) -> tuple[str, str, str]:
|
||||
"""Effective endpoint config: the DB row, seeded from `.env` at first
|
||||
start (see app/llm/overrides.py — "bootstrap, then DB").
|
||||
|
||||
The `or env` fallbacks are a safety net, not the model: they cover the
|
||||
window before `load_config()` has run (early startup, tests that never
|
||||
touch the table) and a field an admin blanked. In a bootstrapped
|
||||
instance the stored value always wins.
|
||||
"""
|
||||
stored = get_config(role)
|
||||
env = env_defaults(role)
|
||||
return (
|
||||
stored.base_url or env.base_url or "",
|
||||
stored.api_key or env.api_key or "",
|
||||
stored.model or env.model or "",
|
||||
)
|
||||
|
||||
|
||||
def _build_client(base_url: str, api_key: str) -> AsyncOpenAI:
|
||||
kwargs: dict[str, Any] = {
|
||||
"base_url": base_url,
|
||||
"api_key": api_key,
|
||||
"timeout": get_settings().llm_timeout_seconds,
|
||||
# One SDK retry: llama.cpp closes idle keep-alive connections, and
|
||||
# the first call on a stale connection fails with APIConnectionError.
|
||||
# (SDK-internal retries are not separately metered.)
|
||||
"max_retries": 1,
|
||||
}
|
||||
http_client = _http_client_factory()
|
||||
if http_client is not None:
|
||||
kwargs["http_client"] = http_client
|
||||
return AsyncOpenAI(**kwargs)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _client_for(role: Role) -> AsyncOpenAI:
|
||||
base_url, api_key, _ = role_config(role)
|
||||
return _build_client(base_url, api_key)
|
||||
|
||||
|
||||
def rebuild_clients() -> None:
|
||||
"""Apply changed endpoint config without a restart: the cached clients
|
||||
hold the old base_url and key, so they must go."""
|
||||
_client_for.cache_clear()
|
||||
|
||||
|
||||
async def probe(
|
||||
role: Role,
|
||||
*,
|
||||
base_url: str | None = None,
|
||||
api_key: str | None = None,
|
||||
model: str | None = None,
|
||||
) -> None:
|
||||
"""Smallest possible call against a candidate config, so an admin can
|
||||
test an endpoint before saving it. Raises LLMError on failure."""
|
||||
effective_url, effective_key, effective_model = role_config(role)
|
||||
client = _build_client(base_url or effective_url, api_key or effective_key)
|
||||
target = model or effective_model
|
||||
started = time.monotonic()
|
||||
try:
|
||||
if role == "embedding":
|
||||
await client.embeddings.create(model=target, input=["ping"])
|
||||
else:
|
||||
stream = await client.chat.completions.create(
|
||||
model=target,
|
||||
messages=[{"role": "user", "content": "ping"}],
|
||||
max_tokens=1,
|
||||
stream=True,
|
||||
)
|
||||
async for _ in stream:
|
||||
break
|
||||
except Exception as exc:
|
||||
raise llm_error("probe", role, exc, started) from None
|
||||
|
||||
|
||||
async def list_models(
|
||||
role: Role,
|
||||
*,
|
||||
base_url: str | None = None,
|
||||
api_key: str | None = None,
|
||||
) -> list[str]:
|
||||
"""Ask an endpoint what it serves (`GET /v1/models`).
|
||||
|
||||
Server-side on purpose: the credentials must never leave the backend,
|
||||
and the browser has no business talking to the model endpoint at all.
|
||||
|
||||
Not every OpenAI-compatible server implements the route, so a failure
|
||||
here is ordinary rather than exceptional — the caller degrades to a
|
||||
free-text model field. Raises LLMError so the caller can distinguish
|
||||
"no such route" from "wrong credentials".
|
||||
"""
|
||||
effective_url, effective_key, _ = role_config(role)
|
||||
client = _build_client(base_url or effective_url, api_key or effective_key)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
page = await client.models.list()
|
||||
except Exception as exc:
|
||||
raise llm_error("list_models", role, exc, started) from None
|
||||
# Ids only, sorted for a stable dropdown. Model ids are configuration,
|
||||
# not content, so they may be returned and logged by count.
|
||||
return sorted({model.id for model in page.data if getattr(model, "id", None)})
|
||||
|
||||
|
||||
def _record(
|
||||
role: Role,
|
||||
kind: str,
|
||||
status: str,
|
||||
started: float,
|
||||
usage: Any = None,
|
||||
**extra_fields: Any,
|
||||
) -> None:
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("llm_calls_total", {"role": role, "kind": kind, "status": status})
|
||||
metrics.observe("llm_call_seconds", duration, {"role": role, "kind": kind})
|
||||
extra: dict[str, Any] = {
|
||||
"event": "llm_call",
|
||||
"role": role,
|
||||
"kind": kind,
|
||||
"status": status,
|
||||
"duration_ms": round(duration * 1000),
|
||||
**extra_fields,
|
||||
}
|
||||
if usage is not None:
|
||||
prompt_tokens = getattr(usage, "prompt_tokens", None)
|
||||
completion_tokens = getattr(usage, "completion_tokens", None)
|
||||
if prompt_tokens:
|
||||
metrics.inc(
|
||||
"llm_tokens_total", {"role": role, "direction": "prompt"}, prompt_tokens
|
||||
)
|
||||
extra["prompt_tokens"] = prompt_tokens
|
||||
if completion_tokens:
|
||||
metrics.inc(
|
||||
"llm_tokens_total",
|
||||
{"role": role, "direction": "completion"},
|
||||
completion_tokens,
|
||||
)
|
||||
extra["completion_tokens"] = completion_tokens
|
||||
logger.info("llm call", extra=extra)
|
||||
|
||||
|
||||
def _debug_log_content(label: str, content: Any) -> None:
|
||||
if get_settings().debug_log_prompts:
|
||||
logger.debug("llm content", extra={"label": label, "content": content})
|
||||
|
||||
|
||||
async def chat_stream(
|
||||
messages: list[ChatMessage],
|
||||
*,
|
||||
role: Role = "chat",
|
||||
temperature: float | None = None,
|
||||
max_tokens: int | None = None,
|
||||
extra_body: dict[str, Any] | None = None,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Stream a chat completion as text deltas.
|
||||
|
||||
`extra_body` is passed through to the endpoint verbatim — used to reach
|
||||
non-standard OpenAI-compatible parameters such as
|
||||
`{"chat_template_kwargs": {"enable_thinking": False}}`, which turns off a
|
||||
reasoning model's hidden thinking for latency-critical calls. Only
|
||||
`delta.content` is ever yielded, so a reasoning channel never leaks into
|
||||
the output regardless.
|
||||
"""
|
||||
base_url, _, model = role_config(role)
|
||||
_debug_log_content("chat_stream.messages", messages)
|
||||
options: dict[str, Any] = {}
|
||||
if temperature is not None:
|
||||
options["temperature"] = temperature
|
||||
if max_tokens is not None:
|
||||
options["max_tokens"] = max_tokens
|
||||
if extra_body is not None:
|
||||
options["extra_body"] = extra_body
|
||||
|
||||
started = time.monotonic()
|
||||
status = "ok"
|
||||
usage = None
|
||||
try:
|
||||
# The slot is held until the last token: a streaming completion
|
||||
# occupies its server slot for its whole life (app/llm/gate.py).
|
||||
async with slot(base_url, role):
|
||||
stream = await _client_for(role).chat.completions.create(
|
||||
model=model,
|
||||
messages=messages, # type: ignore[arg-type]
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
**options,
|
||||
)
|
||||
async for chunk in stream:
|
||||
if chunk.usage is not None:
|
||||
usage = chunk.usage
|
||||
if chunk.choices and chunk.choices[0].delta.content:
|
||||
yield chunk.choices[0].delta.content
|
||||
except GeneratorExit:
|
||||
status = "aborted"
|
||||
raise
|
||||
except LLMError:
|
||||
status = "error"
|
||||
raise
|
||||
except Exception as exc:
|
||||
status = "error"
|
||||
raise llm_error("chat_stream", role, exc, started) from None
|
||||
finally:
|
||||
_record(role, "chat_stream", status, started, usage)
|
||||
|
||||
|
||||
async def chat_json(
|
||||
messages: list[ChatMessage],
|
||||
schema: type[T],
|
||||
*,
|
||||
role: Role = "utility",
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int | None = None,
|
||||
extra_body: dict[str, Any] | None = None,
|
||||
) -> T:
|
||||
"""Structured output: response_format JSON schema + validation + one retry.
|
||||
|
||||
`extra_body` is passed through verbatim (e.g.
|
||||
`{"chat_template_kwargs": {"enable_thinking": False}}` to skip a reasoning
|
||||
model's hidden thinking on latency-sensitive utility calls)."""
|
||||
base_url, _, model = role_config(role)
|
||||
response_format = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": schema.__name__,
|
||||
"schema": schema.model_json_schema(),
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
options: dict[str, Any] = {"temperature": temperature}
|
||||
if max_tokens is not None:
|
||||
options["max_tokens"] = max_tokens
|
||||
if extra_body is not None:
|
||||
options["extra_body"] = extra_body
|
||||
|
||||
attempt_messages = list(messages)
|
||||
for attempt in (1, 2):
|
||||
_debug_log_content("chat_json.messages", attempt_messages)
|
||||
started = time.monotonic()
|
||||
usage = None
|
||||
try:
|
||||
async with slot(base_url, role):
|
||||
response = await _client_for(role).chat.completions.create(
|
||||
model=model,
|
||||
messages=attempt_messages, # type: ignore[arg-type]
|
||||
response_format=response_format, # type: ignore[arg-type]
|
||||
**options,
|
||||
)
|
||||
usage = response.usage
|
||||
content = response.choices[0].message.content or ""
|
||||
result = schema.model_validate_json(content)
|
||||
_record(role, "chat_json", "ok", started, usage, attempt=attempt)
|
||||
return result
|
||||
except ValidationError:
|
||||
_record(role, "chat_json", "invalid", started, usage, attempt=attempt)
|
||||
_debug_log_content("chat_json.invalid_response", content)
|
||||
attempt_messages = [
|
||||
*attempt_messages,
|
||||
{"role": "assistant", "content": content},
|
||||
{"role": "user", "content": _RETRY_INSTRUCTION},
|
||||
]
|
||||
except LLMError:
|
||||
_record(role, "chat_json", "error", started, usage, attempt=attempt)
|
||||
raise
|
||||
except Exception as exc:
|
||||
_record(role, "chat_json", "error", started, usage, attempt=attempt)
|
||||
raise llm_error("chat_json", role, exc, started, attempt=attempt) from None
|
||||
|
||||
raise LLMError(
|
||||
f"chat_json failed (role={role}): response did not match schema "
|
||||
f"{schema.__name__} after retry",
|
||||
role=role,
|
||||
kind="chat_json",
|
||||
status="invalid",
|
||||
cause_type="ValidationError",
|
||||
duration_ms=round((time.monotonic() - started) * 1000),
|
||||
attempt=2,
|
||||
)
|
||||
|
||||
|
||||
async def embed(texts: list[str], *, role: Role = "embedding") -> list[list[float]]:
|
||||
"""Embed a batch of texts; order of results matches the input order."""
|
||||
base_url, _, model = role_config(role)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
async with slot(base_url, role):
|
||||
response = await _client_for(role).embeddings.create(
|
||||
model=model, input=texts
|
||||
)
|
||||
except LLMError:
|
||||
_record(role, "embed", "error", started, batch_size=len(texts))
|
||||
raise
|
||||
except Exception as exc:
|
||||
_record(role, "embed", "error", started, batch_size=len(texts))
|
||||
raise llm_error("embed", role, exc, started) from None
|
||||
_record(role, "embed", "ok", started, response.usage, batch_size=len(texts))
|
||||
ordered = sorted(response.data, key=lambda item: item.index)
|
||||
return [item.embedding for item in ordered]
|
||||
@@ -0,0 +1,101 @@
|
||||
"""What it means when an endpoint does not answer.
|
||||
|
||||
Separate from `client.py` because eight modules catch this and none of them
|
||||
talk to an endpoint: routers, modes and the authoring code only need to know
|
||||
what went wrong and how to say it. The client itself stays the one place that
|
||||
CALLS an endpoint.
|
||||
|
||||
Nothing here ever carries content — not the prompt, not the reply, not the
|
||||
original exception's message. A failure is described by class name, status
|
||||
code and duration, which is everything a log line may hold (rule 12).
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
# Exception class names that mean "nothing answered at the other end" versus
|
||||
# "the other end is there but not ready for us". The SDK wraps both, so the
|
||||
# class name is all we have: APITimeoutError subclasses APIConnectionError,
|
||||
# which is why timeouts are matched first.
|
||||
_TIMEOUT_CAUSES = frozenset(
|
||||
{"APITimeoutError", "ReadTimeout", "PoolTimeout", "TimeoutError"}
|
||||
)
|
||||
_CONNECTION_CAUSES = frozenset(
|
||||
{"APIConnectionError", "ConnectError", "ConnectTimeout", "RemoteProtocolError"}
|
||||
)
|
||||
# Server said "come back later" (rate limit, no free slot, model still loading).
|
||||
_BUSY_STATUS = frozenset({408, 429, 503, 504})
|
||||
# Server said "not with these credentials / not this model".
|
||||
_SETUP_STATUS = frozenset({401, 403, 404})
|
||||
|
||||
|
||||
class LLMError(Exception):
|
||||
"""Sanitized LLM failure: structured metadata for debugging —
|
||||
never content, never original exception messages.
|
||||
|
||||
Fields: role, kind, status ("error" | "invalid"), cause_type (original
|
||||
exception CLASS NAME only), status_code (HTTP, if any), duration_ms,
|
||||
attempt (chat_json: 1 or 2).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
role: str,
|
||||
kind: str,
|
||||
status: str,
|
||||
cause_type: str | None = None,
|
||||
status_code: int | None = None,
|
||||
duration_ms: int | None = None,
|
||||
attempt: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.role = role
|
||||
self.kind = kind
|
||||
self.status = status
|
||||
self.cause_type = cause_type
|
||||
self.status_code = status_code
|
||||
self.duration_ms = duration_ms
|
||||
self.attempt = attempt
|
||||
|
||||
@property
|
||||
def code(self) -> str:
|
||||
"""The API error code for this failure — the ONE place an endpoint
|
||||
failure is classified, so every caller reports the same reason and the
|
||||
frontend can phrase it (`docs/api-protocol.md`).
|
||||
|
||||
`llm_busy` and `llm_unreachable` are worth telling apart: the first is
|
||||
worth retrying in a moment, the second needs someone to start the
|
||||
endpoint.
|
||||
"""
|
||||
if self.status_code in _BUSY_STATUS:
|
||||
return "llm_busy"
|
||||
if self.status_code in _SETUP_STATUS:
|
||||
return "llm_misconfigured"
|
||||
if self.cause_type in _TIMEOUT_CAUSES:
|
||||
return "llm_busy"
|
||||
if self.cause_type in _CONNECTION_CAUSES:
|
||||
return "llm_unreachable"
|
||||
return "llm_failed"
|
||||
|
||||
|
||||
def llm_error(
|
||||
kind: str,
|
||||
role: str,
|
||||
exc: Exception,
|
||||
started: float,
|
||||
*,
|
||||
attempt: int | None = None,
|
||||
) -> LLMError:
|
||||
"""Wrap whatever the SDK raised, keeping only what may be logged."""
|
||||
status_code = getattr(exc, "status_code", None)
|
||||
return LLMError(
|
||||
f"{kind} failed (role={role}): {type(exc).__name__}",
|
||||
role=role,
|
||||
kind=kind,
|
||||
status="error",
|
||||
cause_type=type(exc).__name__,
|
||||
status_code=status_code if isinstance(status_code, int) else None,
|
||||
duration_ms=round((time.monotonic() - started) * 1000),
|
||||
attempt=attempt,
|
||||
)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""How many requests Pablan lets an endpoint see at once.
|
||||
|
||||
A self-hosted llama.cpp server has a fixed number of parallel slots. Sending
|
||||
more than that does not make it faster: the extra requests sit in the server's
|
||||
own queue where Pablan can neither see nor bound them, and every one of them
|
||||
counts against the HTTP timeout. Two colleagues chatting while a reindex runs
|
||||
is enough to turn a working instance into one where everything times out at
|
||||
once.
|
||||
|
||||
So the waiting happens here instead, in front of the endpoint:
|
||||
|
||||
- **One gate per endpoint, not per role.** chat and utility usually point at
|
||||
the same server (they do in the shipped `.env`), and it is the SERVER that
|
||||
has the slots. Keying by base_url is what makes the limit real.
|
||||
- **A bounded wait.** A caller waits at most `llm_queue_wait_seconds` for a
|
||||
slot and then fails as `llm_busy` — a fast, honest "try again" instead of a
|
||||
two-minute timeout that looks like a broken endpoint.
|
||||
- **A bounded queue.** Past `llm_max_queued` waiters the gate stops admitting:
|
||||
when far more work has arrived than the endpoint can absorb, the useful
|
||||
answer is "busy", given immediately, to everyone beyond the line.
|
||||
|
||||
`llm_busy` is already the vocabulary for this (`llm/errors.py`), and the
|
||||
frontend phrases it as "the model is busy" — so a queue rejection reaches the
|
||||
user as the same, correct sentence as a 429 from a cloud provider.
|
||||
|
||||
Admin diagnostics (`probe`, `list_models`) deliberately do NOT pass through
|
||||
the gate: they are single tiny calls, and an admin has to be able to test an
|
||||
endpoint precisely when it is saturated.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from app.config import get_settings
|
||||
from app.llm.errors import LLMError
|
||||
from app.metrics import metrics
|
||||
|
||||
logger = logging.getLogger("pablan.llm")
|
||||
|
||||
|
||||
class _Endpoint:
|
||||
"""The live picture of one endpoint: who is in it, who is waiting."""
|
||||
|
||||
def __init__(self, limit: int) -> None:
|
||||
self.limit = limit
|
||||
self.semaphore = asyncio.Semaphore(limit)
|
||||
self.in_flight = 0
|
||||
self.waiting = 0
|
||||
|
||||
@property
|
||||
def saturated(self) -> bool:
|
||||
return self.in_flight >= self.limit
|
||||
|
||||
|
||||
_endpoints: dict[str, _Endpoint] = {}
|
||||
|
||||
|
||||
def _busy_error(role: str, reason: str, waited: float) -> LLMError:
|
||||
"""A queue rejection, in the same shape as an endpoint's own 503 — the
|
||||
caller classifies it through `LLMError.code` like any other failure."""
|
||||
metrics.inc("llm_queue_rejected_total", {"role": role, "reason": reason})
|
||||
logger.info(
|
||||
"llm request not admitted",
|
||||
extra={
|
||||
"event": "llm_queue_rejected",
|
||||
"role": role,
|
||||
"reason": reason,
|
||||
"waited_ms": round(waited * 1000),
|
||||
},
|
||||
)
|
||||
return LLMError(
|
||||
f"endpoint busy (role={role}): {reason}",
|
||||
role=role,
|
||||
kind="queue",
|
||||
status="error",
|
||||
# 503 is what a saturated endpoint says itself, and what maps to
|
||||
# `llm_busy`. Keeping the queue's own rejection in that vocabulary
|
||||
# means one reason reaches the user, not two.
|
||||
status_code=503,
|
||||
)
|
||||
|
||||
|
||||
def _endpoint_for(base_url: str) -> _Endpoint:
|
||||
limit = max(1, get_settings().llm_max_parallel)
|
||||
endpoint = _endpoints.get(base_url)
|
||||
if endpoint is None or endpoint.limit != limit:
|
||||
# A changed limit (settings reloaded in a test) rebuilds the gate.
|
||||
# In-flight callers hold the old semaphore and still release it.
|
||||
endpoint = _Endpoint(limit)
|
||||
_endpoints[base_url] = endpoint
|
||||
return endpoint
|
||||
|
||||
|
||||
def endpoint_busy(base_url: str) -> bool:
|
||||
"""Is every slot on this endpoint taken right now?
|
||||
|
||||
Read by the query mode so a waiting turn can SAY it is waiting instead of
|
||||
showing a frozen cursor. Advisory: by the time the caller acquires, a slot
|
||||
may well have freed.
|
||||
"""
|
||||
endpoint = _endpoints.get(base_url)
|
||||
return endpoint is not None and endpoint.saturated
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def slot(base_url: str, role: str) -> AsyncIterator[None]:
|
||||
"""Hold one of the endpoint's slots for the whole call.
|
||||
|
||||
For a stream that means until the last token: a streaming completion
|
||||
occupies its server slot until it ends, and releasing early would let the
|
||||
gate admit work the endpoint has no room for.
|
||||
"""
|
||||
settings = get_settings()
|
||||
endpoint = _endpoint_for(base_url)
|
||||
|
||||
if endpoint.saturated and endpoint.waiting >= max(0, settings.llm_max_queued):
|
||||
raise _busy_error(role, "queue_full", 0.0)
|
||||
|
||||
started = time.monotonic()
|
||||
endpoint.waiting += 1
|
||||
metrics.set_gauge("llm_queue_waiting", float(endpoint.waiting), {"role": role})
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
endpoint.semaphore.acquire(), timeout=settings.llm_queue_wait_seconds
|
||||
)
|
||||
except TimeoutError:
|
||||
raise _busy_error(role, "queue_timeout", time.monotonic() - started) from None
|
||||
finally:
|
||||
endpoint.waiting -= 1
|
||||
metrics.set_gauge("llm_queue_waiting", float(endpoint.waiting), {"role": role})
|
||||
|
||||
waited = time.monotonic() - started
|
||||
if waited > 0.01:
|
||||
metrics.observe("llm_queue_wait_seconds", waited, {"role": role})
|
||||
endpoint.in_flight += 1
|
||||
metrics.set_gauge("llm_in_flight", float(endpoint.in_flight), {"role": role})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
endpoint.in_flight -= 1
|
||||
metrics.set_gauge("llm_in_flight", float(endpoint.in_flight), {"role": role})
|
||||
endpoint.semaphore.release()
|
||||
|
||||
|
||||
def reset() -> None:
|
||||
"""Drop every gate. Tests only — a live gate holds waiters."""
|
||||
_endpoints.clear()
|
||||
@@ -0,0 +1,143 @@
|
||||
"""The LLM endpoint configuration, as the process sees it.
|
||||
|
||||
**Bootstrap, then DB.** On the very first start the `PABLAN_*` environment
|
||||
variables are copied into `llm_settings`, one row per role. From that
|
||||
moment the table is the truth: later `.env` edits are ignored, because a
|
||||
configuration an admin can change in the UI and a configuration the
|
||||
deployment can change underneath them cannot both be authoritative. The
|
||||
environment stays reachable as the value a field can be *reset* to, which
|
||||
is what `env_defaults()` is for.
|
||||
|
||||
The configuration lives in a module-level cache because `_role_config` is a
|
||||
hot, synchronous function on every LLM call — it cannot await a query. The
|
||||
cache is filled at startup and refreshed whenever an admin writes, which is
|
||||
also when the OpenAI clients are rebuilt.
|
||||
|
||||
Single-process by design: the customer stack pins `--workers 1` (see
|
||||
architecture.md), so there is exactly one cache to refresh. A multi-worker
|
||||
deployment would need a notification channel instead.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models import LLMSetting
|
||||
|
||||
logger = logging.getLogger("pablan.llm")
|
||||
|
||||
ROLES = ("chat", "utility", "embedding")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoleConfig:
|
||||
"""One role's stored configuration. A None field means the column is
|
||||
empty, which after bootstrap only happens if an admin blanked it."""
|
||||
|
||||
base_url: str | None = None
|
||||
model: str | None = None
|
||||
api_key: str | None = None
|
||||
|
||||
|
||||
_config: dict[str, RoleConfig] = {}
|
||||
|
||||
|
||||
def env_defaults(role: str) -> RoleConfig:
|
||||
"""What `.env` says for this role — the value "reset to .env" restores.
|
||||
|
||||
Read live rather than remembered from bootstrap: an admin who fixes a
|
||||
typo in `.env` and resets the field should get the corrected value, not
|
||||
the one that was wrong at install time.
|
||||
"""
|
||||
settings = get_settings()
|
||||
base_url, api_key, model = {
|
||||
"chat": (settings.chat_base_url, settings.chat_api_key, settings.chat_model),
|
||||
"utility": (
|
||||
settings.utility_base_url,
|
||||
settings.utility_api_key,
|
||||
settings.utility_model,
|
||||
),
|
||||
"embedding": (
|
||||
settings.embedding_base_url,
|
||||
settings.embedding_api_key,
|
||||
settings.embedding_model,
|
||||
),
|
||||
}[role]
|
||||
return RoleConfig(base_url=base_url, model=model, api_key=api_key)
|
||||
|
||||
|
||||
_FIELDS = ("base_url", "model", "api_key")
|
||||
|
||||
|
||||
async def bootstrap_llm_settings(db: AsyncSession) -> int:
|
||||
"""Copy the environment into any field that still defers to it.
|
||||
|
||||
Runs at every startup, but only ever fills blanks: a field is written
|
||||
exactly when it is flagged `*_from_env` AND currently empty. That is
|
||||
true for a fresh install (no rows yet) and for a field an upgrade
|
||||
marked as still belonging to `.env`, and false for anything an admin
|
||||
has typed, which is never touched.
|
||||
|
||||
Returns the number of fields written.
|
||||
"""
|
||||
rows = {row.role: row for row in (await db.execute(select(LLMSetting))).scalars()}
|
||||
written = 0
|
||||
for role in ROLES:
|
||||
row = rows.get(role)
|
||||
if row is None:
|
||||
# Flags set explicitly rather than left to the column defaults:
|
||||
# those only materialise on flush, and the loop below reads them
|
||||
# before that.
|
||||
row = LLMSetting(
|
||||
role=role,
|
||||
base_url_from_env=True,
|
||||
model_from_env=True,
|
||||
api_key_from_env=True,
|
||||
)
|
||||
db.add(row)
|
||||
defaults = env_defaults(role)
|
||||
for field in _FIELDS:
|
||||
if not getattr(row, f"{field}_from_env") or getattr(row, field):
|
||||
continue
|
||||
setattr(row, field, getattr(defaults, field) or None)
|
||||
written += 1
|
||||
if written:
|
||||
await db.commit()
|
||||
# Counts and roles only — never the values, one of which is a key.
|
||||
# NB: not `created` — logging reserves that name on
|
||||
# LogRecord and raises KeyError when an `extra` key collides with it.
|
||||
logger.info(
|
||||
"llm settings bootstrapped",
|
||||
extra={"event": "llm_bootstrap", "fields_written": written},
|
||||
)
|
||||
return written
|
||||
|
||||
|
||||
async def load_config(db: AsyncSession) -> None:
|
||||
"""Re-read every stored row. Call after any write."""
|
||||
rows = (await db.execute(select(LLMSetting))).scalars().all()
|
||||
_config.clear()
|
||||
_config.update(
|
||||
{
|
||||
row.role: RoleConfig(
|
||||
base_url=row.base_url, model=row.model, api_key=row.api_key
|
||||
)
|
||||
for row in rows
|
||||
}
|
||||
)
|
||||
logger.info(
|
||||
"llm settings loaded",
|
||||
extra={"event": "llm_settings_loaded", "roles": sorted(_config)},
|
||||
)
|
||||
|
||||
|
||||
def get_config(role: str) -> RoleConfig:
|
||||
return _config.get(role, RoleConfig())
|
||||
|
||||
|
||||
def clear() -> None:
|
||||
"""Drop the cache — used by tests between cases."""
|
||||
_config.clear()
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Structured JSON logging (stdlib only).
|
||||
|
||||
Logging policy: log lines carry metadata only — never
|
||||
prompts, LLM responses, user messages or document text. Exceptions are
|
||||
reduced to their type plus a sanitized message; SQLAlchemy statement/param
|
||||
dumps are stripped because parameters can contain user content.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from contextvars import ContextVar
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
correlation_id: ContextVar[str | None] = ContextVar("correlation_id", default=None)
|
||||
conversation_id: ContextVar[str | None] = ContextVar("conversation_id", default=None)
|
||||
|
||||
# LogRecord attributes that are not user-supplied extras.
|
||||
_STANDARD_ATTRS = frozenset(
|
||||
{
|
||||
"args",
|
||||
"asctime",
|
||||
"created",
|
||||
"exc_info",
|
||||
"exc_text",
|
||||
"filename",
|
||||
"funcName",
|
||||
"levelname",
|
||||
"levelno",
|
||||
"lineno",
|
||||
"message",
|
||||
"module",
|
||||
"msecs",
|
||||
"msg",
|
||||
"name",
|
||||
"pathname",
|
||||
"process",
|
||||
"processName",
|
||||
"relativeCreated",
|
||||
"stack_info",
|
||||
"taskName",
|
||||
"thread",
|
||||
"threadName",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def safe_error(exc: BaseException, limit: int = 300) -> str:
|
||||
"""Exception text that is safe to log or persist (no content leaks).
|
||||
|
||||
SQLAlchemy appends "[SQL: ...] [parameters: (...)]" to its messages;
|
||||
parameters can contain user content, so everything from "[SQL" on is cut.
|
||||
"""
|
||||
text = str(exc).split("[SQL", 1)[0].strip()
|
||||
return f"{type(exc).__name__}: {text[:limit]}"
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, Any] = {
|
||||
"ts": datetime.fromtimestamp(record.created, tz=UTC).isoformat(
|
||||
timespec="milliseconds"
|
||||
),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
}
|
||||
cid = correlation_id.get()
|
||||
if cid:
|
||||
payload["correlation_id"] = cid
|
||||
conv = conversation_id.get()
|
||||
if conv:
|
||||
payload["conversation_id"] = conv
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in _STANDARD_ATTRS and not key.startswith("_"):
|
||||
payload[key] = value
|
||||
if record.exc_info and record.exc_info[1] is not None:
|
||||
payload["error"] = safe_error(record.exc_info[1])
|
||||
return json.dumps(payload, default=str)
|
||||
|
||||
|
||||
# These third-party loggers dump request/response bodies at DEBUG — with
|
||||
# prompts and user content in them, so they stay capped at INFO unless
|
||||
# content debug logging is explicitly enabled.
|
||||
_CONTENT_DEBUG_LOGGERS = ("openai", "httpx", "httpcore")
|
||||
|
||||
|
||||
def apply_content_log_guard() -> None:
|
||||
level = logging.DEBUG if get_settings().debug_log_prompts else logging.INFO
|
||||
for name in _CONTENT_DEBUG_LOGGERS:
|
||||
logging.getLogger(name).setLevel(level)
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger()
|
||||
root.handlers = [handler]
|
||||
root.setLevel(get_settings().log_level.upper())
|
||||
apply_content_log_guard()
|
||||
@@ -0,0 +1,59 @@
|
||||
import asyncio
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request, Response
|
||||
|
||||
from app.api import api_router
|
||||
from app.db import async_session_factory
|
||||
from app.errors import register_exception_handlers
|
||||
from app.help_import import import_help_documents
|
||||
from app.ingestion.handlers import ensure_retention_scheduled
|
||||
from app.ingestion.queue import run_queue
|
||||
from app.llm.overrides import bootstrap_llm_settings
|
||||
from app.llm.overrides import load_config as load_llm_config
|
||||
from app.log import correlation_id, setup_logging
|
||||
from app.prompts.overrides import load_config as load_prompt_config
|
||||
from app.template_catalog import seed_starter_templates
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
setup_logging()
|
||||
async with async_session_factory() as db:
|
||||
await ensure_retention_scheduled(db)
|
||||
await seed_starter_templates(db)
|
||||
await import_help_documents(db)
|
||||
await bootstrap_llm_settings(db)
|
||||
await load_llm_config(db)
|
||||
await load_prompt_config(db)
|
||||
stop_event = asyncio.Event()
|
||||
queue_task = asyncio.create_task(run_queue(stop_event))
|
||||
yield
|
||||
stop_event.set()
|
||||
try:
|
||||
await asyncio.wait_for(queue_task, timeout=10)
|
||||
except TimeoutError: # pragma: no cover — a handler refused to finish
|
||||
queue_task.cancel()
|
||||
|
||||
|
||||
app = FastAPI(title="Pablan", version="0.1.0", lifespan=lifespan)
|
||||
register_exception_handlers(app)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_correlation_id(
|
||||
request: Request, call_next: Callable[[Request], Awaitable[Response]]
|
||||
) -> Response:
|
||||
cid = request.headers.get("x-request-id") or uuid.uuid4().hex[:16]
|
||||
token = correlation_id.set(cid)
|
||||
try:
|
||||
response = await call_next(request)
|
||||
finally:
|
||||
correlation_id.reset(token)
|
||||
response.headers["x-request-id"] = cid
|
||||
return response
|
||||
|
||||
|
||||
app.include_router(api_router)
|
||||
@@ -0,0 +1,92 @@
|
||||
"""In-process metrics registry — no dependencies, single event loop.
|
||||
|
||||
Counters, gauges and histogram summaries (count/sum/min/max), labeled.
|
||||
Exposed as JSON via GET /api/admin/metrics; a Prometheus text exporter would
|
||||
sit on top of this registry rather than replace it.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
LabelKey = tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
def _key(labels: dict[str, str] | None) -> LabelKey:
|
||||
return tuple(sorted((labels or {}).items()))
|
||||
|
||||
|
||||
@dataclass
|
||||
class HistogramData:
|
||||
count: int = 0
|
||||
total: float = 0.0
|
||||
minimum: float | None = None
|
||||
maximum: float | None = None
|
||||
|
||||
|
||||
class MetricsRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._counters: dict[str, dict[LabelKey, float]] = defaultdict(
|
||||
lambda: defaultdict(float)
|
||||
)
|
||||
self._gauges: dict[str, dict[LabelKey, float]] = defaultdict(dict)
|
||||
self._histograms: dict[str, dict[LabelKey, HistogramData]] = defaultdict(dict)
|
||||
|
||||
def inc(
|
||||
self, name: str, labels: dict[str, str] | None = None, value: float = 1.0
|
||||
) -> None:
|
||||
self._counters[name][_key(labels)] += value
|
||||
|
||||
def set_gauge(
|
||||
self, name: str, value: float, labels: dict[str, str] | None = None
|
||||
) -> None:
|
||||
self._gauges[name][_key(labels)] = value
|
||||
|
||||
def observe(
|
||||
self, name: str, value: float, labels: dict[str, str] | None = None
|
||||
) -> None:
|
||||
data = self._histograms[name].setdefault(_key(labels), HistogramData())
|
||||
data.count += 1
|
||||
data.total += value
|
||||
data.minimum = value if data.minimum is None else min(data.minimum, value)
|
||||
data.maximum = value if data.maximum is None else max(data.maximum, value)
|
||||
|
||||
def snapshot(self) -> dict[str, Any]:
|
||||
return {
|
||||
"counters": {
|
||||
name: [
|
||||
{"labels": dict(key), "value": value}
|
||||
for key, value in sorted(series.items())
|
||||
]
|
||||
for name, series in sorted(self._counters.items())
|
||||
},
|
||||
"gauges": {
|
||||
name: [
|
||||
{"labels": dict(key), "value": value}
|
||||
for key, value in sorted(series.items())
|
||||
]
|
||||
for name, series in sorted(self._gauges.items())
|
||||
},
|
||||
"histograms": {
|
||||
name: [
|
||||
{
|
||||
"labels": dict(key),
|
||||
"count": data.count,
|
||||
"sum": data.total,
|
||||
"min": data.minimum,
|
||||
"max": data.maximum,
|
||||
"avg": data.total / data.count if data.count else None,
|
||||
}
|
||||
for key, data in sorted(series.items())
|
||||
]
|
||||
for name, series in sorted(self._histograms.items())
|
||||
},
|
||||
}
|
||||
|
||||
def reset(self) -> None:
|
||||
self._counters.clear()
|
||||
self._gauges.clear()
|
||||
self._histograms.clear()
|
||||
|
||||
|
||||
metrics = MetricsRegistry()
|
||||
@@ -0,0 +1,58 @@
|
||||
from app.models.auth_session import AuthSession
|
||||
from app.models.base import Base
|
||||
from app.models.conversation import Conversation, Message
|
||||
from app.models.department import Department
|
||||
from app.models.document import (
|
||||
EMBEDDING_DIM,
|
||||
Chunk,
|
||||
DocPermission,
|
||||
Document,
|
||||
DocumentEvent,
|
||||
ReviewRequest,
|
||||
)
|
||||
from app.models.enums import (
|
||||
AccessReason,
|
||||
ConversationMode,
|
||||
ConversationStatus,
|
||||
DocumentEventAction,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
JobStatus,
|
||||
MessageRole,
|
||||
PermissionLevel,
|
||||
UserRole,
|
||||
)
|
||||
from app.models.job import Job
|
||||
from app.models.llm_setting import LLMSetting
|
||||
from app.models.prompt_setting import PromptSetting
|
||||
from app.models.template import Template
|
||||
from app.models.user import User
|
||||
|
||||
__all__ = [
|
||||
"EMBEDDING_DIM",
|
||||
"AuthSession",
|
||||
"Base",
|
||||
"Chunk",
|
||||
"Conversation",
|
||||
"ConversationMode",
|
||||
"ConversationStatus",
|
||||
"Department",
|
||||
"DocPermission",
|
||||
"Document",
|
||||
"DocumentEvent",
|
||||
"ReviewRequest",
|
||||
"DocumentEventAction",
|
||||
"DocumentStatus",
|
||||
"DocumentVisibility",
|
||||
"Job",
|
||||
"LLMSetting",
|
||||
"JobStatus",
|
||||
"Message",
|
||||
"MessageRole",
|
||||
"AccessReason",
|
||||
"PermissionLevel",
|
||||
"PromptSetting",
|
||||
"Template",
|
||||
"User",
|
||||
"UserRole",
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
class AuthSession(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""Server-side login session; the id doubles as the cookie token."""
|
||||
|
||||
__tablename__ = "auth_sessions"
|
||||
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
user: Mapped[User] = relationship(lazy="joined")
|
||||
@@ -0,0 +1,25 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
# Fetch server-generated defaults (created_at/updated_at) via RETURNING
|
||||
# at flush time — otherwise the async session would need a lazy refresh
|
||||
# on attribute access, which raises MissingGreenlet outside a greenlet.
|
||||
__mapper_args__ = {"eager_defaults": True}
|
||||
|
||||
|
||||
class UUIDPrimaryKeyMixin:
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now()
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Enum, ForeignKey, Text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
from app.models.enums import ConversationMode, ConversationStatus, MessageRole
|
||||
|
||||
|
||||
class Conversation(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""Chat thread. The only core mode is query (RAG Q&A); EE adds insight.
|
||||
|
||||
Capture is no longer a conversation — it writes a Document directly (see
|
||||
app/authoring/) — so this table holds no per-turn engine state any more.
|
||||
"""
|
||||
|
||||
__tablename__ = "conversations"
|
||||
|
||||
mode: Mapped[ConversationMode] = mapped_column(
|
||||
Enum(ConversationMode, native_enum=False, length=32)
|
||||
)
|
||||
status: Mapped[ConversationStatus] = mapped_column(
|
||||
Enum(ConversationStatus, native_enum=False, length=32),
|
||||
default=ConversationStatus.active,
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
messages: Mapped[list["Message"]] = relationship(
|
||||
back_populates="conversation",
|
||||
cascade="all, delete-orphan",
|
||||
order_by="Message.created_at",
|
||||
)
|
||||
|
||||
|
||||
class Message(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
__tablename__ = "messages"
|
||||
|
||||
conversation_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("conversations.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
role: Mapped[MessageRole] = mapped_column(
|
||||
Enum(MessageRole, native_enum=False, length=32)
|
||||
)
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
# Assistant turns snapshot their citations here ({"sources": [...]}) so
|
||||
# they survive reload and re-indexing — chunks are disposable, the
|
||||
# rendered citation is not.
|
||||
meta: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSONB, default=dict, server_default="{}"
|
||||
)
|
||||
|
||||
conversation: Mapped[Conversation] = relationship(back_populates="messages")
|
||||
@@ -0,0 +1,10 @@
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
|
||||
|
||||
class Department(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
__tablename__ = "departments"
|
||||
|
||||
name: Mapped[str] = mapped_column(String(200), unique=True)
|
||||
@@ -0,0 +1,199 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
Computed,
|
||||
DateTime,
|
||||
Enum,
|
||||
ForeignKey,
|
||||
Index,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
from app.models.enums import (
|
||||
DocumentEventAction,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
PermissionLevel,
|
||||
)
|
||||
|
||||
# Fixed by the embedding model (bge-m3). Changing the embedding model to a
|
||||
# different dimension requires a migration plus reindex_all.
|
||||
EMBEDDING_DIM = 1024
|
||||
|
||||
|
||||
class Document(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""Markdown is the source of truth; chunks are disposable derivatives."""
|
||||
|
||||
__tablename__ = "documents"
|
||||
|
||||
title: Mapped[str] = mapped_column(String(500))
|
||||
status: Mapped[DocumentStatus] = mapped_column(
|
||||
Enum(DocumentStatus, native_enum=False, length=32),
|
||||
default=DocumentStatus.draft,
|
||||
)
|
||||
visibility: Mapped[DocumentVisibility] = mapped_column(
|
||||
Enum(DocumentVisibility, native_enum=False, length=32),
|
||||
default=DocumentVisibility.department,
|
||||
)
|
||||
content_md: Mapped[str] = mapped_column(Text)
|
||||
meta: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
|
||||
# Built-in help documents: shipped with the product, re-imported from
|
||||
# files on every start, and neither editable nor deletable in the UI.
|
||||
is_builtin: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false"
|
||||
)
|
||||
# SET NULL: documents must survive their author leaving the company.
|
||||
author_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL")
|
||||
)
|
||||
department_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("departments.id", ondelete="SET NULL")
|
||||
)
|
||||
|
||||
chunks: Mapped[list["Chunk"]] = relationship(
|
||||
back_populates="document", cascade="all, delete-orphan"
|
||||
)
|
||||
# Loaded with every document: whether a question is open decides who may
|
||||
# edit it and how it is marked wherever it appears, so it is never a
|
||||
# separate lookup a caller could forget.
|
||||
reviews: Mapped[list["ReviewRequest"]] = relationship(
|
||||
back_populates="document",
|
||||
cascade="all, delete-orphan",
|
||||
lazy="selectin",
|
||||
order_by="ReviewRequest.created_at",
|
||||
)
|
||||
|
||||
@property
|
||||
def open_reviews(self) -> list["ReviewRequest"]:
|
||||
return [review for review in self.reviews if review.resolved_at is None]
|
||||
|
||||
|
||||
class Chunk(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
__tablename__ = "chunks"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("document_id", "chunk_index"),
|
||||
Index(
|
||||
"ix_chunks_embedding_hnsw",
|
||||
"embedding",
|
||||
postgresql_using="hnsw",
|
||||
postgresql_ops={"embedding": "vector_cosine_ops"},
|
||||
),
|
||||
Index("ix_chunks_tsv", "tsv", postgresql_using="gin"),
|
||||
)
|
||||
|
||||
document_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("documents.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
chunk_index: Mapped[int] = mapped_column()
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
embedding: Mapped[list[float]] = mapped_column(Vector(EMBEDDING_DIM))
|
||||
# The heading path is part of what the chunk says: a section reading
|
||||
# "Solldruck 180 bar" never repeats which machine it belongs to, so a
|
||||
# keyword query naming the machine has to reach it through its path.
|
||||
tsv = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
"to_tsvector('german'::regconfig, "
|
||||
"content || ' ' || coalesce(meta->>'heading_path', ''))",
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
meta: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
|
||||
|
||||
document: Mapped[Document] = relationship(back_populates="chunks")
|
||||
|
||||
|
||||
class DocumentEvent(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""An append-only audit record: who did what to a document, and when.
|
||||
|
||||
Content-bearing actions (created / edited) snapshot the Markdown source of
|
||||
truth so a past version can be viewed or diffed; the disposable chunks are
|
||||
never snapshotted. `actor_id` is SET NULL so the record survives its actor
|
||||
leaving the company, exactly like author_id on the document itself. Events
|
||||
cascade with the document (DB-level ON DELETE CASCADE).
|
||||
"""
|
||||
|
||||
__tablename__ = "document_events"
|
||||
|
||||
document_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("documents.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# Who acted. Nullable so the trail outlives the actor's account.
|
||||
actor_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL")
|
||||
)
|
||||
action: Mapped[DocumentEventAction] = mapped_column(
|
||||
Enum(DocumentEventAction, native_enum=False, length=32)
|
||||
)
|
||||
# Frozen Markdown for content-bearing events (created / edited); NULL for
|
||||
# pure transitions (published / archived / a review being asked or
|
||||
# answered).
|
||||
content_md: Mapped[str | None] = mapped_column(Text)
|
||||
title: Mapped[str | None] = mapped_column(String(500))
|
||||
# The document's visibility as of this event — cheap, so always recorded.
|
||||
visibility: Mapped[DocumentVisibility | None] = mapped_column(
|
||||
Enum(DocumentVisibility, native_enum=False, length=32)
|
||||
)
|
||||
meta: Mapped[dict[str, Any] | None] = mapped_column(JSONB)
|
||||
|
||||
|
||||
class ReviewRequest(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
""" "Please look at this" — a question about a document, addressed to a
|
||||
colleague.
|
||||
|
||||
Deliberately not a status. A document can be published AND have an open
|
||||
question about it ("do the holiday numbers still hold?"), which is exactly
|
||||
the case where readers most need to know: an open request marks the
|
||||
document wherever it appears, including the sources under a chat answer.
|
||||
|
||||
Resolving is the reviewer's answer. Editing the document first is normal —
|
||||
being asked to review is what grants the right to edit it.
|
||||
"""
|
||||
|
||||
__tablename__ = "review_requests"
|
||||
|
||||
document_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("documents.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# Both SET NULL: a request outlives the accounts on either side of it.
|
||||
requester_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL")
|
||||
)
|
||||
reviewer_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
# What exactly to look at. Optional: "please check this" is a valid ask.
|
||||
question: Mapped[str | None] = mapped_column(Text)
|
||||
# NULL while open. The pair (resolved_at, resolved_by) is the answer.
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
resolved_by_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="SET NULL")
|
||||
)
|
||||
|
||||
document: Mapped["Document"] = relationship(back_populates="reviews")
|
||||
|
||||
|
||||
class DocPermission(TimestampMixin, Base):
|
||||
"""Additional department read grants on top of documents.visibility."""
|
||||
|
||||
__tablename__ = "doc_permissions"
|
||||
|
||||
document_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("documents.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
department_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey("departments.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
level: Mapped[PermissionLevel] = mapped_column(
|
||||
Enum(PermissionLevel, native_enum=False, length=32),
|
||||
default=PermissionLevel.read,
|
||||
)
|
||||
@@ -0,0 +1,87 @@
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class UserRole(StrEnum):
|
||||
member = "member"
|
||||
admin = "admin"
|
||||
|
||||
|
||||
class ConversationMode(StrEnum):
|
||||
query = "query"
|
||||
insight = "insight" # EE insights mode registers itself
|
||||
|
||||
|
||||
class ConversationStatus(StrEnum):
|
||||
active = "active"
|
||||
completed = "completed"
|
||||
abandoned = "abandoned"
|
||||
|
||||
|
||||
class MessageRole(StrEnum):
|
||||
user = "user"
|
||||
assistant = "assistant"
|
||||
system = "system"
|
||||
|
||||
|
||||
class DocumentStatus(StrEnum):
|
||||
"""Where a document stands.
|
||||
|
||||
Three states, because publishing is the author's own decision: a draft is
|
||||
private, a published document is visible and indexed, an archived one is
|
||||
neither. Uncertainty about CONTENT is not a status — it is an open review
|
||||
request (`ReviewRequest`), which can sit on a published document too.
|
||||
"""
|
||||
|
||||
draft = "draft"
|
||||
published = "published"
|
||||
archived = "archived"
|
||||
|
||||
|
||||
class DocumentVisibility(StrEnum):
|
||||
public = "public"
|
||||
department = "department"
|
||||
restricted = "restricted"
|
||||
|
||||
|
||||
class PermissionLevel(StrEnum):
|
||||
read = "read"
|
||||
|
||||
|
||||
class AccessReason(StrEnum):
|
||||
"""Why a document is visible to the requesting user.
|
||||
|
||||
API-only (never stored): computed per request so the UI can explain
|
||||
access instead of leaving visibility rules implicit.
|
||||
"""
|
||||
|
||||
author = "author"
|
||||
public = "public"
|
||||
department = "department"
|
||||
granted = "granted"
|
||||
# Only reason: somebody asked this user to check the document. It ends
|
||||
# with their answer, which is why it is worth naming separately.
|
||||
review = "review"
|
||||
|
||||
|
||||
class DocumentEventAction(StrEnum):
|
||||
"""A recorded step in a document's audit history.
|
||||
|
||||
Content-bearing actions (created / edited) snapshot the Markdown source of
|
||||
truth; the rest record only who did what and when.
|
||||
"""
|
||||
|
||||
created = "created"
|
||||
edited = "edited"
|
||||
published = "published"
|
||||
archived = "archived"
|
||||
visibility_changed = "visibility_changed"
|
||||
# Someone was asked to check the content, and someone answered.
|
||||
review_requested = "review_requested"
|
||||
review_resolved = "review_resolved"
|
||||
|
||||
|
||||
class JobStatus(StrEnum):
|
||||
pending = "pending"
|
||||
running = "running"
|
||||
done = "done"
|
||||
failed = "failed"
|
||||
@@ -0,0 +1,27 @@
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import DateTime, Enum, Index, String, Text, func
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
from app.models.enums import JobStatus
|
||||
|
||||
|
||||
class Job(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""Postgres-backed background queue, claimed via FOR UPDATE SKIP LOCKED."""
|
||||
|
||||
__tablename__ = "jobs"
|
||||
__table_args__ = (Index("ix_jobs_status_run_after", "status", "run_after"),)
|
||||
|
||||
type: Mapped[str] = mapped_column(String(100))
|
||||
payload: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
|
||||
status: Mapped[JobStatus] = mapped_column(
|
||||
Enum(JobStatus, native_enum=False, length=32), default=JobStatus.pending
|
||||
)
|
||||
run_after: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now()
|
||||
)
|
||||
attempts: Mapped[int] = mapped_column(default=0)
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -0,0 +1,36 @@
|
||||
from sqlalchemy import Boolean, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
|
||||
|
||||
class LLMSetting(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""Per-role endpoint configuration, edited in the admin UI.
|
||||
|
||||
One row per model role. The rows are created once, at first start, from
|
||||
the `PABLAN_CHAT_*` / `PABLAN_UTILITY_*` / `PABLAN_EMBEDDING_*` environment
|
||||
variables; from then on **this table is the truth** and later `.env`
|
||||
edits are ignored (see docs/architecture.md, "bootstrap, then DB").
|
||||
|
||||
The `*_from_env` flags record where each field's current value came
|
||||
from, so the UI can say "taken from .env" or "changed here" per field
|
||||
and offer a reset. They are not a fallback mechanism: the value itself
|
||||
always lives in the column next to them. Tracking the provenance
|
||||
explicitly beats comparing against the current environment, which would
|
||||
mislabel every field the moment someone edits `.env` after bootstrap.
|
||||
|
||||
The api_key is stored in plaintext because it has to be replayed to the
|
||||
endpoint on every call — there is nothing to compare a hash against.
|
||||
It is never returned by the API and never logged.
|
||||
"""
|
||||
|
||||
__tablename__ = "llm_settings"
|
||||
|
||||
role: Mapped[str] = mapped_column(String(32), unique=True)
|
||||
base_url: Mapped[str | None] = mapped_column(String(500))
|
||||
model: Mapped[str | None] = mapped_column(String(200))
|
||||
api_key: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
base_url_from_env: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
model_from_env: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
api_key_from_env: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
@@ -0,0 +1,23 @@
|
||||
from sqlalchemy import String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
|
||||
|
||||
class PromptSetting(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
"""An admin override for a shipped system prompt.
|
||||
|
||||
Every prompt has a CODE default (`app/prompts/defaults.py`); a row here
|
||||
exists only when an admin has changed one. `content` is the full replacement
|
||||
text. Applied without a restart via a module-level cache
|
||||
(`app/prompts/overrides.py`), refreshed on every write — like the LLM
|
||||
settings, and single-process by design (`--workers 1`). Resetting a prompt
|
||||
deletes its row, so the code default takes over again. Unlike LLM settings
|
||||
there is no `.env` layer: prompts have no environment representation, so the
|
||||
reset target is the code default rather than the environment.
|
||||
"""
|
||||
|
||||
__tablename__ = "prompt_settings"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
@@ -0,0 +1,18 @@
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
|
||||
|
||||
class Template(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
__tablename__ = "templates"
|
||||
|
||||
# Every row here belongs to the customer and is editable. Blueprints
|
||||
# shipped with the product stay on disk in templates/ until an admin
|
||||
# adds one (app/template_catalog.py) — there is no read-only template.
|
||||
name: Mapped[str] = mapped_column(String(200))
|
||||
version: Mapped[str] = mapped_column(String(20))
|
||||
config: Mapped[dict[str, Any]] = mapped_column(JSONB)
|
||||
@@ -0,0 +1,29 @@
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import Enum, ForeignKey, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
|
||||
from app.models.department import Department
|
||||
from app.models.enums import UserRole
|
||||
|
||||
|
||||
class User(UUIDPrimaryKeyMixin, TimestampMixin, Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(200))
|
||||
role: Mapped[UserRole] = mapped_column(
|
||||
Enum(UserRole, native_enum=False, length=32), default=UserRole.member
|
||||
)
|
||||
password_hash: Mapped[str] = mapped_column(String(255))
|
||||
# Interface language. NULL follows the browser's Accept-Language; a value
|
||||
# pins it. One column rather than a preferences table: this is the only
|
||||
# preference that has to follow the person across devices — the theme is
|
||||
# per-device and lives in localStorage.
|
||||
locale: Mapped[str | None] = mapped_column(String(5))
|
||||
department_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey("departments.id", ondelete="SET NULL")
|
||||
)
|
||||
|
||||
department: Mapped[Department | None] = relationship(lazy="joined")
|
||||
@@ -0,0 +1,6 @@
|
||||
from app.modes.query import QueryMode
|
||||
from app.modes.registry import get_mode, register_mode, registered_modes
|
||||
|
||||
register_mode(QueryMode())
|
||||
|
||||
__all__ = ["get_mode", "register_mode", "registered_modes"]
|
||||
@@ -0,0 +1,96 @@
|
||||
"""The Mode protocol.
|
||||
|
||||
Every interaction type implements `Mode` and yields `ModeEvent`s; the
|
||||
conversations router converts them 1:1 into SSE. Modes know nothing about
|
||||
HTTP; routers know nothing about mode logic.
|
||||
|
||||
Capture is NOT a Mode: it is writing into a Document directly (see
|
||||
`app/authoring/`), not a conversation. The only core Mode is query (RAG
|
||||
Q&A); EE registers the insights mode.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Conversation
|
||||
|
||||
|
||||
@dataclass
|
||||
class Token:
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceChunk:
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
heading_path: str
|
||||
excerpt: str = ""
|
||||
# Whether this passage was actually passed to the model (grounding the
|
||||
# answer), or only retrieved and then dropped as too weak (the no-answer
|
||||
# path). Drives the "?" context inspector; the cited-source badges show
|
||||
# only `used` chunks.
|
||||
used: bool = True
|
||||
# The document has an unanswered request to check it: readable, but not
|
||||
# settled. Marked on the citation, because trusting an answer means
|
||||
# trusting what it leaned on.
|
||||
review_pending: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Sources:
|
||||
chunks: list[SourceChunk] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StateChanged:
|
||||
"""Progress signal for the UI — metadata only.
|
||||
|
||||
Query mode reports a phase (e.g. "searching" / "no_answer") and a count
|
||||
(documents found). No content ever rides this frame.
|
||||
"""
|
||||
|
||||
phase: str
|
||||
count: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Done:
|
||||
"""Emitted by the ROUTER after persisting the assistant message.
|
||||
Modes normally end their iterator instead of yielding this."""
|
||||
|
||||
message_id: uuid.UUID | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Error:
|
||||
"""Why the turn failed, as a code the frontend phrases (CLAUDE.md: the
|
||||
backend never renders UI-language strings)."""
|
||||
|
||||
code: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Degraded:
|
||||
"""No model could be reached, so this turn has no generated answer: the
|
||||
accompanying `Sources` are what a plain full-text search found, for the
|
||||
user to read themselves. `code` is the endpoint failure that caused it
|
||||
(`LLMError.code`); the frontend says what it means."""
|
||||
|
||||
code: str
|
||||
|
||||
|
||||
ModeEvent = Token | Sources | StateChanged | Done | Error | Degraded
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Mode(Protocol):
|
||||
name: str
|
||||
|
||||
def handle_turn(
|
||||
self, conversation: Conversation, user_message: str, db: AsyncSession
|
||||
) -> AsyncIterator[ModeEvent]: ...
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Prompt rendering for modes — always natural language, never raw YAML or
|
||||
JSON dumps.
|
||||
|
||||
The base texts (the assistant's system prompt, the no-sources note) are
|
||||
admin-editable via `app/prompts/overrides.py::get_prompt`; the query mode reads
|
||||
`query_system` directly and `render_context_turn` reads `query_no_sources`.
|
||||
"""
|
||||
|
||||
from app.prompts.overrides import get_prompt
|
||||
from app.rag.retrieval import SearchResult
|
||||
|
||||
|
||||
def render_context_turn(results: list[SearchResult], question: str) -> str:
|
||||
"""The final user turn: the retrieval for THIS question, then the question.
|
||||
|
||||
Deliberately NOT part of the system prompt: keeping the excerpts here lets
|
||||
the system prompt AND the conversation history stay byte-identical across a
|
||||
conversation's turns, so the endpoint's prompt cache reuses them and only
|
||||
this turn's excerpts are fresh work (docs/architecture.md, prompt caching).
|
||||
"""
|
||||
if not results:
|
||||
# Refusing to answer a greeting because retrieval found nothing makes the
|
||||
# assistant feel broken. It answers from general knowledge, just never as
|
||||
# if that were company policy (the UI labels these source-free).
|
||||
return f"{get_prompt('query_no_sources')}\n\n{question}"
|
||||
blocks = [
|
||||
f"[{index}] {result.heading_path or result.title}\n{result.content}"
|
||||
for index, result in enumerate(results, start=1)
|
||||
]
|
||||
excerpts = "\n\n---\n\n".join(blocks)
|
||||
return f"Knowledge base excerpts:\n\n{excerpts}\n\nQuestion:\n{question}"
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Query mode: permission-filtered retrieval → grounded streamed answer."""
|
||||
|
||||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.authoring.context import summarize_transcript
|
||||
from app.llm.client import chat_stream, role_config
|
||||
from app.llm.errors import LLMError
|
||||
from app.llm.gate import endpoint_busy
|
||||
from app.models import Conversation, MessageRole, User
|
||||
from app.modes.base import (
|
||||
Degraded,
|
||||
ModeEvent,
|
||||
SourceChunk,
|
||||
Sources,
|
||||
StateChanged,
|
||||
Token,
|
||||
)
|
||||
from app.modes.prompts import render_context_turn
|
||||
from app.prompts.overrides import get_prompt
|
||||
from app.rag.retrieval import (
|
||||
SearchResult,
|
||||
results_are_low_confidence,
|
||||
search,
|
||||
text_search,
|
||||
)
|
||||
|
||||
HISTORY_TURNS = 8
|
||||
EXCERPT_CHARS = 280
|
||||
TOP_K = 5
|
||||
|
||||
|
||||
def _topic_transcript(conversation: Conversation, user_message: str) -> str:
|
||||
"""The recent turns plus the current message, as a transcript for the
|
||||
topic summary. Returns '' when there is no earlier context — a first
|
||||
message that misses is a genuine no-answer, not a lost topic."""
|
||||
turns = [
|
||||
(message.role, message.content)
|
||||
for message in conversation.messages
|
||||
if message.role in (MessageRole.user, MessageRole.assistant)
|
||||
]
|
||||
# The current message may or may not already be persisted into
|
||||
# `conversation.messages`; append it only if it is not the last turn.
|
||||
if not turns or turns[-1] != (MessageRole.user, user_message):
|
||||
turns.append((MessageRole.user, user_message))
|
||||
if len(turns) < 2:
|
||||
return ""
|
||||
return "\n".join(
|
||||
f"{'User' if role is MessageRole.user else 'Assistant'}: {content}"
|
||||
for role, content in turns[-HISTORY_TURNS:]
|
||||
)
|
||||
|
||||
|
||||
# Markdown reduced to prose, in the order the rules have to fire.
|
||||
_TABLE_DIVIDER = re.compile(r"^\s*\|?[\s:|-]*-[\s:|-]*\|?\s*$", re.MULTILINE)
|
||||
_BLOCK_MARKER = re.compile(r"^\s{0,3}(#{1,6}|[-*+]|\d+\.|>)\s+", re.MULTILINE)
|
||||
_FENCE = re.compile(r"^\s*```.*$", re.MULTILINE)
|
||||
_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
|
||||
_LINK = re.compile(r"\[([^\]]+)\]\([^)]*\)")
|
||||
_EMPHASIS = re.compile(r"(\*\*\*|\*\*|\*|___|__|`)(?=\S)(.+?)(?<=\S)\1", re.DOTALL)
|
||||
# Single underscores need word boundaries that the asterisk forms do not:
|
||||
# `_kursiv_` is emphasis, but `result_document_id` is an identifier and the
|
||||
# corpus is full of them.
|
||||
_UNDERSCORE_EMPHASIS = re.compile(r"(?<!\w)_(?=\S)(.+?)(?<=\S)_(?!\w)", re.DOTALL)
|
||||
|
||||
|
||||
def excerpt(content: str) -> str:
|
||||
"""Short preview of a cited chunk for the citation popover.
|
||||
|
||||
The user has already passed the permission filter for this chunk, so
|
||||
showing it back is safe, but keep it short: it is a hint, not the
|
||||
document.
|
||||
|
||||
Markdown is reduced to prose rather than rendered. The popover is a
|
||||
~320px hover surface showing a FRAGMENT, and rendering a fragment goes
|
||||
wrong in exactly the cases that matter: a cited table becomes a real
|
||||
table squeezed into the popover, a cited section heading renders at h2
|
||||
size, and a list item arrives without its list. Clean prose answers the
|
||||
only question the popover exists for, "is this the passage I want?".
|
||||
Clicking the badge opens the document in the side panel, which renders
|
||||
the Markdown properly through the sanitizing renderer.
|
||||
|
||||
Keeping this plain text is also what lets `Tooltip` promise that its
|
||||
content is never markup: document text
|
||||
reaches it through here and nowhere else.
|
||||
"""
|
||||
text = _FENCE.sub("", content)
|
||||
# Divider rows first: they are pure punctuation and survive every other
|
||||
# rule as a run of dashes and pipes.
|
||||
text = _TABLE_DIVIDER.sub("", text)
|
||||
text = _IMAGE.sub(r"\1", text)
|
||||
text = _LINK.sub(r"\1", text)
|
||||
text = _BLOCK_MARKER.sub("", text)
|
||||
# Two passes: the outer run of `**bold _and_ italic**` has to go before
|
||||
# the inner one is reachable.
|
||||
for _ in range(2):
|
||||
text = _EMPHASIS.sub(r"\2", text)
|
||||
text = _UNDERSCORE_EMPHASIS.sub(r"\1", text)
|
||||
# Remaining cell walls become sentence-ish separators, so a cited table
|
||||
# reads as "Code · Meaning · Action" instead of a wall of pipes.
|
||||
text = re.sub(r"\s*\|\s*", " · ", text)
|
||||
text = re.sub(r"(?: · )+", " · ", text)
|
||||
# A leading or trailing separator is what an empty first or last table
|
||||
# cell leaves behind. Regex rather than str.strip: the latter treats the
|
||||
# argument as a character set, which is not what this means.
|
||||
text = re.sub(r"^(?:\s|·)+|(?:\s|·)+$", "", text)
|
||||
|
||||
flattened = " ".join(text.split())
|
||||
if len(flattened) <= EXCERPT_CHARS:
|
||||
return flattened
|
||||
cut = flattened[:EXCERPT_CHARS]
|
||||
head, separator, _ = cut.rpartition(" ")
|
||||
return (head if separator else cut) + "…"
|
||||
|
||||
|
||||
def _source(result: SearchResult, *, used: bool) -> SourceChunk:
|
||||
return SourceChunk(
|
||||
document_id=result.document_id,
|
||||
title=result.title,
|
||||
heading_path=result.heading_path,
|
||||
excerpt=excerpt(result.content),
|
||||
used=used,
|
||||
review_pending=result.review_pending,
|
||||
)
|
||||
|
||||
|
||||
class QueryMode:
|
||||
name = "query"
|
||||
|
||||
async def handle_turn(
|
||||
self, conversation: Conversation, user_message: str, db: AsyncSession
|
||||
) -> AsyncIterator[ModeEvent]:
|
||||
user = await db.get(User, conversation.user_id)
|
||||
assert user is not None
|
||||
|
||||
yield StateChanged(phase="searching")
|
||||
try:
|
||||
results = await search(db, user_message, user=user, top_k=TOP_K)
|
||||
except LLMError:
|
||||
# No embedding endpoint. The German full-text index finds documents
|
||||
# on its own (keywords, not meaning), and the chat role is
|
||||
# configured separately, so the turn can still end in a real answer.
|
||||
results = await text_search(db, user_message, user=user, top_k=TOP_K)
|
||||
else:
|
||||
if results_are_low_confidence(results):
|
||||
# A follow-up ("Hi", "and my earlier question?") loses the topic
|
||||
# on its own, but the conversation's subject can recover it.
|
||||
# Runs only on the low-confidence path, so a clear question pays
|
||||
# no extra latency. (Eval: topic-summary 4/4 vs raw message 1/4.)
|
||||
transcript = _topic_transcript(conversation, user_message)
|
||||
if transcript:
|
||||
topic = await summarize_transcript(transcript)
|
||||
if topic and topic != user_message:
|
||||
retry = await search(db, topic, user=user, top_k=TOP_K)
|
||||
if not results_are_low_confidence(retry):
|
||||
results = retry
|
||||
|
||||
grounded = not results_are_low_confidence(results)
|
||||
if grounded:
|
||||
yield StateChanged(phase="results", count=len(results))
|
||||
else:
|
||||
# Nothing solid to ground on: the answer gets no sources and the UI
|
||||
# offers to capture the missing knowledge instead.
|
||||
yield StateChanged(phase="no_answer", count=0)
|
||||
# Every retrieved passage is reported for the "?" context inspector;
|
||||
# `used` marks the ones that actually reached the prompt. On a no-answer
|
||||
# they are all unused, which is exactly what explains "why no answer".
|
||||
yield Sources(chunks=[_source(result, used=grounded) for result in results])
|
||||
|
||||
history = [
|
||||
{"role": message.role.value, "content": message.content}
|
||||
for message in conversation.messages[-HISTORY_TURNS:]
|
||||
if message.role in (MessageRole.user, MessageRole.assistant)
|
||||
]
|
||||
# Only grounded passages ground the model; a no-answer sends none.
|
||||
prompt_results = results if grounded else []
|
||||
# Cache-friendly order: the static system prompt and the history stay
|
||||
# byte-identical across a conversation's turns (so the endpoint's prompt
|
||||
# cache reuses them); only this turn's excerpts + question are new.
|
||||
messages = [
|
||||
{"role": "system", "content": get_prompt("query_system")},
|
||||
*history,
|
||||
{
|
||||
"role": "user",
|
||||
"content": render_context_turn(prompt_results, user_message),
|
||||
},
|
||||
]
|
||||
|
||||
answered = False
|
||||
try:
|
||||
# Someone else may be holding every slot the endpoint has. Saying
|
||||
# so beats a cursor that blinks for twenty seconds; the phase
|
||||
# flips to "answering" the moment the first token arrives.
|
||||
chat_base_url, _, _ = role_config("chat")
|
||||
queued = endpoint_busy(chat_base_url)
|
||||
yield StateChanged(phase="queued" if queued else "answering")
|
||||
async for delta in chat_stream(messages, role="chat"):
|
||||
if queued and not answered:
|
||||
yield StateChanged(phase="answering")
|
||||
answered = True
|
||||
yield Token(text=delta)
|
||||
except LLMError as exc:
|
||||
if answered:
|
||||
# Half an answer is already on screen: the router keeps it and
|
||||
# reports the failure. There is nothing to fall back to.
|
||||
raise
|
||||
# No model at all. What retrieval found IS the reply now, as a plain
|
||||
# list the user opens themselves. Nothing grounded anything, so the
|
||||
# passages are re-sent unused (the second frame replaces the first).
|
||||
yield Sources(chunks=[_source(result, used=False) for result in results])
|
||||
yield Degraded(code=exc.code)
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Mode registration — also the EE extension point (the insights mode
|
||||
registers itself from ee/backend via ee_hooks)."""
|
||||
|
||||
from app.modes.base import Mode
|
||||
|
||||
_MODES: dict[str, Mode] = {}
|
||||
|
||||
|
||||
def register_mode(mode: Mode) -> None:
|
||||
_MODES[mode.name] = mode
|
||||
|
||||
|
||||
def get_mode(name: str) -> Mode | None:
|
||||
return _MODES.get(name)
|
||||
|
||||
|
||||
def registered_modes() -> list[str]:
|
||||
return sorted(_MODES)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""The shipped system prompts, as code defaults.
|
||||
|
||||
Every prompt Pablan sends has its base text here, keyed by a stable id. The
|
||||
render functions in `app/modes/prompts.py` and `app/authoring/prompts.py` read
|
||||
the *effective* value through `app/prompts/overrides.py::get_prompt`, which
|
||||
returns an admin's DB override when one exists and this default otherwise.
|
||||
|
||||
Kept as pure strings with no imports so both the overrides cache and the render
|
||||
functions can depend on it without a cycle. Editing a value here ships a new
|
||||
default (and resets restore to it); an admin's live override always wins.
|
||||
"""
|
||||
|
||||
# The query (RAG Q&A) assistant. Kept byte-identical per turn so the endpoint's
|
||||
# prompt cache reuses it — a DB override only changes on an admin write, so that
|
||||
# still holds (docs/architecture.md, prompt caching).
|
||||
QUERY_SYSTEM = """\
|
||||
You are Pablan, this company's internal knowledge assistant.
|
||||
|
||||
Answer in the language of the question. Company facts — processes, numbers,
|
||||
names, responsibilities — come only from the excerpts you are given; say when
|
||||
something is not documented rather than filling the gap. Be brief and concrete.
|
||||
"""
|
||||
|
||||
# Appended to the final user turn when retrieval found nothing relevant, so the
|
||||
# assistant still answers a greeting or general question without pretending the
|
||||
# answer is company policy.
|
||||
QUERY_NO_SOURCES = """\
|
||||
The knowledge base has nothing relevant for this message.
|
||||
|
||||
Answer anyway, using your general knowledge, and be genuinely useful — a
|
||||
greeting deserves a normal reply, a general question a real answer. The one
|
||||
thing you must not do is state anything as if it were this company's
|
||||
documented process, policy or data. Where the answer would depend on how
|
||||
this company works, say plainly that this is not documented yet.
|
||||
"""
|
||||
|
||||
# The default persona for section refinement (a template may override it per
|
||||
# document); the mechanical rules the refined section must follow.
|
||||
REFINE_PERSONA = (
|
||||
"You are a precise technical editor in a knowledge-management tool. You "
|
||||
"turn rough notes into clear, matter-of-fact documentation."
|
||||
)
|
||||
|
||||
REFINE_RULES = (
|
||||
"Rules: reply in the language the section is written in. Return ONLY the "
|
||||
"refined section as plain Markdown — no preamble, no explanation, no code "
|
||||
"fence around the whole thing, and none of the other sections. Keep a "
|
||||
"heading the section starts with unchanged. Improve clarity, grammar and "
|
||||
"structure (use a list where the content is a sequence of steps), but "
|
||||
"invent no facts: use only what the section already states. If the section "
|
||||
"is already clear, change it little."
|
||||
)
|
||||
|
||||
# The instruction that frames the retrieved grounding block during refinement;
|
||||
# the retrieved excerpts are appended after it.
|
||||
GROUNDING_FRAMING = (
|
||||
"Related knowledge already documented elsewhere (use it only to stay "
|
||||
"consistent and to reference where this section connects to it — do not "
|
||||
"copy it in and add no facts from it that the notes above do not already "
|
||||
"state):"
|
||||
)
|
||||
|
||||
# Condensing a conversation into a short search topic (the topic-summary path).
|
||||
TOPIC_SUMMARY = (
|
||||
"You condense a conversation into a short search topic for a knowledge "
|
||||
"base. Reply with a concise noun phrase (a few words) in the language of "
|
||||
"the conversation, naming what it is about. No sentence, no preamble, no "
|
||||
"quotes."
|
||||
)
|
||||
|
||||
# Suggesting a document title from its content.
|
||||
TITLE = (
|
||||
"You suggest a concise, specific title for a knowledge document, in the "
|
||||
"language of the document. Reply with the title only: a short noun phrase, "
|
||||
"no quotes, no trailing punctuation."
|
||||
)
|
||||
|
||||
DEFAULTS: dict[str, str] = {
|
||||
"query_system": QUERY_SYSTEM,
|
||||
"query_no_sources": QUERY_NO_SOURCES,
|
||||
"refine_persona": REFINE_PERSONA,
|
||||
"refine_rules": REFINE_RULES,
|
||||
"grounding_framing": GROUNDING_FRAMING,
|
||||
"topic_summary": TOPIC_SUMMARY,
|
||||
"title": TITLE,
|
||||
}
|
||||
|
||||
# Stable display/iteration order for the admin panel.
|
||||
PROMPT_KEYS: tuple[str, ...] = tuple(DEFAULTS)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""The effective system prompts, as the process sees them.
|
||||
|
||||
Prompts default to code (`app/prompts/defaults.py`); an admin may override any
|
||||
of them in `prompt_settings`, applied without a restart. This mirrors the LLM
|
||||
settings override pattern, with two simplifications: the reset target is the
|
||||
code default (prompts have no `.env` layer), and there is no bootstrap — a
|
||||
missing row simply means "use the default".
|
||||
|
||||
`get_prompt` is called while rendering a prompt, so it reads a module-level
|
||||
cache rather than awaiting a query. The cache is filled at startup and refreshed
|
||||
on every admin write. Single-process by design (`--workers 1`); a multi-worker
|
||||
deployment would need a notification channel.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import PromptSetting
|
||||
from app.prompts.defaults import DEFAULTS
|
||||
|
||||
logger = logging.getLogger("pablan.prompts")
|
||||
|
||||
# Only holds the keys an admin has actually overridden.
|
||||
_config: dict[str, str] = {}
|
||||
|
||||
|
||||
async def load_config(db: AsyncSession) -> None:
|
||||
"""Re-read every override row into the cache. Call at startup and after any
|
||||
admin write."""
|
||||
rows = (await db.execute(select(PromptSetting))).scalars().all()
|
||||
_config.clear()
|
||||
_config.update({row.key: row.content for row in rows if row.key in DEFAULTS})
|
||||
logger.info(
|
||||
"prompt settings loaded",
|
||||
extra={"event": "prompt_settings_loaded", "overridden": sorted(_config)},
|
||||
)
|
||||
|
||||
|
||||
def get_prompt(key: str) -> str:
|
||||
"""The effective prompt: an admin override if present, else the code default.
|
||||
|
||||
`key` must be a known prompt (a `KeyError` here is a programming error, not
|
||||
user input)."""
|
||||
return _config.get(key) or DEFAULTS[key]
|
||||
|
||||
|
||||
def is_overridden(key: str) -> bool:
|
||||
return key in _config
|
||||
|
||||
|
||||
def clear() -> None:
|
||||
"""Drop the cache — used by tests between cases."""
|
||||
_config.clear()
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Markdown chunking along the heading hierarchy.
|
||||
|
||||
Chunk size uses a character heuristic (~4 chars/token, target ~400 tokens);
|
||||
no tokenizer dependency — precision is not required for chunk sizing, and a
|
||||
real tokenizer would not match local model tokenizers anyway. Sections that
|
||||
exceed the cap are split at paragraph boundaries, never inside code fences.
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
# ~400 tokens at the ~4 chars/token heuristic.
|
||||
TARGET_CHUNK_CHARS = 1600
|
||||
|
||||
HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
|
||||
|
||||
HEADING_PATH_SEPARATOR = " › "
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkData:
|
||||
content: str
|
||||
heading_path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Section:
|
||||
path: list[str]
|
||||
lines: list[str]
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
return "\n".join(self.lines).strip()
|
||||
|
||||
|
||||
def _split_sections(content_md: str, title: str) -> list[_Section]:
|
||||
sections: list[_Section] = [_Section(path=[title], lines=[])]
|
||||
heading_stack: list[tuple[int, str]] = [] # (level, text)
|
||||
in_fence = False
|
||||
|
||||
for line in content_md.splitlines():
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
match = None if in_fence else HEADING_RE.match(line)
|
||||
if match:
|
||||
level = len(match.group(1))
|
||||
text = match.group(2).strip()
|
||||
while heading_stack and heading_stack[-1][0] >= level:
|
||||
heading_stack.pop()
|
||||
heading_stack.append((level, text))
|
||||
path = [title, *(heading for _, heading in heading_stack)]
|
||||
# Drop a leading H1 that just repeats the document title.
|
||||
if len(path) > 1 and path[1] == title:
|
||||
path = [title, *path[2:]]
|
||||
sections.append(_Section(path=path, lines=[line]))
|
||||
else:
|
||||
sections[-1].lines.append(line)
|
||||
|
||||
return [section for section in sections if section.text]
|
||||
|
||||
|
||||
def _split_paragraphs(text: str) -> list[str]:
|
||||
"""Split at blank lines, but never inside a ``` fence."""
|
||||
paragraphs: list[str] = []
|
||||
current: list[str] = []
|
||||
in_fence = False
|
||||
for line in text.splitlines():
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
if not line.strip() and not in_fence:
|
||||
if current:
|
||||
paragraphs.append("\n".join(current))
|
||||
current = []
|
||||
else:
|
||||
current.append(line)
|
||||
if current:
|
||||
paragraphs.append("\n".join(current))
|
||||
return paragraphs
|
||||
|
||||
|
||||
def _split_oversized(text: str) -> list[str]:
|
||||
pieces: list[str] = []
|
||||
current = ""
|
||||
for paragraph in _split_paragraphs(text):
|
||||
candidate = f"{current}\n\n{paragraph}" if current else paragraph
|
||||
if current and len(candidate) > TARGET_CHUNK_CHARS:
|
||||
pieces.append(current)
|
||||
current = paragraph
|
||||
else:
|
||||
current = candidate
|
||||
if current:
|
||||
pieces.append(current)
|
||||
return pieces
|
||||
|
||||
|
||||
def chunk_markdown(content_md: str, title: str) -> list[ChunkData]:
|
||||
"""Split a document into chunks; each chunk has exactly one heading path."""
|
||||
chunks: list[ChunkData] = []
|
||||
for section in _split_sections(content_md, title):
|
||||
heading_path = HEADING_PATH_SEPARATOR.join(section.path)
|
||||
text = section.text
|
||||
if len(text) <= TARGET_CHUNK_CHARS:
|
||||
chunks.append(ChunkData(content=text, heading_path=heading_path))
|
||||
else:
|
||||
for piece in _split_oversized(text):
|
||||
chunks.append(ChunkData(content=piece, heading_path=heading_path))
|
||||
return chunks
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Chunk (re)generation for a document — chunks are disposable derivatives.
|
||||
|
||||
A full re-index is always possible from documents alone; swapping
|
||||
the embedding model is a reindex_all away (same dimension) or a migration
|
||||
plus reindex_all (different dimension).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.llm.client import embed
|
||||
from app.metrics import metrics
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.chunking import chunk_markdown
|
||||
|
||||
logger = logging.getLogger("pablan.rag")
|
||||
|
||||
EMBED_BATCH_SIZE = 32
|
||||
|
||||
|
||||
def embedding_text(heading_path: str, content: str) -> str:
|
||||
"""What is actually embedded for a chunk: its heading path, then its text.
|
||||
|
||||
A section says "Solldruck 180 bar" and never repeats which machine it
|
||||
belongs to, so without its path the chunk is unreachable by the name the
|
||||
asker actually uses. What is STORED as `content` stays the raw section —
|
||||
the path is context for the vector, not part of the document.
|
||||
"""
|
||||
return f"{heading_path}\n\n{content}"
|
||||
|
||||
|
||||
async def reindex_document(db: AsyncSession, document: Document) -> int:
|
||||
"""Delete and regenerate all chunks for one document. Returns the count."""
|
||||
started = time.monotonic()
|
||||
chunks_data = chunk_markdown(document.content_md, document.title)
|
||||
|
||||
vectors: list[list[float]] = []
|
||||
for batch_start in range(0, len(chunks_data), EMBED_BATCH_SIZE):
|
||||
batch = chunks_data[batch_start : batch_start + EMBED_BATCH_SIZE]
|
||||
vectors.extend(
|
||||
await embed(
|
||||
[embedding_text(chunk.heading_path, chunk.content) for chunk in batch]
|
||||
)
|
||||
)
|
||||
|
||||
await db.execute(delete(Chunk).where(Chunk.document_id == document.id))
|
||||
for index, (data, vector) in enumerate(zip(chunks_data, vectors, strict=True)):
|
||||
db.add(
|
||||
Chunk(
|
||||
document_id=document.id,
|
||||
chunk_index=index,
|
||||
content=data.content,
|
||||
embedding=vector,
|
||||
meta={
|
||||
"heading_path": data.heading_path,
|
||||
# Denormalized for display/filtering — NEVER for
|
||||
# permission checks (can be stale until the next reindex).
|
||||
"department_id": (
|
||||
str(document.department_id) if document.department_id else None
|
||||
),
|
||||
"visibility": document.visibility,
|
||||
},
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("chunks_indexed_total", value=len(chunks_data))
|
||||
metrics.observe("indexing_seconds", duration)
|
||||
logger.info(
|
||||
"document indexed",
|
||||
extra={
|
||||
"event": "document_indexed",
|
||||
"document_id": str(document.id),
|
||||
"chunk_count": len(chunks_data),
|
||||
"duration_ms": round(duration * 1000),
|
||||
},
|
||||
)
|
||||
return len(chunks_data)
|
||||
|
||||
|
||||
async def remove_chunks(db: AsyncSession, document_id) -> None:
|
||||
await db.execute(delete(Chunk).where(Chunk.document_id == document_id))
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Single source of truth for who may read which document.
|
||||
|
||||
Used by BOTH the documents API and retrieval, so the permission filter that
|
||||
runs before the LLM can never drift from what the API
|
||||
exposes. Always evaluated against the live documents table — never against
|
||||
denormalized chunk meta, which can be stale between edits and reindexing.
|
||||
"""
|
||||
|
||||
from sqlalchemy import ColumnElement, and_, exists, or_, select, true
|
||||
|
||||
from app.models import (
|
||||
DocPermission,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
ReviewRequest,
|
||||
User,
|
||||
)
|
||||
|
||||
|
||||
def searchable_documents_filter(user: User) -> ColumnElement[bool]:
|
||||
"""Published documents the user may read.
|
||||
|
||||
Rules: public to everyone; department to members of the owning
|
||||
department; restricted only via doc_permissions grants. Authors always
|
||||
see their own documents.
|
||||
"""
|
||||
clauses: list[ColumnElement[bool]] = [
|
||||
Document.visibility == DocumentVisibility.public,
|
||||
Document.author_id == user.id,
|
||||
]
|
||||
if user.department_id is not None:
|
||||
clauses.append(
|
||||
and_(
|
||||
Document.visibility == DocumentVisibility.department,
|
||||
Document.department_id == user.department_id,
|
||||
)
|
||||
)
|
||||
clauses.append(
|
||||
exists(
|
||||
select(DocPermission.document_id).where(
|
||||
DocPermission.document_id == Document.id,
|
||||
DocPermission.department_id == user.department_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
return and_(Document.status == DocumentStatus.published, or_(*clauses))
|
||||
|
||||
|
||||
def readable_documents_filter(user: User) -> ColumnElement[bool]:
|
||||
"""Searchable documents plus the unpublished ones this user owns or was
|
||||
asked to check.
|
||||
|
||||
Being asked IS the grant: a reviewer must be able to open the draft they
|
||||
were pointed at. The clause lives here only, never in
|
||||
`searchable_documents_filter`, so an unpublished document still never
|
||||
reaches chat/search retrieval."""
|
||||
return or_(
|
||||
searchable_documents_filter(user),
|
||||
Document.author_id == user.id,
|
||||
open_review_for(user),
|
||||
)
|
||||
|
||||
|
||||
def open_review_for(user: User) -> ColumnElement[bool]:
|
||||
"""This user has an unanswered request to check the document."""
|
||||
return exists(
|
||||
select(ReviewRequest.document_id).where(
|
||||
ReviewRequest.document_id == Document.id,
|
||||
ReviewRequest.reviewer_id == user.id,
|
||||
ReviewRequest.resolved_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def has_open_review() -> ColumnElement[bool]:
|
||||
"""Anyone has an unanswered question about the document — what marks it as
|
||||
"may not be right yet" wherever it is shown, including chat sources."""
|
||||
return exists(
|
||||
select(ReviewRequest.document_id).where(
|
||||
ReviewRequest.document_id == Document.id,
|
||||
ReviewRequest.resolved_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def document_reader_filter(document: Document) -> ColumnElement[bool]:
|
||||
"""A `User`-table filter for who may read `document` AS IF it were
|
||||
published — the candidate set for assigning a reviewer. Inverts the read
|
||||
rules of `searchable_documents_filter` (author, public, owning department,
|
||||
granted departments), ignoring status so a still-pending document can be
|
||||
handed to a reviewer who will then be able to see it (the reviewer clause
|
||||
in `readable_documents_filter`)."""
|
||||
clauses: list[ColumnElement[bool]] = [User.id == document.author_id]
|
||||
if document.visibility == DocumentVisibility.public:
|
||||
clauses.append(true())
|
||||
elif (
|
||||
document.visibility == DocumentVisibility.department
|
||||
and document.department_id is not None
|
||||
):
|
||||
clauses.append(User.department_id == document.department_id)
|
||||
clauses.append(
|
||||
User.department_id.in_(
|
||||
select(DocPermission.department_id).where(
|
||||
DocPermission.document_id == document.id
|
||||
)
|
||||
)
|
||||
)
|
||||
return or_(*clauses)
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Hybrid retrieval: permission filter BEFORE anything else, then vector +
|
||||
German full-text candidates merged with Reciprocal Rank Fusion.
|
||||
|
||||
There is no search without a user — the permission CTE is part of the one
|
||||
SQL statement, so an unauthorized chunk is structurally impossible to
|
||||
retrieve. Queries are user content and are never logged.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Text, cast, func, literal, select, union
|
||||
from sqlalchemy.dialects.postgresql import REGCONFIG
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.llm.client import embed
|
||||
from app.metrics import metrics
|
||||
from app.models import Chunk, Document, User
|
||||
from app.rag.permissions import has_open_review, searchable_documents_filter
|
||||
|
||||
logger = logging.getLogger("pablan.rag")
|
||||
|
||||
RRF_K = 60
|
||||
CANDIDATES_PER_SOURCE = 20
|
||||
|
||||
# Calibrated against bge-m3 (2026-07): matched top
|
||||
# hits land at cosine distance ~0.34-0.45, unrelated queries at 0.50+.
|
||||
NO_ANSWER_MIN_DISTANCE = 0.45
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
chunk_id: uuid.UUID
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
heading_path: str
|
||||
content: str
|
||||
score: float
|
||||
vector_distance: float | None
|
||||
fts_match: bool
|
||||
# An unanswered request to check this document. Travels with every hit so
|
||||
# an answer can mark the source it leaned on as not-yet-settled.
|
||||
review_pending: bool = False
|
||||
|
||||
|
||||
async def search(
|
||||
db: AsyncSession,
|
||||
query: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 5,
|
||||
) -> list[SearchResult]:
|
||||
started = time.monotonic()
|
||||
query_vector = (await embed([query]))[0]
|
||||
|
||||
allowed = (
|
||||
select(
|
||||
Document.id,
|
||||
Document.title,
|
||||
has_open_review().label("review_pending"),
|
||||
)
|
||||
.where(searchable_documents_filter(user))
|
||||
.cte("allowed")
|
||||
)
|
||||
|
||||
distance = Chunk.embedding.cosine_distance(query_vector)
|
||||
vec = (
|
||||
select(
|
||||
Chunk.id.label("chunk_id"),
|
||||
func.row_number().over(order_by=distance).label("rank"),
|
||||
distance.label("distance"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.order_by(distance)
|
||||
.limit(CANDIDATES_PER_SOURCE)
|
||||
.subquery("vec")
|
||||
)
|
||||
|
||||
tsquery = func.websearch_to_tsquery(cast(literal("german"), REGCONFIG), query)
|
||||
fts_order = func.ts_rank_cd(Chunk.tsv, tsquery).desc()
|
||||
fts = (
|
||||
select(
|
||||
Chunk.id.label("chunk_id"),
|
||||
func.row_number().over(order_by=fts_order).label("rank"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.where(Chunk.tsv.op("@@")(tsquery))
|
||||
.order_by(fts_order)
|
||||
.limit(CANDIDATES_PER_SOURCE)
|
||||
.subquery("fts")
|
||||
)
|
||||
|
||||
candidate_ids = union(select(vec.c.chunk_id), select(fts.c.chunk_id)).subquery(
|
||||
"ids"
|
||||
)
|
||||
score = (
|
||||
func.coalesce(1.0 / (RRF_K + vec.c.rank), 0.0)
|
||||
+ func.coalesce(1.0 / (RRF_K + fts.c.rank), 0.0)
|
||||
).label("score")
|
||||
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Chunk.id,
|
||||
Chunk.document_id,
|
||||
Chunk.content,
|
||||
Chunk.meta,
|
||||
allowed.c.title,
|
||||
allowed.c.review_pending,
|
||||
score,
|
||||
vec.c.distance,
|
||||
fts.c.rank.label("fts_rank"),
|
||||
)
|
||||
.join(candidate_ids, candidate_ids.c.chunk_id == Chunk.id)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.outerjoin(vec, vec.c.chunk_id == Chunk.id)
|
||||
.outerjoin(fts, fts.c.chunk_id == Chunk.id)
|
||||
.order_by(score.desc())
|
||||
.limit(top_k)
|
||||
)
|
||||
).all()
|
||||
|
||||
results = [
|
||||
SearchResult(
|
||||
chunk_id=row.id,
|
||||
document_id=row.document_id,
|
||||
title=row.title,
|
||||
heading_path=heading_path(row.meta),
|
||||
content=row.content,
|
||||
score=float(row.score),
|
||||
vector_distance=float(row.distance) if row.distance is not None else None,
|
||||
fts_match=row.fts_rank is not None,
|
||||
review_pending=bool(row.review_pending),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("retrieval_searches_total")
|
||||
metrics.observe("retrieval_seconds", duration)
|
||||
metrics.observe("retrieval_results", float(len(results)))
|
||||
for result in results:
|
||||
source = (
|
||||
"both"
|
||||
if result.fts_match and result.vector_distance is not None
|
||||
else ("fts" if result.fts_match else "vector")
|
||||
)
|
||||
metrics.inc("retrieval_result_source_total", {"source": source})
|
||||
logger.info(
|
||||
"retrieval",
|
||||
extra={
|
||||
"event": "retrieval",
|
||||
"duration_ms": round(duration * 1000),
|
||||
"result_count": len(results),
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _any_term_tsquery(query: str) -> Any:
|
||||
"""The fallback's query: the lexemes `websearch_to_tsquery` produces, but
|
||||
ORed instead of ANDed.
|
||||
|
||||
With no vector half to carry the recall, an AND query answers a natural
|
||||
question ("Wie läuft die Qualitätsprüfung im Wareneingang?") with nothing
|
||||
at all unless one single chunk happens to contain every word of it. ORing
|
||||
keeps the question usable and leaves the ordering to `ts_rank_cd`, which
|
||||
is what ranks a chunk matching more of the terms higher. Only the AND
|
||||
operators between groups are rewritten, so quoted phrases and exclusions
|
||||
survive. A query of nothing but stop words rewrites to an empty string,
|
||||
and NULLIF turns that into a query that matches nothing rather than a
|
||||
syntax error.
|
||||
"""
|
||||
websearch = func.websearch_to_tsquery(cast(literal("german"), REGCONFIG), query)
|
||||
lexemes = func.nullif(func.replace(cast(websearch, Text), " & ", " | "), "")
|
||||
return func.to_tsquery(cast(literal("german"), REGCONFIG), lexemes)
|
||||
|
||||
|
||||
async def text_search(
|
||||
db: AsyncSession,
|
||||
query: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 5,
|
||||
) -> list[SearchResult]:
|
||||
"""The full-text half of `search()` alone: German `tsvector` matching over
|
||||
the GIN index (an inverted index), with no embedding call.
|
||||
|
||||
This is what keeps the knowledge base searchable when no model answers at
|
||||
the configured endpoint. It finds less than the hybrid path (keywords, not
|
||||
meaning), so it is a fallback the user is told about, never a silent
|
||||
substitute. Same permission CTE as everything else — there is no search
|
||||
without a user.
|
||||
|
||||
Terms are ORed here while the hybrid path ANDs them (`_any_term_tsquery`):
|
||||
alone, a full-sentence question must not come back empty, and in the
|
||||
hybrid path the AND is what makes `fts_match` mean "the words are really
|
||||
in there" for the no-answer signal.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
allowed = (
|
||||
select(
|
||||
Document.id,
|
||||
Document.title,
|
||||
has_open_review().label("review_pending"),
|
||||
)
|
||||
.where(searchable_documents_filter(user))
|
||||
.cte("allowed")
|
||||
)
|
||||
tsquery = _any_term_tsquery(query)
|
||||
rank = func.ts_rank_cd(Chunk.tsv, tsquery)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Chunk.id,
|
||||
Chunk.document_id,
|
||||
Chunk.content,
|
||||
Chunk.meta,
|
||||
allowed.c.title,
|
||||
allowed.c.review_pending,
|
||||
rank.label("rank"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.where(Chunk.tsv.op("@@")(tsquery))
|
||||
.order_by(rank.desc())
|
||||
.limit(top_k)
|
||||
)
|
||||
).all()
|
||||
|
||||
results = [
|
||||
SearchResult(
|
||||
chunk_id=row.id,
|
||||
document_id=row.document_id,
|
||||
title=row.title,
|
||||
heading_path=heading_path(row.meta),
|
||||
content=row.content,
|
||||
score=float(row.rank),
|
||||
vector_distance=None,
|
||||
fts_match=True,
|
||||
review_pending=bool(row.review_pending),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("retrieval_text_searches_total")
|
||||
metrics.observe("retrieval_seconds", duration)
|
||||
logger.info(
|
||||
"retrieval (text only)",
|
||||
extra={
|
||||
"event": "retrieval_text",
|
||||
"duration_ms": round(duration * 1000),
|
||||
"result_count": len(results),
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def heading_path(meta: dict[str, Any] | None) -> str:
|
||||
return (meta or {}).get("heading_path", "")
|
||||
|
||||
|
||||
def results_are_low_confidence(results: list[SearchResult]) -> bool:
|
||||
"""No-answer signal: no keyword match anywhere and the best vector
|
||||
candidate is far away. Callers should not present such results as
|
||||
grounding."""
|
||||
if not results:
|
||||
return True
|
||||
top = results[0]
|
||||
return not top.fts_match and (
|
||||
top.vector_distance is None or top.vector_distance >= NO_ANSWER_MIN_DISTANCE
|
||||
)
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Similarity: "what else is close to this text", with a threshold.
|
||||
|
||||
Deliberately NOT the hybrid path. RRF produces a fusion rank, not a
|
||||
similarity, and its `vector_distance` is None for hits that surfaced only
|
||||
through full text — a threshold needs a comparable number. What both paths DO
|
||||
share is the permission filter: the same `allowed` CTE, so a suggestion can
|
||||
never point at something the caller may not read.
|
||||
|
||||
Two callers, two calibrated limits: a refinement grounds on loosely related
|
||||
material, while a duplicate check may only propose merging on a close match.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import and_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.llm.client import embed
|
||||
from app.metrics import metrics
|
||||
from app.models import Chunk, Document, User
|
||||
from app.rag.permissions import searchable_documents_filter
|
||||
from app.rag.retrieval import CANDIDATES_PER_SOURCE, heading_path
|
||||
|
||||
logger = logging.getLogger("pablan.rag")
|
||||
|
||||
# One mechanic at two moments: during capture a loose limit is useful
|
||||
# because a near-miss still makes good context, while at review time only a
|
||||
# high-confidence match may propose merging into an existing document.
|
||||
# CHANGING THE EMBEDDING MODEL MEANS RE-MEASURING ALL THREE constants;
|
||||
# tests/evals/test_duplicate_eval.py prints the numbers to do it with.
|
||||
#
|
||||
# Measured against bge-m3 on the fixture corpus (2026-07-20): drafts that
|
||||
# duplicate an existing document land at 0.116-0.274, genuinely new topics
|
||||
# at 0.402-0.486. 0.35 sits in that gap. The upper end of the duplicate
|
||||
# range comes from REAL capture drafts, which are compressed notes rather
|
||||
# than full prose and therefore sit further from their source than a
|
||||
# hand-written paraphrase does — calibrating on paraphrases alone gives a
|
||||
# threshold that misses real duplicates (it did: 0.25 missed one at 0.256).
|
||||
CAPTURE_CONTEXT_MAX_DISTANCE = 0.45
|
||||
DUPLICATE_MAX_DISTANCE = 0.35
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimilarChunk:
|
||||
chunk_id: uuid.UUID
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
heading_path: str
|
||||
content: str
|
||||
distance: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimilarDocument:
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
distance: float # the closest chunk of that document
|
||||
|
||||
|
||||
async def similar_chunks(
|
||||
db: AsyncSession,
|
||||
text: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 5,
|
||||
max_distance: float,
|
||||
exclude_builtin: bool = False,
|
||||
exclude_document_id: uuid.UUID | None = None,
|
||||
) -> list[SimilarChunk]:
|
||||
"""Pure vector neighbours of a text, permission-filtered like everything
|
||||
else — the same `allowed` CTE `search()` uses.
|
||||
|
||||
Deliberately NOT the hybrid path: RRF produces a fusion rank, not a
|
||||
similarity, and its `vector_distance` is None for hits that surfaced
|
||||
only through full text. A threshold needs a comparable number.
|
||||
|
||||
`max_distance` is keyword-only and has no default on purpose: every
|
||||
caller names one of the two calibrated constants, so "similar" means
|
||||
exactly two things in this product and both are written down.
|
||||
|
||||
`exclude_builtin` drops Pablan's own help pages. They are answerable
|
||||
through query mode on purpose (the product documents itself), but a
|
||||
capture or duplicate check asks "what does the COMPANY already know" —
|
||||
proposing to extend a help page, or telling an author their topic is
|
||||
"already documented" because a help page mentions it, is wrong.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
vector = (await embed([text]))[0]
|
||||
|
||||
allowed_filter = searchable_documents_filter(user)
|
||||
if exclude_builtin:
|
||||
allowed_filter = and_(allowed_filter, Document.is_builtin.is_(False))
|
||||
if exclude_document_id is not None:
|
||||
# A document must never ground on itself (the extend flow re-opens a
|
||||
# published document and would otherwise retrieve its own chunks).
|
||||
allowed_filter = and_(allowed_filter, Document.id != exclude_document_id)
|
||||
allowed = select(Document.id, Document.title).where(allowed_filter).cte("allowed")
|
||||
distance = Chunk.embedding.cosine_distance(vector)
|
||||
|
||||
# Threshold in Python, after ORDER BY ... LIMIT: a distance predicate in
|
||||
# WHERE fights the HNSW index, ordering and limiting is what it serves.
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Chunk.id,
|
||||
Chunk.document_id,
|
||||
Chunk.content,
|
||||
Chunk.meta,
|
||||
allowed.c.title,
|
||||
distance.label("distance"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.order_by(distance)
|
||||
.limit(top_k)
|
||||
)
|
||||
).all()
|
||||
|
||||
results = [
|
||||
SimilarChunk(
|
||||
chunk_id=row.id,
|
||||
document_id=row.document_id,
|
||||
title=row.title,
|
||||
heading_path=heading_path(row.meta),
|
||||
content=row.content,
|
||||
distance=float(row.distance),
|
||||
)
|
||||
for row in rows
|
||||
if float(row.distance) <= max_distance
|
||||
]
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("similarity_searches_total")
|
||||
metrics.observe("similarity_seconds", duration)
|
||||
logger.info(
|
||||
"similarity",
|
||||
extra={
|
||||
"event": "similarity",
|
||||
"duration_ms": round(duration * 1000),
|
||||
"candidate_count": len(rows),
|
||||
"result_count": len(results),
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def similar_documents(
|
||||
db: AsyncSession,
|
||||
text: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 3,
|
||||
max_distance: float,
|
||||
exclude_builtin: bool = False,
|
||||
) -> list[SimilarDocument]:
|
||||
"""Documents near a text, ranked by their closest chunk.
|
||||
|
||||
Overfetches chunks and groups them, so this is literally the same search
|
||||
as `similar_chunks` — one notion of "similar" in the product, not two
|
||||
implementations that drift apart.
|
||||
"""
|
||||
chunks = await similar_chunks(
|
||||
db,
|
||||
text,
|
||||
user=user,
|
||||
top_k=CANDIDATES_PER_SOURCE,
|
||||
max_distance=max_distance,
|
||||
exclude_builtin=exclude_builtin,
|
||||
)
|
||||
best: dict[uuid.UUID, SimilarDocument] = {}
|
||||
for chunk in chunks: # distance-ordered, so the first hit per document wins
|
||||
best.setdefault(
|
||||
chunk.document_id,
|
||||
SimilarDocument(
|
||||
document_id=chunk.document_id,
|
||||
title=chunk.title,
|
||||
distance=chunk.distance,
|
||||
),
|
||||
)
|
||||
return list(best.values())[:top_k]
|
||||
@@ -0,0 +1,364 @@
|
||||
"""Dev seed data: departments, users and the fixture corpus as documents with
|
||||
a plausible past. Never runs in production.
|
||||
|
||||
Seeded documents are not inserted as finished rows — they are given the
|
||||
history they would have if someone had written them in the app: an empty
|
||||
draft, one edit per section as the author works down the page, the publish,
|
||||
and the questions colleagues asked afterwards. Without that, every history
|
||||
view, diff and "recently changed" list in the dev stack is empty or lies.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth.passwords import hash_password
|
||||
from app.config import get_settings
|
||||
from app.db import async_session_factory, engine
|
||||
from app.ingestion.handlers import INDEX_DOCUMENT
|
||||
from app.ingestion.queue import enqueue
|
||||
from app.models import (
|
||||
Department,
|
||||
DocPermission,
|
||||
Document,
|
||||
DocumentEvent,
|
||||
DocumentEventAction,
|
||||
DocumentStatus,
|
||||
ReviewRequest,
|
||||
User,
|
||||
UserRole,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING: # the corpus is a dev-only import, see _seed_corpus
|
||||
from tests.fixtures.loader import CorpusDoc
|
||||
|
||||
DEV_PASSWORD = "pablan-dev"
|
||||
|
||||
DEPARTMENTS = ["Engineering", "Sales", "Administration"]
|
||||
|
||||
# The dev team, one per department, so the permission scenarios the e2e
|
||||
# suite relies on stay intact: an admin in Administration, a member in
|
||||
# Engineering, a member in Sales.
|
||||
USERS = [
|
||||
("florian@pablan.dev", "Florian", UserRole.admin, "Administration"),
|
||||
("pablo@pablan.dev", "Pablo", UserRole.member, "Engineering"),
|
||||
("max@pablan.dev", "Max", UserRole.member, "Sales"),
|
||||
]
|
||||
|
||||
# Which seeded user authors a department's corpus documents.
|
||||
AUTHORS_BY_DEPARTMENT = {
|
||||
"Engineering": "pablo@pablan.dev",
|
||||
"Sales": "max@pablan.dev",
|
||||
"Administration": "florian@pablan.dev",
|
||||
}
|
||||
|
||||
# Deliberately authorless: it doubles as the demo for the "department" access
|
||||
# reason (a member sees it without owning it) and, being the knowledge of
|
||||
# someone who has since left, fits the offboarding theme.
|
||||
AUTHORLESS_SLUG = "wartungsplan-cnc-f350"
|
||||
|
||||
# Documents whose life stops before the publish: unpublished work in progress,
|
||||
# one per author, so whoever logs in finds their own drafts waiting on the
|
||||
# home page.
|
||||
DRAFT_SLUGS = {
|
||||
"netzwerk-produktions-it", # Engineering
|
||||
"messevorbereitung", # Sales
|
||||
"it-onboarding-arbeitsplatz", # Administration
|
||||
}
|
||||
|
||||
# Published once, then retired — so the archive is not an empty concept in the
|
||||
# dev stack.
|
||||
ARCHIVED_SLUGS = {"edi-rechnungen"}
|
||||
|
||||
# How far back the corpus starts and how far apart the documents were written.
|
||||
# Deterministic rather than random: the "recently changed" order stays stable
|
||||
# across re-seeds, and the oldest documents are the ones that look oldest.
|
||||
CORPUS_STARTS_DAYS_AGO = 120
|
||||
DAYS_BETWEEN_DOCUMENTS = 6
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SeededReview:
|
||||
"""A "please check this" the author sent a colleague.
|
||||
|
||||
Open ones are the interesting case: the document is published and readable
|
||||
and still carries an unanswered question, which is exactly what every
|
||||
surface — list, detail page, chat sources — has to mark.
|
||||
"""
|
||||
|
||||
reviewer: str
|
||||
question: str
|
||||
answered: bool = False
|
||||
# What the reviewer corrected, as (current text, the text it replaced).
|
||||
# Applied in reverse to every version before the answer, so the history
|
||||
# holds a real diff and the answer is visibly a fix, not a rubber stamp.
|
||||
correction: tuple[str, str] | None = None
|
||||
|
||||
|
||||
REVIEWS = {
|
||||
# Published, public, unanswered: the case a reader most needs to see.
|
||||
"urlaubsantrag-prozess": SeededReview(
|
||||
reviewer="pablo@pablan.dev",
|
||||
question=(
|
||||
"Stimmt das für die Fertigung noch so, dass pro Schicht maximal "
|
||||
"zwei Personen gleichzeitig Urlaub haben dürfen?"
|
||||
),
|
||||
),
|
||||
# On a draft: being asked is what lets a colleague see it at all.
|
||||
"messevorbereitung": SeededReview(
|
||||
reviewer="florian@pablan.dev",
|
||||
question="Passt der Budgetrahmen so, bevor ich das veröffentliche?",
|
||||
),
|
||||
# Answered — and the reviewer fixed the number before answering.
|
||||
"rabattrichtlinie": SeededReview(
|
||||
reviewer="florian@pablan.dev",
|
||||
question="Gilt für Ersatzteile weiterhin die 3-%-Grenze?",
|
||||
answered=True,
|
||||
correction=(
|
||||
"- Bis 5 %: eigenverantwortlich durch den Vertriebsmitarbeiter",
|
||||
"- Bis 3 %: eigenverantwortlich durch den Vertriebsmitarbeiter",
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _seed_order(doc: "CorpusDoc") -> tuple[int, str]:
|
||||
"""The order the corpus was "written" in, oldest first.
|
||||
|
||||
Not alphabetical: the knowledge of someone who has left is the oldest
|
||||
thing in the base, and work still in draft has to be the most recent.
|
||||
"""
|
||||
if doc.slug == AUTHORLESS_SLUG:
|
||||
return (0, doc.slug)
|
||||
if doc.slug in DRAFT_SLUGS:
|
||||
return (2, doc.slug)
|
||||
return (1, doc.slug)
|
||||
|
||||
|
||||
def _writing_steps(content_md: str) -> list[str]:
|
||||
"""The document as it grew: the empty draft it starts as, then one state
|
||||
per section — the shape the writing editor produces, where a section is
|
||||
refined and saved before the next one is started."""
|
||||
sections = re.split(r"(?m)^(?=## )", content_md)
|
||||
return [""] + ["".join(sections[: index + 1]) for index in range(len(sections))]
|
||||
|
||||
|
||||
async def _get_or_create_department(db: AsyncSession, name: str) -> Department:
|
||||
existing = (
|
||||
await db.execute(select(Department).where(Department.name == name))
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
department = Department(name=name)
|
||||
db.add(department)
|
||||
await db.flush()
|
||||
return department
|
||||
|
||||
|
||||
async def seed() -> None:
|
||||
async with async_session_factory() as db:
|
||||
departments = {
|
||||
name: await _get_or_create_department(db, name) for name in DEPARTMENTS
|
||||
}
|
||||
|
||||
created = 0
|
||||
for email, name, role, department_name in USERS:
|
||||
existing = (
|
||||
await db.execute(select(User).where(User.email == email))
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
continue
|
||||
db.add(
|
||||
User(
|
||||
email=email,
|
||||
name=name,
|
||||
role=role,
|
||||
password_hash=hash_password(DEV_PASSWORD),
|
||||
department_id=departments[department_name].id,
|
||||
)
|
||||
)
|
||||
created += 1
|
||||
|
||||
documents_created = await _seed_corpus(db, departments)
|
||||
await db.commit()
|
||||
|
||||
await engine.dispose()
|
||||
print(f"Seeded {len(DEPARTMENTS)} departments, {created} new users.")
|
||||
print(f"Dev logins (password: {DEV_PASSWORD!r}):")
|
||||
for email, _, role, department in USERS:
|
||||
print(f" {email} ({role}, {department})")
|
||||
print(
|
||||
f"Seeded {documents_created} new corpus documents with their history "
|
||||
f"({len(DRAFT_SLUGS)} drafts, {len(REVIEWS)} review requests); "
|
||||
"index jobs enqueued (processed once the backend runs)."
|
||||
)
|
||||
|
||||
|
||||
async def _seed_corpus(db: AsyncSession, departments: dict[str, Department]) -> int:
|
||||
# Dev-only import: the corpus lives with the test fixtures on purpose —
|
||||
# seeds and tests draw from the same product asset.
|
||||
from tests.fixtures.loader import load_corpus
|
||||
|
||||
users_by_email = {u.email: u for u in (await db.execute(select(User))).scalars()}
|
||||
created = 0
|
||||
for index, doc in enumerate(sorted(load_corpus(), key=_seed_order)):
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(Document.id).where(Document.meta["slug"].astext == doc.slug)
|
||||
)
|
||||
).first()
|
||||
if existing is not None:
|
||||
continue
|
||||
document = await _seed_document(
|
||||
db,
|
||||
doc,
|
||||
department=departments[doc.department],
|
||||
users_by_email=users_by_email,
|
||||
started_at=datetime.now(UTC)
|
||||
- timedelta(days=CORPUS_STARTS_DAYS_AGO - index * DAYS_BETWEEN_DOCUMENTS),
|
||||
)
|
||||
for grant in doc.grants:
|
||||
db.add(
|
||||
DocPermission(
|
||||
document_id=document.id,
|
||||
department_id=departments[grant].id,
|
||||
)
|
||||
)
|
||||
if document.status == DocumentStatus.published:
|
||||
await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document.id)})
|
||||
created += 1
|
||||
return created
|
||||
|
||||
|
||||
async def _seed_document(
|
||||
db: AsyncSession,
|
||||
doc: "CorpusDoc",
|
||||
*,
|
||||
department: Department,
|
||||
users_by_email: dict[str, User],
|
||||
started_at: datetime,
|
||||
) -> Document:
|
||||
"""One corpus document plus the trail of everything that happened to it."""
|
||||
author: User | None = users_by_email[AUTHORS_BY_DEPARTMENT[doc.department]]
|
||||
if doc.slug == AUTHORLESS_SLUG:
|
||||
author = None
|
||||
|
||||
slug = doc.slug
|
||||
content = doc.content_md
|
||||
review = REVIEWS.get(slug)
|
||||
is_draft = slug in DRAFT_SLUGS
|
||||
|
||||
document = Document(
|
||||
title=doc.title,
|
||||
status=DocumentStatus.draft if is_draft else DocumentStatus.published,
|
||||
visibility=doc.visibility,
|
||||
content_md=content,
|
||||
meta={"slug": slug},
|
||||
author_id=author.id if author else None,
|
||||
department_id=department.id,
|
||||
reviews=[],
|
||||
)
|
||||
db.add(document)
|
||||
await db.flush()
|
||||
|
||||
at = started_at
|
||||
|
||||
def happened(
|
||||
action: DocumentEventAction,
|
||||
actor: User | None,
|
||||
snapshot: str | None = None,
|
||||
*,
|
||||
after: timedelta = timedelta(),
|
||||
) -> datetime:
|
||||
nonlocal at
|
||||
at += after
|
||||
db.add(
|
||||
DocumentEvent(
|
||||
document_id=document.id,
|
||||
actor_id=actor.id if actor else None,
|
||||
action=action,
|
||||
content_md=snapshot,
|
||||
title=document.title if snapshot is not None else None,
|
||||
visibility=document.visibility,
|
||||
meta=document.meta if snapshot is not None else None,
|
||||
created_at=at,
|
||||
updated_at=at,
|
||||
)
|
||||
)
|
||||
return at
|
||||
|
||||
# The text as it stood before the reviewer's correction — everything up to
|
||||
# their answer holds the old wording.
|
||||
written = content
|
||||
if review and review.correction:
|
||||
current, previous = review.correction
|
||||
if current not in content:
|
||||
raise ValueError(f"correction text not found in {slug}: {current!r}")
|
||||
written = content.replace(current, previous)
|
||||
|
||||
steps = _writing_steps(written)
|
||||
happened(DocumentEventAction.created, author, steps[0])
|
||||
for step in steps[1:]:
|
||||
happened(DocumentEventAction.edited, author, step, after=timedelta(minutes=40))
|
||||
|
||||
if not is_draft:
|
||||
happened(DocumentEventAction.published, author, after=timedelta(days=1))
|
||||
|
||||
if review:
|
||||
reviewer = users_by_email[review.reviewer]
|
||||
asked_at = happened(
|
||||
DocumentEventAction.review_requested, author, after=timedelta(days=3)
|
||||
)
|
||||
resolved_at = None
|
||||
if review.answered:
|
||||
if review.correction:
|
||||
happened(
|
||||
DocumentEventAction.edited,
|
||||
reviewer,
|
||||
content,
|
||||
after=timedelta(days=1),
|
||||
)
|
||||
resolved_at = happened(
|
||||
DocumentEventAction.review_resolved,
|
||||
reviewer,
|
||||
after=timedelta(minutes=20),
|
||||
)
|
||||
db.add(
|
||||
ReviewRequest(
|
||||
document_id=document.id,
|
||||
requester_id=author.id if author else None,
|
||||
reviewer_id=reviewer.id,
|
||||
question=review.question,
|
||||
resolved_at=resolved_at,
|
||||
resolved_by_id=reviewer.id if resolved_at else None,
|
||||
created_at=asked_at,
|
||||
updated_at=resolved_at or asked_at,
|
||||
)
|
||||
)
|
||||
|
||||
if slug in ARCHIVED_SLUGS:
|
||||
document.status = DocumentStatus.archived
|
||||
happened(DocumentEventAction.archived, author, after=timedelta(days=30))
|
||||
|
||||
document.created_at = started_at
|
||||
document.updated_at = at
|
||||
return document
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if get_settings().env == "production":
|
||||
sys.exit(
|
||||
"Refusing to seed: PABLAN_ENV=production. "
|
||||
"Seed data contains known dev credentials."
|
||||
)
|
||||
asyncio.run(seed())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,181 @@
|
||||
"""The shipped template catalog.
|
||||
|
||||
`templates/` holds blueprints, not active templates. A blueprint is inert
|
||||
product content: it sits in the catalog until an admin adds it, and adding
|
||||
it produces an ordinary row in `templates` that is theirs, editable,
|
||||
renameable, deletable, and never overwritten by a later deploy.
|
||||
|
||||
That is the difference to the built-in help documents (`help_import.py`),
|
||||
which ARE re-imported on every start and stay read-only: those describe how
|
||||
Pablan works, so the product owns them. A template describes how a company
|
||||
documents its own knowledge, so the company owns it.
|
||||
|
||||
**The only automatic write to `templates` is `seed_starter_templates`, and
|
||||
it runs exclusively against an empty table.** Everything else goes through
|
||||
an explicit admin action in `api/templates.py`. This is load-bearing: a
|
||||
startup upsert from the catalog would silently discard an admin's edits the
|
||||
next time we improved a shipped blueprint.
|
||||
|
||||
File naming: `<id>.<locale>.yaml` (`prozess.de.yaml`). A file
|
||||
without a locale suffix is treated as belonging to the default locale, so
|
||||
a customer can drop their own YAML into the directory without learning the
|
||||
convention. See docs/authoring-templates.md.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.models import Template
|
||||
from app.template_import import TemplateImportError, parse_template, upsert_template
|
||||
|
||||
logger = logging.getLogger("pablan.templates")
|
||||
|
||||
SUPPORTED_LOCALES = ("de", "en")
|
||||
|
||||
# What a brand-new instance starts with, in its default language: the four
|
||||
# occasions on which anyone actually writes something down. Write it down
|
||||
# (no structure at all — a blank page beats three generic headings), how we
|
||||
# do this, what broke, and who you are. Everything more specific — a
|
||||
# machine, a decision, a project review — is a deliberate add from the
|
||||
# catalog, because a picker of ten options is a picker nobody reads.
|
||||
STARTER_TEMPLATE_IDS = (
|
||||
"notiz",
|
||||
"prozess",
|
||||
"stoerung",
|
||||
"person",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CatalogEntry:
|
||||
"""A blueprint as it sits on disk, never a DB row."""
|
||||
|
||||
id: str
|
||||
locale: str
|
||||
name: str
|
||||
description: str
|
||||
version: str
|
||||
sections: int
|
||||
source: str
|
||||
|
||||
|
||||
def _split_stem(stem: str) -> tuple[str, str]:
|
||||
"""`prozess.de` -> (`prozess`, `de`); a stem without a
|
||||
known locale suffix belongs to the default locale."""
|
||||
base, _, suffix = stem.rpartition(".")
|
||||
if base and suffix in SUPPORTED_LOCALES:
|
||||
return base, suffix
|
||||
return stem, get_settings().default_locale
|
||||
|
||||
|
||||
def load_catalog() -> list[CatalogEntry]:
|
||||
"""Parse every blueprint in the catalog directory.
|
||||
|
||||
Read per call rather than cached: the directory is small, and a
|
||||
self-hosted admin who drops a YAML file in it should not have to
|
||||
restart the server to see it.
|
||||
"""
|
||||
directory = Path(get_settings().templates_dir)
|
||||
if not directory.is_dir():
|
||||
logger.info(
|
||||
"no template catalog directory",
|
||||
extra={"event": "catalog_missing", "directory": str(directory)},
|
||||
)
|
||||
return []
|
||||
|
||||
entries: list[CatalogEntry] = []
|
||||
for path in sorted(directory.glob("*.yaml")):
|
||||
source = path.read_text()
|
||||
try:
|
||||
template = parse_template(source)
|
||||
except TemplateImportError as exc:
|
||||
# A broken blueprint must not take the catalog down with it.
|
||||
logger.error(
|
||||
"catalog blueprint invalid",
|
||||
extra={
|
||||
"event": "catalog_invalid",
|
||||
"file": path.name,
|
||||
"error": str(exc),
|
||||
},
|
||||
)
|
||||
continue
|
||||
_base, from_name = _split_stem(path.stem)
|
||||
entries.append(
|
||||
CatalogEntry(
|
||||
id=template.id,
|
||||
# The YAML says what language it is written in; the filename
|
||||
# suffix is the human-facing convention and the fallback.
|
||||
locale=template.locale or from_name,
|
||||
name=template.name,
|
||||
description=template.description or "",
|
||||
version=template.version,
|
||||
sections=len(template.sections),
|
||||
source=source,
|
||||
)
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def catalog_for_locale(locale: str | None = None) -> list[CatalogEntry]:
|
||||
"""One entry per blueprint id, in `locale` where a variant exists.
|
||||
|
||||
A blueprint with no variant in the requested language still shows up in
|
||||
whatever language it has — a missing translation must not hide a
|
||||
template from the admin who needs it.
|
||||
"""
|
||||
wanted = locale or get_settings().default_locale
|
||||
best: dict[str, CatalogEntry] = {}
|
||||
for entry in load_catalog():
|
||||
current = best.get(entry.id)
|
||||
if current is None or (entry.locale == wanted and current.locale != wanted):
|
||||
best[entry.id] = entry
|
||||
return sorted(best.values(), key=lambda entry: entry.name)
|
||||
|
||||
|
||||
def get_catalog_entry(
|
||||
catalog_id: str, locale: str | None = None
|
||||
) -> CatalogEntry | None:
|
||||
return next(
|
||||
(entry for entry in catalog_for_locale(locale) if entry.id == catalog_id),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
async def seed_starter_templates(db: AsyncSession) -> int:
|
||||
"""Give a brand-new instance something to capture with.
|
||||
|
||||
Only ever runs against an EMPTY templates table. Once an admin has
|
||||
curated the list, added blueprints, deleted a starter, renamed things,
|
||||
that curation is the truth and startup must not re-litigate it. This
|
||||
is the ONLY automatic write to the table.
|
||||
"""
|
||||
existing = (await db.execute(select(func.count(Template.id)))).scalar_one()
|
||||
if existing:
|
||||
return 0
|
||||
|
||||
locale = get_settings().default_locale
|
||||
by_id = {entry.id: entry for entry in catalog_for_locale(locale)}
|
||||
|
||||
seeded = 0
|
||||
for catalog_id in STARTER_TEMPLATE_IDS:
|
||||
entry = by_id.get(catalog_id)
|
||||
if entry is None:
|
||||
logger.error(
|
||||
"starter template missing from catalog",
|
||||
extra={"event": "catalog_starter_missing", "template": catalog_id},
|
||||
)
|
||||
continue
|
||||
await upsert_template(db, parse_template(entry.source))
|
||||
seeded += 1
|
||||
|
||||
await db.commit()
|
||||
logger.info(
|
||||
"starter templates seeded",
|
||||
extra={"event": "templates_seeded", "count": seeded, "locale": locale},
|
||||
)
|
||||
return seeded
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Template import: YAML → validated config → templates table.
|
||||
|
||||
Every row in `templates` is the customer's own, whichever way it got there
|
||||
— pasted YAML, a fork, or added from the shipped catalog
|
||||
(`template_catalog.py`). None of them is read-only.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
import yaml
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.authoring.schema import AuthoringTemplate
|
||||
from app.models import Template
|
||||
|
||||
logger = logging.getLogger("pablan.templates")
|
||||
|
||||
|
||||
class TemplateImportError(Exception):
|
||||
"""Sanitized import failure — safe to surface to an admin."""
|
||||
|
||||
|
||||
def parse_template(source: str) -> AuthoringTemplate:
|
||||
try:
|
||||
raw = yaml.safe_load(source)
|
||||
except yaml.YAMLError as exc:
|
||||
raise TemplateImportError(f"Invalid YAML: {type(exc).__name__}") from None
|
||||
if not isinstance(raw, dict):
|
||||
raise TemplateImportError("Template must be a YAML mapping.")
|
||||
try:
|
||||
return AuthoringTemplate.model_validate(raw)
|
||||
except ValueError as exc:
|
||||
first_error = str(exc).splitlines()[1] if "\n" in str(exc) else str(exc)
|
||||
raise TemplateImportError(
|
||||
f"Template failed validation: {first_error.strip()}"
|
||||
) from None
|
||||
|
||||
|
||||
async def upsert_template(
|
||||
db: AsyncSession, template: AuthoringTemplate
|
||||
) -> tuple[Template, bool]:
|
||||
"""Insert or update by the template's config id. Returns (row, created)."""
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(Template).where(Template.config["id"].astext == template.id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
config = template.model_dump()
|
||||
if existing is not None:
|
||||
existing.name = template.name
|
||||
existing.version = template.version
|
||||
existing.config = config
|
||||
return existing, False
|
||||
row = Template(
|
||||
name=template.name,
|
||||
version=template.version,
|
||||
config=config,
|
||||
)
|
||||
db.add(row)
|
||||
await db.flush()
|
||||
return row, True
|
||||
Reference in New Issue
Block a user