Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
co-authored by
Claude Opus 5
Claude Opus 5
parent
68d3a43191
commit
784b76baf7
@@ -0,0 +1,115 @@
|
||||
"""Job handlers. Importing this module registers them with the queue."""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import get_settings
|
||||
from app.ingestion.queue import enqueue, job_handler
|
||||
from app.models import (
|
||||
AuthSession,
|
||||
Conversation,
|
||||
ConversationMode,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
Job,
|
||||
JobStatus,
|
||||
)
|
||||
from app.rag.indexing import reindex_document, remove_chunks
|
||||
|
||||
logger = logging.getLogger("pablan.queue")
|
||||
|
||||
RETENTION_CLEANUP = "retention_cleanup"
|
||||
INDEX_DOCUMENT = "index_document"
|
||||
REINDEX_ALL = "reindex_all"
|
||||
|
||||
|
||||
@job_handler(INDEX_DOCUMENT)
|
||||
async def index_document(db: AsyncSession, job: Job) -> None:
|
||||
"""(Re)build the chunks of one document; drop them if it is not published."""
|
||||
document_id = uuid.UUID(job.payload["document_id"])
|
||||
document = await db.get(Document, document_id)
|
||||
if document is None:
|
||||
logger.info(
|
||||
"index skipped, document gone",
|
||||
extra={"event": "index_skipped", "document_id": str(document_id)},
|
||||
)
|
||||
return
|
||||
if document.status == DocumentStatus.published:
|
||||
await reindex_document(db, document)
|
||||
else:
|
||||
await remove_chunks(db, document.id)
|
||||
|
||||
|
||||
@job_handler(REINDEX_ALL)
|
||||
async def reindex_all(db: AsyncSession, job: Job) -> None:
|
||||
"""Fan out one index_document job per published document.
|
||||
|
||||
Never embeds the corpus in this handler itself — the queue holds the
|
||||
claim transaction open for the whole handler run.
|
||||
"""
|
||||
document_ids = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document.id).where(Document.status == DocumentStatus.published)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for document_id in document_ids:
|
||||
await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document_id)})
|
||||
logger.info(
|
||||
"reindex fan-out",
|
||||
extra={"event": "reindex_all", "document_count": len(document_ids)},
|
||||
)
|
||||
|
||||
|
||||
@job_handler(RETENTION_CLEANUP)
|
||||
async def retention_cleanup(db: AsyncSession, job: Job) -> None:
|
||||
"""GDPR retention: drop old query conversations and expired auth sessions.
|
||||
|
||||
Messages go with their conversation via ON DELETE CASCADE. Reschedules
|
||||
itself daily.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(days=get_settings().query_retention_days)
|
||||
|
||||
conversations_deleted = (
|
||||
await db.execute(
|
||||
delete(Conversation).where(
|
||||
Conversation.mode == ConversationMode.query,
|
||||
Conversation.updated_at < cutoff,
|
||||
)
|
||||
)
|
||||
).rowcount
|
||||
sessions_deleted = (
|
||||
await db.execute(delete(AuthSession).where(AuthSession.expires_at < now))
|
||||
).rowcount
|
||||
|
||||
await enqueue(db, RETENTION_CLEANUP, run_after=now + timedelta(days=1))
|
||||
logger.info(
|
||||
"retention cleanup",
|
||||
extra={
|
||||
"event": "retention_cleanup",
|
||||
"conversations_deleted": conversations_deleted,
|
||||
"auth_sessions_deleted": sessions_deleted,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def ensure_retention_scheduled(db: AsyncSession) -> None:
|
||||
"""Idempotent startup bootstrap: exactly one pending retention job."""
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(Job.id).where(
|
||||
Job.type == RETENTION_CLEANUP, Job.status == JobStatus.pending
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if existing is None:
|
||||
await enqueue(db, RETENTION_CLEANUP)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Postgres-backed background queue.
|
||||
|
||||
One asyncio loop in the app lifespan claims jobs via
|
||||
SELECT … FOR UPDATE SKIP LOCKED. The claim transaction stays open while the
|
||||
handler runs: a crash rolls everything back and the job remains pending and
|
||||
claimable after restart — handler writes are atomic with job completion.
|
||||
Failure bookkeeping (attempts, backoff, last_error) happens in a follow-up
|
||||
transaction. Single worker per process; the loop moves into a worker
|
||||
container unchanged when scale demands it (Variant B).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import async_session_factory
|
||||
from app.log import safe_error
|
||||
from app.metrics import metrics
|
||||
from app.models import Job, JobStatus
|
||||
|
||||
logger = logging.getLogger("pablan.queue")
|
||||
|
||||
JobHandler = Callable[[AsyncSession, Job], Awaitable[None]]
|
||||
|
||||
_HANDLERS: dict[str, JobHandler] = {}
|
||||
|
||||
MAX_ATTEMPTS = 5
|
||||
BACKOFF_BASE_SECONDS = 30.0 # 30s, 1m, 2m, 4m between retries
|
||||
|
||||
|
||||
def job_handler(job_type: str) -> Callable[[JobHandler], JobHandler]:
|
||||
def register(fn: JobHandler) -> JobHandler:
|
||||
_HANDLERS[job_type] = fn
|
||||
return fn
|
||||
|
||||
return register
|
||||
|
||||
|
||||
def backoff_delay(attempts: int) -> timedelta:
|
||||
return timedelta(seconds=BACKOFF_BASE_SECONDS * 2 ** (attempts - 1))
|
||||
|
||||
|
||||
async def enqueue(
|
||||
db: AsyncSession,
|
||||
job_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
run_after: datetime | None = None,
|
||||
) -> Job:
|
||||
job = Job(type=job_type, payload=payload or {})
|
||||
if run_after is not None:
|
||||
job.run_after = run_after
|
||||
db.add(job)
|
||||
await db.flush()
|
||||
return job
|
||||
|
||||
|
||||
async def process_one(
|
||||
session_factory: async_sessionmaker[AsyncSession] = async_session_factory,
|
||||
) -> bool:
|
||||
"""Claim and process a single due job. Returns True if one was processed."""
|
||||
started = asyncio.get_running_loop().time()
|
||||
async with session_factory() as db:
|
||||
job = (
|
||||
await db.execute(
|
||||
select(Job)
|
||||
.where(Job.status == JobStatus.pending, Job.run_after <= func.now())
|
||||
.order_by(Job.run_after)
|
||||
.limit(1)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
await db.rollback()
|
||||
return False
|
||||
|
||||
job_id, job_type, attempts_before = job.id, job.type, job.attempts
|
||||
try:
|
||||
handler = _HANDLERS.get(job_type)
|
||||
if handler is None:
|
||||
raise LookupError(f"no handler registered for job type {job_type!r}")
|
||||
await handler(db, job)
|
||||
job.status = JobStatus.done
|
||||
job.attempts = attempts_before + 1
|
||||
await db.commit()
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
await _record_failure(session_factory, job_id, exc)
|
||||
duration = asyncio.get_running_loop().time() - started
|
||||
metrics.inc("jobs_processed_total", {"type": job_type, "status": "failed"})
|
||||
metrics.observe("job_seconds", duration, {"type": job_type})
|
||||
logger.warning(
|
||||
"job failed",
|
||||
extra={
|
||||
"event": "job_failed",
|
||||
"job_id": str(job_id),
|
||||
"job_type": job_type,
|
||||
"attempt": attempts_before + 1,
|
||||
"error": safe_error(exc),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
duration = asyncio.get_running_loop().time() - started
|
||||
metrics.inc("jobs_processed_total", {"type": job_type, "status": "done"})
|
||||
metrics.observe("job_seconds", duration, {"type": job_type})
|
||||
logger.info(
|
||||
"job done",
|
||||
extra={
|
||||
"event": "job_done",
|
||||
"job_id": str(job_id),
|
||||
"job_type": job_type,
|
||||
"attempt": attempts_before + 1,
|
||||
"duration_ms": round(duration * 1000),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _record_failure(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
job_id: Any,
|
||||
exc: Exception,
|
||||
) -> None:
|
||||
async with session_factory() as db:
|
||||
job = await db.get(Job, job_id, with_for_update=True)
|
||||
if job is None: # pragma: no cover — job deleted underneath us
|
||||
return
|
||||
job.attempts += 1
|
||||
job.last_error = safe_error(exc, limit=500)
|
||||
if job.attempts >= MAX_ATTEMPTS:
|
||||
job.status = JobStatus.failed
|
||||
metrics.inc("jobs_exhausted_total", {"type": job.type})
|
||||
else:
|
||||
job.status = JobStatus.pending
|
||||
job.run_after = datetime.now(UTC) + backoff_delay(job.attempts)
|
||||
metrics.inc("jobs_retried_total", {"type": job.type})
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def _update_depth_gauge(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
async with session_factory() as db:
|
||||
depth = (
|
||||
await db.execute(
|
||||
select(func.count(Job.id)).where(Job.status == JobStatus.pending)
|
||||
)
|
||||
).scalar_one()
|
||||
metrics.set_gauge("jobs_queue_depth", float(depth))
|
||||
|
||||
|
||||
async def run_queue(
|
||||
stop_event: asyncio.Event,
|
||||
session_factory: async_sessionmaker[AsyncSession] = async_session_factory,
|
||||
) -> None:
|
||||
poll_seconds = get_settings().job_poll_seconds
|
||||
logger.info("job queue started", extra={"event": "queue_started"})
|
||||
while not stop_event.is_set():
|
||||
worked = False
|
||||
try:
|
||||
worked = await process_one(session_factory)
|
||||
await _update_depth_gauge(session_factory)
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"queue iteration failed",
|
||||
extra={"event": "queue_error", "error": safe_error(exc)},
|
||||
)
|
||||
if not worked:
|
||||
try:
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
logger.info("job queue stopped", extra={"event": "queue_stopped"})
|
||||
Reference in New Issue
Block a user