Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
2026-09-04 09:21:37 +02:00
parent 68d3a43191
commit 784b76baf7
346 changed files with 43430 additions and 0 deletions
@@ -0,0 +1,179 @@
+"""Postgres-backed background queue.
+
+One asyncio loop in the app lifespan claims jobs via
+SELECT … FOR UPDATE SKIP LOCKED. The claim transaction stays open while the
+handler runs: a crash rolls everything back and the job remains pending and
+claimable after restart — handler writes are atomic with job completion.
+Failure bookkeeping (attempts, backoff, last_error) happens in a follow-up
+transaction. Single worker per process; the loop moves into a worker
+container unchanged when scale demands it (Variant B).
+"""
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from app.config import get_settings
+from app.db import async_session_factory
+from app.log import safe_error
+from app.metrics import metrics
+from app.models import Job, JobStatus
+
+logger = logging.getLogger("pablan.queue")
+
+JobHandler = Callable[[AsyncSession, Job], Awaitable[None]]
+
+_HANDLERS: dict[str, JobHandler] = {}
+
+MAX_ATTEMPTS = 5
+BACKOFF_BASE_SECONDS = 30.0  # 30s, 1m, 2m, 4m between retries
+
+
+def job_handler(job_type: str) -> Callable[[JobHandler], JobHandler]:
+    def register(fn: JobHandler) -> JobHandler:
+        _HANDLERS[job_type] = fn
+        return fn
+
+    return register
+
+
+def backoff_delay(attempts: int) -> timedelta:
+    return timedelta(seconds=BACKOFF_BASE_SECONDS * 2 ** (attempts - 1))
+
+
+async def enqueue(
+    db: AsyncSession,
+    job_type: str,
+    payload: dict[str, Any] | None = None,
+    run_after: datetime | None = None,
+) -> Job:
+    job = Job(type=job_type, payload=payload or {})
+    if run_after is not None:
+        job.run_after = run_after
+    db.add(job)
+    await db.flush()
+    return job
+
+
+async def process_one(
+    session_factory: async_sessionmaker[AsyncSession] = async_session_factory,
+) -> bool:
+    """Claim and process a single due job. Returns True if one was processed."""
+    started = asyncio.get_running_loop().time()
+    async with session_factory() as db:
+        job = (
+            await db.execute(
+                select(Job)
+                .where(Job.status == JobStatus.pending, Job.run_after <= func.now())
+                .order_by(Job.run_after)
+                .limit(1)
+                .with_for_update(skip_locked=True)
+            )
+        ).scalar_one_or_none()
+        if job is None:
+            await db.rollback()
+            return False
+
+        job_id, job_type, attempts_before = job.id, job.type, job.attempts
+        try:
+            handler = _HANDLERS.get(job_type)
+            if handler is None:
+                raise LookupError(f"no handler registered for job type {job_type!r}")
+            await handler(db, job)
+            job.status = JobStatus.done
+            job.attempts = attempts_before + 1
+            await db.commit()
+        except Exception as exc:
+            await db.rollback()
+            await _record_failure(session_factory, job_id, exc)
+            duration = asyncio.get_running_loop().time() - started
+            metrics.inc("jobs_processed_total", {"type": job_type, "status": "failed"})
+            metrics.observe("job_seconds", duration, {"type": job_type})
+            logger.warning(
+                "job failed",
+                extra={
+                    "event": "job_failed",
+                    "job_id": str(job_id),
+                    "job_type": job_type,
+                    "attempt": attempts_before + 1,
+                    "error": safe_error(exc),
+                },
+            )
+            return True
+
+    duration = asyncio.get_running_loop().time() - started
+    metrics.inc("jobs_processed_total", {"type": job_type, "status": "done"})
+    metrics.observe("job_seconds", duration, {"type": job_type})
+    logger.info(
+        "job done",
+        extra={
+            "event": "job_done",
+            "job_id": str(job_id),
+            "job_type": job_type,
+            "attempt": attempts_before + 1,
+            "duration_ms": round(duration * 1000),
+        },
+    )
+    return True
+
+
+async def _record_failure(
+    session_factory: async_sessionmaker[AsyncSession],
+    job_id: Any,
+    exc: Exception,
+) -> None:
+    async with session_factory() as db:
+        job = await db.get(Job, job_id, with_for_update=True)
+        if job is None:  # pragma: no cover — job deleted underneath us
+            return
+        job.attempts += 1
+        job.last_error = safe_error(exc, limit=500)
+        if job.attempts >= MAX_ATTEMPTS:
+            job.status = JobStatus.failed
+            metrics.inc("jobs_exhausted_total", {"type": job.type})
+        else:
+            job.status = JobStatus.pending
+            job.run_after = datetime.now(UTC) + backoff_delay(job.attempts)
+            metrics.inc("jobs_retried_total", {"type": job.type})
+        await db.commit()
+
+
+async def _update_depth_gauge(
+    session_factory: async_sessionmaker[AsyncSession],
+) -> None:
+    async with session_factory() as db:
+        depth = (
+            await db.execute(
+                select(func.count(Job.id)).where(Job.status == JobStatus.pending)
+            )
+        ).scalar_one()
+    metrics.set_gauge("jobs_queue_depth", float(depth))
+
+
+async def run_queue(
+    stop_event: asyncio.Event,
+    session_factory: async_sessionmaker[AsyncSession] = async_session_factory,
+) -> None:
+    poll_seconds = get_settings().job_poll_seconds
+    logger.info("job queue started", extra={"event": "queue_started"})
+    while not stop_event.is_set():
+        worked = False
+        try:
+            worked = await process_one(session_factory)
+            await _update_depth_gauge(session_factory)
+        except Exception as exc:
+            logger.error(
+                "queue iteration failed",
+                extra={"event": "queue_error", "error": safe_error(exc)},
+            )
+        if not worked:
+            try:
+                await asyncio.wait_for(stop_event.wait(), timeout=poll_seconds)
+            except TimeoutError:
+                pass
+    logger.info("job queue stopped", extra={"event": "queue_stopped"})