Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
2026-09-04 08:36:17 +02:00
commit 97dbff309c
346 changed files with 43430 additions and 0 deletions
@@ -0,0 +1,115 @@
+"""Job handlers. Importing this module registers them with the queue."""
+
+import logging
+import uuid
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import get_settings
+from app.ingestion.queue import enqueue, job_handler
+from app.models import (
+    AuthSession,
+    Conversation,
+    ConversationMode,
+    Document,
+    DocumentStatus,
+    Job,
+    JobStatus,
+)
+from app.rag.indexing import reindex_document, remove_chunks
+
+logger = logging.getLogger("pablan.queue")
+
+RETENTION_CLEANUP = "retention_cleanup"
+INDEX_DOCUMENT = "index_document"
+REINDEX_ALL = "reindex_all"
+
+
+@job_handler(INDEX_DOCUMENT)
+async def index_document(db: AsyncSession, job: Job) -> None:
+    """(Re)build the chunks of one document; drop them if it is not published."""
+    document_id = uuid.UUID(job.payload["document_id"])
+    document = await db.get(Document, document_id)
+    if document is None:
+        logger.info(
+            "index skipped, document gone",
+            extra={"event": "index_skipped", "document_id": str(document_id)},
+        )
+        return
+    if document.status == DocumentStatus.published:
+        await reindex_document(db, document)
+    else:
+        await remove_chunks(db, document.id)
+
+
+@job_handler(REINDEX_ALL)
+async def reindex_all(db: AsyncSession, job: Job) -> None:
+    """Fan out one index_document job per published document.
+
+    Never embeds the corpus in this handler itself — the queue holds the
+    claim transaction open for the whole handler run.
+    """
+    document_ids = (
+        (
+            await db.execute(
+                select(Document.id).where(Document.status == DocumentStatus.published)
+            )
+        )
+        .scalars()
+        .all()
+    )
+    for document_id in document_ids:
+        await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document_id)})
+    logger.info(
+        "reindex fan-out",
+        extra={"event": "reindex_all", "document_count": len(document_ids)},
+    )
+
+
+@job_handler(RETENTION_CLEANUP)
+async def retention_cleanup(db: AsyncSession, job: Job) -> None:
+    """GDPR retention: drop old query conversations and expired auth sessions.
+
+    Messages go with their conversation via ON DELETE CASCADE. Reschedules
+    itself daily.
+    """
+    now = datetime.now(UTC)
+    cutoff = now - timedelta(days=get_settings().query_retention_days)
+
+    conversations_deleted = (
+        await db.execute(
+            delete(Conversation).where(
+                Conversation.mode == ConversationMode.query,
+                Conversation.updated_at < cutoff,
+            )
+        )
+    ).rowcount
+    sessions_deleted = (
+        await db.execute(delete(AuthSession).where(AuthSession.expires_at < now))
+    ).rowcount
+
+    await enqueue(db, RETENTION_CLEANUP, run_after=now + timedelta(days=1))
+    logger.info(
+        "retention cleanup",
+        extra={
+            "event": "retention_cleanup",
+            "conversations_deleted": conversations_deleted,
+            "auth_sessions_deleted": sessions_deleted,
+        },
+    )
+
+
+async def ensure_retention_scheduled(db: AsyncSession) -> None:
+    """Idempotent startup bootstrap: exactly one pending retention job."""
+    existing = (
+        await db.execute(
+            select(Job.id).where(
+                Job.type == RETENTION_CLEANUP, Job.status == JobStatus.pending
+            )
+        )
+    ).first()
+    if existing is None:
+        await enqueue(db, RETENTION_CLEANUP)
+        await db.commit()