ProfessorNovaandClaude Opus 5
97dbff309c
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
116 lines
3.5 KiB
Python
116 lines
3.5 KiB
Python
"""Job handlers. Importing this module registers them with the queue."""
|
|
|
|
import logging
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from sqlalchemy import delete, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.config import get_settings
|
|
from app.ingestion.queue import enqueue, job_handler
|
|
from app.models import (
|
|
AuthSession,
|
|
Conversation,
|
|
ConversationMode,
|
|
Document,
|
|
DocumentStatus,
|
|
Job,
|
|
JobStatus,
|
|
)
|
|
from app.rag.indexing import reindex_document, remove_chunks
|
|
|
|
logger = logging.getLogger("pablan.queue")
|
|
|
|
RETENTION_CLEANUP = "retention_cleanup"
|
|
INDEX_DOCUMENT = "index_document"
|
|
REINDEX_ALL = "reindex_all"
|
|
|
|
|
|
@job_handler(INDEX_DOCUMENT)
|
|
async def index_document(db: AsyncSession, job: Job) -> None:
|
|
"""(Re)build the chunks of one document; drop them if it is not published."""
|
|
document_id = uuid.UUID(job.payload["document_id"])
|
|
document = await db.get(Document, document_id)
|
|
if document is None:
|
|
logger.info(
|
|
"index skipped, document gone",
|
|
extra={"event": "index_skipped", "document_id": str(document_id)},
|
|
)
|
|
return
|
|
if document.status == DocumentStatus.published:
|
|
await reindex_document(db, document)
|
|
else:
|
|
await remove_chunks(db, document.id)
|
|
|
|
|
|
@job_handler(REINDEX_ALL)
|
|
async def reindex_all(db: AsyncSession, job: Job) -> None:
|
|
"""Fan out one index_document job per published document.
|
|
|
|
Never embeds the corpus in this handler itself — the queue holds the
|
|
claim transaction open for the whole handler run.
|
|
"""
|
|
document_ids = (
|
|
(
|
|
await db.execute(
|
|
select(Document.id).where(Document.status == DocumentStatus.published)
|
|
)
|
|
)
|
|
.scalars()
|
|
.all()
|
|
)
|
|
for document_id in document_ids:
|
|
await enqueue(db, INDEX_DOCUMENT, {"document_id": str(document_id)})
|
|
logger.info(
|
|
"reindex fan-out",
|
|
extra={"event": "reindex_all", "document_count": len(document_ids)},
|
|
)
|
|
|
|
|
|
@job_handler(RETENTION_CLEANUP)
|
|
async def retention_cleanup(db: AsyncSession, job: Job) -> None:
|
|
"""GDPR retention: drop old query conversations and expired auth sessions.
|
|
|
|
Messages go with their conversation via ON DELETE CASCADE. Reschedules
|
|
itself daily.
|
|
"""
|
|
now = datetime.now(UTC)
|
|
cutoff = now - timedelta(days=get_settings().query_retention_days)
|
|
|
|
conversations_deleted = (
|
|
await db.execute(
|
|
delete(Conversation).where(
|
|
Conversation.mode == ConversationMode.query,
|
|
Conversation.updated_at < cutoff,
|
|
)
|
|
)
|
|
).rowcount
|
|
sessions_deleted = (
|
|
await db.execute(delete(AuthSession).where(AuthSession.expires_at < now))
|
|
).rowcount
|
|
|
|
await enqueue(db, RETENTION_CLEANUP, run_after=now + timedelta(days=1))
|
|
logger.info(
|
|
"retention cleanup",
|
|
extra={
|
|
"event": "retention_cleanup",
|
|
"conversations_deleted": conversations_deleted,
|
|
"auth_sessions_deleted": sessions_deleted,
|
|
},
|
|
)
|
|
|
|
|
|
async def ensure_retention_scheduled(db: AsyncSession) -> None:
|
|
"""Idempotent startup bootstrap: exactly one pending retention job."""
|
|
existing = (
|
|
await db.execute(
|
|
select(Job.id).where(
|
|
Job.type == RETENTION_CLEANUP, Job.status == JobStatus.pending
|
|
)
|
|
)
|
|
).first()
|
|
if existing is None:
|
|
await enqueue(db, RETENTION_CLEANUP)
|
|
await db.commit()
|