Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/app/rag/indexing.py

@@ -0,0 +1,86 @@
+"""Chunk (re)generation for a document — chunks are disposable derivatives.
+
+A full re-index is always possible from documents alone; swapping
+the embedding model is a reindex_all away (same dimension) or a migration
+plus reindex_all (different dimension).
+"""
+
+import logging
+import time
+
+from sqlalchemy import delete
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.llm.client import embed
+from app.metrics import metrics
+from app.models import Chunk, Document
+from app.rag.chunking import chunk_markdown
+
+logger = logging.getLogger("pablan.rag")
+
+EMBED_BATCH_SIZE = 32
+
+
+def embedding_text(heading_path: str, content: str) -> str:
+    """What is actually embedded for a chunk: its heading path, then its text.
+
+    A section says "Solldruck 180 bar" and never repeats which machine it
+    belongs to, so without its path the chunk is unreachable by the name the
+    asker actually uses. What is STORED as `content` stays the raw section —
+    the path is context for the vector, not part of the document.
+    """
+    return f"{heading_path}\n\n{content}"
+
+
+async def reindex_document(db: AsyncSession, document: Document) -> int:
+    """Delete and regenerate all chunks for one document. Returns the count."""
+    started = time.monotonic()
+    chunks_data = chunk_markdown(document.content_md, document.title)
+
+    vectors: list[list[float]] = []
+    for batch_start in range(0, len(chunks_data), EMBED_BATCH_SIZE):
+        batch = chunks_data[batch_start : batch_start + EMBED_BATCH_SIZE]
+        vectors.extend(
+            await embed(
+                [embedding_text(chunk.heading_path, chunk.content) for chunk in batch]
+            )
+        )
+
+    await db.execute(delete(Chunk).where(Chunk.document_id == document.id))
+    for index, (data, vector) in enumerate(zip(chunks_data, vectors, strict=True)):
+        db.add(
+            Chunk(
+                document_id=document.id,
+                chunk_index=index,
+                content=data.content,
+                embedding=vector,
+                meta={
+                    "heading_path": data.heading_path,
+                    # Denormalized for display/filtering — NEVER for
+                    # permission checks (can be stale until the next reindex).
+                    "department_id": (
+                        str(document.department_id) if document.department_id else None
+                    ),
+                    "visibility": document.visibility,
+                },
+            )
+        )
+    await db.flush()
+
+    duration = time.monotonic() - started
+    metrics.inc("chunks_indexed_total", value=len(chunks_data))
+    metrics.observe("indexing_seconds", duration)
+    logger.info(
+        "document indexed",
+        extra={
+            "event": "document_indexed",
+            "document_id": str(document.id),
+            "chunk_count": len(chunks_data),
+            "duration_ms": round(duration * 1000),
+        },
+    )
+    return len(chunks_data)
+
+
+async def remove_chunks(db: AsyncSession, document_id) -> None:
+    await db.execute(delete(Chunk).where(Chunk.document_id == document_id))