"""Chunk (re)generation for a document — chunks are disposable derivatives. A full re-index is always possible from documents alone; swapping the embedding model is a reindex_all away (same dimension) or a migration plus reindex_all (different dimension). """ import logging import time from sqlalchemy import delete from sqlalchemy.ext.asyncio import AsyncSession from app.llm.client import embed from app.metrics import metrics from app.models import Chunk, Document from app.rag.chunking import chunk_markdown logger = logging.getLogger("pablan.rag") EMBED_BATCH_SIZE = 32 def embedding_text(heading_path: str, content: str) -> str: """What is actually embedded for a chunk: its heading path, then its text. A section says "Solldruck 180 bar" and never repeats which machine it belongs to, so without its path the chunk is unreachable by the name the asker actually uses. What is STORED as `content` stays the raw section — the path is context for the vector, not part of the document. """ return f"{heading_path}\n\n{content}" async def reindex_document(db: AsyncSession, document: Document) -> int: """Delete and regenerate all chunks for one document. Returns the count.""" started = time.monotonic() chunks_data = chunk_markdown(document.content_md, document.title) vectors: list[list[float]] = [] for batch_start in range(0, len(chunks_data), EMBED_BATCH_SIZE): batch = chunks_data[batch_start : batch_start + EMBED_BATCH_SIZE] vectors.extend( await embed( [embedding_text(chunk.heading_path, chunk.content) for chunk in batch] ) ) await db.execute(delete(Chunk).where(Chunk.document_id == document.id)) for index, (data, vector) in enumerate(zip(chunks_data, vectors, strict=True)): db.add( Chunk( document_id=document.id, chunk_index=index, content=data.content, embedding=vector, meta={ "heading_path": data.heading_path, # Denormalized for display/filtering — NEVER for # permission checks (can be stale until the next reindex). "department_id": ( str(document.department_id) if document.department_id else None ), "visibility": document.visibility, }, ) ) await db.flush() duration = time.monotonic() - started metrics.inc("chunks_indexed_total", value=len(chunks_data)) metrics.observe("indexing_seconds", duration) logger.info( "document indexed", extra={ "event": "document_indexed", "document_id": str(document.id), "chunk_count": len(chunks_data), "duration_ms": round(duration * 1000), }, ) return len(chunks_data) async def remove_chunks(db: AsyncSession, document_id) -> None: await db.execute(delete(Chunk).where(Chunk.document_id == document_id))