ProfessorNovaandClaude Opus 5
784b76baf7
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
87 lines
3.0 KiB
Python
87 lines
3.0 KiB
Python
"""Chunk (re)generation for a document — chunks are disposable derivatives.
|
|
|
|
A full re-index is always possible from documents alone; swapping
|
|
the embedding model is a reindex_all away (same dimension) or a migration
|
|
plus reindex_all (different dimension).
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
|
|
from sqlalchemy import delete
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.llm.client import embed
|
|
from app.metrics import metrics
|
|
from app.models import Chunk, Document
|
|
from app.rag.chunking import chunk_markdown
|
|
|
|
logger = logging.getLogger("pablan.rag")
|
|
|
|
EMBED_BATCH_SIZE = 32
|
|
|
|
|
|
def embedding_text(heading_path: str, content: str) -> str:
|
|
"""What is actually embedded for a chunk: its heading path, then its text.
|
|
|
|
A section says "Solldruck 180 bar" and never repeats which machine it
|
|
belongs to, so without its path the chunk is unreachable by the name the
|
|
asker actually uses. What is STORED as `content` stays the raw section —
|
|
the path is context for the vector, not part of the document.
|
|
"""
|
|
return f"{heading_path}\n\n{content}"
|
|
|
|
|
|
async def reindex_document(db: AsyncSession, document: Document) -> int:
|
|
"""Delete and regenerate all chunks for one document. Returns the count."""
|
|
started = time.monotonic()
|
|
chunks_data = chunk_markdown(document.content_md, document.title)
|
|
|
|
vectors: list[list[float]] = []
|
|
for batch_start in range(0, len(chunks_data), EMBED_BATCH_SIZE):
|
|
batch = chunks_data[batch_start : batch_start + EMBED_BATCH_SIZE]
|
|
vectors.extend(
|
|
await embed(
|
|
[embedding_text(chunk.heading_path, chunk.content) for chunk in batch]
|
|
)
|
|
)
|
|
|
|
await db.execute(delete(Chunk).where(Chunk.document_id == document.id))
|
|
for index, (data, vector) in enumerate(zip(chunks_data, vectors, strict=True)):
|
|
db.add(
|
|
Chunk(
|
|
document_id=document.id,
|
|
chunk_index=index,
|
|
content=data.content,
|
|
embedding=vector,
|
|
meta={
|
|
"heading_path": data.heading_path,
|
|
# Denormalized for display/filtering — NEVER for
|
|
# permission checks (can be stale until the next reindex).
|
|
"department_id": (
|
|
str(document.department_id) if document.department_id else None
|
|
),
|
|
"visibility": document.visibility,
|
|
},
|
|
)
|
|
)
|
|
await db.flush()
|
|
|
|
duration = time.monotonic() - started
|
|
metrics.inc("chunks_indexed_total", value=len(chunks_data))
|
|
metrics.observe("indexing_seconds", duration)
|
|
logger.info(
|
|
"document indexed",
|
|
extra={
|
|
"event": "document_indexed",
|
|
"document_id": str(document.id),
|
|
"chunk_count": len(chunks_data),
|
|
"duration_ms": round(duration * 1000),
|
|
},
|
|
)
|
|
return len(chunks_data)
|
|
|
|
|
|
async def remove_chunks(db: AsyncSession, document_id) -> None:
|
|
await db.execute(delete(Chunk).where(Chunk.document_id == document_id))
|