Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
@@ -0,0 +1,107 @@
|
||||
"""Markdown chunking along the heading hierarchy.
|
||||
|
||||
Chunk size uses a character heuristic (~4 chars/token, target ~400 tokens);
|
||||
no tokenizer dependency — precision is not required for chunk sizing, and a
|
||||
real tokenizer would not match local model tokenizers anyway. Sections that
|
||||
exceed the cap are split at paragraph boundaries, never inside code fences.
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
# ~400 tokens at the ~4 chars/token heuristic.
|
||||
TARGET_CHUNK_CHARS = 1600
|
||||
|
||||
HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
|
||||
|
||||
HEADING_PATH_SEPARATOR = " › "
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkData:
|
||||
content: str
|
||||
heading_path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Section:
|
||||
path: list[str]
|
||||
lines: list[str]
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
return "\n".join(self.lines).strip()
|
||||
|
||||
|
||||
def _split_sections(content_md: str, title: str) -> list[_Section]:
|
||||
sections: list[_Section] = [_Section(path=[title], lines=[])]
|
||||
heading_stack: list[tuple[int, str]] = [] # (level, text)
|
||||
in_fence = False
|
||||
|
||||
for line in content_md.splitlines():
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
match = None if in_fence else HEADING_RE.match(line)
|
||||
if match:
|
||||
level = len(match.group(1))
|
||||
text = match.group(2).strip()
|
||||
while heading_stack and heading_stack[-1][0] >= level:
|
||||
heading_stack.pop()
|
||||
heading_stack.append((level, text))
|
||||
path = [title, *(heading for _, heading in heading_stack)]
|
||||
# Drop a leading H1 that just repeats the document title.
|
||||
if len(path) > 1 and path[1] == title:
|
||||
path = [title, *path[2:]]
|
||||
sections.append(_Section(path=path, lines=[line]))
|
||||
else:
|
||||
sections[-1].lines.append(line)
|
||||
|
||||
return [section for section in sections if section.text]
|
||||
|
||||
|
||||
def _split_paragraphs(text: str) -> list[str]:
|
||||
"""Split at blank lines, but never inside a ``` fence."""
|
||||
paragraphs: list[str] = []
|
||||
current: list[str] = []
|
||||
in_fence = False
|
||||
for line in text.splitlines():
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
if not line.strip() and not in_fence:
|
||||
if current:
|
||||
paragraphs.append("\n".join(current))
|
||||
current = []
|
||||
else:
|
||||
current.append(line)
|
||||
if current:
|
||||
paragraphs.append("\n".join(current))
|
||||
return paragraphs
|
||||
|
||||
|
||||
def _split_oversized(text: str) -> list[str]:
|
||||
pieces: list[str] = []
|
||||
current = ""
|
||||
for paragraph in _split_paragraphs(text):
|
||||
candidate = f"{current}\n\n{paragraph}" if current else paragraph
|
||||
if current and len(candidate) > TARGET_CHUNK_CHARS:
|
||||
pieces.append(current)
|
||||
current = paragraph
|
||||
else:
|
||||
current = candidate
|
||||
if current:
|
||||
pieces.append(current)
|
||||
return pieces
|
||||
|
||||
|
||||
def chunk_markdown(content_md: str, title: str) -> list[ChunkData]:
|
||||
"""Split a document into chunks; each chunk has exactly one heading path."""
|
||||
chunks: list[ChunkData] = []
|
||||
for section in _split_sections(content_md, title):
|
||||
heading_path = HEADING_PATH_SEPARATOR.join(section.path)
|
||||
text = section.text
|
||||
if len(text) <= TARGET_CHUNK_CHARS:
|
||||
chunks.append(ChunkData(content=text, heading_path=heading_path))
|
||||
else:
|
||||
for piece in _split_oversized(text):
|
||||
chunks.append(ChunkData(content=piece, heading_path=heading_path))
|
||||
return chunks
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Chunk (re)generation for a document — chunks are disposable derivatives.
|
||||
|
||||
A full re-index is always possible from documents alone; swapping
|
||||
the embedding model is a reindex_all away (same dimension) or a migration
|
||||
plus reindex_all (different dimension).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.llm.client import embed
|
||||
from app.metrics import metrics
|
||||
from app.models import Chunk, Document
|
||||
from app.rag.chunking import chunk_markdown
|
||||
|
||||
logger = logging.getLogger("pablan.rag")
|
||||
|
||||
EMBED_BATCH_SIZE = 32
|
||||
|
||||
|
||||
def embedding_text(heading_path: str, content: str) -> str:
|
||||
"""What is actually embedded for a chunk: its heading path, then its text.
|
||||
|
||||
A section says "Solldruck 180 bar" and never repeats which machine it
|
||||
belongs to, so without its path the chunk is unreachable by the name the
|
||||
asker actually uses. What is STORED as `content` stays the raw section —
|
||||
the path is context for the vector, not part of the document.
|
||||
"""
|
||||
return f"{heading_path}\n\n{content}"
|
||||
|
||||
|
||||
async def reindex_document(db: AsyncSession, document: Document) -> int:
|
||||
"""Delete and regenerate all chunks for one document. Returns the count."""
|
||||
started = time.monotonic()
|
||||
chunks_data = chunk_markdown(document.content_md, document.title)
|
||||
|
||||
vectors: list[list[float]] = []
|
||||
for batch_start in range(0, len(chunks_data), EMBED_BATCH_SIZE):
|
||||
batch = chunks_data[batch_start : batch_start + EMBED_BATCH_SIZE]
|
||||
vectors.extend(
|
||||
await embed(
|
||||
[embedding_text(chunk.heading_path, chunk.content) for chunk in batch]
|
||||
)
|
||||
)
|
||||
|
||||
await db.execute(delete(Chunk).where(Chunk.document_id == document.id))
|
||||
for index, (data, vector) in enumerate(zip(chunks_data, vectors, strict=True)):
|
||||
db.add(
|
||||
Chunk(
|
||||
document_id=document.id,
|
||||
chunk_index=index,
|
||||
content=data.content,
|
||||
embedding=vector,
|
||||
meta={
|
||||
"heading_path": data.heading_path,
|
||||
# Denormalized for display/filtering — NEVER for
|
||||
# permission checks (can be stale until the next reindex).
|
||||
"department_id": (
|
||||
str(document.department_id) if document.department_id else None
|
||||
),
|
||||
"visibility": document.visibility,
|
||||
},
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("chunks_indexed_total", value=len(chunks_data))
|
||||
metrics.observe("indexing_seconds", duration)
|
||||
logger.info(
|
||||
"document indexed",
|
||||
extra={
|
||||
"event": "document_indexed",
|
||||
"document_id": str(document.id),
|
||||
"chunk_count": len(chunks_data),
|
||||
"duration_ms": round(duration * 1000),
|
||||
},
|
||||
)
|
||||
return len(chunks_data)
|
||||
|
||||
|
||||
async def remove_chunks(db: AsyncSession, document_id) -> None:
|
||||
await db.execute(delete(Chunk).where(Chunk.document_id == document_id))
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Single source of truth for who may read which document.
|
||||
|
||||
Used by BOTH the documents API and retrieval, so the permission filter that
|
||||
runs before the LLM can never drift from what the API
|
||||
exposes. Always evaluated against the live documents table — never against
|
||||
denormalized chunk meta, which can be stale between edits and reindexing.
|
||||
"""
|
||||
|
||||
from sqlalchemy import ColumnElement, and_, exists, or_, select, true
|
||||
|
||||
from app.models import (
|
||||
DocPermission,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
DocumentVisibility,
|
||||
ReviewRequest,
|
||||
User,
|
||||
)
|
||||
|
||||
|
||||
def searchable_documents_filter(user: User) -> ColumnElement[bool]:
|
||||
"""Published documents the user may read.
|
||||
|
||||
Rules: public to everyone; department to members of the owning
|
||||
department; restricted only via doc_permissions grants. Authors always
|
||||
see their own documents.
|
||||
"""
|
||||
clauses: list[ColumnElement[bool]] = [
|
||||
Document.visibility == DocumentVisibility.public,
|
||||
Document.author_id == user.id,
|
||||
]
|
||||
if user.department_id is not None:
|
||||
clauses.append(
|
||||
and_(
|
||||
Document.visibility == DocumentVisibility.department,
|
||||
Document.department_id == user.department_id,
|
||||
)
|
||||
)
|
||||
clauses.append(
|
||||
exists(
|
||||
select(DocPermission.document_id).where(
|
||||
DocPermission.document_id == Document.id,
|
||||
DocPermission.department_id == user.department_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
return and_(Document.status == DocumentStatus.published, or_(*clauses))
|
||||
|
||||
|
||||
def readable_documents_filter(user: User) -> ColumnElement[bool]:
|
||||
"""Searchable documents plus the unpublished ones this user owns or was
|
||||
asked to check.
|
||||
|
||||
Being asked IS the grant: a reviewer must be able to open the draft they
|
||||
were pointed at. The clause lives here only, never in
|
||||
`searchable_documents_filter`, so an unpublished document still never
|
||||
reaches chat/search retrieval."""
|
||||
return or_(
|
||||
searchable_documents_filter(user),
|
||||
Document.author_id == user.id,
|
||||
open_review_for(user),
|
||||
)
|
||||
|
||||
|
||||
def open_review_for(user: User) -> ColumnElement[bool]:
|
||||
"""This user has an unanswered request to check the document."""
|
||||
return exists(
|
||||
select(ReviewRequest.document_id).where(
|
||||
ReviewRequest.document_id == Document.id,
|
||||
ReviewRequest.reviewer_id == user.id,
|
||||
ReviewRequest.resolved_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def has_open_review() -> ColumnElement[bool]:
|
||||
"""Anyone has an unanswered question about the document — what marks it as
|
||||
"may not be right yet" wherever it is shown, including chat sources."""
|
||||
return exists(
|
||||
select(ReviewRequest.document_id).where(
|
||||
ReviewRequest.document_id == Document.id,
|
||||
ReviewRequest.resolved_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def document_reader_filter(document: Document) -> ColumnElement[bool]:
|
||||
"""A `User`-table filter for who may read `document` AS IF it were
|
||||
published — the candidate set for assigning a reviewer. Inverts the read
|
||||
rules of `searchable_documents_filter` (author, public, owning department,
|
||||
granted departments), ignoring status so a still-pending document can be
|
||||
handed to a reviewer who will then be able to see it (the reviewer clause
|
||||
in `readable_documents_filter`)."""
|
||||
clauses: list[ColumnElement[bool]] = [User.id == document.author_id]
|
||||
if document.visibility == DocumentVisibility.public:
|
||||
clauses.append(true())
|
||||
elif (
|
||||
document.visibility == DocumentVisibility.department
|
||||
and document.department_id is not None
|
||||
):
|
||||
clauses.append(User.department_id == document.department_id)
|
||||
clauses.append(
|
||||
User.department_id.in_(
|
||||
select(DocPermission.department_id).where(
|
||||
DocPermission.document_id == document.id
|
||||
)
|
||||
)
|
||||
)
|
||||
return or_(*clauses)
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Hybrid retrieval: permission filter BEFORE anything else, then vector +
|
||||
German full-text candidates merged with Reciprocal Rank Fusion.
|
||||
|
||||
There is no search without a user — the permission CTE is part of the one
|
||||
SQL statement, so an unauthorized chunk is structurally impossible to
|
||||
retrieve. Queries are user content and are never logged.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Text, cast, func, literal, select, union
|
||||
from sqlalchemy.dialects.postgresql import REGCONFIG
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.llm.client import embed
|
||||
from app.metrics import metrics
|
||||
from app.models import Chunk, Document, User
|
||||
from app.rag.permissions import has_open_review, searchable_documents_filter
|
||||
|
||||
logger = logging.getLogger("pablan.rag")
|
||||
|
||||
RRF_K = 60
|
||||
CANDIDATES_PER_SOURCE = 20
|
||||
|
||||
# Calibrated against bge-m3 (2026-07): matched top
|
||||
# hits land at cosine distance ~0.34-0.45, unrelated queries at 0.50+.
|
||||
NO_ANSWER_MIN_DISTANCE = 0.45
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
chunk_id: uuid.UUID
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
heading_path: str
|
||||
content: str
|
||||
score: float
|
||||
vector_distance: float | None
|
||||
fts_match: bool
|
||||
# An unanswered request to check this document. Travels with every hit so
|
||||
# an answer can mark the source it leaned on as not-yet-settled.
|
||||
review_pending: bool = False
|
||||
|
||||
|
||||
async def search(
|
||||
db: AsyncSession,
|
||||
query: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 5,
|
||||
) -> list[SearchResult]:
|
||||
started = time.monotonic()
|
||||
query_vector = (await embed([query]))[0]
|
||||
|
||||
allowed = (
|
||||
select(
|
||||
Document.id,
|
||||
Document.title,
|
||||
has_open_review().label("review_pending"),
|
||||
)
|
||||
.where(searchable_documents_filter(user))
|
||||
.cte("allowed")
|
||||
)
|
||||
|
||||
distance = Chunk.embedding.cosine_distance(query_vector)
|
||||
vec = (
|
||||
select(
|
||||
Chunk.id.label("chunk_id"),
|
||||
func.row_number().over(order_by=distance).label("rank"),
|
||||
distance.label("distance"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.order_by(distance)
|
||||
.limit(CANDIDATES_PER_SOURCE)
|
||||
.subquery("vec")
|
||||
)
|
||||
|
||||
tsquery = func.websearch_to_tsquery(cast(literal("german"), REGCONFIG), query)
|
||||
fts_order = func.ts_rank_cd(Chunk.tsv, tsquery).desc()
|
||||
fts = (
|
||||
select(
|
||||
Chunk.id.label("chunk_id"),
|
||||
func.row_number().over(order_by=fts_order).label("rank"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.where(Chunk.tsv.op("@@")(tsquery))
|
||||
.order_by(fts_order)
|
||||
.limit(CANDIDATES_PER_SOURCE)
|
||||
.subquery("fts")
|
||||
)
|
||||
|
||||
candidate_ids = union(select(vec.c.chunk_id), select(fts.c.chunk_id)).subquery(
|
||||
"ids"
|
||||
)
|
||||
score = (
|
||||
func.coalesce(1.0 / (RRF_K + vec.c.rank), 0.0)
|
||||
+ func.coalesce(1.0 / (RRF_K + fts.c.rank), 0.0)
|
||||
).label("score")
|
||||
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Chunk.id,
|
||||
Chunk.document_id,
|
||||
Chunk.content,
|
||||
Chunk.meta,
|
||||
allowed.c.title,
|
||||
allowed.c.review_pending,
|
||||
score,
|
||||
vec.c.distance,
|
||||
fts.c.rank.label("fts_rank"),
|
||||
)
|
||||
.join(candidate_ids, candidate_ids.c.chunk_id == Chunk.id)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.outerjoin(vec, vec.c.chunk_id == Chunk.id)
|
||||
.outerjoin(fts, fts.c.chunk_id == Chunk.id)
|
||||
.order_by(score.desc())
|
||||
.limit(top_k)
|
||||
)
|
||||
).all()
|
||||
|
||||
results = [
|
||||
SearchResult(
|
||||
chunk_id=row.id,
|
||||
document_id=row.document_id,
|
||||
title=row.title,
|
||||
heading_path=heading_path(row.meta),
|
||||
content=row.content,
|
||||
score=float(row.score),
|
||||
vector_distance=float(row.distance) if row.distance is not None else None,
|
||||
fts_match=row.fts_rank is not None,
|
||||
review_pending=bool(row.review_pending),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("retrieval_searches_total")
|
||||
metrics.observe("retrieval_seconds", duration)
|
||||
metrics.observe("retrieval_results", float(len(results)))
|
||||
for result in results:
|
||||
source = (
|
||||
"both"
|
||||
if result.fts_match and result.vector_distance is not None
|
||||
else ("fts" if result.fts_match else "vector")
|
||||
)
|
||||
metrics.inc("retrieval_result_source_total", {"source": source})
|
||||
logger.info(
|
||||
"retrieval",
|
||||
extra={
|
||||
"event": "retrieval",
|
||||
"duration_ms": round(duration * 1000),
|
||||
"result_count": len(results),
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _any_term_tsquery(query: str) -> Any:
|
||||
"""The fallback's query: the lexemes `websearch_to_tsquery` produces, but
|
||||
ORed instead of ANDed.
|
||||
|
||||
With no vector half to carry the recall, an AND query answers a natural
|
||||
question ("Wie läuft die Qualitätsprüfung im Wareneingang?") with nothing
|
||||
at all unless one single chunk happens to contain every word of it. ORing
|
||||
keeps the question usable and leaves the ordering to `ts_rank_cd`, which
|
||||
is what ranks a chunk matching more of the terms higher. Only the AND
|
||||
operators between groups are rewritten, so quoted phrases and exclusions
|
||||
survive. A query of nothing but stop words rewrites to an empty string,
|
||||
and NULLIF turns that into a query that matches nothing rather than a
|
||||
syntax error.
|
||||
"""
|
||||
websearch = func.websearch_to_tsquery(cast(literal("german"), REGCONFIG), query)
|
||||
lexemes = func.nullif(func.replace(cast(websearch, Text), " & ", " | "), "")
|
||||
return func.to_tsquery(cast(literal("german"), REGCONFIG), lexemes)
|
||||
|
||||
|
||||
async def text_search(
|
||||
db: AsyncSession,
|
||||
query: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 5,
|
||||
) -> list[SearchResult]:
|
||||
"""The full-text half of `search()` alone: German `tsvector` matching over
|
||||
the GIN index (an inverted index), with no embedding call.
|
||||
|
||||
This is what keeps the knowledge base searchable when no model answers at
|
||||
the configured endpoint. It finds less than the hybrid path (keywords, not
|
||||
meaning), so it is a fallback the user is told about, never a silent
|
||||
substitute. Same permission CTE as everything else — there is no search
|
||||
without a user.
|
||||
|
||||
Terms are ORed here while the hybrid path ANDs them (`_any_term_tsquery`):
|
||||
alone, a full-sentence question must not come back empty, and in the
|
||||
hybrid path the AND is what makes `fts_match` mean "the words are really
|
||||
in there" for the no-answer signal.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
allowed = (
|
||||
select(
|
||||
Document.id,
|
||||
Document.title,
|
||||
has_open_review().label("review_pending"),
|
||||
)
|
||||
.where(searchable_documents_filter(user))
|
||||
.cte("allowed")
|
||||
)
|
||||
tsquery = _any_term_tsquery(query)
|
||||
rank = func.ts_rank_cd(Chunk.tsv, tsquery)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Chunk.id,
|
||||
Chunk.document_id,
|
||||
Chunk.content,
|
||||
Chunk.meta,
|
||||
allowed.c.title,
|
||||
allowed.c.review_pending,
|
||||
rank.label("rank"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.where(Chunk.tsv.op("@@")(tsquery))
|
||||
.order_by(rank.desc())
|
||||
.limit(top_k)
|
||||
)
|
||||
).all()
|
||||
|
||||
results = [
|
||||
SearchResult(
|
||||
chunk_id=row.id,
|
||||
document_id=row.document_id,
|
||||
title=row.title,
|
||||
heading_path=heading_path(row.meta),
|
||||
content=row.content,
|
||||
score=float(row.rank),
|
||||
vector_distance=None,
|
||||
fts_match=True,
|
||||
review_pending=bool(row.review_pending),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("retrieval_text_searches_total")
|
||||
metrics.observe("retrieval_seconds", duration)
|
||||
logger.info(
|
||||
"retrieval (text only)",
|
||||
extra={
|
||||
"event": "retrieval_text",
|
||||
"duration_ms": round(duration * 1000),
|
||||
"result_count": len(results),
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def heading_path(meta: dict[str, Any] | None) -> str:
|
||||
return (meta or {}).get("heading_path", "")
|
||||
|
||||
|
||||
def results_are_low_confidence(results: list[SearchResult]) -> bool:
|
||||
"""No-answer signal: no keyword match anywhere and the best vector
|
||||
candidate is far away. Callers should not present such results as
|
||||
grounding."""
|
||||
if not results:
|
||||
return True
|
||||
top = results[0]
|
||||
return not top.fts_match and (
|
||||
top.vector_distance is None or top.vector_distance >= NO_ANSWER_MIN_DISTANCE
|
||||
)
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Similarity: "what else is close to this text", with a threshold.
|
||||
|
||||
Deliberately NOT the hybrid path. RRF produces a fusion rank, not a
|
||||
similarity, and its `vector_distance` is None for hits that surfaced only
|
||||
through full text — a threshold needs a comparable number. What both paths DO
|
||||
share is the permission filter: the same `allowed` CTE, so a suggestion can
|
||||
never point at something the caller may not read.
|
||||
|
||||
Two callers, two calibrated limits: a refinement grounds on loosely related
|
||||
material, while a duplicate check may only propose merging on a close match.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import and_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.llm.client import embed
|
||||
from app.metrics import metrics
|
||||
from app.models import Chunk, Document, User
|
||||
from app.rag.permissions import searchable_documents_filter
|
||||
from app.rag.retrieval import CANDIDATES_PER_SOURCE, heading_path
|
||||
|
||||
logger = logging.getLogger("pablan.rag")
|
||||
|
||||
# One mechanic at two moments: during capture a loose limit is useful
|
||||
# because a near-miss still makes good context, while at review time only a
|
||||
# high-confidence match may propose merging into an existing document.
|
||||
# CHANGING THE EMBEDDING MODEL MEANS RE-MEASURING ALL THREE constants;
|
||||
# tests/evals/test_duplicate_eval.py prints the numbers to do it with.
|
||||
#
|
||||
# Measured against bge-m3 on the fixture corpus (2026-07-20): drafts that
|
||||
# duplicate an existing document land at 0.116-0.274, genuinely new topics
|
||||
# at 0.402-0.486. 0.35 sits in that gap. The upper end of the duplicate
|
||||
# range comes from REAL capture drafts, which are compressed notes rather
|
||||
# than full prose and therefore sit further from their source than a
|
||||
# hand-written paraphrase does — calibrating on paraphrases alone gives a
|
||||
# threshold that misses real duplicates (it did: 0.25 missed one at 0.256).
|
||||
CAPTURE_CONTEXT_MAX_DISTANCE = 0.45
|
||||
DUPLICATE_MAX_DISTANCE = 0.35
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimilarChunk:
|
||||
chunk_id: uuid.UUID
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
heading_path: str
|
||||
content: str
|
||||
distance: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class SimilarDocument:
|
||||
document_id: uuid.UUID
|
||||
title: str
|
||||
distance: float # the closest chunk of that document
|
||||
|
||||
|
||||
async def similar_chunks(
|
||||
db: AsyncSession,
|
||||
text: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 5,
|
||||
max_distance: float,
|
||||
exclude_builtin: bool = False,
|
||||
exclude_document_id: uuid.UUID | None = None,
|
||||
) -> list[SimilarChunk]:
|
||||
"""Pure vector neighbours of a text, permission-filtered like everything
|
||||
else — the same `allowed` CTE `search()` uses.
|
||||
|
||||
Deliberately NOT the hybrid path: RRF produces a fusion rank, not a
|
||||
similarity, and its `vector_distance` is None for hits that surfaced
|
||||
only through full text. A threshold needs a comparable number.
|
||||
|
||||
`max_distance` is keyword-only and has no default on purpose: every
|
||||
caller names one of the two calibrated constants, so "similar" means
|
||||
exactly two things in this product and both are written down.
|
||||
|
||||
`exclude_builtin` drops Pablan's own help pages. They are answerable
|
||||
through query mode on purpose (the product documents itself), but a
|
||||
capture or duplicate check asks "what does the COMPANY already know" —
|
||||
proposing to extend a help page, or telling an author their topic is
|
||||
"already documented" because a help page mentions it, is wrong.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
vector = (await embed([text]))[0]
|
||||
|
||||
allowed_filter = searchable_documents_filter(user)
|
||||
if exclude_builtin:
|
||||
allowed_filter = and_(allowed_filter, Document.is_builtin.is_(False))
|
||||
if exclude_document_id is not None:
|
||||
# A document must never ground on itself (the extend flow re-opens a
|
||||
# published document and would otherwise retrieve its own chunks).
|
||||
allowed_filter = and_(allowed_filter, Document.id != exclude_document_id)
|
||||
allowed = select(Document.id, Document.title).where(allowed_filter).cte("allowed")
|
||||
distance = Chunk.embedding.cosine_distance(vector)
|
||||
|
||||
# Threshold in Python, after ORDER BY ... LIMIT: a distance predicate in
|
||||
# WHERE fights the HNSW index, ordering and limiting is what it serves.
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
Chunk.id,
|
||||
Chunk.document_id,
|
||||
Chunk.content,
|
||||
Chunk.meta,
|
||||
allowed.c.title,
|
||||
distance.label("distance"),
|
||||
)
|
||||
.join(allowed, allowed.c.id == Chunk.document_id)
|
||||
.order_by(distance)
|
||||
.limit(top_k)
|
||||
)
|
||||
).all()
|
||||
|
||||
results = [
|
||||
SimilarChunk(
|
||||
chunk_id=row.id,
|
||||
document_id=row.document_id,
|
||||
title=row.title,
|
||||
heading_path=heading_path(row.meta),
|
||||
content=row.content,
|
||||
distance=float(row.distance),
|
||||
)
|
||||
for row in rows
|
||||
if float(row.distance) <= max_distance
|
||||
]
|
||||
|
||||
duration = time.monotonic() - started
|
||||
metrics.inc("similarity_searches_total")
|
||||
metrics.observe("similarity_seconds", duration)
|
||||
logger.info(
|
||||
"similarity",
|
||||
extra={
|
||||
"event": "similarity",
|
||||
"duration_ms": round(duration * 1000),
|
||||
"candidate_count": len(rows),
|
||||
"result_count": len(results),
|
||||
"top_k": top_k,
|
||||
},
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
async def similar_documents(
|
||||
db: AsyncSession,
|
||||
text: str,
|
||||
*,
|
||||
user: User,
|
||||
top_k: int = 3,
|
||||
max_distance: float,
|
||||
exclude_builtin: bool = False,
|
||||
) -> list[SimilarDocument]:
|
||||
"""Documents near a text, ranked by their closest chunk.
|
||||
|
||||
Overfetches chunks and groups them, so this is literally the same search
|
||||
as `similar_chunks` — one notion of "similar" in the product, not two
|
||||
implementations that drift apart.
|
||||
"""
|
||||
chunks = await similar_chunks(
|
||||
db,
|
||||
text,
|
||||
user=user,
|
||||
top_k=CANDIDATES_PER_SOURCE,
|
||||
max_distance=max_distance,
|
||||
exclude_builtin=exclude_builtin,
|
||||
)
|
||||
best: dict[uuid.UUID, SimilarDocument] = {}
|
||||
for chunk in chunks: # distance-ordered, so the first hit per document wins
|
||||
best.setdefault(
|
||||
chunk.document_id,
|
||||
SimilarDocument(
|
||||
document_id=chunk.document_id,
|
||||
title=chunk.title,
|
||||
distance=chunk.distance,
|
||||
),
|
||||
)
|
||||
return list(best.values())[:top_k]
|
||||
Reference in New Issue
Block a user