Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
@@ -0,0 +1,293 @@
|
||||
"""Finding documents: the paged list, ranked search, the ZIP export, and the
|
||||
company-wide counts.
|
||||
|
||||
Every route here has a static path, so this router is included FIRST: after
|
||||
`/{document_id}` is registered, "search" would be parsed as a document id.
|
||||
"""
|
||||
|
||||
import io
|
||||
import re
|
||||
import uuid
|
||||
import zipfile
|
||||
from typing import Annotated
|
||||
|
||||
import yaml
|
||||
from fastapi import Depends, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import exists, func, or_, select, true
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.documents.routing import documents_router
|
||||
from app.api.documents.schemas import (
|
||||
DocumentPage,
|
||||
DocumentSearchHit,
|
||||
DocumentSort,
|
||||
DocumentStats,
|
||||
)
|
||||
from app.api.documents.view import document_fields, summary
|
||||
from app.auth.deps import get_current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
Department,
|
||||
DocPermission,
|
||||
Document,
|
||||
DocumentStatus,
|
||||
User,
|
||||
)
|
||||
from app.rag.permissions import open_review_for, readable_documents_filter
|
||||
|
||||
# aliased: `search` is also a query parameter on the list endpoint
|
||||
from app.rag.retrieval import search as hybrid_search
|
||||
|
||||
router = documents_router()
|
||||
|
||||
# Chunks retrieved before grouping, and the most documents a search returns.
|
||||
SEARCH_CANDIDATES = 20
|
||||
SEARCH_LIMIT = 20
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_documents(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
department: uuid.UUID | None = None,
|
||||
status: DocumentStatus | None = None,
|
||||
assigned_to_me: bool = False,
|
||||
search: str | None = Query(None, max_length=200),
|
||||
sort: DocumentSort = DocumentSort.updated,
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(30, ge=1, le=100),
|
||||
) -> DocumentPage:
|
||||
"""Browse readable documents.
|
||||
|
||||
Paginated server-side: the list is the one screen that grows without
|
||||
bound as a knowledge base fills up. Search has its own endpoint and is
|
||||
ranked rather than paged.
|
||||
"""
|
||||
filters = [readable_documents_filter(user)]
|
||||
if department is not None:
|
||||
# A department filter matches the owning department OR a shared grant,
|
||||
# so a document shared with a department shows up under it too.
|
||||
filters.append(
|
||||
or_(
|
||||
Document.department_id == department,
|
||||
exists(
|
||||
select(DocPermission.document_id).where(
|
||||
DocPermission.document_id == Document.id,
|
||||
DocPermission.department_id == department,
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
if status is not None:
|
||||
filters.append(Document.status == status)
|
||||
if assigned_to_me:
|
||||
# "Waiting for me": documents someone asked THIS user to check.
|
||||
filters.append(open_review_for(user))
|
||||
if search:
|
||||
filters.append(Document.title.ilike(f"%{search}%"))
|
||||
|
||||
total = (
|
||||
await db.execute(select(func.count(Document.id)).where(*filters))
|
||||
).scalar_one()
|
||||
|
||||
order = (
|
||||
Document.created_at.desc()
|
||||
if sort is DocumentSort.created
|
||||
else Document.updated_at.desc()
|
||||
)
|
||||
documents = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document)
|
||||
.where(*filters)
|
||||
# Built-in help is reference material and belongs after the
|
||||
# team's own documents — sorted in SQL so it holds across page
|
||||
# boundaries, which a client-side sort could not manage.
|
||||
# `Document.id` breaks ties. Without it the order is only
|
||||
# partial: the corpus is seeded in one transaction, so many
|
||||
# rows share a timestamp to the microsecond, and Postgres is
|
||||
# free to return them in a different order per query. Two pages
|
||||
# then overlap and a document is shown twice while another is
|
||||
# never reachable.
|
||||
.order_by(Document.is_builtin.asc(), order, Document.id)
|
||||
.offset((page - 1) * per_page)
|
||||
.limit(per_page)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return DocumentPage(
|
||||
items=[summary(document, user) for document in documents],
|
||||
total=total,
|
||||
per_page=per_page,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
async def search_documents(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
q: Annotated[str, Query(min_length=1, max_length=200)],
|
||||
) -> list[DocumentSearchHit]:
|
||||
"""Find documents through the same hybrid retrieval the chat uses.
|
||||
|
||||
Permission-safe by construction: `search()` requires a user and applies
|
||||
the shared filter. Drafts and pending documents are readable
|
||||
but never indexed, so a title fallback covers them — the one asymmetry
|
||||
between this endpoint and chat retrieval.
|
||||
"""
|
||||
results = await hybrid_search(db, q, user=user, top_k=SEARCH_CANDIDATES)
|
||||
|
||||
# Group chunks per document, keeping the best-scoring chunk's heading.
|
||||
best_heading: dict[uuid.UUID, str] = {}
|
||||
for result in results:
|
||||
best_heading.setdefault(result.document_id, result.heading_path)
|
||||
|
||||
hits: list[DocumentSearchHit] = []
|
||||
if best_heading:
|
||||
documents = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document).where(
|
||||
Document.id.in_(best_heading),
|
||||
readable_documents_filter(user),
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
by_id = {document.id: document for document in documents}
|
||||
# Preserve retrieval order — relevance, not insertion order.
|
||||
for document_id, heading in best_heading.items():
|
||||
document = by_id.get(document_id)
|
||||
if document is not None:
|
||||
hits.append(
|
||||
DocumentSearchHit(
|
||||
**document_fields(document, user),
|
||||
heading_path=heading,
|
||||
)
|
||||
)
|
||||
|
||||
# Title fallback for everything retrieval cannot see.
|
||||
remaining = SEARCH_LIMIT - len(hits)
|
||||
if remaining > 0:
|
||||
by_title = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document)
|
||||
.where(
|
||||
readable_documents_filter(user),
|
||||
Document.title.ilike(f"%{q}%"),
|
||||
Document.id.notin_(best_heading) if best_heading else true(),
|
||||
)
|
||||
.order_by(Document.updated_at.desc())
|
||||
.limit(remaining)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
hits.extend(
|
||||
DocumentSearchHit(**document_fields(document, user))
|
||||
for document in by_title
|
||||
)
|
||||
return hits[:SEARCH_LIMIT]
|
||||
|
||||
|
||||
def _export_name(document: Document, used: set[str]) -> str:
|
||||
"""A stable, de-duplicated `.md` filename for a document in the export."""
|
||||
slug = (document.meta or {}).get("slug")
|
||||
base = slug or re.sub(r"[^a-z0-9]+", "-", document.title.lower()).strip("-")
|
||||
base = base or str(document.id)
|
||||
name = f"{base}.md"
|
||||
counter = 2
|
||||
while name in used:
|
||||
name = f"{base}-{counter}.md"
|
||||
counter += 1
|
||||
used.add(name)
|
||||
return name
|
||||
|
||||
|
||||
@router.get("/export")
|
||||
async def export_documents(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> StreamingResponse:
|
||||
"""The readable knowledge base as a ZIP of Markdown files with YAML
|
||||
frontmatter. Permission-filtered by construction (`readable_documents_filter`
|
||||
— an admin exports what they can read, anyone else the same); built-in help
|
||||
pages are excluded (product content, not the company's knowledge). stdlib
|
||||
only, streamed, no temp files."""
|
||||
documents = (
|
||||
(
|
||||
await db.execute(
|
||||
select(Document)
|
||||
.where(readable_documents_filter(user), Document.is_builtin.is_(False))
|
||||
.order_by(Document.title)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
# Resolve department names once for the frontmatter: the owning department
|
||||
# plus any shared grants, so an export records the full reach of a document.
|
||||
dept_names = dict((await db.execute(select(Department.id, Department.name))).all())
|
||||
shared: dict[uuid.UUID, list[str]] = {}
|
||||
for doc_id, dept_id in (
|
||||
await db.execute(select(DocPermission.document_id, DocPermission.department_id))
|
||||
).all():
|
||||
shared.setdefault(doc_id, []).append(dept_names.get(dept_id, ""))
|
||||
|
||||
buffer = io.BytesIO()
|
||||
used: set[str] = set()
|
||||
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
for document in documents:
|
||||
departments = [
|
||||
*(
|
||||
[dept_names[document.department_id]]
|
||||
if document.department_id in dept_names
|
||||
else []
|
||||
),
|
||||
*sorted(shared.get(document.id, [])),
|
||||
]
|
||||
frontmatter = yaml.safe_dump(
|
||||
{
|
||||
"title": document.title,
|
||||
"status": str(document.status),
|
||||
"visibility": str(document.visibility),
|
||||
"departments": departments,
|
||||
},
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
)
|
||||
body = f"---\n{frontmatter}---\n\n{document.content_md.rstrip()}\n"
|
||||
archive.writestr(_export_name(document, used), body)
|
||||
|
||||
buffer.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([buffer.getvalue()]),
|
||||
media_type="application/zip",
|
||||
headers={"content-disposition": 'attachment; filename="pablan-export.zip"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
async def document_stats(
|
||||
user: Annotated[User, Depends(get_current_user)],
|
||||
db: Annotated[AsyncSession, Depends(get_db)],
|
||||
) -> DocumentStats:
|
||||
"""Is this a fresh install or a filled one? Read by the landing page's
|
||||
first-run guide."""
|
||||
published = Document.status == DocumentStatus.published
|
||||
return DocumentStats(
|
||||
documents_total=(
|
||||
await db.execute(select(func.count(Document.id)).where(published))
|
||||
).scalar_one(),
|
||||
departments_total=(
|
||||
await db.execute(select(func.count(Department.id)))
|
||||
).scalar_one(),
|
||||
)
|
||||
Reference in New Issue
Block a user