"""Finding documents: the paged list, ranked search, the ZIP export, and the company-wide counts. Every route here has a static path, so this router is included FIRST: after `/{document_id}` is registered, "search" would be parsed as a document id. """ import io import re import uuid import zipfile from typing import Annotated import yaml from fastapi import Depends, Query from fastapi.responses import StreamingResponse from sqlalchemy import exists, func, or_, select, true from sqlalchemy.ext.asyncio import AsyncSession from app.api.documents.routing import documents_router from app.api.documents.schemas import ( DocumentPage, DocumentSearchHit, DocumentSort, DocumentStats, ) from app.api.documents.view import document_fields, summary from app.auth.deps import get_current_user from app.db import get_db from app.models import ( Department, DocPermission, Document, DocumentStatus, User, ) from app.rag.permissions import open_review_for, readable_documents_filter # aliased: `search` is also a query parameter on the list endpoint from app.rag.retrieval import search as hybrid_search router = documents_router() # Chunks retrieved before grouping, and the most documents a search returns. SEARCH_CANDIDATES = 20 SEARCH_LIMIT = 20 @router.get("") async def list_documents( user: Annotated[User, Depends(get_current_user)], db: Annotated[AsyncSession, Depends(get_db)], department: uuid.UUID | None = None, status: DocumentStatus | None = None, assigned_to_me: bool = False, search: str | None = Query(None, max_length=200), sort: DocumentSort = DocumentSort.updated, page: int = Query(1, ge=1), per_page: int = Query(30, ge=1, le=100), ) -> DocumentPage: """Browse readable documents. Paginated server-side: the list is the one screen that grows without bound as a knowledge base fills up. Search has its own endpoint and is ranked rather than paged. """ filters = [readable_documents_filter(user)] if department is not None: # A department filter matches the owning department OR a shared grant, # so a document shared with a department shows up under it too. filters.append( or_( Document.department_id == department, exists( select(DocPermission.document_id).where( DocPermission.document_id == Document.id, DocPermission.department_id == department, ) ), ) ) if status is not None: filters.append(Document.status == status) if assigned_to_me: # "Waiting for me": documents someone asked THIS user to check. filters.append(open_review_for(user)) if search: filters.append(Document.title.ilike(f"%{search}%")) total = ( await db.execute(select(func.count(Document.id)).where(*filters)) ).scalar_one() order = ( Document.created_at.desc() if sort is DocumentSort.created else Document.updated_at.desc() ) documents = ( ( await db.execute( select(Document) .where(*filters) # Built-in help is reference material and belongs after the # team's own documents — sorted in SQL so it holds across page # boundaries, which a client-side sort could not manage. # `Document.id` breaks ties. Without it the order is only # partial: the corpus is seeded in one transaction, so many # rows share a timestamp to the microsecond, and Postgres is # free to return them in a different order per query. Two pages # then overlap and a document is shown twice while another is # never reachable. .order_by(Document.is_builtin.asc(), order, Document.id) .offset((page - 1) * per_page) .limit(per_page) ) ) .scalars() .all() ) return DocumentPage( items=[summary(document, user) for document in documents], total=total, per_page=per_page, ) @router.get("/search") async def search_documents( user: Annotated[User, Depends(get_current_user)], db: Annotated[AsyncSession, Depends(get_db)], q: Annotated[str, Query(min_length=1, max_length=200)], ) -> list[DocumentSearchHit]: """Find documents through the same hybrid retrieval the chat uses. Permission-safe by construction: `search()` requires a user and applies the shared filter. Drafts and pending documents are readable but never indexed, so a title fallback covers them — the one asymmetry between this endpoint and chat retrieval. """ results = await hybrid_search(db, q, user=user, top_k=SEARCH_CANDIDATES) # Group chunks per document, keeping the best-scoring chunk's heading. best_heading: dict[uuid.UUID, str] = {} for result in results: best_heading.setdefault(result.document_id, result.heading_path) hits: list[DocumentSearchHit] = [] if best_heading: documents = ( ( await db.execute( select(Document).where( Document.id.in_(best_heading), readable_documents_filter(user), ) ) ) .scalars() .all() ) by_id = {document.id: document for document in documents} # Preserve retrieval order — relevance, not insertion order. for document_id, heading in best_heading.items(): document = by_id.get(document_id) if document is not None: hits.append( DocumentSearchHit( **document_fields(document, user), heading_path=heading, ) ) # Title fallback for everything retrieval cannot see. remaining = SEARCH_LIMIT - len(hits) if remaining > 0: by_title = ( ( await db.execute( select(Document) .where( readable_documents_filter(user), Document.title.ilike(f"%{q}%"), Document.id.notin_(best_heading) if best_heading else true(), ) .order_by(Document.updated_at.desc()) .limit(remaining) ) ) .scalars() .all() ) hits.extend( DocumentSearchHit(**document_fields(document, user)) for document in by_title ) return hits[:SEARCH_LIMIT] def _export_name(document: Document, used: set[str]) -> str: """A stable, de-duplicated `.md` filename for a document in the export.""" slug = (document.meta or {}).get("slug") base = slug or re.sub(r"[^a-z0-9]+", "-", document.title.lower()).strip("-") base = base or str(document.id) name = f"{base}.md" counter = 2 while name in used: name = f"{base}-{counter}.md" counter += 1 used.add(name) return name @router.get("/export") async def export_documents( user: Annotated[User, Depends(get_current_user)], db: Annotated[AsyncSession, Depends(get_db)], ) -> StreamingResponse: """The readable knowledge base as a ZIP of Markdown files with YAML frontmatter. Permission-filtered by construction (`readable_documents_filter` — an admin exports what they can read, anyone else the same); built-in help pages are excluded (product content, not the company's knowledge). stdlib only, streamed, no temp files.""" documents = ( ( await db.execute( select(Document) .where(readable_documents_filter(user), Document.is_builtin.is_(False)) .order_by(Document.title) ) ) .scalars() .all() ) # Resolve department names once for the frontmatter: the owning department # plus any shared grants, so an export records the full reach of a document. dept_names = dict((await db.execute(select(Department.id, Department.name))).all()) shared: dict[uuid.UUID, list[str]] = {} for doc_id, dept_id in ( await db.execute(select(DocPermission.document_id, DocPermission.department_id)) ).all(): shared.setdefault(doc_id, []).append(dept_names.get(dept_id, "")) buffer = io.BytesIO() used: set[str] = set() with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: for document in documents: departments = [ *( [dept_names[document.department_id]] if document.department_id in dept_names else [] ), *sorted(shared.get(document.id, [])), ] frontmatter = yaml.safe_dump( { "title": document.title, "status": str(document.status), "visibility": str(document.visibility), "departments": departments, }, allow_unicode=True, sort_keys=False, ) body = f"---\n{frontmatter}---\n\n{document.content_md.rstrip()}\n" archive.writestr(_export_name(document, used), body) buffer.seek(0) return StreamingResponse( iter([buffer.getvalue()]), media_type="application/zip", headers={"content-disposition": 'attachment; filename="pablan-export.zip"'}, ) @router.get("/stats") async def document_stats( user: Annotated[User, Depends(get_current_user)], db: Annotated[AsyncSession, Depends(get_db)], ) -> DocumentStats: """Is this a fresh install or a filled one? Read by the landing page's first-run guide.""" published = Document.status == DocumentStatus.published return DocumentStats( documents_total=( await db.execute(select(func.count(Document.id)).where(published)) ).scalar_one(), departments_total=( await db.execute(select(func.count(Department.id))) ).scalar_one(), )