pablan/backend/app/api/documents/browse.py

"""Finding documents: the paged list, ranked search, the ZIP export, and the
company-wide counts.

Every route here has a static path, so this router is included FIRST: after
`/{document_id}` is registered, "search" would be parsed as a document id.
"""

import io
import re
import uuid
import zipfile
from typing import Annotated

import yaml
from fastapi import Depends, Query
from fastapi.responses import StreamingResponse
from sqlalchemy import exists, func, or_, select, true
from sqlalchemy.ext.asyncio import AsyncSession

from app.api.documents.routing import documents_router
from app.api.documents.schemas import (
    DocumentPage,
    DocumentSearchHit,
    DocumentSort,
    DocumentStats,
)
from app.api.documents.view import document_fields, summary
from app.auth.deps import get_current_user
from app.db import get_db
from app.models import (
    Department,
    DocPermission,
    Document,
    DocumentStatus,
    User,
)
from app.rag.permissions import open_review_for, readable_documents_filter

# aliased: `search` is also a query parameter on the list endpoint
from app.rag.retrieval import search as hybrid_search

router = documents_router()

# Chunks retrieved before grouping, and the most documents a search returns.
SEARCH_CANDIDATES = 20
SEARCH_LIMIT = 20


@router.get("")
async def list_documents(
    user: Annotated[User, Depends(get_current_user)],
    db: Annotated[AsyncSession, Depends(get_db)],
    department: uuid.UUID | None = None,
    status: DocumentStatus | None = None,
    assigned_to_me: bool = False,
    search: str | None = Query(None, max_length=200),
    sort: DocumentSort = DocumentSort.updated,
    page: int = Query(1, ge=1),
    per_page: int = Query(30, ge=1, le=100),
) -> DocumentPage:
    """Browse readable documents.

    Paginated server-side: the list is the one screen that grows without
    bound as a knowledge base fills up. Search has its own endpoint and is
    ranked rather than paged.
    """
    filters = [readable_documents_filter(user)]
    if department is not None:
        # A department filter matches the owning department OR a shared grant,
        # so a document shared with a department shows up under it too.
        filters.append(
            or_(
                Document.department_id == department,
                exists(
                    select(DocPermission.document_id).where(
                        DocPermission.document_id == Document.id,
                        DocPermission.department_id == department,
                    )
                ),
            )
        )
    if status is not None:
        filters.append(Document.status == status)
    if assigned_to_me:
        # "Waiting for me": documents someone asked THIS user to check.
        filters.append(open_review_for(user))
    if search:
        filters.append(Document.title.ilike(f"%{search}%"))

    total = (
        await db.execute(select(func.count(Document.id)).where(*filters))
    ).scalar_one()

    order = (
        Document.created_at.desc()
        if sort is DocumentSort.created
        else Document.updated_at.desc()
    )
    documents = (
        (
            await db.execute(
                select(Document)
                .where(*filters)
                # Built-in help is reference material and belongs after the
                # team's own documents — sorted in SQL so it holds across page
                # boundaries, which a client-side sort could not manage.
                # `Document.id` breaks ties. Without it the order is only
                # partial: the corpus is seeded in one transaction, so many
                # rows share a timestamp to the microsecond, and Postgres is
                # free to return them in a different order per query. Two pages
                # then overlap and a document is shown twice while another is
                # never reachable.
                .order_by(Document.is_builtin.asc(), order, Document.id)
                .offset((page - 1) * per_page)
                .limit(per_page)
            )
        )
        .scalars()
        .all()
    )
    return DocumentPage(
        items=[summary(document, user) for document in documents],
        total=total,
        per_page=per_page,
    )


@router.get("/search")
async def search_documents(
    user: Annotated[User, Depends(get_current_user)],
    db: Annotated[AsyncSession, Depends(get_db)],
    q: Annotated[str, Query(min_length=1, max_length=200)],
) -> list[DocumentSearchHit]:
    """Find documents through the same hybrid retrieval the chat uses.

    Permission-safe by construction: `search()` requires a user and applies
    the shared filter. Drafts and pending documents are readable
    but never indexed, so a title fallback covers them — the one asymmetry
    between this endpoint and chat retrieval.
    """
    results = await hybrid_search(db, q, user=user, top_k=SEARCH_CANDIDATES)

    # Group chunks per document, keeping the best-scoring chunk's heading.
    best_heading: dict[uuid.UUID, str] = {}
    for result in results:
        best_heading.setdefault(result.document_id, result.heading_path)

    hits: list[DocumentSearchHit] = []
    if best_heading:
        documents = (
            (
                await db.execute(
                    select(Document).where(
                        Document.id.in_(best_heading),
                        readable_documents_filter(user),
                    )
                )
            )
            .scalars()
            .all()
        )
        by_id = {document.id: document for document in documents}
        # Preserve retrieval order — relevance, not insertion order.
        for document_id, heading in best_heading.items():
            document = by_id.get(document_id)
            if document is not None:
                hits.append(
                    DocumentSearchHit(
                        **document_fields(document, user),
                        heading_path=heading,
                    )
                )

    # Title fallback for everything retrieval cannot see.
    remaining = SEARCH_LIMIT - len(hits)
    if remaining > 0:
        by_title = (
            (
                await db.execute(
                    select(Document)
                    .where(
                        readable_documents_filter(user),
                        Document.title.ilike(f"%{q}%"),
                        Document.id.notin_(best_heading) if best_heading else true(),
                    )
                    .order_by(Document.updated_at.desc())
                    .limit(remaining)
                )
            )
            .scalars()
            .all()
        )
        hits.extend(
            DocumentSearchHit(**document_fields(document, user))
            for document in by_title
        )
    return hits[:SEARCH_LIMIT]


def _export_name(document: Document, used: set[str]) -> str:
    """A stable, de-duplicated `.md` filename for a document in the export."""
    slug = (document.meta or {}).get("slug")
    base = slug or re.sub(r"[^a-z0-9]+", "-", document.title.lower()).strip("-")
    base = base or str(document.id)
    name = f"{base}.md"
    counter = 2
    while name in used:
        name = f"{base}-{counter}.md"
        counter += 1
    used.add(name)
    return name


@router.get("/export")
async def export_documents(
    user: Annotated[User, Depends(get_current_user)],
    db: Annotated[AsyncSession, Depends(get_db)],
) -> StreamingResponse:
    """The readable knowledge base as a ZIP of Markdown files with YAML
    frontmatter. Permission-filtered by construction (`readable_documents_filter`
    — an admin exports what they can read, anyone else the same); built-in help
    pages are excluded (product content, not the company's knowledge). stdlib
    only, streamed, no temp files."""
    documents = (
        (
            await db.execute(
                select(Document)
                .where(readable_documents_filter(user), Document.is_builtin.is_(False))
                .order_by(Document.title)
            )
        )
        .scalars()
        .all()
    )

    # Resolve department names once for the frontmatter: the owning department
    # plus any shared grants, so an export records the full reach of a document.
    dept_names = dict((await db.execute(select(Department.id, Department.name))).all())
    shared: dict[uuid.UUID, list[str]] = {}
    for doc_id, dept_id in (
        await db.execute(select(DocPermission.document_id, DocPermission.department_id))
    ).all():
        shared.setdefault(doc_id, []).append(dept_names.get(dept_id, ""))

    buffer = io.BytesIO()
    used: set[str] = set()
    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
        for document in documents:
            departments = [
                *(
                    [dept_names[document.department_id]]
                    if document.department_id in dept_names
                    else []
                ),
                *sorted(shared.get(document.id, [])),
            ]
            frontmatter = yaml.safe_dump(
                {
                    "title": document.title,
                    "status": str(document.status),
                    "visibility": str(document.visibility),
                    "departments": departments,
                },
                allow_unicode=True,
                sort_keys=False,
            )
            body = f"---\n{frontmatter}---\n\n{document.content_md.rstrip()}\n"
            archive.writestr(_export_name(document, used), body)

    buffer.seek(0)
    return StreamingResponse(
        iter([buffer.getvalue()]),
        media_type="application/zip",
        headers={"content-disposition": 'attachment; filename="pablan-export.zip"'},
    )


@router.get("/stats")
async def document_stats(
    user: Annotated[User, Depends(get_current_user)],
    db: Annotated[AsyncSession, Depends(get_db)],
) -> DocumentStats:
    """Is this a fresh install or a filled one? Read by the landing page's
    first-run guide."""
    published = Document.status == DocumentStatus.published
    return DocumentStats(
        documents_total=(
            await db.execute(select(func.count(Document.id)).where(published))
        ).scalar_one(),
        departments_total=(
            await db.execute(select(func.count(Department.id)))
        ).scalar_one(),
    )