Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
2026-09-04 09:21:37 +02:00
co-authored by Claude Opus 5
parent 68d3a43191
commit 784b76baf7
346 changed files with 43430 additions and 0 deletions
+58
View File
@@ -0,0 +1,58 @@
from app.models.auth_session import AuthSession
from app.models.base import Base
from app.models.conversation import Conversation, Message
from app.models.department import Department
from app.models.document import (
EMBEDDING_DIM,
Chunk,
DocPermission,
Document,
DocumentEvent,
ReviewRequest,
)
from app.models.enums import (
AccessReason,
ConversationMode,
ConversationStatus,
DocumentEventAction,
DocumentStatus,
DocumentVisibility,
JobStatus,
MessageRole,
PermissionLevel,
UserRole,
)
from app.models.job import Job
from app.models.llm_setting import LLMSetting
from app.models.prompt_setting import PromptSetting
from app.models.template import Template
from app.models.user import User
__all__ = [
"EMBEDDING_DIM",
"AuthSession",
"Base",
"Chunk",
"Conversation",
"ConversationMode",
"ConversationStatus",
"Department",
"DocPermission",
"Document",
"DocumentEvent",
"ReviewRequest",
"DocumentEventAction",
"DocumentStatus",
"DocumentVisibility",
"Job",
"LLMSetting",
"JobStatus",
"Message",
"MessageRole",
"AccessReason",
"PermissionLevel",
"PromptSetting",
"Template",
"User",
"UserRole",
]
+21
View File
@@ -0,0 +1,21 @@
import uuid
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
from app.models.user import User
class AuthSession(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""Server-side login session; the id doubles as the cookie token."""
__tablename__ = "auth_sessions"
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
user: Mapped[User] = relationship(lazy="joined")
+25
View File
@@ -0,0 +1,25 @@
import uuid
from datetime import datetime
from sqlalchemy import DateTime, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
# Fetch server-generated defaults (created_at/updated_at) via RETURNING
# at flush time — otherwise the async session would need a lazy refresh
# on attribute access, which raises MissingGreenlet outside a greenlet.
__mapper_args__ = {"eager_defaults": True}
class UUIDPrimaryKeyMixin:
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
class TimestampMixin:
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
+56
View File
@@ -0,0 +1,56 @@
import uuid
from typing import Any
from sqlalchemy import Enum, ForeignKey, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
from app.models.enums import ConversationMode, ConversationStatus, MessageRole
class Conversation(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""Chat thread. The only core mode is query (RAG Q&A); EE adds insight.
Capture is no longer a conversation — it writes a Document directly (see
app/authoring/) — so this table holds no per-turn engine state any more.
"""
__tablename__ = "conversations"
mode: Mapped[ConversationMode] = mapped_column(
Enum(ConversationMode, native_enum=False, length=32)
)
status: Mapped[ConversationStatus] = mapped_column(
Enum(ConversationStatus, native_enum=False, length=32),
default=ConversationStatus.active,
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"), index=True
)
messages: Mapped[list["Message"]] = relationship(
back_populates="conversation",
cascade="all, delete-orphan",
order_by="Message.created_at",
)
class Message(UUIDPrimaryKeyMixin, TimestampMixin, Base):
__tablename__ = "messages"
conversation_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("conversations.id", ondelete="CASCADE"), index=True
)
role: Mapped[MessageRole] = mapped_column(
Enum(MessageRole, native_enum=False, length=32)
)
content: Mapped[str] = mapped_column(Text)
# Assistant turns snapshot their citations here ({"sources": [...]}) so
# they survive reload and re-indexing — chunks are disposable, the
# rendered citation is not.
meta: Mapped[dict[str, Any]] = mapped_column(
JSONB, default=dict, server_default="{}"
)
conversation: Mapped[Conversation] = relationship(back_populates="messages")
+10
View File
@@ -0,0 +1,10 @@
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
class Department(UUIDPrimaryKeyMixin, TimestampMixin, Base):
__tablename__ = "departments"
name: Mapped[str] = mapped_column(String(200), unique=True)
+199
View File
@@ -0,0 +1,199 @@
import uuid
from datetime import datetime
from typing import Any
from pgvector.sqlalchemy import Vector
from sqlalchemy import (
Boolean,
Computed,
DateTime,
Enum,
ForeignKey,
Index,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
from app.models.enums import (
DocumentEventAction,
DocumentStatus,
DocumentVisibility,
PermissionLevel,
)
# Fixed by the embedding model (bge-m3). Changing the embedding model to a
# different dimension requires a migration plus reindex_all.
EMBEDDING_DIM = 1024
class Document(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""Markdown is the source of truth; chunks are disposable derivatives."""
__tablename__ = "documents"
title: Mapped[str] = mapped_column(String(500))
status: Mapped[DocumentStatus] = mapped_column(
Enum(DocumentStatus, native_enum=False, length=32),
default=DocumentStatus.draft,
)
visibility: Mapped[DocumentVisibility] = mapped_column(
Enum(DocumentVisibility, native_enum=False, length=32),
default=DocumentVisibility.department,
)
content_md: Mapped[str] = mapped_column(Text)
meta: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
# Built-in help documents: shipped with the product, re-imported from
# files on every start, and neither editable nor deletable in the UI.
is_builtin: Mapped[bool] = mapped_column(
Boolean, default=False, server_default="false"
)
# SET NULL: documents must survive their author leaving the company.
author_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL")
)
department_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("departments.id", ondelete="SET NULL")
)
chunks: Mapped[list["Chunk"]] = relationship(
back_populates="document", cascade="all, delete-orphan"
)
# Loaded with every document: whether a question is open decides who may
# edit it and how it is marked wherever it appears, so it is never a
# separate lookup a caller could forget.
reviews: Mapped[list["ReviewRequest"]] = relationship(
back_populates="document",
cascade="all, delete-orphan",
lazy="selectin",
order_by="ReviewRequest.created_at",
)
@property
def open_reviews(self) -> list["ReviewRequest"]:
return [review for review in self.reviews if review.resolved_at is None]
class Chunk(UUIDPrimaryKeyMixin, TimestampMixin, Base):
__tablename__ = "chunks"
__table_args__ = (
UniqueConstraint("document_id", "chunk_index"),
Index(
"ix_chunks_embedding_hnsw",
"embedding",
postgresql_using="hnsw",
postgresql_ops={"embedding": "vector_cosine_ops"},
),
Index("ix_chunks_tsv", "tsv", postgresql_using="gin"),
)
document_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("documents.id", ondelete="CASCADE"), index=True
)
chunk_index: Mapped[int] = mapped_column()
content: Mapped[str] = mapped_column(Text)
embedding: Mapped[list[float]] = mapped_column(Vector(EMBEDDING_DIM))
# The heading path is part of what the chunk says: a section reading
# "Solldruck 180 bar" never repeats which machine it belongs to, so a
# keyword query naming the machine has to reach it through its path.
tsv = mapped_column(
TSVECTOR,
Computed(
"to_tsvector('german'::regconfig, "
"content || ' ' || coalesce(meta->>'heading_path', ''))",
persisted=True,
),
)
meta: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
document: Mapped[Document] = relationship(back_populates="chunks")
class DocumentEvent(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""An append-only audit record: who did what to a document, and when.
Content-bearing actions (created / edited) snapshot the Markdown source of
truth so a past version can be viewed or diffed; the disposable chunks are
never snapshotted. `actor_id` is SET NULL so the record survives its actor
leaving the company, exactly like author_id on the document itself. Events
cascade with the document (DB-level ON DELETE CASCADE).
"""
__tablename__ = "document_events"
document_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("documents.id", ondelete="CASCADE"), index=True
)
# Who acted. Nullable so the trail outlives the actor's account.
actor_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL")
)
action: Mapped[DocumentEventAction] = mapped_column(
Enum(DocumentEventAction, native_enum=False, length=32)
)
# Frozen Markdown for content-bearing events (created / edited); NULL for
# pure transitions (published / archived / a review being asked or
# answered).
content_md: Mapped[str | None] = mapped_column(Text)
title: Mapped[str | None] = mapped_column(String(500))
# The document's visibility as of this event — cheap, so always recorded.
visibility: Mapped[DocumentVisibility | None] = mapped_column(
Enum(DocumentVisibility, native_enum=False, length=32)
)
meta: Mapped[dict[str, Any] | None] = mapped_column(JSONB)
class ReviewRequest(UUIDPrimaryKeyMixin, TimestampMixin, Base):
""" "Please look at this" — a question about a document, addressed to a
colleague.
Deliberately not a status. A document can be published AND have an open
question about it ("do the holiday numbers still hold?"), which is exactly
the case where readers most need to know: an open request marks the
document wherever it appears, including the sources under a chat answer.
Resolving is the reviewer's answer. Editing the document first is normal —
being asked to review is what grants the right to edit it.
"""
__tablename__ = "review_requests"
document_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("documents.id", ondelete="CASCADE"), index=True
)
# Both SET NULL: a request outlives the accounts on either side of it.
requester_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL")
)
reviewer_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL"), index=True
)
# What exactly to look at. Optional: "please check this" is a valid ask.
question: Mapped[str | None] = mapped_column(Text)
# NULL while open. The pair (resolved_at, resolved_by) is the answer.
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
resolved_by_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("users.id", ondelete="SET NULL")
)
document: Mapped["Document"] = relationship(back_populates="reviews")
class DocPermission(TimestampMixin, Base):
"""Additional department read grants on top of documents.visibility."""
__tablename__ = "doc_permissions"
document_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("documents.id", ondelete="CASCADE"), primary_key=True
)
department_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey("departments.id", ondelete="CASCADE"), primary_key=True
)
level: Mapped[PermissionLevel] = mapped_column(
Enum(PermissionLevel, native_enum=False, length=32),
default=PermissionLevel.read,
)
+87
View File
@@ -0,0 +1,87 @@
from enum import StrEnum
class UserRole(StrEnum):
member = "member"
admin = "admin"
class ConversationMode(StrEnum):
query = "query"
insight = "insight" # EE insights mode registers itself
class ConversationStatus(StrEnum):
active = "active"
completed = "completed"
abandoned = "abandoned"
class MessageRole(StrEnum):
user = "user"
assistant = "assistant"
system = "system"
class DocumentStatus(StrEnum):
"""Where a document stands.
Three states, because publishing is the author's own decision: a draft is
private, a published document is visible and indexed, an archived one is
neither. Uncertainty about CONTENT is not a status — it is an open review
request (`ReviewRequest`), which can sit on a published document too.
"""
draft = "draft"
published = "published"
archived = "archived"
class DocumentVisibility(StrEnum):
public = "public"
department = "department"
restricted = "restricted"
class PermissionLevel(StrEnum):
read = "read"
class AccessReason(StrEnum):
"""Why a document is visible to the requesting user.
API-only (never stored): computed per request so the UI can explain
access instead of leaving visibility rules implicit.
"""
author = "author"
public = "public"
department = "department"
granted = "granted"
# Only reason: somebody asked this user to check the document. It ends
# with their answer, which is why it is worth naming separately.
review = "review"
class DocumentEventAction(StrEnum):
"""A recorded step in a document's audit history.
Content-bearing actions (created / edited) snapshot the Markdown source of
truth; the rest record only who did what and when.
"""
created = "created"
edited = "edited"
published = "published"
archived = "archived"
visibility_changed = "visibility_changed"
# Someone was asked to check the content, and someone answered.
review_requested = "review_requested"
review_resolved = "review_resolved"
class JobStatus(StrEnum):
pending = "pending"
running = "running"
done = "done"
failed = "failed"
+27
View File
@@ -0,0 +1,27 @@
from datetime import datetime
from typing import Any
from sqlalchemy import DateTime, Enum, Index, String, Text, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
from app.models.enums import JobStatus
class Job(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""Postgres-backed background queue, claimed via FOR UPDATE SKIP LOCKED."""
__tablename__ = "jobs"
__table_args__ = (Index("ix_jobs_status_run_after", "status", "run_after"),)
type: Mapped[str] = mapped_column(String(100))
payload: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
status: Mapped[JobStatus] = mapped_column(
Enum(JobStatus, native_enum=False, length=32), default=JobStatus.pending
)
run_after: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
attempts: Mapped[int] = mapped_column(default=0)
last_error: Mapped[str | None] = mapped_column(Text)
+36
View File
@@ -0,0 +1,36 @@
from sqlalchemy import Boolean, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
class LLMSetting(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""Per-role endpoint configuration, edited in the admin UI.
One row per model role. The rows are created once, at first start, from
the `PABLAN_CHAT_*` / `PABLAN_UTILITY_*` / `PABLAN_EMBEDDING_*` environment
variables; from then on **this table is the truth** and later `.env`
edits are ignored (see docs/architecture.md, "bootstrap, then DB").
The `*_from_env` flags record where each field's current value came
from, so the UI can say "taken from .env" or "changed here" per field
and offer a reset. They are not a fallback mechanism: the value itself
always lives in the column next to them. Tracking the provenance
explicitly beats comparing against the current environment, which would
mislabel every field the moment someone edits `.env` after bootstrap.
The api_key is stored in plaintext because it has to be replayed to the
endpoint on every call — there is nothing to compare a hash against.
It is never returned by the API and never logged.
"""
__tablename__ = "llm_settings"
role: Mapped[str] = mapped_column(String(32), unique=True)
base_url: Mapped[str | None] = mapped_column(String(500))
model: Mapped[str | None] = mapped_column(String(200))
api_key: Mapped[str | None] = mapped_column(Text)
base_url_from_env: Mapped[bool] = mapped_column(Boolean, default=True)
model_from_env: Mapped[bool] = mapped_column(Boolean, default=True)
api_key_from_env: Mapped[bool] = mapped_column(Boolean, default=True)
+23
View File
@@ -0,0 +1,23 @@
from sqlalchemy import String, Text
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
class PromptSetting(UUIDPrimaryKeyMixin, TimestampMixin, Base):
"""An admin override for a shipped system prompt.
Every prompt has a CODE default (`app/prompts/defaults.py`); a row here
exists only when an admin has changed one. `content` is the full replacement
text. Applied without a restart via a module-level cache
(`app/prompts/overrides.py`), refreshed on every write — like the LLM
settings, and single-process by design (`--workers 1`). Resetting a prompt
deletes its row, so the code default takes over again. Unlike LLM settings
there is no `.env` layer: prompts have no environment representation, so the
reset target is the code default rather than the environment.
"""
__tablename__ = "prompt_settings"
key: Mapped[str] = mapped_column(String(64), unique=True)
content: Mapped[str] = mapped_column(Text)
+18
View File
@@ -0,0 +1,18 @@
from typing import Any
from sqlalchemy import String
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
class Template(UUIDPrimaryKeyMixin, TimestampMixin, Base):
__tablename__ = "templates"
# Every row here belongs to the customer and is editable. Blueprints
# shipped with the product stay on disk in templates/ until an admin
# adds one (app/template_catalog.py) — there is no read-only template.
name: Mapped[str] = mapped_column(String(200))
version: Mapped[str] = mapped_column(String(20))
config: Mapped[dict[str, Any]] = mapped_column(JSONB)
+29
View File
@@ -0,0 +1,29 @@
import uuid
from sqlalchemy import Enum, ForeignKey, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.base import Base, TimestampMixin, UUIDPrimaryKeyMixin
from app.models.department import Department
from app.models.enums import UserRole
class User(UUIDPrimaryKeyMixin, TimestampMixin, Base):
__tablename__ = "users"
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
name: Mapped[str] = mapped_column(String(200))
role: Mapped[UserRole] = mapped_column(
Enum(UserRole, native_enum=False, length=32), default=UserRole.member
)
password_hash: Mapped[str] = mapped_column(String(255))
# Interface language. NULL follows the browser's Accept-Language; a value
# pins it. One column rather than a preferences table: this is the only
# preference that has to follow the person across devices — the theme is
# per-device and lives in localStorage.
locale: Mapped[str | None] = mapped_column(String(5))
department_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey("departments.id", ondelete="SET NULL")
)
department: Mapped[Department | None] = relationship(lazy="joined")