Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
co-authored by
Claude Opus 5
Claude Opus 5
parent
68d3a43191
commit
784b76baf7
@@ -0,0 +1,8 @@
|
||||
"""Writing-first knowledge capture.
|
||||
|
||||
Capture is not a conversation Mode: the artifact is a Document the user
|
||||
writes directly (Markdown is the source of truth), and the model
|
||||
refines one section at a time (FIM-style). This package holds the template
|
||||
schema, the skeleton/title rendering, the active-section boundary and the
|
||||
refinement prompt. The HTTP surface lives in `app/api/authoring.py`.
|
||||
"""
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Conversation context for a capture.
|
||||
|
||||
When a document is written out of a chat, that chat's subject is useful twice:
|
||||
to find existing documents the user might extend, and as background for section
|
||||
refinement. Both use a short LLM topic summary of the conversation. All LLM
|
||||
traffic goes through `llm/client.py`; nothing here logs content — a failure
|
||||
degrades quietly rather than breaking the capture.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.authoring.prompts import render_topic_summary_prompt
|
||||
from app.llm.client import chat_json
|
||||
from app.llm.errors import LLMError
|
||||
from app.models import Conversation, MessageRole
|
||||
|
||||
logger = logging.getLogger("pablan.authoring")
|
||||
|
||||
# How many recent turns feed the summary — enough for the subject, bounded so
|
||||
# a long thread cannot blow up the utility prompt.
|
||||
MAX_CONTEXT_MESSAGES = 12
|
||||
|
||||
# Skip the reasoning model's hidden thinking: this is a short, latency-
|
||||
# sensitive utility call.
|
||||
_NO_THINKING = {"chat_template_kwargs": {"enable_thinking": False}}
|
||||
|
||||
|
||||
class _TopicSummary(BaseModel):
|
||||
topic: str
|
||||
|
||||
|
||||
def conversation_transcript(conversation: Conversation) -> str:
|
||||
"""The recent user/assistant turns as a plain transcript."""
|
||||
turns = [
|
||||
message
|
||||
for message in conversation.messages
|
||||
if message.role in (MessageRole.user, MessageRole.assistant)
|
||||
][-MAX_CONTEXT_MESSAGES:]
|
||||
return "\n".join(
|
||||
f"{'User' if message.role == MessageRole.user else 'Assistant'}: "
|
||||
f"{message.content}"
|
||||
for message in turns
|
||||
)
|
||||
|
||||
|
||||
async def summarize_transcript(transcript: str) -> str:
|
||||
"""A short topic summary of a conversation transcript, or '' if none can
|
||||
be made. Failures degrade quietly."""
|
||||
if not transcript.strip():
|
||||
return ""
|
||||
try:
|
||||
result = await chat_json(
|
||||
render_topic_summary_prompt(transcript),
|
||||
_TopicSummary,
|
||||
extra_body=_NO_THINKING,
|
||||
)
|
||||
except LLMError:
|
||||
logger.info("topic summary failed", extra={"event": "topic_summary_failed"})
|
||||
return ""
|
||||
return result.topic.strip()
|
||||
|
||||
|
||||
async def summarize_conversation(conversation: Conversation) -> str:
|
||||
"""A short topic summary of the conversation, or '' if none can be made."""
|
||||
return await summarize_transcript(conversation_transcript(conversation))
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Render a template's Markdown skeleton and title into a new draft document."""
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.authoring.schema import AuthoringTemplate
|
||||
from app.models import User
|
||||
|
||||
|
||||
def render_title(template: AuthoringTemplate, user: User) -> str:
|
||||
today = datetime.now(UTC).date().isoformat()
|
||||
return (
|
||||
template.title_template.replace("{{user.name}}", user.name)
|
||||
.replace("{{date}}", today)
|
||||
.strip()
|
||||
)
|
||||
|
||||
|
||||
def render_skeleton(template: AuthoringTemplate) -> str:
|
||||
"""The Markdown the editor opens with: the skeleton verbatim, normalized
|
||||
to a single trailing newline. An empty skeleton yields an empty document
|
||||
the author fills from scratch."""
|
||||
skeleton = template.skeleton.strip()
|
||||
return f"{skeleton}\n" if skeleton else ""
|
||||
@@ -0,0 +1,40 @@
|
||||
"""The document audit trail.
|
||||
|
||||
Every content edit and lifecycle transition is appended to `document_events`
|
||||
as an immutable record of who did what, when. Content-bearing actions snapshot
|
||||
the Markdown source of truth (never the disposable chunks) so a past version
|
||||
can later be viewed or diffed.
|
||||
"""
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models import Document, DocumentEvent, DocumentEventAction, User
|
||||
|
||||
|
||||
def record_event(
|
||||
db: AsyncSession,
|
||||
document: Document,
|
||||
actor: User,
|
||||
action: DocumentEventAction,
|
||||
*,
|
||||
snapshot: bool = False,
|
||||
) -> None:
|
||||
"""Append an audit record for `document`.
|
||||
|
||||
`snapshot` freezes the current Markdown, title and meta so the version can
|
||||
be reconstructed later — pass it for content-bearing events (created /
|
||||
edited). Visibility is small, so it is always recorded. Leave `snapshot`
|
||||
False for pure transitions that carry no new content. The document must
|
||||
already have an id (flush a freshly created document first).
|
||||
"""
|
||||
db.add(
|
||||
DocumentEvent(
|
||||
document_id=document.id,
|
||||
actor_id=actor.id,
|
||||
action=action,
|
||||
content_md=document.content_md if snapshot else None,
|
||||
title=document.title if snapshot else None,
|
||||
visibility=document.visibility,
|
||||
meta=document.meta if snapshot else None,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Refinement prompt for the writing editor — natural language only.
|
||||
|
||||
The model refines exactly ONE section of a document the user is writing. The
|
||||
rest of the document travels as prefix/suffix context so the section stays
|
||||
coherent with its surroundings, but the model regenerates ONLY the section —
|
||||
a large document is never re-emitted whole (FIM-style).
|
||||
|
||||
The base texts (persona, rules, framings) are admin-editable: they come from
|
||||
`app/prompts/overrides.py::get_prompt`, which returns a DB override when one
|
||||
exists and the code default (`app/prompts/defaults.py`) otherwise.
|
||||
"""
|
||||
|
||||
from app.llm.client import ChatMessage
|
||||
from app.prompts.overrides import get_prompt
|
||||
|
||||
|
||||
def render_refine_prompt(
|
||||
section: str,
|
||||
*,
|
||||
prefix: str,
|
||||
suffix: str,
|
||||
persona: str | None,
|
||||
hint: str | None,
|
||||
context: str | None = None,
|
||||
knowledge: list[str] | None = None,
|
||||
) -> list[ChatMessage]:
|
||||
# A template may carry its own persona; otherwise the admin-editable default.
|
||||
system_parts = [persona.strip() if persona else get_prompt("refine_persona")]
|
||||
if hint:
|
||||
system_parts.append(f"What this section should convey: {hint}")
|
||||
if context:
|
||||
# Background from the chat this capture came from, so the refinement
|
||||
# is on-topic — but only as orientation, never a source of new facts.
|
||||
system_parts.append(
|
||||
f"Background (the conversation this document came from, for "
|
||||
f"orientation only — do not invent facts from it): {context}"
|
||||
)
|
||||
system_parts.append(get_prompt("refine_rules"))
|
||||
|
||||
# The document being edited leads the user turn (prefix/suffix/section);
|
||||
# the retrieved knowledge trails it, because it changes on every call and
|
||||
# keeping it last leaves the stable prompt prefix reusable between calls.
|
||||
user_parts: list[str] = []
|
||||
if prefix.strip():
|
||||
user_parts.append(
|
||||
f"Text before the section (context only, do not repeat it):\n{prefix}"
|
||||
)
|
||||
if suffix.strip():
|
||||
user_parts.append(
|
||||
f"Text after the section (context only, do not repeat it):\n{suffix}"
|
||||
)
|
||||
user_parts.append(f"Refine only this section:\n{section}")
|
||||
if knowledge:
|
||||
# What the company has already documented elsewhere. It is grounding,
|
||||
# not source material: it keeps terminology and facts consistent and
|
||||
# lets the section point at related documents, but it must not be
|
||||
# copied in or become a way to add facts the section's notes do not
|
||||
# support.
|
||||
joined = "\n\n".join(knowledge)
|
||||
user_parts.append(f"{get_prompt('grounding_framing')}\n{joined}")
|
||||
|
||||
return [
|
||||
{"role": "system", "content": "\n\n".join(system_parts)},
|
||||
{"role": "user", "content": "\n\n".join(user_parts)},
|
||||
]
|
||||
|
||||
|
||||
def render_topic_summary_prompt(transcript: str) -> list[ChatMessage]:
|
||||
"""Condense a conversation into a short search topic (a few words)."""
|
||||
return [
|
||||
{"role": "system", "content": get_prompt("topic_summary")},
|
||||
{"role": "user", "content": f"Conversation:\n{transcript}"},
|
||||
]
|
||||
|
||||
|
||||
def render_title_prompt(content_md: str) -> list[ChatMessage]:
|
||||
"""Suggest a concise document title from its written content."""
|
||||
return [
|
||||
{"role": "system", "content": get_prompt("title")},
|
||||
{"role": "user", "content": f"Document:\n{content_md}"},
|
||||
]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Pydantic schema for authoring templates (schema version 1.0).
|
||||
|
||||
A template is a **Markdown skeleton** — a starting document with headings the
|
||||
author fills in — plus a persona and optional per-section hints that steer the
|
||||
section-refinement model. It is declarative configuration, not code (see
|
||||
docs/authoring-templates.md), stored in `templates.config` (JSONB) and
|
||||
validated on load.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, field_validator
|
||||
|
||||
|
||||
class TemplateModelHints(BaseModel):
|
||||
temperature: float = 0.4
|
||||
# UI warning when the configured endpoint is weaker than the template
|
||||
# expects; read by api/templates.py.
|
||||
min_class_hint: str | None = None
|
||||
|
||||
|
||||
class SectionHint(BaseModel):
|
||||
"""Steers what the refinement model should draw out of one section.
|
||||
|
||||
`heading` is matched to a skeleton heading by its exact text, so the hint
|
||||
only reaches the model while the author is writing under that heading.
|
||||
"""
|
||||
|
||||
heading: str
|
||||
hint: str
|
||||
|
||||
|
||||
class TemplateMetadata(BaseModel):
|
||||
visibility: Literal["public", "department", "restricted"] = "department"
|
||||
|
||||
|
||||
class AuthoringTemplate(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
version: str
|
||||
# Names the template's shape, so a differently-shaped config is rejected
|
||||
# rather than silently loaded as an authoring template.
|
||||
kind: Literal["authoring"] = "authoring"
|
||||
# The language this template's CONTENT is written in — persona, skeleton
|
||||
# and hints, not the UI. The picker lists matching templates first.
|
||||
locale: Literal["de", "en"] | None = None
|
||||
description: str = ""
|
||||
model: TemplateModelHints = TemplateModelHints()
|
||||
persona: str
|
||||
# The Markdown the editor opens with: headings the author fills in. This
|
||||
# IS the starting content, not a description of it.
|
||||
skeleton: str
|
||||
sections: list[SectionHint] = []
|
||||
title_template: str
|
||||
metadata: TemplateMetadata = TemplateMetadata()
|
||||
|
||||
@field_validator("version", mode="before")
|
||||
@classmethod
|
||||
def _version_to_string(cls, value: object) -> str:
|
||||
# YAML reads an unquoted 1.0 as a float.
|
||||
return str(value)
|
||||
|
||||
def hint_for(self, heading: str) -> str | None:
|
||||
for section in self.sections:
|
||||
if section.heading == heading:
|
||||
return section.hint
|
||||
return None
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Find the section of a Markdown document the cursor sits in.
|
||||
|
||||
The refinement endpoint refines exactly one section at a time (FIM-style),
|
||||
so this is the AUTHORITATIVE boundary computation — the client mirrors it for
|
||||
a visual highlight, but the server owns it. A section runs from the nearest
|
||||
heading at or above the cursor down to the line before the next heading of
|
||||
the same or higher level; content before the first heading is its own
|
||||
section. A section whose body exceeds the chunk cap narrows to the blank-line
|
||||
paragraph at the cursor, so a large document never refines as one giant block.
|
||||
|
||||
Shares the heading regex, fence-awareness and cap with `rag/chunking.py` so
|
||||
"what is a section" means the same thing to refinement and to indexing.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.rag.chunking import HEADING_RE, TARGET_CHUNK_CHARS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActiveSection:
|
||||
start_line: int # 1-based, inclusive, into content_md
|
||||
end_line: int # 1-based, inclusive
|
||||
|
||||
|
||||
def _heading_lines(lines: list[str]) -> list[tuple[int, int]]:
|
||||
"""(line_index_0based, level) for every heading line, ignoring fences."""
|
||||
headings: list[tuple[int, int]] = []
|
||||
in_fence = False
|
||||
for i, line in enumerate(lines):
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
if in_fence:
|
||||
continue
|
||||
match = HEADING_RE.match(line)
|
||||
if match:
|
||||
headings.append((i, len(match.group(1))))
|
||||
return headings
|
||||
|
||||
|
||||
def _paragraph_at(
|
||||
lines: list[str], start0: int, end0: int, cursor0: int
|
||||
) -> tuple[int, int] | None:
|
||||
"""The blank-line-delimited block (fence-aware) at the cursor, within
|
||||
[start0, end0]. Falls back to the block just before the cursor when it
|
||||
sits on a blank gap, else the first block."""
|
||||
blocks: list[tuple[int, int]] = []
|
||||
block_start: int | None = None
|
||||
in_fence = False
|
||||
for i in range(start0, end0 + 1):
|
||||
line = lines[i]
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
if block_start is None:
|
||||
block_start = i
|
||||
continue
|
||||
if not line.strip() and not in_fence:
|
||||
if block_start is not None:
|
||||
blocks.append((block_start, i - 1))
|
||||
block_start = None
|
||||
elif block_start is None:
|
||||
block_start = i
|
||||
if block_start is not None:
|
||||
blocks.append((block_start, end0))
|
||||
if not blocks:
|
||||
return None
|
||||
for b_start, b_end in blocks:
|
||||
if b_start <= cursor0 <= b_end:
|
||||
return b_start, b_end
|
||||
for b_start, b_end in reversed(blocks):
|
||||
if b_end < cursor0:
|
||||
return b_start, b_end
|
||||
return blocks[0]
|
||||
|
||||
|
||||
def active_section(content_md: str, cursor_line: int) -> ActiveSection:
|
||||
lines = content_md.splitlines()
|
||||
n = len(lines)
|
||||
if n == 0:
|
||||
return ActiveSection(1, 1)
|
||||
cursor0 = max(1, min(cursor_line, n)) - 1
|
||||
|
||||
headings = _heading_lines(lines)
|
||||
owner: tuple[int, int] | None = None
|
||||
for idx, level in headings:
|
||||
if idx <= cursor0:
|
||||
owner = (idx, level)
|
||||
else:
|
||||
break
|
||||
|
||||
if owner is None:
|
||||
# Preamble before the first heading (or a document with no headings).
|
||||
start0 = 0
|
||||
end0 = headings[0][0] - 1 if headings else n - 1
|
||||
else:
|
||||
start0, owner_level = owner
|
||||
end0 = n - 1
|
||||
for idx, level in headings:
|
||||
if idx > start0 and level <= owner_level:
|
||||
end0 = idx - 1
|
||||
break
|
||||
|
||||
# Trailing blank lines belong to the separation before the next section,
|
||||
# not to this one: keeping them in the range would let an accepted
|
||||
# suggestion swallow the blank line above the next heading.
|
||||
while end0 > start0 and not lines[end0].strip():
|
||||
end0 -= 1
|
||||
|
||||
body = "\n".join(lines[start0 : end0 + 1])
|
||||
if len(body) > TARGET_CHUNK_CHARS:
|
||||
narrowed = _paragraph_at(lines, start0, end0, cursor0)
|
||||
if narrowed is not None:
|
||||
start0, end0 = narrowed
|
||||
|
||||
return ActiveSection(start_line=start0 + 1, end_line=end0 + 1)
|
||||
|
||||
|
||||
def slice_lines(
|
||||
content_md: str, start_line: int, end_line: int
|
||||
) -> tuple[str, str, str]:
|
||||
"""(prefix, section, suffix) split at the 1-based inclusive line range.
|
||||
|
||||
The section is the lines the model refines; prefix/suffix are the rest of
|
||||
the document, handed to the model as context it must not re-emit.
|
||||
"""
|
||||
lines = content_md.splitlines()
|
||||
prefix = "\n".join(lines[: start_line - 1])
|
||||
section = "\n".join(lines[start_line - 1 : end_line])
|
||||
suffix = "\n".join(lines[end_line:])
|
||||
return prefix, section, suffix
|
||||
Reference in New Issue
Block a user