Pablan, as it stands

Self-hosted knowledge management for SMEs: a split-screen Markdown editor
whose sections an LLM refines while you write, and RAG question answering
over the documents that result. FastAPI + Postgres/pgvector on the back,
SvelteKit on the front, everything OpenAI-compatible and self-hostable.

Squashed into a single commit; the development history stays local.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b

backend/app/rag/chunking.py

@@ -0,0 +1,107 @@
+"""Markdown chunking along the heading hierarchy.
+
+Chunk size uses a character heuristic (~4 chars/token, target ~400 tokens);
+no tokenizer dependency — precision is not required for chunk sizing, and a
+real tokenizer would not match local model tokenizers anyway. Sections that
+exceed the cap are split at paragraph boundaries, never inside code fences.
+"""
+
+import re
+from dataclasses import dataclass
+
+# ~400 tokens at the ~4 chars/token heuristic.
+TARGET_CHUNK_CHARS = 1600
+
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
+
+HEADING_PATH_SEPARATOR = " › "
+
+
+@dataclass
+class ChunkData:
+    content: str
+    heading_path: str
+
+
+@dataclass
+class _Section:
+    path: list[str]
+    lines: list[str]
+
+    @property
+    def text(self) -> str:
+        return "\n".join(self.lines).strip()
+
+
+def _split_sections(content_md: str, title: str) -> list[_Section]:
+    sections: list[_Section] = [_Section(path=[title], lines=[])]
+    heading_stack: list[tuple[int, str]] = []  # (level, text)
+    in_fence = False
+
+    for line in content_md.splitlines():
+        if line.lstrip().startswith("```"):
+            in_fence = not in_fence
+        match = None if in_fence else HEADING_RE.match(line)
+        if match:
+            level = len(match.group(1))
+            text = match.group(2).strip()
+            while heading_stack and heading_stack[-1][0] >= level:
+                heading_stack.pop()
+            heading_stack.append((level, text))
+            path = [title, *(heading for _, heading in heading_stack)]
+            # Drop a leading H1 that just repeats the document title.
+            if len(path) > 1 and path[1] == title:
+                path = [title, *path[2:]]
+            sections.append(_Section(path=path, lines=[line]))
+        else:
+            sections[-1].lines.append(line)
+
+    return [section for section in sections if section.text]
+
+
+def _split_paragraphs(text: str) -> list[str]:
+    """Split at blank lines, but never inside a ``` fence."""
+    paragraphs: list[str] = []
+    current: list[str] = []
+    in_fence = False
+    for line in text.splitlines():
+        if line.lstrip().startswith("```"):
+            in_fence = not in_fence
+        if not line.strip() and not in_fence:
+            if current:
+                paragraphs.append("\n".join(current))
+                current = []
+        else:
+            current.append(line)
+    if current:
+        paragraphs.append("\n".join(current))
+    return paragraphs
+
+
+def _split_oversized(text: str) -> list[str]:
+    pieces: list[str] = []
+    current = ""
+    for paragraph in _split_paragraphs(text):
+        candidate = f"{current}\n\n{paragraph}" if current else paragraph
+        if current and len(candidate) > TARGET_CHUNK_CHARS:
+            pieces.append(current)
+            current = paragraph
+        else:
+            current = candidate
+    if current:
+        pieces.append(current)
+    return pieces
+
+
+def chunk_markdown(content_md: str, title: str) -> list[ChunkData]:
+    """Split a document into chunks; each chunk has exactly one heading path."""
+    chunks: list[ChunkData] = []
+    for section in _split_sections(content_md, title):
+        heading_path = HEADING_PATH_SEPARATOR.join(section.path)
+        text = section.text
+        if len(text) <= TARGET_CHUNK_CHARS:
+            chunks.append(ChunkData(content=text, heading_path=heading_path))
+        else:
+            for piece in _split_oversized(text):
+                chunks.append(ChunkData(content=piece, heading_path=heading_path))
+    return chunks