ProfessorNovaandClaude Opus 5
784b76baf7
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
108 lines
3.4 KiB
Python
108 lines
3.4 KiB
Python
"""Markdown chunking along the heading hierarchy.
|
||
|
||
Chunk size uses a character heuristic (~4 chars/token, target ~400 tokens);
|
||
no tokenizer dependency — precision is not required for chunk sizing, and a
|
||
real tokenizer would not match local model tokenizers anyway. Sections that
|
||
exceed the cap are split at paragraph boundaries, never inside code fences.
|
||
"""
|
||
|
||
import re
|
||
from dataclasses import dataclass
|
||
|
||
# ~400 tokens at the ~4 chars/token heuristic.
|
||
TARGET_CHUNK_CHARS = 1600
|
||
|
||
HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$")
|
||
|
||
HEADING_PATH_SEPARATOR = " › "
|
||
|
||
|
||
@dataclass
|
||
class ChunkData:
|
||
content: str
|
||
heading_path: str
|
||
|
||
|
||
@dataclass
|
||
class _Section:
|
||
path: list[str]
|
||
lines: list[str]
|
||
|
||
@property
|
||
def text(self) -> str:
|
||
return "\n".join(self.lines).strip()
|
||
|
||
|
||
def _split_sections(content_md: str, title: str) -> list[_Section]:
|
||
sections: list[_Section] = [_Section(path=[title], lines=[])]
|
||
heading_stack: list[tuple[int, str]] = [] # (level, text)
|
||
in_fence = False
|
||
|
||
for line in content_md.splitlines():
|
||
if line.lstrip().startswith("```"):
|
||
in_fence = not in_fence
|
||
match = None if in_fence else HEADING_RE.match(line)
|
||
if match:
|
||
level = len(match.group(1))
|
||
text = match.group(2).strip()
|
||
while heading_stack and heading_stack[-1][0] >= level:
|
||
heading_stack.pop()
|
||
heading_stack.append((level, text))
|
||
path = [title, *(heading for _, heading in heading_stack)]
|
||
# Drop a leading H1 that just repeats the document title.
|
||
if len(path) > 1 and path[1] == title:
|
||
path = [title, *path[2:]]
|
||
sections.append(_Section(path=path, lines=[line]))
|
||
else:
|
||
sections[-1].lines.append(line)
|
||
|
||
return [section for section in sections if section.text]
|
||
|
||
|
||
def _split_paragraphs(text: str) -> list[str]:
|
||
"""Split at blank lines, but never inside a ``` fence."""
|
||
paragraphs: list[str] = []
|
||
current: list[str] = []
|
||
in_fence = False
|
||
for line in text.splitlines():
|
||
if line.lstrip().startswith("```"):
|
||
in_fence = not in_fence
|
||
if not line.strip() and not in_fence:
|
||
if current:
|
||
paragraphs.append("\n".join(current))
|
||
current = []
|
||
else:
|
||
current.append(line)
|
||
if current:
|
||
paragraphs.append("\n".join(current))
|
||
return paragraphs
|
||
|
||
|
||
def _split_oversized(text: str) -> list[str]:
|
||
pieces: list[str] = []
|
||
current = ""
|
||
for paragraph in _split_paragraphs(text):
|
||
candidate = f"{current}\n\n{paragraph}" if current else paragraph
|
||
if current and len(candidate) > TARGET_CHUNK_CHARS:
|
||
pieces.append(current)
|
||
current = paragraph
|
||
else:
|
||
current = candidate
|
||
if current:
|
||
pieces.append(current)
|
||
return pieces
|
||
|
||
|
||
def chunk_markdown(content_md: str, title: str) -> list[ChunkData]:
|
||
"""Split a document into chunks; each chunk has exactly one heading path."""
|
||
chunks: list[ChunkData] = []
|
||
for section in _split_sections(content_md, title):
|
||
heading_path = HEADING_PATH_SEPARATOR.join(section.path)
|
||
text = section.text
|
||
if len(text) <= TARGET_CHUNK_CHARS:
|
||
chunks.append(ChunkData(content=text, heading_path=heading_path))
|
||
else:
|
||
for piece in _split_oversized(text):
|
||
chunks.append(ChunkData(content=piece, heading_path=heading_path))
|
||
return chunks
|