"""Markdown chunking along the heading hierarchy. Chunk size uses a character heuristic (~4 chars/token, target ~400 tokens); no tokenizer dependency — precision is not required for chunk sizing, and a real tokenizer would not match local model tokenizers anyway. Sections that exceed the cap are split at paragraph boundaries, never inside code fences. """ import re from dataclasses import dataclass # ~400 tokens at the ~4 chars/token heuristic. TARGET_CHUNK_CHARS = 1600 HEADING_RE = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$") HEADING_PATH_SEPARATOR = " › " @dataclass class ChunkData: content: str heading_path: str @dataclass class _Section: path: list[str] lines: list[str] @property def text(self) -> str: return "\n".join(self.lines).strip() def _split_sections(content_md: str, title: str) -> list[_Section]: sections: list[_Section] = [_Section(path=[title], lines=[])] heading_stack: list[tuple[int, str]] = [] # (level, text) in_fence = False for line in content_md.splitlines(): if line.lstrip().startswith("```"): in_fence = not in_fence match = None if in_fence else HEADING_RE.match(line) if match: level = len(match.group(1)) text = match.group(2).strip() while heading_stack and heading_stack[-1][0] >= level: heading_stack.pop() heading_stack.append((level, text)) path = [title, *(heading for _, heading in heading_stack)] # Drop a leading H1 that just repeats the document title. if len(path) > 1 and path[1] == title: path = [title, *path[2:]] sections.append(_Section(path=path, lines=[line])) else: sections[-1].lines.append(line) return [section for section in sections if section.text] def _split_paragraphs(text: str) -> list[str]: """Split at blank lines, but never inside a ``` fence.""" paragraphs: list[str] = [] current: list[str] = [] in_fence = False for line in text.splitlines(): if line.lstrip().startswith("```"): in_fence = not in_fence if not line.strip() and not in_fence: if current: paragraphs.append("\n".join(current)) current = [] else: current.append(line) if current: paragraphs.append("\n".join(current)) return paragraphs def _split_oversized(text: str) -> list[str]: pieces: list[str] = [] current = "" for paragraph in _split_paragraphs(text): candidate = f"{current}\n\n{paragraph}" if current else paragraph if current and len(candidate) > TARGET_CHUNK_CHARS: pieces.append(current) current = paragraph else: current = candidate if current: pieces.append(current) return pieces def chunk_markdown(content_md: str, title: str) -> list[ChunkData]: """Split a document into chunks; each chunk has exactly one heading path.""" chunks: list[ChunkData] = [] for section in _split_sections(content_md, title): heading_path = HEADING_PATH_SEPARATOR.join(section.path) text = section.text if len(text) <= TARGET_CHUNK_CHARS: chunks.append(ChunkData(content=text, heading_path=heading_path)) else: for piece in _split_oversized(text): chunks.append(ChunkData(content=piece, heading_path=heading_path)) return chunks