trueref/docs/features/TRUEREF-0005.md

TRUEREF-0005 — Document Parser & Chunker

Priority: P0 Status: Pending Depends On: TRUEREF-0001 Blocks: TRUEREF-0006, TRUEREF-0007, TRUEREF-0009

---

Overview

Implement the document parsing and chunking pipeline that transforms raw file contents (from the crawlers) into structured, searchable Snippet records. This is the core intellectual layer of TrueRef — the quality of the chunks directly determines the quality of documentation retrieval.

---

Acceptance Criteria

• Parse Markdown files into heading-based sections (info snippets)
• Extract fenced code blocks from Markdown as separate code snippets
• Parse standalone code files into function/class-level chunks
• Respect token limits per chunk (max 512 tokens, with 50-token overlap)
• Assign breadcrumb paths based on heading hierarchy (Markdown) or file path (code)
• Detect programming language from file extension
• Produce both code and info type snippets
• Calculate approximate token counts using character-based estimation
• Skip empty or trivially short content (< 20 chars)
• Unit tests with representative samples of each file type

---

Supported File Types

Extension	Parser Strategy
.md, .mdx	Heading-based section splitting + code block extraction
.txt, .rst	Paragraph-based splitting
.ts, .tsx, .js, .jsx	AST-free: function/class boundary detection via regex
.py	def/class boundary detection
.go	func/type boundary detection
.rs	fn/impl/struct boundary detection
.java, .cs, .kt, .swift	Class/method boundary detection
.rb	def/class boundary detection
.json, .yaml, .yml, .toml	Structural chunking (top-level keys)
.html, .svelte, .vue	Text content extraction + script block splitting
Other code	Line-count-based sliding window (200 lines per chunk)

---

Token Counting

Use a simple character-based approximation (no tokenizer library needed for v1):

function estimateTokens(text: string): number {
  // Empirically: ~4 chars per token for English prose
  // ~3 chars per token for code (more symbols)
  return Math.ceil(text.length / 3.5);
}

---

Markdown Parser

The Markdown parser is the most important parser as most documentation is Markdown.

Algorithm

1. Split the file into lines.
2. Track current heading stack (H1 > H2 > H3 > H4).
3. When a new heading is encountered, emit the accumulated content as an info snippet.
4. Fenced code blocks (```) within sections are extracted as separate code snippets.
5. The breadcrumb is built from the heading stack: "Getting Started > Installation".

interface MarkdownSection {
  headings: string[];    // heading stack at this point
  content: string;       // text content (sans code blocks)
  codeBlocks: { language: string; code: string }[];
}

function parseMarkdown(content: string, filePath: string): Snippet[] {
  const sections = splitIntoSections(content);
  const snippets: Snippet[] = [];

  for (const section of sections) {
    const breadcrumb = section.headings.join(' > ');
    const title = section.headings.at(-1) ?? path.basename(filePath);

    // Emit info snippet for text content
    if (section.content.trim().length >= 20) {
      const chunks = chunkText(section.content, MAX_TOKENS, OVERLAP_TOKENS);
      for (const chunk of chunks) {
        snippets.push({
          type: 'info',
          title,
          content: chunk,
          breadcrumb,
          tokenCount: estimateTokens(chunk),
        });
      }
    }

    // Emit code snippets for each code block
    for (const block of section.codeBlocks) {
      if (block.code.trim().length >= 20) {
        snippets.push({
          type: 'code',
          title,
          content: block.code,
          language: block.language || detectLanguage('.' + block.language),
          breadcrumb,
          tokenCount: estimateTokens(block.code),
        });
      }
    }
  }

  return snippets;
}

---

Code File Parser

For non-Markdown code files, use regex-based function/class boundary detection.

Algorithm

1. Detect language-specific top-level declaration patterns.
2. Split the file at those boundaries.
3. Each chunk: the declaration line(s) + body up to the next declaration.
4. If a chunk exceeds MAX_TOKENS, apply sliding window splitting with overlap.

const BOUNDARY_PATTERNS: Record<string, RegExp> = {
  typescript: /^(export\s+)?(async\s+)?(function|class|interface|type|const|let|var)\s+\w+/m,
  python: /^(async\s+)?(def|class)\s+\w+/m,
  go: /^(func|type|var|const)\s+\w+/m,
  rust: /^(pub\s+)?(fn|impl|struct|enum|trait)\s+\w+/m,
  java: /^(public|private|protected|static).*?(class|interface|enum|void|\w+)\s+\w+\s*[({]/m,
};

function parseCodeFile(
  content: string,
  filePath: string,
  language: string
): Snippet[] {
  const pattern = BOUNDARY_PATTERNS[language];
  const breadcrumb = filePath;
  const title = path.basename(filePath);

  if (!pattern) {
    // Fallback: sliding window
    return slidingWindowChunks(content, filePath, language);
  }

  const chunks = splitAtBoundaries(content, pattern);
  return chunks
    .filter(chunk => chunk.trim().length >= 20)
    .flatMap(chunk => {
      if (estimateTokens(chunk) <= MAX_TOKENS) {
        return [{
          type: 'code' as const,
          title,
          content: chunk,
          language,
          breadcrumb,
          tokenCount: estimateTokens(chunk),
        }];
      }
      return slidingWindowChunks(chunk, filePath, language);
    });
}

---

Chunking Constants

const MAX_TOKENS = 512;
const OVERLAP_TOKENS = 50;
const MIN_CONTENT_LENGTH = 20; // characters

Sliding Window Chunker

function chunkText(
  text: string,
  maxTokens: number,
  overlapTokens: number
): string[] {
  const words = text.split(/\s+/);
  const wordsPerToken = 0.75; // ~0.75 words per token
  const maxWords = Math.floor(maxTokens * wordsPerToken);
  const overlapWords = Math.floor(overlapTokens * wordsPerToken);

  const chunks: string[] = [];
  let start = 0;

  while (start < words.length) {
    const end = Math.min(start + maxWords, words.length);
    chunks.push(words.slice(start, end).join(' '));
    if (end === words.length) break;
    start = end - overlapWords;
  }

  return chunks;
}

---

Language Detection

const LANGUAGE_MAP: Record<string, string> = {
  '.ts': 'typescript', '.tsx': 'typescript',
  '.js': 'javascript', '.jsx': 'javascript',
  '.py': 'python',
  '.rb': 'ruby',
  '.go': 'go',
  '.rs': 'rust',
  '.java': 'java',
  '.cs': 'csharp',
  '.cpp': 'cpp', '.c': 'c', '.h': 'c',
  '.swift': 'swift',
  '.kt': 'kotlin',
  '.php': 'php',
  '.scala': 'scala',
  '.sh': 'bash', '.bash': 'bash', '.zsh': 'bash',
  '.md': 'markdown', '.mdx': 'markdown',
  '.json': 'json',
  '.yaml': 'yaml', '.yml': 'yaml',
  '.toml': 'toml',
  '.html': 'html',
  '.css': 'css',
  '.svelte': 'svelte',
  '.vue': 'vue',
  '.sql': 'sql',
};

function detectLanguage(filePath: string): string {
  const ext = path.extname(filePath).toLowerCase();
  return LANGUAGE_MAP[ext] ?? 'text';
}

---

Main Entry Point

export interface ParseOptions {
  repositoryId: string;
  documentId: string;
  versionId?: string;
}

export function parseFile(
  file: CrawledFile,
  options: ParseOptions
): NewSnippet[] {
  const language = detectLanguage(file.path);
  let rawSnippets: Omit<NewSnippet, 'id' | 'repositoryId' | 'documentId' | 'versionId' | 'createdAt'>[];

  if (language === 'markdown') {
    rawSnippets = parseMarkdown(file.content, file.path);
  } else {
    rawSnippets = parseCodeFile(file.content, file.path, language);
  }

  return rawSnippets.map(s => ({
    ...s,
    id: crypto.randomUUID(),
    repositoryId: options.repositoryId,
    documentId: options.documentId,
    versionId: options.versionId ?? null,
    createdAt: new Date(),
  }));
}

---

Files to Create

• src/lib/server/parser/markdown.parser.ts
• src/lib/server/parser/code.parser.ts
• src/lib/server/parser/chunker.ts
• src/lib/server/parser/language.ts
• src/lib/server/parser/index.ts — exports parseFile
• src/lib/server/parser/markdown.parser.test.ts
• src/lib/server/parser/code.parser.test.ts