/**
 * Text chunking utilities for the document parser (TRUEREF-0005).
 *
 * Provides sliding-window chunking with overlap and token estimation.
 */

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

export const MAX_TOKENS = 512;
export const OVERLAP_TOKENS = 50;
export const MIN_CONTENT_LENGTH = 20; // characters

// ---------------------------------------------------------------------------
// Token estimation
// ---------------------------------------------------------------------------

/**
 * Estimate the token count for a piece of text using a character-based
 * approximation (~3.5 chars per token on average for mixed prose/code).
 */
export function estimateTokens(text: string): number {
	return Math.ceil(text.length / 3.5);
}

// ---------------------------------------------------------------------------
// Sliding-window chunker
// ---------------------------------------------------------------------------

/**
 * Split `text` into overlapping word-based chunks that stay within the token
 * budget.  Returns at least one chunk even when the text fits in a single
 * window.
 */
export function chunkText(
	text: string,
	maxTokens: number = MAX_TOKENS,
	overlapTokens: number = OVERLAP_TOKENS
): string[] {
	const words = text.split(/\s+/).filter((w) => w.length > 0);
	if (words.length === 0) return [];

	// ~0.75 words per token
	const maxWords = Math.max(1, Math.floor(maxTokens * 0.75));
	const overlapWords = Math.max(0, Math.floor(overlapTokens * 0.75));

	if (words.length <= maxWords) {
		return [words.join(' ')];
	}

	const chunks: string[] = [];
	let start = 0;

	while (start < words.length) {
		const end = Math.min(start + maxWords, words.length);
		chunks.push(words.slice(start, end).join(' '));
		if (end === words.length) break;
		start = end - overlapWords;
		// Guard against infinite loop when overlapWords >= maxWords
		if (start <= 0) start = end;
	}

	return chunks;
}

// ---------------------------------------------------------------------------
// Line-count sliding window (for code files without recognised boundaries)
// ---------------------------------------------------------------------------

/** Split `lines` into groups of at most `maxLines` with `overlapLines` overlap. */
export function chunkLines(
	lines: string[],
	maxLines: number = 200,
	overlapLines: number = 20
): string[] {
	if (lines.length === 0) return [];
	if (lines.length <= maxLines) return [lines.join('\n')];

	const chunks: string[] = [];
	let start = 0;

	while (start < lines.length) {
		const end = Math.min(start + maxLines, lines.length);
		chunks.push(lines.slice(start, end).join('\n'));
		if (end === lines.length) break;
		start = end - overlapLines;
		if (start <= 0) start = end;
	}

	return chunks;
}
