/**
 * GitHub Repository Crawler (TRUEREF-0003).
 *
 * Fetches repository file trees via the GitHub Trees API and downloads file
 * contents in parallel while respecting rate limits and applying
 * include/exclude filtering rules from trueref.json.
 *
 * Download strategy:
 *  - Uses raw.githubusercontent.com for file content — faster and counts less
 *    against the REST API rate limit.
 *  - Falls back to the GitHub Contents API if raw download fails.
 *
 * Error handling:
 *  - 404  → RepositoryNotFoundError
 *  - 401  → AuthenticationError
 *  - 403  → waits for rate-limit reset if X-RateLimit-Remaining is 0; else PermissionError
 *  - 422  → tree too large; switches to directory-by-directory traversal (depth pagination)
 *  - Network errors → retried up to 3 times with exponential backoff
 *  - Bad base64 content → file skipped with a console warning
 */

import { shouldIndexFile, detectLanguage } from './file-filter.js';
import { GitHubRateLimiter, Semaphore, withRetry } from './rate-limiter.js';
import {
	AuthenticationError,
	PermissionError,
	RateLimitError,
	RepositoryNotFoundError
} from './types.js';

// Domain errors should not be retried — they are permanent HTTP status codes.
function isDomainError(err: unknown): boolean {
	return (
		err instanceof RepositoryNotFoundError ||
		err instanceof AuthenticationError ||
		err instanceof PermissionError ||
		err instanceof RateLimitError
	);
}

function isRetryable(err: unknown): boolean {
	return !isDomainError(err);
}
import type {
	CrawlOptions,
	CrawlResult,
	CrawledFile,
	GitHubContentResponse,
	GitHubRepoResponse,
	GitHubTreeItem,
	GitHubTreeResponse
} from './types.js';

// ---------------------------------------------------------------------------
// Internal constants
// ---------------------------------------------------------------------------

const GITHUB_API = 'https://api.github.com';
const RAW_CONTENT = 'https://raw.githubusercontent.com';

/** Maximum parallel file downloads. */
const DOWNLOAD_CONCURRENCY = 10;

/** Config file names that should be fetched first so their filtering rules
 *  apply to all subsequent downloads. */
const CONFIG_FILE_NAMES = new Set(['trueref.json', 'context7.json']);

// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------

/**
 * Build standard GitHub API request headers.
 */
function buildHeaders(token?: string): Record<string, string> {
	const headers: Record<string, string> = {
		Accept: 'application/vnd.github+json',
		'X-GitHub-Api-Version': '2022-11-28'
	};
	if (token) {
		headers.Authorization = `Bearer ${token}`;
	}
	return headers;
}

/**
 * Throw a domain error for non-2xx GitHub API responses.
 * Mutates the rate limiter with header data before throwing.
 */
async function throwForStatus(response: Response, rateLimiter: GitHubRateLimiter): Promise<void> {
	if (response.ok) return;

	rateLimiter.updateFromHeaders(response.headers);

	switch (response.status) {
		case 401:
			throw new AuthenticationError('GitHub authentication failed — check your PAT.');
		case 403: {
			const remaining = response.headers.get('X-RateLimit-Remaining');
			if (remaining === '0') {
				const reset = parseInt(response.headers.get('X-RateLimit-Reset') ?? '0', 10) * 1000;
				throw new RateLimitError('GitHub rate limit exceeded.', reset);
			}
			throw new PermissionError(
				'GitHub returned 403 Forbidden — insufficient permissions for this resource.'
			);
		}
		case 404:
			throw new RepositoryNotFoundError(
				`Repository not found or not accessible: ${response.url}`
			);
		default: {
			const body = await response.text().catch(() => '');
			throw new Error(`GitHub API error ${response.status}: ${body}`);
		}
	}
}

// ---------------------------------------------------------------------------
// GitHub API calls
// ---------------------------------------------------------------------------

/**
 * Fetch repository metadata (default branch, stars, etc.).
 */
async function fetchRepoInfo(
	owner: string,
	repo: string,
	token: string | undefined,
	rateLimiter: GitHubRateLimiter
): Promise<GitHubRepoResponse> {
	return withRetry(async () => {
		await rateLimiter.waitIfNeeded();

		const response = await fetch(`${GITHUB_API}/repos/${owner}/${repo}`, {
			headers: buildHeaders(token)
		});

		rateLimiter.updateFromHeaders(response.headers);
		await throwForStatus(response, rateLimiter);

		return (await response.json()) as GitHubRepoResponse;
	}, 3, isRetryable);
}

/**
 * Fetch the recursive file tree for a given ref.
 * Returns null when the tree is truncated (>100k items), signalling that we
 * should fall back to directory-level traversal.
 */
async function fetchTree(
	owner: string,
	repo: string,
	ref: string,
	token: string | undefined,
	rateLimiter: GitHubRateLimiter
): Promise<GitHubTreeResponse | null> {
	return withRetry(async () => {
		await rateLimiter.waitIfNeeded();

		const url = `${GITHUB_API}/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`;
		const response = await fetch(url, { headers: buildHeaders(token) });

		rateLimiter.updateFromHeaders(response.headers);

		// 422 means the tree is too large for a single recursive call.
		if (response.status === 422) return null;

		await throwForStatus(response, rateLimiter);

		return (await response.json()) as GitHubTreeResponse;
	}, 3, isRetryable);
}

/**
 * Fetch a subtree (non-recursive) for a single directory path.
 * Used when the full recursive tree is truncated.
 */
async function fetchSubTree(
	owner: string,
	repo: string,
	ref: string,
	treeSha: string,
	token: string | undefined,
	rateLimiter: GitHubRateLimiter
): Promise<GitHubTreeResponse> {
	return withRetry(async () => {
		await rateLimiter.waitIfNeeded();

		const url = `${GITHUB_API}/repos/${owner}/${repo}/git/trees/${treeSha}`;
		const response = await fetch(url, { headers: buildHeaders(token) });

		rateLimiter.updateFromHeaders(response.headers);
		await throwForStatus(response, rateLimiter);

		return (await response.json()) as GitHubTreeResponse;
	}, 3, isRetryable);
}

/**
 * Resolve the HEAD commit SHA from a branch/tag ref by fetching the
 * commit object at the ref tip.
 */
async function fetchCommitSha(
	owner: string,
	repo: string,
	ref: string,
	token: string | undefined,
	rateLimiter: GitHubRateLimiter
): Promise<string> {
	return withRetry(async () => {
		await rateLimiter.waitIfNeeded();

		const url = `${GITHUB_API}/repos/${owner}/${repo}/commits/${ref}`;
		const response = await fetch(url, {
			headers: { ...buildHeaders(token), Accept: 'application/vnd.github.sha' }
		});

		rateLimiter.updateFromHeaders(response.headers);
		await throwForStatus(response, rateLimiter);

		// When Accept is 'application/vnd.github.sha', the response body is the
		// bare SHA string.
		return (await response.text()).trim();
	}, 3, isRetryable);
}

/**
 * Download raw file content via raw.githubusercontent.com.
 * Returns null on any failure (the caller will skip or fall back).
 */
async function downloadRawFile(
	owner: string,
	repo: string,
	ref: string,
	filePath: string,
	token: string | undefined
): Promise<string | null> {
	try {
		const url = `${RAW_CONTENT}/${owner}/${repo}/${ref}/${filePath}`;
		const headers: Record<string, string> = {};
		if (token) headers.Authorization = `Bearer ${token}`;

		const response = await fetch(url, { headers });
		if (!response.ok) return null;

		return await response.text();
	} catch {
		return null;
	}
}

/**
 * Download file content via the GitHub Contents API (fallback).
 */
async function downloadViaContentsApi(
	owner: string,
	repo: string,
	ref: string,
	filePath: string,
	token: string | undefined,
	rateLimiter: GitHubRateLimiter
): Promise<string | null> {
	try {
		return await withRetry(async () => {
			await rateLimiter.waitIfNeeded();

			const url = `${GITHUB_API}/repos/${owner}/${repo}/contents/${filePath}?ref=${ref}`;
			const response = await fetch(url, { headers: buildHeaders(token) });

			rateLimiter.updateFromHeaders(response.headers);
			if (!response.ok) return null;

			const data = (await response.json()) as GitHubContentResponse;
			if (data.encoding !== 'base64') return null;

			// Node.js Buffer handles both padded and unpadded base64.
			return Buffer.from(data.content.replace(/\n/g, ''), 'base64').toString('utf-8');
		});
	} catch {
		return null;
	}
}

// ---------------------------------------------------------------------------
// Directory-level traversal (fallback for truncated trees)
// ---------------------------------------------------------------------------

/**
 * Recursively collect all blob items from sub-trees when the top-level
 * recursive tree is truncated (>100k items).
 */
async function collectBlobsFromSubTrees(
	owner: string,
	repo: string,
	ref: string,
	token: string | undefined,
	rateLimiter: GitHubRateLimiter
): Promise<GitHubTreeItem[]> {
	const allBlobs: GitHubTreeItem[] = [];
	const queue: Array<{ sha: string; prefix: string }> = [{ sha: ref, prefix: '' }];

	while (queue.length > 0) {
		const batch = queue.splice(0, DOWNLOAD_CONCURRENCY);

		await Promise.all(
			batch.map(async ({ sha, prefix }) => {
				const subTree = await fetchSubTree(owner, repo, ref, sha, token, rateLimiter).catch(
					() => null
				);
				if (!subTree) return;

				for (const item of subTree.tree) {
					const fullPath = prefix ? `${prefix}/${item.path}` : item.path;
					if (item.type === 'blob') {
						allBlobs.push({ ...item, path: fullPath });
					} else if (item.type === 'tree') {
						queue.push({ sha: item.sha, prefix: fullPath });
					}
				}
			})
		);
	}

	return allBlobs;
}

// ---------------------------------------------------------------------------
// Config file detection
// ---------------------------------------------------------------------------

/**
 * Try to download and parse a trueref.json / context7.json config from the
 * repository root. Returns undefined if not found or unparseable.
 */
async function fetchRepoConfig(
	owner: string,
	repo: string,
	ref: string,
	token: string | undefined,
	blobs: GitHubTreeItem[],
	rateLimiter: GitHubRateLimiter
): Promise<CrawlOptions['config'] | undefined> {
	// Look for config files only at the repo root (no directory prefix).
	const configItem = blobs.find((b) => CONFIG_FILE_NAMES.has(b.path));
	if (!configItem) return undefined;

	const content =
		(await downloadRawFile(owner, repo, ref, configItem.path, token)) ??
		(await downloadViaContentsApi(
			owner,
			repo,
			ref,
			configItem.path,
			token,
			rateLimiter
		));

	if (!content) return undefined;

	try {
		return JSON.parse(content) as CrawlOptions['config'];
	} catch {
		console.warn(`[GitHubCrawler] Failed to parse config file: ${configItem.path}`);
		return undefined;
	}
}

// ---------------------------------------------------------------------------
// Public crawl() function
// ---------------------------------------------------------------------------

/**
 * Crawl a GitHub repository and return structured file objects.
 *
 * @param options - Repository coordinates, auth token, filter config, and
 *                  optional progress callback.
 * @returns CrawlResult with all downloaded files and summary statistics.
 */
export async function crawl(options: CrawlOptions): Promise<CrawlResult> {
	const { owner, repo, token, onProgress } = options;
	const rateLimiter = new GitHubRateLimiter();
	const semaphore = new Semaphore(DOWNLOAD_CONCURRENCY);

	// ---- Step 1: Resolve the ref (default branch if not provided) ----------
	let ref = options.ref;
	let commitSha = '';

	const repoInfo = await fetchRepoInfo(owner, repo, token, rateLimiter);
	if (!ref) {
		ref = repoInfo.default_branch;
	}

	// ---- Step 2: Fetch the file tree ---------------------------------------
	let blobs: GitHubTreeItem[];

	const treeResponse = await fetchTree(owner, repo, ref, token, rateLimiter);

	if (treeResponse === null) {
		// Tree truncated — fall back to directory-by-directory traversal.
		console.warn(
			`[GitHubCrawler] Tree for ${owner}/${repo}@${ref} is truncated; using sub-tree traversal.`
		);
		blobs = await collectBlobsFromSubTrees(owner, repo, ref, token, rateLimiter);
	} else {
		blobs = treeResponse.tree.filter((item) => item.type === 'blob');
	}

	// Resolve HEAD commit SHA (best-effort; empty string on failure).
	commitSha = await fetchCommitSha(owner, repo, ref, token, rateLimiter).catch(() => '');

	// ---- Step 3: Detect and download config file first ---------------------
	// Merge caller-supplied config with any discovered repo config.
	let effectiveConfig = options.config;
	if (!effectiveConfig) {
		effectiveConfig = await fetchRepoConfig(owner, repo, ref, token, blobs, rateLimiter);
	}

	// ---- Step 4: Filter blobs according to config --------------------------
	const filteredBlobs = blobs.filter((item) =>
		shouldIndexFile(item.path, item.size ?? 0, effectiveConfig)
	);

	const totalFiles = filteredBlobs.length;
	const skippedFiles = blobs.length - totalFiles;

	// ---- Step 5: Download file contents in parallel -------------------------
	const files: CrawledFile[] = [];
	let processed = 0;

	await Promise.all(
		filteredBlobs.map((item) =>
			semaphore.run(async () => {
				try {
					// Prefer raw download (cheaper on rate limit); fall back to API.
					const content =
						(await downloadRawFile(owner, repo, ref!, item.path, token)) ??
						(await downloadViaContentsApi(
							owner,
							repo,
							ref!,
							item.path,
							token,
							rateLimiter
						));

					if (content === null) {
						console.warn(`[GitHubCrawler] Could not download: ${item.path} — skipping.`);
					} else {
						files.push({
							path: item.path,
							content,
							size: item.size ?? Buffer.byteLength(content, 'utf-8'),
							sha: item.sha,
							language: detectLanguage(item.path)
						});
					}
				} catch (err) {
					console.warn(
						`[GitHubCrawler] Error downloading ${item.path}: ${err instanceof Error ? err.message : String(err)}`
					);
				} finally {
					processed++;
					onProgress?.(processed, totalFiles);
				}
			})
		)
	);

	return {
		files,
		totalFiles,
		skippedFiles,
		branch: ref,
		commitSha
	};
}
