TRUEREF-0023 rewrite indexing pipeline - parallel reads - serialized writes

src/routes/repos/[id]/+page.svelte

@@ -39,8 +39,11 @@
 		indexedAt: string | null;
 		createdAt: string;
 	}
+	type VersionStateFilter = VersionDto['state'] | 'all';
 	let versions = $state<VersionDto[]>([]);
 	let versionsLoading = $state(false);
+	let activeVersionFilter = $state<VersionStateFilter>('all');
+	let bulkReprocessBusy = $state(false);
 
 	// Add version form
 	let addVersionTag = $state('');
@@ -49,7 +52,7 @@
 	// Discover tags state
 	let discoverBusy = $state(false);
 	let discoveredTags = $state<Array<{ tag: string; commitHash: string }>>([]);
-	let selectedDiscoveredTags = new SvelteSet<string>();
+	const selectedDiscoveredTags = new SvelteSet<string>();
 	let showDiscoverPanel = $state(false);
 	let registerBusy = $state(false);
 
@@ -76,6 +79,14 @@
 		error: 'Error'
 	};
 
+	const versionFilterOptions: Array<{ value: VersionStateFilter; label: string }> = [
+		{ value: 'all', label: 'All' },
+		{ value: 'pending', label: stateLabels.pending },
+		{ value: 'indexing', label: stateLabels.indexing },
+		{ value: 'indexed', label: stateLabels.indexed },
+		{ value: 'error', label: stateLabels.error }
+	];
+
 	const stageLabels: Record<string, string> = {
 		queued: 'Queued',
 		differential: 'Diff',
@@ -88,6 +99,20 @@
 		failed: 'Failed'
 	};
 
+	const filteredVersions = $derived(
+		activeVersionFilter === 'all'
+			? versions
+			: versions.filter((version) => version.state === activeVersionFilter)
+	);
+	const actionableErroredTags = $derived(
+		versions
+			.filter((version) => version.state === 'error' && !activeVersionJobs[version.tag])
+			.map((version) => version.tag)
+	);
+	const activeVersionFilterLabel = $derived(
+		versionFilterOptions.find((option) => option.value === activeVersionFilter)?.label ?? 'All'
+	);
+
 	async function refreshRepo() {
 		try {
 			const res = await fetch(`/api/v1/libs/${encodeURIComponent(repo.id)}`);
@@ -123,9 +148,7 @@
 		if (!repo.id) return;
 
 		let stopped = false;
-		const es = new EventSource(
-			`/api/v1/jobs/stream?repositoryId=${encodeURIComponent(repo.id)}`
-		);
+		const es = new EventSource(`/api/v1/jobs/stream?repositoryId=${encodeURIComponent(repo.id)}`);
 
 		es.addEventListener('job-progress', (event) => {
 			if (stopped) return;
@@ -277,23 +300,58 @@
 	async function handleIndexVersion(tag: string) {
 		errorMessage = null;
 		try {
-			const res = await fetch(
-				`/api/v1/libs/${encodeURIComponent(repo.id)}/versions/${encodeURIComponent(tag)}/index`,
-				{ method: 'POST' }
-			);
-			if (!res.ok) {
-				const d = await res.json();
-				throw new Error(d.error ?? 'Failed to queue version indexing');
-			}
-			const d = await res.json();
-			if (d.job?.id) {
-				activeVersionJobs = { ...activeVersionJobs, [tag]: d.job.id };
+			const jobId = await queueVersionIndex(tag);
+			if (jobId) {
+				activeVersionJobs = { ...activeVersionJobs, [tag]: jobId };
 			}
 		} catch (e) {
 			errorMessage = (e as Error).message;
 		}
 	}
 
+	async function queueVersionIndex(tag: string): Promise<string | null> {
+		const res = await fetch(
+			`/api/v1/libs/${encodeURIComponent(repo.id)}/versions/${encodeURIComponent(tag)}/index`,
+			{ method: 'POST' }
+		);
+		if (!res.ok) {
+			const d = await res.json();
+			throw new Error(d.error ?? 'Failed to queue version indexing');
+		}
+		const d = await res.json();
+		return d.job?.id ?? null;
+	}
+
+	async function handleBulkReprocessErroredVersions() {
+		if (actionableErroredTags.length === 0) return;
+		bulkReprocessBusy = true;
+		errorMessage = null;
+		successMessage = null;
+		try {
+			const tags = [...actionableErroredTags];
+			const BATCH_SIZE = 5;
+			let next = { ...activeVersionJobs };
+
+			for (let i = 0; i < tags.length; i += BATCH_SIZE) {
+				const batch = tags.slice(i, i + BATCH_SIZE);
+				const jobIds = await Promise.all(batch.map((versionTag) => queueVersionIndex(versionTag)));
+				for (let j = 0; j < batch.length; j++) {
+					if (jobIds[j]) {
+						next = { ...next, [batch[j]]: jobIds[j] ?? undefined };
+					}
+				}
+				activeVersionJobs = next;
+			}
+
+			successMessage = `Queued ${tags.length} errored tag${tags.length === 1 ? '' : 's'} for reprocessing.`;
+			await loadVersions();
+		} catch (e) {
+			errorMessage = (e as Error).message;
+		} finally {
+			bulkReprocessBusy = false;
+		}
+	}
+
 	async function handleRemoveVersion() {
 		if (!removeTag) return;
 		const tag = removeTag;
@@ -318,10 +376,9 @@
 		discoverBusy = true;
 		errorMessage = null;
 		try {
-			const res = await fetch(
-				`/api/v1/libs/${encodeURIComponent(repo.id)}/versions/discover`,
-				{ method: 'POST' }
-			);
+			const res = await fetch(`/api/v1/libs/${encodeURIComponent(repo.id)}/versions/discover`, {
+				method: 'POST'
+			});
 			if (!res.ok) {
 				const d = await res.json();
 				throw new Error(d.error ?? 'Failed to discover tags');
@@ -331,7 +388,10 @@
 			discoveredTags = (d.tags ?? []).filter(
 				(t: { tag: string; commitHash: string }) => !registeredTags.has(t.tag)
 			);
-			selectedDiscoveredTags = new SvelteSet(discoveredTags.map((t) => t.tag));
+			selectedDiscoveredTags.clear();
+			for (const discoveredTag of discoveredTags) {
+				selectedDiscoveredTags.add(discoveredTag.tag);
+			}
 			showDiscoverPanel = true;
 		} catch (e) {
 			errorMessage = (e as Error).message;
@@ -380,7 +440,7 @@
 			activeVersionJobs = next;
 			showDiscoverPanel = false;
 			discoveredTags = [];
-			selectedDiscoveredTags = new SvelteSet();
+			selectedDiscoveredTags.clear();
 			await loadVersions();
 		} catch (e) {
 			errorMessage = (e as Error).message;
@@ -498,41 +558,69 @@
 
 <!-- Versions -->
 <div class="mt-6 rounded-xl border border-gray-200 bg-white p-5">
-	<div class="mb-4 flex flex-wrap items-center justify-between gap-3">
-		<h2 class="text-sm font-semibold text-gray-700">Versions</h2>
-		<div class="flex flex-wrap items-center gap-2">
-			<!-- Add version inline form -->
-			<form
-				onsubmit={(e) => {
-					e.preventDefault();
-					handleAddVersion();
-				}}
-				class="flex items-center gap-1.5"
-			>
-				<input
-					type="text"
-					bind:value={addVersionTag}
-					placeholder="e.g. v2.0.0"
-					class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm text-gray-900 placeholder-gray-400 focus:border-blue-400 focus:outline-none"
-				/>
+	<div class="mb-4 flex flex-col gap-3">
+		<div class="flex flex-wrap items-center justify-between gap-3">
+			<div class="flex flex-wrap items-center gap-3">
+				<h2 class="text-sm font-semibold text-gray-700">Versions</h2>
+				<div class="flex flex-wrap items-center gap-1 rounded-lg bg-gray-100 p-1">
+					{#each versionFilterOptions as option (option.value)}
+						<button
+							type="button"
+							onclick={() => (activeVersionFilter = option.value)}
+							class="rounded-md px-2.5 py-1 text-xs font-medium transition-colors {activeVersionFilter ===
+							option.value
+								? 'bg-white text-gray-900 shadow-sm'
+								: 'text-gray-500 hover:text-gray-700'}"
+						>
+							{option.label}
+						</button>
+					{/each}
+				</div>
+			</div>
+			<div class="flex flex-wrap items-center gap-2">
 				<button
-					type="submit"
-					disabled={addVersionBusy || !addVersionTag.trim()}
-					class="rounded-lg bg-blue-600 px-3 py-1.5 text-sm font-medium text-white hover:bg-blue-700 disabled:cursor-not-allowed disabled:opacity-50"
+					type="button"
+					onclick={handleBulkReprocessErroredVersions}
+					disabled={bulkReprocessBusy || actionableErroredTags.length === 0}
+					class="rounded-lg border border-red-200 px-3 py-1.5 text-sm font-medium text-red-600 hover:bg-red-50 disabled:cursor-not-allowed disabled:opacity-50"
 				>
-					Add
+					{bulkReprocessBusy
+						? 'Reprocessing...'
+						: `Reprocess errored${actionableErroredTags.length > 0 ? ` (${actionableErroredTags.length})` : ''}`}
 				</button>
-			</form>
-			<!-- Discover tags button — local repos only -->
-			{#if repo.source === 'local'}
-				<button
-					onclick={handleDiscoverTags}
-					disabled={discoverBusy}
-					class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm font-medium text-gray-700 hover:bg-gray-50 disabled:cursor-not-allowed disabled:opacity-50"
+				<!-- Add version inline form -->
+				<form
+					onsubmit={(e) => {
+						e.preventDefault();
+						handleAddVersion();
+					}}
+					class="flex items-center gap-1.5"
 				>
-					{discoverBusy ? 'Discovering...' : 'Discover tags'}
-				</button>
-			{/if}
+					<input
+						type="text"
+						bind:value={addVersionTag}
+						placeholder="e.g. v2.0.0"
+						class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm text-gray-900 placeholder-gray-400 focus:border-blue-400 focus:outline-none"
+					/>
+					<button
+						type="submit"
+						disabled={addVersionBusy || !addVersionTag.trim()}
+						class="rounded-lg bg-blue-600 px-3 py-1.5 text-sm font-medium text-white hover:bg-blue-700 disabled:cursor-not-allowed disabled:opacity-50"
+					>
+						Add
+					</button>
+				</form>
+				<!-- Discover tags button — local repos only -->
+				{#if repo.source === 'local'}
+					<button
+						onclick={handleDiscoverTags}
+						disabled={discoverBusy}
+						class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm font-medium text-gray-700 hover:bg-gray-50 disabled:cursor-not-allowed disabled:opacity-50"
+					>
+						{discoverBusy ? 'Discovering...' : 'Discover tags'}
+					</button>
+				{/if}
+			</div>
 		</div>
 	</div>
 
@@ -549,7 +637,7 @@
 					onclick={() => {
 						showDiscoverPanel = false;
 						discoveredTags = [];
-					selectedDiscoveredTags = new SvelteSet();
+						selectedDiscoveredTags.clear();
 					}}
 					class="text-xs text-blue-600 hover:underline"
 				>
@@ -567,7 +655,9 @@
 								class="rounded border-gray-300"
 							/>
 							<span class="font-mono text-gray-800">{discovered.tag}</span>
-							<span class="font-mono text-xs text-gray-400">{discovered.commitHash.slice(0, 8)}</span>
+							<span class="font-mono text-xs text-gray-400"
+								>{discovered.commitHash.slice(0, 8)}</span
+							>
 						</label>
 					{/each}
 				</div>
@@ -576,9 +666,7 @@
 					disabled={registerBusy || selectedDiscoveredTags.size === 0}
 					class="rounded-lg bg-blue-600 px-3 py-1.5 text-sm font-medium text-white hover:bg-blue-700 disabled:cursor-not-allowed disabled:opacity-50"
 				>
-					{registerBusy
-						? 'Registering...'
-						: `Register ${selectedDiscoveredTags.size} selected`}
+					{registerBusy ? 'Registering...' : `Register ${selectedDiscoveredTags.size} selected`}
 				</button>
 			{/if}
 		</div>
@@ -589,9 +677,15 @@
 		<p class="text-sm text-gray-400">Loading versions...</p>
 	{:else if versions.length === 0}
 		<p class="text-sm text-gray-400">No versions registered. Add a tag above to get started.</p>
+	{:else if filteredVersions.length === 0}
+		<div class="rounded-lg border border-dashed border-gray-200 bg-gray-50 px-4 py-5">
+			<p class="text-sm text-gray-500">
+				No versions match the {activeVersionFilterLabel.toLowerCase()} filter.
+			</p>
+		</div>
 	{:else}
 		<div class="divide-y divide-gray-100">
-			{#each versions as version (version.id)}
+			{#each filteredVersions as version (version.id)}
 				<div class="py-2.5">
 					<div class="flex items-center justify-between">
 						<div class="flex items-center gap-3">
@@ -609,7 +703,9 @@
 								disabled={version.state === 'indexing' || !!activeVersionJobs[version.tag]}
 								class="rounded-lg border border-blue-200 px-3 py-1 text-xs font-medium text-blue-600 hover:bg-blue-50 disabled:cursor-not-allowed disabled:opacity-50"
 							>
-								{version.state === 'indexing' || !!activeVersionJobs[version.tag] ? 'Indexing...' : 'Index'}
+								{version.state === 'indexing' || !!activeVersionJobs[version.tag]
+									? 'Indexing...'
+									: 'Index'}
 							</button>
 							<button
 								onclick={() => (removeTag = version.tag)}
@@ -625,12 +721,8 @@
 								version.totalSnippets > 0
 									? { text: `${version.totalSnippets} snippets`, mono: false }
 									: null,
-								version.commitHash
-									? { text: version.commitHash.slice(0, 8), mono: true }
-									: null,
-								version.indexedAt
-									? { text: formatDate(version.indexedAt), mono: false }
-									: null
+								version.commitHash ? { text: version.commitHash.slice(0, 8), mono: true } : null,
+								version.indexedAt ? { text: formatDate(version.indexedAt), mono: false } : null
 							] as Array<{ text: string; mono: boolean } | null>
 						).filter((p): p is { text: string; mono: boolean } => p !== null)}
 						<div class="mt-1 flex items-center gap-1.5">
@@ -638,7 +730,8 @@
 								{#if i > 0}
 									<span class="text-xs text-gray-300">·</span>
 								{/if}
-								<span class="text-xs text-gray-400{part.mono ? ' font-mono' : ''}">{part.text}</span>
+								<span class="text-xs text-gray-400{part.mono ? ' font-mono' : ''}">{part.text}</span
+								>
 							{/each}
 						</div>
 					{/if}
@@ -646,10 +739,12 @@
 						{@const job = versionJobProgress[activeVersionJobs[version.tag]!]}
 						<div class="mt-2">
 							<div class="flex justify-between text-xs text-gray-500">
-							<span>
-								{#if job?.stageDetail}{job.stageDetail}{:else}{(job?.processedFiles ?? 0).toLocaleString()} / {(job?.totalFiles ?? 0).toLocaleString()} files{/if}
-								{#if job?.stage}{' - ' + (stageLabels[job.stage] ?? job.stage)}{/if}
-							</span>
+								<span>
+									{#if job?.stageDetail}{job.stageDetail}{:else}{(
+											job?.processedFiles ?? 0
+										).toLocaleString()} / {(job?.totalFiles ?? 0).toLocaleString()} files{/if}
+									{#if job?.stage}{' - ' + (stageLabels[job.stage] ?? job.stage)}{/if}
+								</span>
 								<span>{job?.progress ?? 0}%</span>
 							</div>
 							<div class="mt-1 h-1.5 w-full rounded-full bg-gray-200">