feat(TRUEREF-0022): complete iteration 0 — worker-thread indexing, parallel jobs, SSE progress

- Move IndexingPipeline.run() into Worker Threads via WorkerPool
- Add dedicated embedding worker thread with single model instance
- Add stage/stageDetail columns to indexing_jobs schema
- Create ProgressBroadcaster for SSE channel management
- Add SSE endpoints: GET /api/v1/jobs/:id/stream, GET /api/v1/jobs/stream
- Replace UI polling with EventSource on repo detail and admin pages
- Add concurrency settings UI and API endpoint
- Build worker entries separately via esbuild

src/routes/api/v1/jobs/[id]/stream/+server.ts

@@ -0,0 +1,115 @@
+/**
+ * GET /api/v1/jobs/:id/stream — stream real-time job progress via SSE.
+ *
+ * Headers:
+ *   Last-Event-ID (optional) — triggers replay of last cached event
+ */
+
+import type { RequestHandler } from './$types';
+import { getClient } from '$lib/server/db/client.js';
+import { JobQueue } from '$lib/server/pipeline/job-queue.js';
+import { getBroadcaster } from '$lib/server/pipeline/progress-broadcaster.js';
+import { handleServiceError } from '$lib/server/utils/validation.js';
+
+export const GET: RequestHandler = ({ params, request }) => {
+	try {
+		const db = getClient();
+		const queue = new JobQueue(db);
+		const jobId = params.id;
+
+		// Get the job from the queue
+		const job = queue.getJob(jobId);
+		if (!job) {
+			return new Response('Not found', { status: 404 });
+		}
+
+		// Get broadcaster
+		const broadcaster = getBroadcaster();
+		if (!broadcaster) {
+			return new Response('Service unavailable', { status: 503 });
+		}
+
+		// Create a new readable stream for SSE
+		const stream = new ReadableStream<string>({
+			async start(controller) {
+				try {
+					// Send initial job state as first event
+					const initialData = {
+						jobId,
+						stage: job.stage,
+						stageDetail: job.stageDetail,
+						progress: job.progress,
+						processedFiles: job.processedFiles,
+						totalFiles: job.totalFiles,
+						status: job.status,
+						error: job.error
+					};
+					controller.enqueue(`data: ${JSON.stringify(initialData)}\n\n`);
+
+					// Check for Last-Event-ID header for reconnect
+					const lastEventId = request.headers.get('Last-Event-ID');
+					if (lastEventId) {
+						const lastEvent = broadcaster.getLastEvent(jobId);
+						if (lastEvent && lastEvent.id >= parseInt(lastEventId, 10)) {
+							controller.enqueue(`id: ${lastEvent.id}\nevent: ${lastEvent.event}\ndata: ${lastEvent.data}\n\n`);
+						}
+					}
+
+					// Check if job is already done or failed - close immediately after first event
+					if (job.status === 'done' || job.status === 'failed') {
+						controller.close();
+						return;
+					}
+
+					// Subscribe to broadcaster for live events
+					const eventStream = broadcaster.subscribe(jobId);
+					const reader = eventStream.getReader();
+
+					// Pipe broadcaster events to the response
+					try {
+						while (true) {
+							const { done, value } = await reader.read();
+							if (done) break;
+							controller.enqueue(value);
+
+							// Check if the incoming event indicates job completion
+							if (value.includes('event: done') || value.includes('event: failed')) {
+								controller.close();
+								break;
+							}
+						}
+					} finally {
+						reader.releaseLock();
+						controller.close();
+					}
+				} catch (err) {
+					console.error('SSE stream error:', err);
+					controller.close();
+				}
+			}
+		});
+
+		return new Response(stream, {
+			headers: {
+				'Content-Type': 'text/event-stream',
+				'Cache-Control': 'no-cache',
+				'Connection': 'keep-alive',
+				'X-Accel-Buffering': 'no',
+				'Access-Control-Allow-Origin': '*'
+			}
+		});
+	} catch (err) {
+		return handleServiceError(err);
+	}
+};
+
+export const OPTIONS: RequestHandler = () => {
+	return new Response(null, {
+		status: 204,
+		headers: {
+			'Access-Control-Allow-Origin': '*',
+			'Access-Control-Allow-Methods': 'GET, OPTIONS',
+			'Access-Control-Allow-Headers': 'Content-Type, Authorization, Last-Event-ID'
+		}
+	});
+};