feat(TRUEREF-0023): add sqlite-vec search pipeline

This commit is contained in:
Giancarmine Salucci
2026-04-01 14:09:19 +02:00
parent 0752636847
commit 9525c58e9a
45 changed files with 4009 additions and 614 deletions

View File

@@ -1,5 +1,10 @@
<script lang="ts">
import { onMount } from 'svelte';
import { SvelteURLSearchParams } from 'svelte/reactivity';
import JobSkeleton from '$lib/components/admin/JobSkeleton.svelte';
import JobStatusBadge from '$lib/components/admin/JobStatusBadge.svelte';
import Toast from '$lib/components/admin/Toast.svelte';
import WorkerStatusPanel from '$lib/components/admin/WorkerStatusPanel.svelte';
import type { IndexingJobDto } from '$lib/server/models/indexing-job.js';
interface JobResponse {
@@ -7,174 +12,16 @@
total: number;
}
let jobs = $state<IndexingJobDto[]>([]);
let loading = $state(true);
let error = $state<string | null>(null);
let actionInProgress = $state<string | null>(null);
// Fetch jobs from API
async function fetchJobs() {
try {
const response = await fetch('/api/v1/jobs?limit=50');
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const data: JobResponse = await response.json();
jobs = data.jobs;
error = null;
} catch (err) {
error = err instanceof Error ? err.message : 'Failed to fetch jobs';
console.error('Failed to fetch jobs:', err);
} finally {
loading = false;
}
interface ToastItem {
id: string;
message: string;
type: 'success' | 'error' | 'info';
}
// Action handlers
async function pauseJob(id: string) {
actionInProgress = id;
try {
const response = await fetch(`/api/v1/jobs/${id}/pause`, { method: 'POST' });
if (!response.ok) {
const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
throw new Error(errorData.message || `HTTP ${response.status}`);
}
// Optimistic update
jobs = jobs.map((j) => (j.id === id ? { ...j, status: 'paused' as const } : j));
// Show success message
showToast('Job paused successfully');
} catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to pause job';
showToast(msg, 'error');
console.error('Failed to pause job:', err);
} finally {
actionInProgress = null;
// Refresh after a short delay to get the actual state
setTimeout(fetchJobs, 500);
}
}
type FilterStatus = 'queued' | 'running' | 'done' | 'failed';
type JobAction = 'pause' | 'resume' | 'cancel';
async function resumeJob(id: string) {
actionInProgress = id;
try {
const response = await fetch(`/api/v1/jobs/${id}/resume`, { method: 'POST' });
if (!response.ok) {
const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
throw new Error(errorData.message || `HTTP ${response.status}`);
}
// Optimistic update
jobs = jobs.map((j) => (j.id === id ? { ...j, status: 'queued' as const } : j));
showToast('Job resumed successfully');
} catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to resume job';
showToast(msg, 'error');
console.error('Failed to resume job:', err);
} finally {
actionInProgress = null;
setTimeout(fetchJobs, 500);
}
}
async function cancelJob(id: string) {
if (!confirm('Are you sure you want to cancel this job?')) {
return;
}
actionInProgress = id;
try {
const response = await fetch(`/api/v1/jobs/${id}/cancel`, { method: 'POST' });
if (!response.ok) {
const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
throw new Error(errorData.message || `HTTP ${response.status}`);
}
// Optimistic update
jobs = jobs.map((j) => (j.id === id ? { ...j, status: 'cancelled' as const } : j));
showToast('Job cancelled successfully');
} catch (err) {
const msg = err instanceof Error ? err.message : 'Failed to cancel job';
showToast(msg, 'error');
console.error('Failed to cancel job:', err);
} finally {
actionInProgress = null;
setTimeout(fetchJobs, 500);
}
}
// Simple toast notification (using alert for v1, can be enhanced later)
function showToast(message: string, type: 'success' | 'error' = 'success') {
// For v1, just use alert. In production, integrate with a toast library.
if (type === 'error') {
alert(`Error: ${message}`);
} else {
console.log(`✓ ${message}`);
}
}
// Auto-refresh with EventSource streaming + fallback polling
$effect(() => {
fetchJobs();
const es = new EventSource('/api/v1/jobs/stream');
let fallbackInterval: ReturnType<typeof setInterval> | null = null;
es.addEventListener('job-progress', (event) => {
const data = JSON.parse(event.data);
jobs = jobs.map((j) =>
j.id === data.jobId
? {
...j,
progress: data.progress,
stage: data.stage,
stageDetail: data.stageDetail,
processedFiles: data.processedFiles,
totalFiles: data.totalFiles
}
: j
);
});
es.addEventListener('job-done', () => {
void fetchJobs();
});
es.addEventListener('job-failed', () => {
void fetchJobs();
});
es.onerror = () => {
es.close();
// Fall back to polling on error
fallbackInterval = setInterval(fetchJobs, 3000);
};
return () => {
es.close();
if (fallbackInterval) {
clearInterval(fallbackInterval);
}
};
});
// Format date for display
function formatDate(date: Date | null): string {
if (!date) return '—';
return new Date(date).toLocaleString();
}
// Determine which actions are available for a job
function canPause(status: IndexingJobDto['status']): boolean {
return status === 'queued' || status === 'running';
}
function canResume(status: IndexingJobDto['status']): boolean {
return status === 'paused';
}
function canCancel(status: IndexingJobDto['status']): boolean {
return status !== 'done' && status !== 'failed';
}
// Map IndexingStage values to display labels
const filterStatuses: FilterStatus[] = ['queued', 'running', 'done', 'failed'];
const stageLabels: Record<string, string> = {
queued: 'Queued',
differential: 'Diff',
@@ -187,9 +34,274 @@
failed: 'Failed'
};
let jobs = $state<IndexingJobDto[]>([]);
let total = $state(0);
let loading = $state(true);
let refreshing = $state(false);
let error = $state<string | null>(null);
let repositoryInput = $state('');
let selectedStatuses = $state<FilterStatus[]>([]);
let appliedRepositoryFilter = $state('');
let appliedStatuses = $state<FilterStatus[]>([]);
let pendingCancelJobId = $state<string | null>(null);
let rowActions = $state<Record<string, JobAction | undefined>>({});
let toasts = $state<ToastItem[]>([]);
let refreshTimer: ReturnType<typeof setTimeout> | null = null;
function buildJobsUrl(): string {
const params = new SvelteURLSearchParams({ limit: '50' });
if (appliedRepositoryFilter) {
params.set('repositoryId', appliedRepositoryFilter);
}
if (appliedStatuses.length > 0) {
params.set('status', appliedStatuses.join(','));
}
return `/api/v1/jobs?${params.toString()}`;
}
function pushToast(message: string, type: ToastItem['type'] = 'success') {
toasts = [...toasts, { id: crypto.randomUUID(), message, type }];
}
function clearRowAction(jobId: string) {
const next = { ...rowActions };
delete next[jobId];
rowActions = next;
}
function setRowAction(jobId: string, action: JobAction) {
rowActions = { ...rowActions, [jobId]: action };
}
function scheduleRefresh(delayMs = 500) {
if (refreshTimer) {
clearTimeout(refreshTimer);
}
refreshTimer = setTimeout(() => {
void fetchJobs({ background: true });
}, delayMs);
}
function hasAppliedFilters(): boolean {
return appliedRepositoryFilter.length > 0 || appliedStatuses.length > 0;
}
function sameStatuses(left: FilterStatus[], right: FilterStatus[]): boolean {
return left.length === right.length && left.every((status, index) => status === right[index]);
}
function filtersDirty(): boolean {
return repositoryInput.trim() !== appliedRepositoryFilter || !sameStatuses(selectedStatuses, appliedStatuses);
}
function isSpecificRepositoryId(repositoryId: string): boolean {
return repositoryId.split('/').filter(Boolean).length >= 2;
}
function matchesAppliedFilters(job: IndexingJobDto): boolean {
if (appliedRepositoryFilter) {
const repositoryFilter = appliedRepositoryFilter;
const repositoryMatches = isSpecificRepositoryId(repositoryFilter)
? job.repositoryId === repositoryFilter
: job.repositoryId === repositoryFilter || job.repositoryId.startsWith(`${repositoryFilter}/`);
if (!repositoryMatches) {
return false;
}
}
if (appliedStatuses.length === 0) {
return true;
}
return appliedStatuses.includes(job.status as FilterStatus);
}
function syncCancelState(nextJobs: IndexingJobDto[]) {
if (!pendingCancelJobId) {
return;
}
const pendingJob = nextJobs.find((job) => job.id === pendingCancelJobId);
if (!pendingJob || !canCancel(pendingJob.status)) {
pendingCancelJobId = null;
}
}
async function fetchJobs(options: { background?: boolean } = {}) {
const background = options.background ?? false;
if (background) {
refreshing = true;
} else {
loading = true;
}
try {
const response = await fetch(buildJobsUrl());
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const data: JobResponse = await response.json();
jobs = data.jobs;
total = data.total;
error = null;
syncCancelState(data.jobs);
} catch (err) {
error = err instanceof Error ? err.message : 'Failed to fetch jobs';
console.error('Failed to fetch jobs:', err);
} finally {
loading = false;
refreshing = false;
}
}
async function runJobAction(job: IndexingJobDto, action: JobAction) {
setRowAction(job.id, action);
try {
const response = await fetch(`/api/v1/jobs/${job.id}/${action}`, { method: 'POST' });
const payload = await response.json().catch(() => ({ message: 'Unknown error' }));
if (!response.ok) {
throw new Error(payload.message || `HTTP ${response.status}`);
}
const updatedJob = payload.job as IndexingJobDto | undefined;
if (updatedJob) {
if (matchesAppliedFilters(updatedJob)) {
jobs = jobs.map((currentJob) =>
currentJob.id === updatedJob.id ? updatedJob : currentJob
);
} else {
jobs = jobs.filter((currentJob) => currentJob.id !== updatedJob.id);
}
}
pendingCancelJobId = null;
pushToast(`Job ${action}d successfully`);
} catch (err) {
const message = err instanceof Error ? err.message : `Failed to ${action} job`;
pushToast(message, 'error');
console.error(`Failed to ${action} job:`, err);
} finally {
clearRowAction(job.id);
scheduleRefresh();
}
}
function toggleStatusFilter(status: FilterStatus) {
selectedStatuses = selectedStatuses.includes(status)
? selectedStatuses.filter((candidate) => candidate !== status)
: [...selectedStatuses, status].sort(
(left, right) => filterStatuses.indexOf(left) - filterStatuses.indexOf(right)
);
}
function applyFilters(event?: SubmitEvent) {
event?.preventDefault();
appliedRepositoryFilter = repositoryInput.trim();
appliedStatuses = [...selectedStatuses];
pendingCancelJobId = null;
void fetchJobs();
}
function resetFilters() {
repositoryInput = '';
selectedStatuses = [];
appliedRepositoryFilter = '';
appliedStatuses = [];
pendingCancelJobId = null;
void fetchJobs();
}
function requestCancel(jobId: string) {
pendingCancelJobId = pendingCancelJobId === jobId ? null : jobId;
}
function formatDate(date: Date | string | null): string {
if (!date) {
return '—';
}
return new Date(date).toLocaleString();
}
function canPause(status: IndexingJobDto['status']): boolean {
return status === 'queued' || status === 'running';
}
function canResume(status: IndexingJobDto['status']): boolean {
return status === 'paused';
}
function canCancel(status: IndexingJobDto['status']): boolean {
return status !== 'done' && status !== 'failed' && status !== 'cancelled';
}
function isRowBusy(jobId: string): boolean {
return Boolean(rowActions[jobId]);
}
function getStageLabel(stage: string | undefined): string {
return stage ? (stageLabels[stage] ?? stage) : '—';
}
onMount(() => {
void fetchJobs();
const es = new EventSource('/api/v1/jobs/stream');
let fallbackInterval: ReturnType<typeof setInterval> | null = null;
const refreshJobs = () => {
void fetchJobs({ background: true });
};
es.addEventListener('job-progress', (event) => {
const data = JSON.parse(event.data) as Partial<IndexingJobDto> & { jobId?: string };
if (!data.jobId) {
return;
}
jobs = jobs.map((job) =>
job.id === data.jobId
? {
...job,
progress: data.progress ?? job.progress,
stage: data.stage ?? job.stage,
stageDetail: data.stageDetail ?? job.stageDetail,
processedFiles: data.processedFiles ?? job.processedFiles,
totalFiles: data.totalFiles ?? job.totalFiles,
status: data.status ?? job.status
}
: job
);
});
es.addEventListener('job-done', refreshJobs);
es.addEventListener('job-failed', refreshJobs);
es.onerror = () => {
es.close();
if (!fallbackInterval) {
fallbackInterval = setInterval(refreshJobs, 3000);
}
};
return () => {
es.close();
if (fallbackInterval) {
clearInterval(fallbackInterval);
}
if (refreshTimer) {
clearTimeout(refreshTimer);
}
};
});
</script>
<svelte:head>
@@ -202,23 +314,92 @@
<p class="mt-2 text-gray-600">Monitor and control indexing jobs</p>
</div>
{#if loading && jobs.length === 0}
<div class="flex items-center justify-center py-12">
<div class="text-center">
<div
class="inline-block h-8 w-8 animate-spin rounded-full border-4 border-solid border-blue-600 border-r-transparent"
></div>
<p class="mt-2 text-gray-600">Loading jobs...</p>
<WorkerStatusPanel />
<form class="mb-6 rounded-lg border border-gray-200 bg-white p-4 shadow-sm" onsubmit={applyFilters}>
<div class="flex flex-col gap-4 lg:flex-row lg:items-end lg:justify-between">
<div class="flex-1">
<label class="mb-2 block text-sm font-medium text-gray-700" for="repository-filter">
Repository filter
</label>
<input
id="repository-filter"
type="text"
bind:value={repositoryInput}
placeholder="/owner or /owner/repo"
class="w-full rounded-md border border-gray-300 px-3 py-2 text-sm text-gray-900 shadow-sm focus:border-blue-500 focus:outline-none focus:ring-2 focus:ring-blue-200"
/>
<p class="mt-2 text-xs text-gray-500">
Use an owner prefix like <code>/facebook</code> or a full repository ID like <code>/facebook/react</code>.
</p>
</div>
<div class="lg:min-w-72">
<span class="mb-2 block text-sm font-medium text-gray-700">Statuses</span>
<div class="flex flex-wrap gap-2">
{#each filterStatuses as status (status)}
<button
type="button"
onclick={() => toggleStatusFilter(status)}
class="rounded-full border px-3 py-1 text-xs font-semibold uppercase transition {selectedStatuses.includes(status)
? 'border-blue-600 bg-blue-50 text-blue-700'
: 'border-gray-300 text-gray-600 hover:border-gray-400 hover:text-gray-900'}"
>
{status}
</button>
{/each}
</div>
</div>
<div class="flex gap-2">
<button
type="submit"
disabled={!filtersDirty()}
class="rounded bg-blue-600 px-4 py-2 text-sm font-semibold text-white hover:bg-blue-700 disabled:cursor-not-allowed disabled:opacity-50"
>
Apply filters
</button>
<button
type="button"
onclick={resetFilters}
class="rounded border border-gray-300 px-4 py-2 text-sm font-semibold text-gray-700 hover:border-gray-400 hover:text-gray-900"
>
Reset
</button>
</div>
</div>
{:else if error && jobs.length === 0}
<div class="rounded-md bg-red-50 p-4">
<p class="text-sm text-red-800">Error: {error}</p>
</form>
<div class="mb-4 flex flex-col gap-2 text-sm text-gray-600 md:flex-row md:items-center md:justify-between">
<p>
Showing <span class="font-semibold text-gray-900">{jobs.length}</span> of
<span class="font-semibold text-gray-900">{total}</span> jobs
</p>
{#if hasAppliedFilters()}
<p class="text-xs text-gray-500">
Active filters:
{appliedRepositoryFilter || 'all repositories'}
{#if appliedStatuses.length > 0}
· {appliedStatuses.join(', ')}
{:else}
· all statuses
{/if}
</p>
{/if}
</div>
{#if error}
<div class="mb-4 rounded-md border border-red-200 bg-red-50 px-4 py-3 text-sm text-red-800">
{error}
</div>
{:else if jobs.length === 0}
{/if}
{#if !loading && jobs.length === 0}
<div class="rounded-md bg-gray-50 p-8 text-center">
<p class="text-gray-600">
No jobs found. Jobs will appear here when repositories are indexed.
{hasAppliedFilters()
? 'No jobs match the current filters.'
: 'No jobs found. Jobs will appear here when repositories are indexed.'}
</p>
</div>
{:else}
@@ -259,86 +440,117 @@
</tr>
</thead>
<tbody class="divide-y divide-gray-200 bg-white">
{#each jobs as job (job.id)}
{#if loading && jobs.length === 0}
<JobSkeleton rows={6} />
{:else}
{#each jobs as job (job.id)}
<tr class="hover:bg-gray-50">
<td class="px-6 py-4 text-sm font-medium whitespace-nowrap text-gray-900">
{job.repositoryId}
{#if job.versionId}
<span class="ml-1 text-xs text-gray-500">@{job.versionId}</span>
{/if}
<div class="mt-1 text-xs text-gray-400">{job.id}</div>
</td>
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
<JobStatusBadge status={job.status} />
<JobStatusBadge status={job.status} spinning={job.status === 'running'} />
</td>
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
<div class="flex items-center gap-2">
<span>{getStageLabel(job.stage)}</span>
{#if job.stageDetail}
<span class="text-xs text-gray-400">{job.stageDetail}</span>
{/if}
</div>
</td>
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
<div class="flex items-center">
<span class="mr-2">{job.progress}%</span>
<div class="h-2 w-32 rounded-full bg-gray-200">
<div
class="h-2 rounded-full bg-blue-600 transition-all"
style="width: {job.progress}%"
></div>
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
<div class="flex items-center gap-2">
<span>{getStageLabel(job.stage)}</span>
{#if job.stageDetail}
<span class="text-xs text-gray-400">{job.stageDetail}</span>
{/if}
</div>
{#if job.totalFiles > 0}
<span class="ml-2 text-xs text-gray-400">
{job.processedFiles}/{job.totalFiles} files
</span>
{/if}
</div>
</td>
</td>
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
<div class="space-y-2">
<div class="flex items-center gap-2">
<span class="w-12 text-right text-xs font-semibold text-gray-600">{job.progress}%</span>
<div class="h-2 w-32 rounded-full bg-gray-200">
<div
class="h-2 rounded-full bg-blue-600 transition-all"
style="width: {job.progress}%"
></div>
</div>
</div>
{#if job.totalFiles > 0}
<div class="text-xs text-gray-400">
{job.processedFiles}/{job.totalFiles} files processed
</div>
{/if}
</div>
</td>
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
{formatDate(job.createdAt)}
</td>
<td class="px-6 py-4 text-right text-sm font-medium whitespace-nowrap">
<div class="flex justify-end gap-2">
{#if canPause(job.status)}
{#if pendingCancelJobId === job.id}
<button
onclick={() => pauseJob(job.id)}
disabled={actionInProgress === job.id}
class="rounded bg-yellow-600 px-3 py-1 text-xs font-semibold text-white hover:bg-yellow-700 disabled:opacity-50"
type="button"
onclick={() => void runJobAction(job, 'cancel')}
disabled={isRowBusy(job.id)}
class="rounded bg-red-600 px-3 py-1 text-xs font-semibold text-white hover:bg-red-700 disabled:cursor-not-allowed disabled:opacity-50"
>
Pause
{rowActions[job.id] === 'cancel' ? 'Cancelling...' : 'Confirm cancel'}
</button>
{/if}
{#if canResume(job.status)}
<button
onclick={() => resumeJob(job.id)}
disabled={actionInProgress === job.id}
class="rounded bg-green-600 px-3 py-1 text-xs font-semibold text-white hover:bg-green-700 disabled:opacity-50"
type="button"
onclick={() => requestCancel(job.id)}
disabled={isRowBusy(job.id)}
class="rounded border border-gray-300 px-3 py-1 text-xs font-semibold text-gray-700 hover:border-gray-400 hover:text-gray-900 disabled:cursor-not-allowed disabled:opacity-50"
>
Resume
Keep job
</button>
{/if}
{#if canCancel(job.status)}
<button
onclick={() => cancelJob(job.id)}
disabled={actionInProgress === job.id}
class="rounded bg-red-600 px-3 py-1 text-xs font-semibold text-white hover:bg-red-700 disabled:opacity-50"
>
Cancel
</button>
{/if}
{#if !canPause(job.status) && !canResume(job.status) && !canCancel(job.status)}
<span class="text-xs text-gray-400"></span>
{:else}
{#if canPause(job.status)}
<button
type="button"
onclick={() => void runJobAction(job, 'pause')}
disabled={isRowBusy(job.id)}
class="rounded bg-yellow-600 px-3 py-1 text-xs font-semibold text-white hover:bg-yellow-700 disabled:cursor-not-allowed disabled:opacity-50"
>
{rowActions[job.id] === 'pause' ? 'Pausing...' : 'Pause'}
</button>
{/if}
{#if canResume(job.status)}
<button
type="button"
onclick={() => void runJobAction(job, 'resume')}
disabled={isRowBusy(job.id)}
class="rounded bg-green-600 px-3 py-1 text-xs font-semibold text-white hover:bg-green-700 disabled:cursor-not-allowed disabled:opacity-50"
>
{rowActions[job.id] === 'resume' ? 'Resuming...' : 'Resume'}
</button>
{/if}
{#if canCancel(job.status)}
<button
type="button"
onclick={() => requestCancel(job.id)}
disabled={isRowBusy(job.id)}
class="rounded bg-red-600 px-3 py-1 text-xs font-semibold text-white hover:bg-red-700 disabled:cursor-not-allowed disabled:opacity-50"
>
Cancel
</button>
{/if}
{#if !canPause(job.status) && !canResume(job.status) && !canCancel(job.status)}
<span class="text-xs text-gray-400"></span>
{/if}
{/if}
</div>
</td>
</tr>
{/each}
{/each}
{/if}
</tbody>
</table>
</div>
{#if loading}
{#if refreshing}
<div class="mt-4 text-center text-sm text-gray-500">Refreshing...</div>
{/if}
{/if}
</div>
<Toast bind:toasts={toasts} />

View File

@@ -15,15 +15,39 @@ import { JobQueue } from '$lib/server/pipeline/job-queue.js';
import { handleServiceError } from '$lib/server/utils/validation.js';
import type { IndexingJob } from '$lib/types';
const VALID_JOB_STATUSES: ReadonlySet<IndexingJob['status']> = new Set([
'queued',
'running',
'done',
'failed'
]);
function parseStatusFilter(searchValue: string | null): IndexingJob['status'] | Array<IndexingJob['status']> | undefined {
if (!searchValue) {
return undefined;
}
const statuses = [...new Set(
searchValue
.split(',')
.map((value) => value.trim())
.filter((value): value is IndexingJob['status'] => VALID_JOB_STATUSES.has(value as IndexingJob['status']))
)];
if (statuses.length === 0) {
return undefined;
}
return statuses.length === 1 ? statuses[0] : statuses;
}
export const GET: RequestHandler = ({ url }) => {
try {
const db = getClient();
const queue = new JobQueue(db);
const repositoryId = url.searchParams.get('repositoryId') ?? undefined;
const status = (url.searchParams.get('status') ?? undefined) as
| IndexingJob['status']
| undefined;
const repositoryId = url.searchParams.get('repositoryId')?.trim() || undefined;
const status = parseStatusFilter(url.searchParams.get('status'));
const limit = Math.min(parseInt(url.searchParams.get('limit') ?? '20', 10) || 20, 1000);
const jobs = queue.listJobs({ repositoryId, status, limit });

View File

@@ -44,7 +44,7 @@ export const GET: RequestHandler = ({ params, request }) => {
status: job.status,
error: job.error
};
controller.enqueue(`data: ${JSON.stringify(initialData)}\n\n`);
controller.enqueue(`event: job-progress\ndata: ${JSON.stringify(initialData)}\n\n`);
// Check for Last-Event-ID header for reconnect
const lastEventId = request.headers.get('Last-Event-ID');
@@ -57,6 +57,13 @@ export const GET: RequestHandler = ({ params, request }) => {
// Check if job is already done or failed - close immediately after first event
if (job.status === 'done' || job.status === 'failed') {
if (job.status === 'done') {
controller.enqueue(`event: job-done\ndata: ${JSON.stringify({ jobId })}\n\n`);
} else {
controller.enqueue(
`event: job-failed\ndata: ${JSON.stringify({ jobId, error: job.error })}\n\n`
);
}
controller.close();
return;
}
@@ -73,18 +80,29 @@ export const GET: RequestHandler = ({ params, request }) => {
controller.enqueue(value);
// Check if the incoming event indicates job completion
if (value.includes('event: done') || value.includes('event: failed')) {
if (
value.includes('event: job-done') ||
value.includes('event: job-failed')
) {
controller.close();
break;
}
}
} finally {
reader.releaseLock();
controller.close();
try {
controller.close();
} catch {
// Stream may already be closed after a terminal event.
}
}
} catch (err) {
console.error('SSE stream error:', err);
controller.close();
try {
controller.close();
} catch {
// Stream may already be closed.
}
}
}
});

View File

@@ -18,6 +18,7 @@ import type { ProgressBroadcaster as BroadcasterType } from '$lib/server/pipelin
let db: Database.Database;
// Closed over by the vi.mock factory below.
let mockBroadcaster: BroadcasterType | null = null;
let mockPool: { getStatus: () => object; setMaxConcurrency?: (value: number) => void } | null = null;
vi.mock('$lib/server/db/client', () => ({
getClient: () => db
@@ -29,12 +30,12 @@ vi.mock('$lib/server/db/client.js', () => ({
vi.mock('$lib/server/pipeline/startup', () => ({
getQueue: () => null,
getPool: () => null
getPool: () => mockPool
}));
vi.mock('$lib/server/pipeline/startup.js', () => ({
getQueue: () => null,
getPool: () => null
getPool: () => mockPool
}));
vi.mock('$lib/server/pipeline/progress-broadcaster', async (importOriginal) => {
@@ -58,9 +59,11 @@ vi.mock('$lib/server/pipeline/progress-broadcaster.js', async (importOriginal) =
// ---------------------------------------------------------------------------
import { ProgressBroadcaster } from '$lib/server/pipeline/progress-broadcaster.js';
import { GET as getJobsList } from './jobs/+server.js';
import { GET as getJobStream } from './jobs/[id]/stream/+server.js';
import { GET as getJobsStream } from './jobs/stream/+server.js';
import { GET as getIndexingSettings, PUT as putIndexingSettings } from './settings/indexing/+server.js';
import { GET as getWorkers } from './workers/+server.js';
// ---------------------------------------------------------------------------
// DB factory
@@ -306,6 +309,25 @@ describe('GET /api/v1/jobs/:id/stream', () => {
// The replay event should include the cached event data
expect(fullText).toContain('progress');
});
it('closes after receiving the broadcaster job-done event', async () => {
seedRepo(db);
const jobId = seedJob(db, { status: 'running', stage: 'parsing', progress: 10 });
const response = await getJobStream(makeEvent({ params: { id: jobId } }));
const reader = response.body!.getReader();
const initialChunk = await reader.read();
expect(String(initialChunk.value ?? '')).toContain('event: job-progress');
mockBroadcaster!.broadcast(jobId, '/test/repo', 'job-done', { jobId, status: 'done' });
const completionChunk = await reader.read();
expect(String(completionChunk.value ?? '')).toContain('event: job-done');
const closed = await reader.read();
expect(closed.done).toBe(true);
});
});
// ---------------------------------------------------------------------------
@@ -377,12 +399,125 @@ describe('GET /api/v1/jobs/stream', () => {
});
// ---------------------------------------------------------------------------
// Test group 3: GET /api/v1/settings/indexing
// Test group 3: GET /api/v1/jobs
// ---------------------------------------------------------------------------
describe('GET /api/v1/jobs', () => {
beforeEach(() => {
db = createTestDb();
});
it('supports repository prefix and comma-separated status filters', async () => {
seedRepo(db, '/facebook/react');
seedRepo(db, '/facebook/react-native');
seedRepo(db, '/vitejs/vite');
seedJob(db, { repository_id: '/facebook/react', status: 'queued' });
seedJob(db, { repository_id: '/facebook/react-native', status: 'running' });
seedJob(db, { repository_id: '/facebook/react', status: 'done' });
seedJob(db, { repository_id: '/vitejs/vite', status: 'queued' });
const response = await getJobsList(
makeEvent<Parameters<typeof getJobsList>[0]>({
url: 'http://localhost/api/v1/jobs?repositoryId=%2Ffacebook&status=queued,%20running'
})
);
const body = await response.json();
expect(response.status).toBe(200);
expect(body.total).toBe(2);
expect(body.jobs).toHaveLength(2);
expect(body.jobs.map((job: { repositoryId: string }) => job.repositoryId).sort()).toEqual([
'/facebook/react',
'/facebook/react-native'
]);
expect(body.jobs.map((job: { status: string }) => job.status).sort()).toEqual([
'queued',
'running'
]);
});
it('keeps exact-match behavior for specific repository IDs', async () => {
seedRepo(db, '/facebook/react');
seedRepo(db, '/facebook/react-native');
seedJob(db, { repository_id: '/facebook/react', status: 'queued' });
seedJob(db, { repository_id: '/facebook/react-native', status: 'queued' });
const response = await getJobsList(
makeEvent<Parameters<typeof getJobsList>[0]>({
url: 'http://localhost/api/v1/jobs?repositoryId=%2Ffacebook%2Freact&status=queued'
})
);
const body = await response.json();
expect(response.status).toBe(200);
expect(body.total).toBe(1);
expect(body.jobs).toHaveLength(1);
expect(body.jobs[0].repositoryId).toBe('/facebook/react');
});
});
// ---------------------------------------------------------------------------
// Test group 4: GET /api/v1/workers
// ---------------------------------------------------------------------------
describe('GET /api/v1/workers', () => {
beforeEach(() => {
mockPool = null;
});
it('returns 503 when the worker pool is not initialized', async () => {
const response = await getWorkers(makeEvent<Parameters<typeof getWorkers>[0]>({}));
expect(response.status).toBe(503);
});
it('returns the current worker status snapshot', async () => {
mockPool = {
getStatus: () => ({
concurrency: 2,
active: 1,
idle: 1,
workers: [
{
index: 0,
state: 'running',
jobId: 'job-1',
repositoryId: '/test/repo',
versionId: null
},
{
index: 1,
state: 'idle',
jobId: null,
repositoryId: null,
versionId: null
}
]
})
};
const response = await getWorkers(makeEvent<Parameters<typeof getWorkers>[0]>({}));
const body = await response.json();
expect(response.status).toBe(200);
expect(body.active).toBe(1);
expect(body.workers[0].jobId).toBe('job-1');
});
});
// ---------------------------------------------------------------------------
// Test group 5: GET /api/v1/settings/indexing
// ---------------------------------------------------------------------------
describe('GET /api/v1/settings/indexing', () => {
beforeEach(() => {
db = createTestDb();
mockPool = {
getStatus: () => ({ concurrency: 2, active: 0, idle: 2, workers: [] }),
setMaxConcurrency: vi.fn()
};
});
it('returns { concurrency: 2 } when no setting exists in DB', async () => {
@@ -417,12 +552,16 @@ describe('GET /api/v1/settings/indexing', () => {
});
// ---------------------------------------------------------------------------
// Test group 4: PUT /api/v1/settings/indexing
// Test group 6: PUT /api/v1/settings/indexing
// ---------------------------------------------------------------------------
describe('PUT /api/v1/settings/indexing', () => {
beforeEach(() => {
db = createTestDb();
mockPool = {
getStatus: () => ({ concurrency: 2, active: 0, idle: 2, workers: [] }),
setMaxConcurrency: vi.fn()
};
});
function makePutEvent(body: unknown) {

View File

@@ -0,0 +1,16 @@
import type { RequestHandler } from './$types';
import { getPool } from '$lib/server/pipeline/startup.js';
import { handleServiceError } from '$lib/server/utils/validation.js';
export const GET: RequestHandler = () => {
try {
const pool = getPool();
if (!pool) {
return new Response('Service unavailable', { status: 503 });
}
return Response.json(pool.getStatus());
} catch (error) {
return handleServiceError(error);
}
};