feat(TRUEREF-0023): add sqlite-vec search pipeline
This commit is contained in:
@@ -1,5 +1,10 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import { SvelteURLSearchParams } from 'svelte/reactivity';
|
||||
import JobSkeleton from '$lib/components/admin/JobSkeleton.svelte';
|
||||
import JobStatusBadge from '$lib/components/admin/JobStatusBadge.svelte';
|
||||
import Toast from '$lib/components/admin/Toast.svelte';
|
||||
import WorkerStatusPanel from '$lib/components/admin/WorkerStatusPanel.svelte';
|
||||
import type { IndexingJobDto } from '$lib/server/models/indexing-job.js';
|
||||
|
||||
interface JobResponse {
|
||||
@@ -7,174 +12,16 @@
|
||||
total: number;
|
||||
}
|
||||
|
||||
let jobs = $state<IndexingJobDto[]>([]);
|
||||
let loading = $state(true);
|
||||
let error = $state<string | null>(null);
|
||||
let actionInProgress = $state<string | null>(null);
|
||||
|
||||
// Fetch jobs from API
|
||||
async function fetchJobs() {
|
||||
try {
|
||||
const response = await fetch('/api/v1/jobs?limit=50');
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
const data: JobResponse = await response.json();
|
||||
jobs = data.jobs;
|
||||
error = null;
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : 'Failed to fetch jobs';
|
||||
console.error('Failed to fetch jobs:', err);
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
interface ToastItem {
|
||||
id: string;
|
||||
message: string;
|
||||
type: 'success' | 'error' | 'info';
|
||||
}
|
||||
|
||||
// Action handlers
|
||||
async function pauseJob(id: string) {
|
||||
actionInProgress = id;
|
||||
try {
|
||||
const response = await fetch(`/api/v1/jobs/${id}/pause`, { method: 'POST' });
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
|
||||
throw new Error(errorData.message || `HTTP ${response.status}`);
|
||||
}
|
||||
// Optimistic update
|
||||
jobs = jobs.map((j) => (j.id === id ? { ...j, status: 'paused' as const } : j));
|
||||
// Show success message
|
||||
showToast('Job paused successfully');
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'Failed to pause job';
|
||||
showToast(msg, 'error');
|
||||
console.error('Failed to pause job:', err);
|
||||
} finally {
|
||||
actionInProgress = null;
|
||||
// Refresh after a short delay to get the actual state
|
||||
setTimeout(fetchJobs, 500);
|
||||
}
|
||||
}
|
||||
type FilterStatus = 'queued' | 'running' | 'done' | 'failed';
|
||||
type JobAction = 'pause' | 'resume' | 'cancel';
|
||||
|
||||
async function resumeJob(id: string) {
|
||||
actionInProgress = id;
|
||||
try {
|
||||
const response = await fetch(`/api/v1/jobs/${id}/resume`, { method: 'POST' });
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
|
||||
throw new Error(errorData.message || `HTTP ${response.status}`);
|
||||
}
|
||||
// Optimistic update
|
||||
jobs = jobs.map((j) => (j.id === id ? { ...j, status: 'queued' as const } : j));
|
||||
showToast('Job resumed successfully');
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'Failed to resume job';
|
||||
showToast(msg, 'error');
|
||||
console.error('Failed to resume job:', err);
|
||||
} finally {
|
||||
actionInProgress = null;
|
||||
setTimeout(fetchJobs, 500);
|
||||
}
|
||||
}
|
||||
|
||||
async function cancelJob(id: string) {
|
||||
if (!confirm('Are you sure you want to cancel this job?')) {
|
||||
return;
|
||||
}
|
||||
|
||||
actionInProgress = id;
|
||||
try {
|
||||
const response = await fetch(`/api/v1/jobs/${id}/cancel`, { method: 'POST' });
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json().catch(() => ({ message: 'Unknown error' }));
|
||||
throw new Error(errorData.message || `HTTP ${response.status}`);
|
||||
}
|
||||
// Optimistic update
|
||||
jobs = jobs.map((j) => (j.id === id ? { ...j, status: 'cancelled' as const } : j));
|
||||
showToast('Job cancelled successfully');
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'Failed to cancel job';
|
||||
showToast(msg, 'error');
|
||||
console.error('Failed to cancel job:', err);
|
||||
} finally {
|
||||
actionInProgress = null;
|
||||
setTimeout(fetchJobs, 500);
|
||||
}
|
||||
}
|
||||
|
||||
// Simple toast notification (using alert for v1, can be enhanced later)
|
||||
function showToast(message: string, type: 'success' | 'error' = 'success') {
|
||||
// For v1, just use alert. In production, integrate with a toast library.
|
||||
if (type === 'error') {
|
||||
alert(`Error: ${message}`);
|
||||
} else {
|
||||
console.log(`✓ ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-refresh with EventSource streaming + fallback polling
|
||||
$effect(() => {
|
||||
fetchJobs();
|
||||
|
||||
const es = new EventSource('/api/v1/jobs/stream');
|
||||
let fallbackInterval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
es.addEventListener('job-progress', (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
jobs = jobs.map((j) =>
|
||||
j.id === data.jobId
|
||||
? {
|
||||
...j,
|
||||
progress: data.progress,
|
||||
stage: data.stage,
|
||||
stageDetail: data.stageDetail,
|
||||
processedFiles: data.processedFiles,
|
||||
totalFiles: data.totalFiles
|
||||
}
|
||||
: j
|
||||
);
|
||||
});
|
||||
|
||||
es.addEventListener('job-done', () => {
|
||||
void fetchJobs();
|
||||
});
|
||||
|
||||
es.addEventListener('job-failed', () => {
|
||||
void fetchJobs();
|
||||
});
|
||||
|
||||
es.onerror = () => {
|
||||
es.close();
|
||||
// Fall back to polling on error
|
||||
fallbackInterval = setInterval(fetchJobs, 3000);
|
||||
};
|
||||
|
||||
return () => {
|
||||
es.close();
|
||||
if (fallbackInterval) {
|
||||
clearInterval(fallbackInterval);
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
// Format date for display
|
||||
function formatDate(date: Date | null): string {
|
||||
if (!date) return '—';
|
||||
return new Date(date).toLocaleString();
|
||||
}
|
||||
|
||||
// Determine which actions are available for a job
|
||||
function canPause(status: IndexingJobDto['status']): boolean {
|
||||
return status === 'queued' || status === 'running';
|
||||
}
|
||||
|
||||
function canResume(status: IndexingJobDto['status']): boolean {
|
||||
return status === 'paused';
|
||||
}
|
||||
|
||||
function canCancel(status: IndexingJobDto['status']): boolean {
|
||||
return status !== 'done' && status !== 'failed';
|
||||
}
|
||||
|
||||
// Map IndexingStage values to display labels
|
||||
const filterStatuses: FilterStatus[] = ['queued', 'running', 'done', 'failed'];
|
||||
const stageLabels: Record<string, string> = {
|
||||
queued: 'Queued',
|
||||
differential: 'Diff',
|
||||
@@ -187,9 +34,274 @@
|
||||
failed: 'Failed'
|
||||
};
|
||||
|
||||
let jobs = $state<IndexingJobDto[]>([]);
|
||||
let total = $state(0);
|
||||
let loading = $state(true);
|
||||
let refreshing = $state(false);
|
||||
let error = $state<string | null>(null);
|
||||
let repositoryInput = $state('');
|
||||
let selectedStatuses = $state<FilterStatus[]>([]);
|
||||
let appliedRepositoryFilter = $state('');
|
||||
let appliedStatuses = $state<FilterStatus[]>([]);
|
||||
let pendingCancelJobId = $state<string | null>(null);
|
||||
let rowActions = $state<Record<string, JobAction | undefined>>({});
|
||||
let toasts = $state<ToastItem[]>([]);
|
||||
let refreshTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
function buildJobsUrl(): string {
|
||||
const params = new SvelteURLSearchParams({ limit: '50' });
|
||||
|
||||
if (appliedRepositoryFilter) {
|
||||
params.set('repositoryId', appliedRepositoryFilter);
|
||||
}
|
||||
|
||||
if (appliedStatuses.length > 0) {
|
||||
params.set('status', appliedStatuses.join(','));
|
||||
}
|
||||
|
||||
return `/api/v1/jobs?${params.toString()}`;
|
||||
}
|
||||
|
||||
function pushToast(message: string, type: ToastItem['type'] = 'success') {
|
||||
toasts = [...toasts, { id: crypto.randomUUID(), message, type }];
|
||||
}
|
||||
|
||||
function clearRowAction(jobId: string) {
|
||||
const next = { ...rowActions };
|
||||
delete next[jobId];
|
||||
rowActions = next;
|
||||
}
|
||||
|
||||
function setRowAction(jobId: string, action: JobAction) {
|
||||
rowActions = { ...rowActions, [jobId]: action };
|
||||
}
|
||||
|
||||
function scheduleRefresh(delayMs = 500) {
|
||||
if (refreshTimer) {
|
||||
clearTimeout(refreshTimer);
|
||||
}
|
||||
|
||||
refreshTimer = setTimeout(() => {
|
||||
void fetchJobs({ background: true });
|
||||
}, delayMs);
|
||||
}
|
||||
|
||||
function hasAppliedFilters(): boolean {
|
||||
return appliedRepositoryFilter.length > 0 || appliedStatuses.length > 0;
|
||||
}
|
||||
|
||||
function sameStatuses(left: FilterStatus[], right: FilterStatus[]): boolean {
|
||||
return left.length === right.length && left.every((status, index) => status === right[index]);
|
||||
}
|
||||
|
||||
function filtersDirty(): boolean {
|
||||
return repositoryInput.trim() !== appliedRepositoryFilter || !sameStatuses(selectedStatuses, appliedStatuses);
|
||||
}
|
||||
|
||||
function isSpecificRepositoryId(repositoryId: string): boolean {
|
||||
return repositoryId.split('/').filter(Boolean).length >= 2;
|
||||
}
|
||||
|
||||
function matchesAppliedFilters(job: IndexingJobDto): boolean {
|
||||
if (appliedRepositoryFilter) {
|
||||
const repositoryFilter = appliedRepositoryFilter;
|
||||
const repositoryMatches = isSpecificRepositoryId(repositoryFilter)
|
||||
? job.repositoryId === repositoryFilter
|
||||
: job.repositoryId === repositoryFilter || job.repositoryId.startsWith(`${repositoryFilter}/`);
|
||||
|
||||
if (!repositoryMatches) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (appliedStatuses.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return appliedStatuses.includes(job.status as FilterStatus);
|
||||
}
|
||||
|
||||
function syncCancelState(nextJobs: IndexingJobDto[]) {
|
||||
if (!pendingCancelJobId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pendingJob = nextJobs.find((job) => job.id === pendingCancelJobId);
|
||||
if (!pendingJob || !canCancel(pendingJob.status)) {
|
||||
pendingCancelJobId = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchJobs(options: { background?: boolean } = {}) {
|
||||
const background = options.background ?? false;
|
||||
|
||||
if (background) {
|
||||
refreshing = true;
|
||||
} else {
|
||||
loading = true;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(buildJobsUrl());
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const data: JobResponse = await response.json();
|
||||
jobs = data.jobs;
|
||||
total = data.total;
|
||||
error = null;
|
||||
syncCancelState(data.jobs);
|
||||
} catch (err) {
|
||||
error = err instanceof Error ? err.message : 'Failed to fetch jobs';
|
||||
console.error('Failed to fetch jobs:', err);
|
||||
} finally {
|
||||
loading = false;
|
||||
refreshing = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runJobAction(job: IndexingJobDto, action: JobAction) {
|
||||
setRowAction(job.id, action);
|
||||
|
||||
try {
|
||||
const response = await fetch(`/api/v1/jobs/${job.id}/${action}`, { method: 'POST' });
|
||||
const payload = await response.json().catch(() => ({ message: 'Unknown error' }));
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(payload.message || `HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
const updatedJob = payload.job as IndexingJobDto | undefined;
|
||||
if (updatedJob) {
|
||||
if (matchesAppliedFilters(updatedJob)) {
|
||||
jobs = jobs.map((currentJob) =>
|
||||
currentJob.id === updatedJob.id ? updatedJob : currentJob
|
||||
);
|
||||
} else {
|
||||
jobs = jobs.filter((currentJob) => currentJob.id !== updatedJob.id);
|
||||
}
|
||||
}
|
||||
|
||||
pendingCancelJobId = null;
|
||||
pushToast(`Job ${action}d successfully`);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : `Failed to ${action} job`;
|
||||
pushToast(message, 'error');
|
||||
console.error(`Failed to ${action} job:`, err);
|
||||
} finally {
|
||||
clearRowAction(job.id);
|
||||
scheduleRefresh();
|
||||
}
|
||||
}
|
||||
|
||||
function toggleStatusFilter(status: FilterStatus) {
|
||||
selectedStatuses = selectedStatuses.includes(status)
|
||||
? selectedStatuses.filter((candidate) => candidate !== status)
|
||||
: [...selectedStatuses, status].sort(
|
||||
(left, right) => filterStatuses.indexOf(left) - filterStatuses.indexOf(right)
|
||||
);
|
||||
}
|
||||
|
||||
function applyFilters(event?: SubmitEvent) {
|
||||
event?.preventDefault();
|
||||
appliedRepositoryFilter = repositoryInput.trim();
|
||||
appliedStatuses = [...selectedStatuses];
|
||||
pendingCancelJobId = null;
|
||||
void fetchJobs();
|
||||
}
|
||||
|
||||
function resetFilters() {
|
||||
repositoryInput = '';
|
||||
selectedStatuses = [];
|
||||
appliedRepositoryFilter = '';
|
||||
appliedStatuses = [];
|
||||
pendingCancelJobId = null;
|
||||
void fetchJobs();
|
||||
}
|
||||
|
||||
function requestCancel(jobId: string) {
|
||||
pendingCancelJobId = pendingCancelJobId === jobId ? null : jobId;
|
||||
}
|
||||
|
||||
function formatDate(date: Date | string | null): string {
|
||||
if (!date) {
|
||||
return '—';
|
||||
}
|
||||
|
||||
return new Date(date).toLocaleString();
|
||||
}
|
||||
|
||||
function canPause(status: IndexingJobDto['status']): boolean {
|
||||
return status === 'queued' || status === 'running';
|
||||
}
|
||||
|
||||
function canResume(status: IndexingJobDto['status']): boolean {
|
||||
return status === 'paused';
|
||||
}
|
||||
|
||||
function canCancel(status: IndexingJobDto['status']): boolean {
|
||||
return status !== 'done' && status !== 'failed' && status !== 'cancelled';
|
||||
}
|
||||
|
||||
function isRowBusy(jobId: string): boolean {
|
||||
return Boolean(rowActions[jobId]);
|
||||
}
|
||||
|
||||
function getStageLabel(stage: string | undefined): string {
|
||||
return stage ? (stageLabels[stage] ?? stage) : '—';
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
void fetchJobs();
|
||||
|
||||
const es = new EventSource('/api/v1/jobs/stream');
|
||||
let fallbackInterval: ReturnType<typeof setInterval> | null = null;
|
||||
const refreshJobs = () => {
|
||||
void fetchJobs({ background: true });
|
||||
};
|
||||
|
||||
es.addEventListener('job-progress', (event) => {
|
||||
const data = JSON.parse(event.data) as Partial<IndexingJobDto> & { jobId?: string };
|
||||
if (!data.jobId) {
|
||||
return;
|
||||
}
|
||||
|
||||
jobs = jobs.map((job) =>
|
||||
job.id === data.jobId
|
||||
? {
|
||||
...job,
|
||||
progress: data.progress ?? job.progress,
|
||||
stage: data.stage ?? job.stage,
|
||||
stageDetail: data.stageDetail ?? job.stageDetail,
|
||||
processedFiles: data.processedFiles ?? job.processedFiles,
|
||||
totalFiles: data.totalFiles ?? job.totalFiles,
|
||||
status: data.status ?? job.status
|
||||
}
|
||||
: job
|
||||
);
|
||||
});
|
||||
|
||||
es.addEventListener('job-done', refreshJobs);
|
||||
es.addEventListener('job-failed', refreshJobs);
|
||||
|
||||
es.onerror = () => {
|
||||
es.close();
|
||||
if (!fallbackInterval) {
|
||||
fallbackInterval = setInterval(refreshJobs, 3000);
|
||||
}
|
||||
};
|
||||
|
||||
return () => {
|
||||
es.close();
|
||||
if (fallbackInterval) {
|
||||
clearInterval(fallbackInterval);
|
||||
}
|
||||
if (refreshTimer) {
|
||||
clearTimeout(refreshTimer);
|
||||
}
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
@@ -202,23 +314,92 @@
|
||||
<p class="mt-2 text-gray-600">Monitor and control indexing jobs</p>
|
||||
</div>
|
||||
|
||||
{#if loading && jobs.length === 0}
|
||||
<div class="flex items-center justify-center py-12">
|
||||
<div class="text-center">
|
||||
<div
|
||||
class="inline-block h-8 w-8 animate-spin rounded-full border-4 border-solid border-blue-600 border-r-transparent"
|
||||
></div>
|
||||
<p class="mt-2 text-gray-600">Loading jobs...</p>
|
||||
<WorkerStatusPanel />
|
||||
|
||||
<form class="mb-6 rounded-lg border border-gray-200 bg-white p-4 shadow-sm" onsubmit={applyFilters}>
|
||||
<div class="flex flex-col gap-4 lg:flex-row lg:items-end lg:justify-between">
|
||||
<div class="flex-1">
|
||||
<label class="mb-2 block text-sm font-medium text-gray-700" for="repository-filter">
|
||||
Repository filter
|
||||
</label>
|
||||
<input
|
||||
id="repository-filter"
|
||||
type="text"
|
||||
bind:value={repositoryInput}
|
||||
placeholder="/owner or /owner/repo"
|
||||
class="w-full rounded-md border border-gray-300 px-3 py-2 text-sm text-gray-900 shadow-sm focus:border-blue-500 focus:outline-none focus:ring-2 focus:ring-blue-200"
|
||||
/>
|
||||
<p class="mt-2 text-xs text-gray-500">
|
||||
Use an owner prefix like <code>/facebook</code> or a full repository ID like <code>/facebook/react</code>.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="lg:min-w-72">
|
||||
<span class="mb-2 block text-sm font-medium text-gray-700">Statuses</span>
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{#each filterStatuses as status (status)}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => toggleStatusFilter(status)}
|
||||
class="rounded-full border px-3 py-1 text-xs font-semibold uppercase transition {selectedStatuses.includes(status)
|
||||
? 'border-blue-600 bg-blue-50 text-blue-700'
|
||||
: 'border-gray-300 text-gray-600 hover:border-gray-400 hover:text-gray-900'}"
|
||||
>
|
||||
{status}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex gap-2">
|
||||
<button
|
||||
type="submit"
|
||||
disabled={!filtersDirty()}
|
||||
class="rounded bg-blue-600 px-4 py-2 text-sm font-semibold text-white hover:bg-blue-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
Apply filters
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onclick={resetFilters}
|
||||
class="rounded border border-gray-300 px-4 py-2 text-sm font-semibold text-gray-700 hover:border-gray-400 hover:text-gray-900"
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{:else if error && jobs.length === 0}
|
||||
<div class="rounded-md bg-red-50 p-4">
|
||||
<p class="text-sm text-red-800">Error: {error}</p>
|
||||
</form>
|
||||
|
||||
<div class="mb-4 flex flex-col gap-2 text-sm text-gray-600 md:flex-row md:items-center md:justify-between">
|
||||
<p>
|
||||
Showing <span class="font-semibold text-gray-900">{jobs.length}</span> of
|
||||
<span class="font-semibold text-gray-900">{total}</span> jobs
|
||||
</p>
|
||||
{#if hasAppliedFilters()}
|
||||
<p class="text-xs text-gray-500">
|
||||
Active filters:
|
||||
{appliedRepositoryFilter || 'all repositories'}
|
||||
{#if appliedStatuses.length > 0}
|
||||
· {appliedStatuses.join(', ')}
|
||||
{:else}
|
||||
· all statuses
|
||||
{/if}
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if error}
|
||||
<div class="mb-4 rounded-md border border-red-200 bg-red-50 px-4 py-3 text-sm text-red-800">
|
||||
{error}
|
||||
</div>
|
||||
{:else if jobs.length === 0}
|
||||
{/if}
|
||||
|
||||
{#if !loading && jobs.length === 0}
|
||||
<div class="rounded-md bg-gray-50 p-8 text-center">
|
||||
<p class="text-gray-600">
|
||||
No jobs found. Jobs will appear here when repositories are indexed.
|
||||
{hasAppliedFilters()
|
||||
? 'No jobs match the current filters.'
|
||||
: 'No jobs found. Jobs will appear here when repositories are indexed.'}
|
||||
</p>
|
||||
</div>
|
||||
{:else}
|
||||
@@ -259,86 +440,117 @@
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-gray-200 bg-white">
|
||||
{#each jobs as job (job.id)}
|
||||
{#if loading && jobs.length === 0}
|
||||
<JobSkeleton rows={6} />
|
||||
{:else}
|
||||
{#each jobs as job (job.id)}
|
||||
<tr class="hover:bg-gray-50">
|
||||
<td class="px-6 py-4 text-sm font-medium whitespace-nowrap text-gray-900">
|
||||
{job.repositoryId}
|
||||
{#if job.versionId}
|
||||
<span class="ml-1 text-xs text-gray-500">@{job.versionId}</span>
|
||||
{/if}
|
||||
<div class="mt-1 text-xs text-gray-400">{job.id}</div>
|
||||
</td>
|
||||
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
|
||||
<JobStatusBadge status={job.status} />
|
||||
<JobStatusBadge status={job.status} spinning={job.status === 'running'} />
|
||||
</td>
|
||||
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
|
||||
<div class="flex items-center gap-2">
|
||||
<span>{getStageLabel(job.stage)}</span>
|
||||
{#if job.stageDetail}
|
||||
<span class="text-xs text-gray-400">{job.stageDetail}</span>
|
||||
{/if}
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
|
||||
<div class="flex items-center">
|
||||
<span class="mr-2">{job.progress}%</span>
|
||||
<div class="h-2 w-32 rounded-full bg-gray-200">
|
||||
<div
|
||||
class="h-2 rounded-full bg-blue-600 transition-all"
|
||||
style="width: {job.progress}%"
|
||||
></div>
|
||||
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
|
||||
<div class="flex items-center gap-2">
|
||||
<span>{getStageLabel(job.stage)}</span>
|
||||
{#if job.stageDetail}
|
||||
<span class="text-xs text-gray-400">{job.stageDetail}</span>
|
||||
{/if}
|
||||
</div>
|
||||
{#if job.totalFiles > 0}
|
||||
<span class="ml-2 text-xs text-gray-400">
|
||||
{job.processedFiles}/{job.totalFiles} files
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
</td>
|
||||
</td>
|
||||
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
|
||||
<div class="space-y-2">
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="w-12 text-right text-xs font-semibold text-gray-600">{job.progress}%</span>
|
||||
<div class="h-2 w-32 rounded-full bg-gray-200">
|
||||
<div
|
||||
class="h-2 rounded-full bg-blue-600 transition-all"
|
||||
style="width: {job.progress}%"
|
||||
></div>
|
||||
</div>
|
||||
</div>
|
||||
{#if job.totalFiles > 0}
|
||||
<div class="text-xs text-gray-400">
|
||||
{job.processedFiles}/{job.totalFiles} files processed
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</td>
|
||||
<td class="px-6 py-4 text-sm whitespace-nowrap text-gray-500">
|
||||
{formatDate(job.createdAt)}
|
||||
</td>
|
||||
<td class="px-6 py-4 text-right text-sm font-medium whitespace-nowrap">
|
||||
<div class="flex justify-end gap-2">
|
||||
{#if canPause(job.status)}
|
||||
{#if pendingCancelJobId === job.id}
|
||||
<button
|
||||
onclick={() => pauseJob(job.id)}
|
||||
disabled={actionInProgress === job.id}
|
||||
class="rounded bg-yellow-600 px-3 py-1 text-xs font-semibold text-white hover:bg-yellow-700 disabled:opacity-50"
|
||||
type="button"
|
||||
onclick={() => void runJobAction(job, 'cancel')}
|
||||
disabled={isRowBusy(job.id)}
|
||||
class="rounded bg-red-600 px-3 py-1 text-xs font-semibold text-white hover:bg-red-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
Pause
|
||||
{rowActions[job.id] === 'cancel' ? 'Cancelling...' : 'Confirm cancel'}
|
||||
</button>
|
||||
{/if}
|
||||
{#if canResume(job.status)}
|
||||
<button
|
||||
onclick={() => resumeJob(job.id)}
|
||||
disabled={actionInProgress === job.id}
|
||||
class="rounded bg-green-600 px-3 py-1 text-xs font-semibold text-white hover:bg-green-700 disabled:opacity-50"
|
||||
type="button"
|
||||
onclick={() => requestCancel(job.id)}
|
||||
disabled={isRowBusy(job.id)}
|
||||
class="rounded border border-gray-300 px-3 py-1 text-xs font-semibold text-gray-700 hover:border-gray-400 hover:text-gray-900 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
Resume
|
||||
Keep job
|
||||
</button>
|
||||
{/if}
|
||||
{#if canCancel(job.status)}
|
||||
<button
|
||||
onclick={() => cancelJob(job.id)}
|
||||
disabled={actionInProgress === job.id}
|
||||
class="rounded bg-red-600 px-3 py-1 text-xs font-semibold text-white hover:bg-red-700 disabled:opacity-50"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
{/if}
|
||||
{#if !canPause(job.status) && !canResume(job.status) && !canCancel(job.status)}
|
||||
<span class="text-xs text-gray-400">—</span>
|
||||
{:else}
|
||||
{#if canPause(job.status)}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => void runJobAction(job, 'pause')}
|
||||
disabled={isRowBusy(job.id)}
|
||||
class="rounded bg-yellow-600 px-3 py-1 text-xs font-semibold text-white hover:bg-yellow-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{rowActions[job.id] === 'pause' ? 'Pausing...' : 'Pause'}
|
||||
</button>
|
||||
{/if}
|
||||
{#if canResume(job.status)}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => void runJobAction(job, 'resume')}
|
||||
disabled={isRowBusy(job.id)}
|
||||
class="rounded bg-green-600 px-3 py-1 text-xs font-semibold text-white hover:bg-green-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{rowActions[job.id] === 'resume' ? 'Resuming...' : 'Resume'}
|
||||
</button>
|
||||
{/if}
|
||||
{#if canCancel(job.status)}
|
||||
<button
|
||||
type="button"
|
||||
onclick={() => requestCancel(job.id)}
|
||||
disabled={isRowBusy(job.id)}
|
||||
class="rounded bg-red-600 px-3 py-1 text-xs font-semibold text-white hover:bg-red-700 disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
{/if}
|
||||
{#if !canPause(job.status) && !canResume(job.status) && !canCancel(job.status)}
|
||||
<span class="text-xs text-gray-400">—</span>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{/each}
|
||||
{/each}
|
||||
{/if}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{#if loading}
|
||||
{#if refreshing}
|
||||
<div class="mt-4 text-center text-sm text-gray-500">Refreshing...</div>
|
||||
{/if}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<Toast bind:toasts={toasts} />
|
||||
|
||||
@@ -15,15 +15,39 @@ import { JobQueue } from '$lib/server/pipeline/job-queue.js';
|
||||
import { handleServiceError } from '$lib/server/utils/validation.js';
|
||||
import type { IndexingJob } from '$lib/types';
|
||||
|
||||
const VALID_JOB_STATUSES: ReadonlySet<IndexingJob['status']> = new Set([
|
||||
'queued',
|
||||
'running',
|
||||
'done',
|
||||
'failed'
|
||||
]);
|
||||
|
||||
function parseStatusFilter(searchValue: string | null): IndexingJob['status'] | Array<IndexingJob['status']> | undefined {
|
||||
if (!searchValue) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const statuses = [...new Set(
|
||||
searchValue
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter((value): value is IndexingJob['status'] => VALID_JOB_STATUSES.has(value as IndexingJob['status']))
|
||||
)];
|
||||
|
||||
if (statuses.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return statuses.length === 1 ? statuses[0] : statuses;
|
||||
}
|
||||
|
||||
export const GET: RequestHandler = ({ url }) => {
|
||||
try {
|
||||
const db = getClient();
|
||||
const queue = new JobQueue(db);
|
||||
|
||||
const repositoryId = url.searchParams.get('repositoryId') ?? undefined;
|
||||
const status = (url.searchParams.get('status') ?? undefined) as
|
||||
| IndexingJob['status']
|
||||
| undefined;
|
||||
const repositoryId = url.searchParams.get('repositoryId')?.trim() || undefined;
|
||||
const status = parseStatusFilter(url.searchParams.get('status'));
|
||||
const limit = Math.min(parseInt(url.searchParams.get('limit') ?? '20', 10) || 20, 1000);
|
||||
|
||||
const jobs = queue.listJobs({ repositoryId, status, limit });
|
||||
|
||||
@@ -44,7 +44,7 @@ export const GET: RequestHandler = ({ params, request }) => {
|
||||
status: job.status,
|
||||
error: job.error
|
||||
};
|
||||
controller.enqueue(`data: ${JSON.stringify(initialData)}\n\n`);
|
||||
controller.enqueue(`event: job-progress\ndata: ${JSON.stringify(initialData)}\n\n`);
|
||||
|
||||
// Check for Last-Event-ID header for reconnect
|
||||
const lastEventId = request.headers.get('Last-Event-ID');
|
||||
@@ -57,6 +57,13 @@ export const GET: RequestHandler = ({ params, request }) => {
|
||||
|
||||
// Check if job is already done or failed - close immediately after first event
|
||||
if (job.status === 'done' || job.status === 'failed') {
|
||||
if (job.status === 'done') {
|
||||
controller.enqueue(`event: job-done\ndata: ${JSON.stringify({ jobId })}\n\n`);
|
||||
} else {
|
||||
controller.enqueue(
|
||||
`event: job-failed\ndata: ${JSON.stringify({ jobId, error: job.error })}\n\n`
|
||||
);
|
||||
}
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
@@ -73,18 +80,29 @@ export const GET: RequestHandler = ({ params, request }) => {
|
||||
controller.enqueue(value);
|
||||
|
||||
// Check if the incoming event indicates job completion
|
||||
if (value.includes('event: done') || value.includes('event: failed')) {
|
||||
if (
|
||||
value.includes('event: job-done') ||
|
||||
value.includes('event: job-failed')
|
||||
) {
|
||||
controller.close();
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
controller.close();
|
||||
try {
|
||||
controller.close();
|
||||
} catch {
|
||||
// Stream may already be closed after a terminal event.
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('SSE stream error:', err);
|
||||
controller.close();
|
||||
try {
|
||||
controller.close();
|
||||
} catch {
|
||||
// Stream may already be closed.
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@ import type { ProgressBroadcaster as BroadcasterType } from '$lib/server/pipelin
|
||||
let db: Database.Database;
|
||||
// Closed over by the vi.mock factory below.
|
||||
let mockBroadcaster: BroadcasterType | null = null;
|
||||
let mockPool: { getStatus: () => object; setMaxConcurrency?: (value: number) => void } | null = null;
|
||||
|
||||
vi.mock('$lib/server/db/client', () => ({
|
||||
getClient: () => db
|
||||
@@ -29,12 +30,12 @@ vi.mock('$lib/server/db/client.js', () => ({
|
||||
|
||||
vi.mock('$lib/server/pipeline/startup', () => ({
|
||||
getQueue: () => null,
|
||||
getPool: () => null
|
||||
getPool: () => mockPool
|
||||
}));
|
||||
|
||||
vi.mock('$lib/server/pipeline/startup.js', () => ({
|
||||
getQueue: () => null,
|
||||
getPool: () => null
|
||||
getPool: () => mockPool
|
||||
}));
|
||||
|
||||
vi.mock('$lib/server/pipeline/progress-broadcaster', async (importOriginal) => {
|
||||
@@ -58,9 +59,11 @@ vi.mock('$lib/server/pipeline/progress-broadcaster.js', async (importOriginal) =
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
import { ProgressBroadcaster } from '$lib/server/pipeline/progress-broadcaster.js';
|
||||
import { GET as getJobsList } from './jobs/+server.js';
|
||||
import { GET as getJobStream } from './jobs/[id]/stream/+server.js';
|
||||
import { GET as getJobsStream } from './jobs/stream/+server.js';
|
||||
import { GET as getIndexingSettings, PUT as putIndexingSettings } from './settings/indexing/+server.js';
|
||||
import { GET as getWorkers } from './workers/+server.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DB factory
|
||||
@@ -306,6 +309,25 @@ describe('GET /api/v1/jobs/:id/stream', () => {
|
||||
// The replay event should include the cached event data
|
||||
expect(fullText).toContain('progress');
|
||||
});
|
||||
|
||||
it('closes after receiving the broadcaster job-done event', async () => {
|
||||
seedRepo(db);
|
||||
const jobId = seedJob(db, { status: 'running', stage: 'parsing', progress: 10 });
|
||||
|
||||
const response = await getJobStream(makeEvent({ params: { id: jobId } }));
|
||||
const reader = response.body!.getReader();
|
||||
|
||||
const initialChunk = await reader.read();
|
||||
expect(String(initialChunk.value ?? '')).toContain('event: job-progress');
|
||||
|
||||
mockBroadcaster!.broadcast(jobId, '/test/repo', 'job-done', { jobId, status: 'done' });
|
||||
|
||||
const completionChunk = await reader.read();
|
||||
expect(String(completionChunk.value ?? '')).toContain('event: job-done');
|
||||
|
||||
const closed = await reader.read();
|
||||
expect(closed.done).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -377,12 +399,125 @@ describe('GET /api/v1/jobs/stream', () => {
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test group 3: GET /api/v1/settings/indexing
|
||||
// Test group 3: GET /api/v1/jobs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('GET /api/v1/jobs', () => {
|
||||
beforeEach(() => {
|
||||
db = createTestDb();
|
||||
});
|
||||
|
||||
it('supports repository prefix and comma-separated status filters', async () => {
|
||||
seedRepo(db, '/facebook/react');
|
||||
seedRepo(db, '/facebook/react-native');
|
||||
seedRepo(db, '/vitejs/vite');
|
||||
|
||||
seedJob(db, { repository_id: '/facebook/react', status: 'queued' });
|
||||
seedJob(db, { repository_id: '/facebook/react-native', status: 'running' });
|
||||
seedJob(db, { repository_id: '/facebook/react', status: 'done' });
|
||||
seedJob(db, { repository_id: '/vitejs/vite', status: 'queued' });
|
||||
|
||||
const response = await getJobsList(
|
||||
makeEvent<Parameters<typeof getJobsList>[0]>({
|
||||
url: 'http://localhost/api/v1/jobs?repositoryId=%2Ffacebook&status=queued,%20running'
|
||||
})
|
||||
);
|
||||
const body = await response.json();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body.total).toBe(2);
|
||||
expect(body.jobs).toHaveLength(2);
|
||||
expect(body.jobs.map((job: { repositoryId: string }) => job.repositoryId).sort()).toEqual([
|
||||
'/facebook/react',
|
||||
'/facebook/react-native'
|
||||
]);
|
||||
expect(body.jobs.map((job: { status: string }) => job.status).sort()).toEqual([
|
||||
'queued',
|
||||
'running'
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps exact-match behavior for specific repository IDs', async () => {
|
||||
seedRepo(db, '/facebook/react');
|
||||
seedRepo(db, '/facebook/react-native');
|
||||
|
||||
seedJob(db, { repository_id: '/facebook/react', status: 'queued' });
|
||||
seedJob(db, { repository_id: '/facebook/react-native', status: 'queued' });
|
||||
|
||||
const response = await getJobsList(
|
||||
makeEvent<Parameters<typeof getJobsList>[0]>({
|
||||
url: 'http://localhost/api/v1/jobs?repositoryId=%2Ffacebook%2Freact&status=queued'
|
||||
})
|
||||
);
|
||||
const body = await response.json();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body.total).toBe(1);
|
||||
expect(body.jobs).toHaveLength(1);
|
||||
expect(body.jobs[0].repositoryId).toBe('/facebook/react');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test group 4: GET /api/v1/workers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('GET /api/v1/workers', () => {
|
||||
beforeEach(() => {
|
||||
mockPool = null;
|
||||
});
|
||||
|
||||
it('returns 503 when the worker pool is not initialized', async () => {
|
||||
const response = await getWorkers(makeEvent<Parameters<typeof getWorkers>[0]>({}));
|
||||
|
||||
expect(response.status).toBe(503);
|
||||
});
|
||||
|
||||
it('returns the current worker status snapshot', async () => {
|
||||
mockPool = {
|
||||
getStatus: () => ({
|
||||
concurrency: 2,
|
||||
active: 1,
|
||||
idle: 1,
|
||||
workers: [
|
||||
{
|
||||
index: 0,
|
||||
state: 'running',
|
||||
jobId: 'job-1',
|
||||
repositoryId: '/test/repo',
|
||||
versionId: null
|
||||
},
|
||||
{
|
||||
index: 1,
|
||||
state: 'idle',
|
||||
jobId: null,
|
||||
repositoryId: null,
|
||||
versionId: null
|
||||
}
|
||||
]
|
||||
})
|
||||
};
|
||||
|
||||
const response = await getWorkers(makeEvent<Parameters<typeof getWorkers>[0]>({}));
|
||||
const body = await response.json();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(body.active).toBe(1);
|
||||
expect(body.workers[0].jobId).toBe('job-1');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test group 5: GET /api/v1/settings/indexing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('GET /api/v1/settings/indexing', () => {
|
||||
beforeEach(() => {
|
||||
db = createTestDb();
|
||||
mockPool = {
|
||||
getStatus: () => ({ concurrency: 2, active: 0, idle: 2, workers: [] }),
|
||||
setMaxConcurrency: vi.fn()
|
||||
};
|
||||
});
|
||||
|
||||
it('returns { concurrency: 2 } when no setting exists in DB', async () => {
|
||||
@@ -417,12 +552,16 @@ describe('GET /api/v1/settings/indexing', () => {
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test group 4: PUT /api/v1/settings/indexing
|
||||
// Test group 6: PUT /api/v1/settings/indexing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('PUT /api/v1/settings/indexing', () => {
|
||||
beforeEach(() => {
|
||||
db = createTestDb();
|
||||
mockPool = {
|
||||
getStatus: () => ({ concurrency: 2, active: 0, idle: 2, workers: [] }),
|
||||
setMaxConcurrency: vi.fn()
|
||||
};
|
||||
});
|
||||
|
||||
function makePutEvent(body: unknown) {
|
||||
|
||||
16
src/routes/api/v1/workers/+server.ts
Normal file
16
src/routes/api/v1/workers/+server.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getPool } from '$lib/server/pipeline/startup.js';
|
||||
import { handleServiceError } from '$lib/server/utils/validation.js';
|
||||
|
||||
export const GET: RequestHandler = () => {
|
||||
try {
|
||||
const pool = getPool();
|
||||
if (!pool) {
|
||||
return new Response('Service unavailable', { status: 503 });
|
||||
}
|
||||
|
||||
return Response.json(pool.getStatus());
|
||||
} catch (error) {
|
||||
return handleServiceError(error);
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user