feat(TRUEREF-0009): implement indexing pipeline and job queue

Implements the end-to-end indexing pipeline with a SQLite-backed job
queue, startup recovery, and REST API endpoints for job status.

- IndexingPipeline: orchestrates crawl → parse → atomic replace → embed
  → repo stats update with progress tracking at each stage
- JobQueue: sequential SQLite-backed queue (no external broker), deduplicates
  active jobs per repository, drains queued jobs on startup
- startup.ts: stale job recovery (running→failed), repo state reset, singleton
  initialization wired from hooks.server.ts
- GET /api/v1/jobs with repositoryId/status/limit filtering
- GET /api/v1/jobs/[id] single job lookup
- hooks.server.ts: initializes DB and pipeline on server start
- 18 unit tests covering queue, pipeline stages, recovery, and atomicity

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Giancarmine Salucci
2026-03-22 18:22:20 +01:00
parent bf4caf5e3b
commit 956b2a3a62
7 changed files with 1342 additions and 0 deletions

View File

@@ -0,0 +1,46 @@
/**
* GET /api/v1/jobs — list recent indexing jobs.
*
* Query parameters:
* repositoryId (optional) — filter by repository
* status (optional) — filter by status: queued|running|done|failed
* limit (optional, default 20, max 200)
*/
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { getClient } from '$lib/server/db/client.js';
import { JobQueue } from '$lib/server/pipeline/job-queue.js';
import { handleServiceError } from '$lib/server/utils/validation.js';
import type { IndexingJob } from '$lib/types';
export const GET: RequestHandler = ({ url }) => {
try {
const db = getClient();
const queue = new JobQueue(db);
const repositoryId = url.searchParams.get('repositoryId') ?? undefined;
const status = (url.searchParams.get('status') ?? undefined) as
| IndexingJob['status']
| undefined;
const limit = Math.min(parseInt(url.searchParams.get('limit') ?? '20', 10) || 20, 200);
const jobs = queue.listJobs({ repositoryId, status, limit });
const total = queue.countJobs({ repositoryId, status });
return json({ jobs, total });
} catch (err) {
return handleServiceError(err);
}
};
export const OPTIONS: RequestHandler = () => {
return new Response(null, {
status: 204,
headers: {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET, OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type, Authorization'
}
});
};