feat(TRUEREF-0009): implement indexing pipeline and job queue
Implements the end-to-end indexing pipeline with a SQLite-backed job queue, startup recovery, and REST API endpoints for job status. - IndexingPipeline: orchestrates crawl → parse → atomic replace → embed → repo stats update with progress tracking at each stage - JobQueue: sequential SQLite-backed queue (no external broker), deduplicates active jobs per repository, drains queued jobs on startup - startup.ts: stale job recovery (running→failed), repo state reset, singleton initialization wired from hooks.server.ts - GET /api/v1/jobs with repositoryId/status/limit filtering - GET /api/v1/jobs/[id] single job lookup - hooks.server.ts: initializes DB and pipeline on server start - 18 unit tests covering queue, pipeline stages, recovery, and atomicity Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
46
src/routes/api/v1/jobs/+server.ts
Normal file
46
src/routes/api/v1/jobs/+server.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* GET /api/v1/jobs — list recent indexing jobs.
|
||||
*
|
||||
* Query parameters:
|
||||
* repositoryId (optional) — filter by repository
|
||||
* status (optional) — filter by status: queued|running|done|failed
|
||||
* limit (optional, default 20, max 200)
|
||||
*/
|
||||
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getClient } from '$lib/server/db/client.js';
|
||||
import { JobQueue } from '$lib/server/pipeline/job-queue.js';
|
||||
import { handleServiceError } from '$lib/server/utils/validation.js';
|
||||
import type { IndexingJob } from '$lib/types';
|
||||
|
||||
export const GET: RequestHandler = ({ url }) => {
|
||||
try {
|
||||
const db = getClient();
|
||||
const queue = new JobQueue(db);
|
||||
|
||||
const repositoryId = url.searchParams.get('repositoryId') ?? undefined;
|
||||
const status = (url.searchParams.get('status') ?? undefined) as
|
||||
| IndexingJob['status']
|
||||
| undefined;
|
||||
const limit = Math.min(parseInt(url.searchParams.get('limit') ?? '20', 10) || 20, 200);
|
||||
|
||||
const jobs = queue.listJobs({ repositoryId, status, limit });
|
||||
const total = queue.countJobs({ repositoryId, status });
|
||||
|
||||
return json({ jobs, total });
|
||||
} catch (err) {
|
||||
return handleServiceError(err);
|
||||
}
|
||||
};
|
||||
|
||||
export const OPTIONS: RequestHandler = () => {
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, Authorization'
|
||||
}
|
||||
});
|
||||
};
|
||||
34
src/routes/api/v1/jobs/[id]/+server.ts
Normal file
34
src/routes/api/v1/jobs/[id]/+server.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* GET /api/v1/jobs/:id — retrieve a single indexing job by ID.
|
||||
*/
|
||||
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getClient } from '$lib/server/db/client.js';
|
||||
import { JobQueue } from '$lib/server/pipeline/job-queue.js';
|
||||
import { handleServiceError, NotFoundError } from '$lib/server/utils/validation.js';
|
||||
|
||||
export const GET: RequestHandler = ({ params }) => {
|
||||
try {
|
||||
const db = getClient();
|
||||
const queue = new JobQueue(db);
|
||||
|
||||
const job = queue.getJob(params.id);
|
||||
if (!job) throw new NotFoundError(`Job ${params.id} not found`);
|
||||
|
||||
return json({ job });
|
||||
} catch (err) {
|
||||
return handleServiceError(err);
|
||||
}
|
||||
};
|
||||
|
||||
export const OPTIONS: RequestHandler = () => {
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, Authorization'
|
||||
}
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user