refactor: introduce domain model classes and mapper layer
Replace ad-hoc inline row casting (snake_case → camelCase) spread across services, routes, and the indexing pipeline with explicit model classes (Repository, IndexingJob, RepositoryVersion, Snippet, SearchResult) and dedicated mapper classes that own the DB → domain conversion. - Add src/lib/server/models/ with typed model classes for all domain entities - Add src/lib/server/mappers/ with mapper classes per entity - Remove duplicated RawRow interfaces and inline map functions from job-queue, repository.service, indexing.pipeline, and all API routes - Add dtoJsonResponse helper to standardise JSON responses via SvelteKit json() - Add api-contract.integration.test.ts as a regression baseline Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -458,6 +458,31 @@ describe('IndexingPipeline', () => {
|
||||
expect(updated.progress).toBe(100);
|
||||
});
|
||||
|
||||
it('uses the repository source_url when crawling local repositories', async () => {
|
||||
const crawl = vi.fn().mockResolvedValue({
|
||||
files: [],
|
||||
totalFiles: 0,
|
||||
skippedFiles: 0,
|
||||
branch: 'local',
|
||||
commitSha: 'abc'
|
||||
});
|
||||
|
||||
const pipeline = new IndexingPipeline(
|
||||
db,
|
||||
vi.fn() as never,
|
||||
{ crawl } as never,
|
||||
null
|
||||
);
|
||||
|
||||
const job = makeJob();
|
||||
await pipeline.run(job as never);
|
||||
|
||||
expect(crawl).toHaveBeenCalledWith({
|
||||
rootPath: '/tmp/test-repo',
|
||||
ref: undefined
|
||||
});
|
||||
});
|
||||
|
||||
it('integration: handles unchanged, modified, added, and deleted files in one run', async () => {
|
||||
// ---- First run: index three files -----------------------------------
|
||||
const firstFiles = [
|
||||
|
||||
@@ -15,10 +15,13 @@
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { Document, IndexingJob, NewDocument, NewSnippet, Repository } from '$lib/types';
|
||||
import type { Document, NewDocument, NewSnippet } from '$lib/types';
|
||||
import type { crawl as GithubCrawlFn } from '$lib/server/crawler/github.crawler.js';
|
||||
import type { LocalCrawler } from '$lib/server/crawler/local.crawler.js';
|
||||
import type { EmbeddingService } from '$lib/server/embeddings/embedding.service.js';
|
||||
import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js';
|
||||
import { IndexingJob } from '$lib/server/models/indexing-job.js';
|
||||
import { Repository, RepositoryEntity } from '$lib/server/models/repository.js';
|
||||
import { parseFile } from '$lib/server/parser/index.js';
|
||||
import { computeTrustScore } from '$lib/server/search/trust-score.js';
|
||||
import { computeDiff } from './diff.js';
|
||||
@@ -399,11 +402,10 @@ export class IndexingPipeline {
|
||||
}
|
||||
|
||||
private getRepository(id: string): Repository | null {
|
||||
return (
|
||||
(this.db
|
||||
.prepare<[string], Repository>(`SELECT * FROM repositories WHERE id = ?`)
|
||||
.get(id) as Repository | undefined) ?? null
|
||||
);
|
||||
const raw = this.db
|
||||
.prepare<[string], RepositoryEntity>(`SELECT * FROM repositories WHERE id = ?`)
|
||||
.get(id);
|
||||
return raw ? RepositoryMapper.fromEntity(new RepositoryEntity(raw)) : null;
|
||||
}
|
||||
|
||||
private updateJob(id: string, fields: Record<string, unknown>): void {
|
||||
|
||||
@@ -7,48 +7,15 @@
|
||||
*/
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { IndexingJob, NewIndexingJob } from '$lib/types';
|
||||
import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js';
|
||||
import { IndexingJob, IndexingJobEntity } from '$lib/server/models/indexing-job.js';
|
||||
import type { IndexingPipeline } from './indexing.pipeline.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SQL projection + row mapper (mirrors repository.service.ts pattern)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const JOB_SELECT = `
|
||||
SELECT id,
|
||||
repository_id AS repositoryId,
|
||||
version_id AS versionId,
|
||||
status, progress,
|
||||
total_files AS totalFiles,
|
||||
processed_files AS processedFiles,
|
||||
error,
|
||||
started_at AS startedAt,
|
||||
completed_at AS completedAt,
|
||||
created_at AS createdAt
|
||||
FROM indexing_jobs`;
|
||||
|
||||
interface RawJob {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
totalFiles: number;
|
||||
processedFiles: number;
|
||||
error: string | null;
|
||||
startedAt: number | null;
|
||||
completedAt: number | null;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
function mapJob(raw: RawJob): IndexingJob {
|
||||
return {
|
||||
...raw,
|
||||
startedAt: raw.startedAt != null ? new Date(raw.startedAt * 1000) : null,
|
||||
completedAt: raw.completedAt != null ? new Date(raw.completedAt * 1000) : null,
|
||||
createdAt: new Date(raw.createdAt * 1000)
|
||||
};
|
||||
}
|
||||
const JOB_SELECT = `SELECT * FROM indexing_jobs`;
|
||||
|
||||
export class JobQueue {
|
||||
private isRunning = false;
|
||||
@@ -71,7 +38,7 @@ export class JobQueue {
|
||||
enqueue(repositoryId: string, versionId?: string): IndexingJob {
|
||||
// Return early if there's already an active job for this repo.
|
||||
const activeRaw = this.db
|
||||
.prepare<[string], RawJob>(
|
||||
.prepare<[string], IndexingJobEntity>(
|
||||
`${JOB_SELECT}
|
||||
WHERE repository_id = ? AND status IN ('queued', 'running')
|
||||
ORDER BY created_at DESC LIMIT 1`
|
||||
@@ -81,11 +48,11 @@ export class JobQueue {
|
||||
if (activeRaw) {
|
||||
// Ensure the queue is draining even if enqueue was called concurrently.
|
||||
if (!this.isRunning) setImmediate(() => this.processNext());
|
||||
return mapJob(activeRaw);
|
||||
return IndexingJobMapper.fromEntity(new IndexingJobEntity(activeRaw));
|
||||
}
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const job: NewIndexingJob = {
|
||||
const job = new IndexingJob({
|
||||
id: crypto.randomUUID(),
|
||||
repositoryId,
|
||||
versionId: versionId ?? null,
|
||||
@@ -97,7 +64,7 @@ export class JobQueue {
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
createdAt: new Date(now * 1000)
|
||||
};
|
||||
});
|
||||
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -125,9 +92,10 @@ export class JobQueue {
|
||||
setImmediate(() => this.processNext());
|
||||
}
|
||||
|
||||
return mapJob(
|
||||
this.db.prepare<[string], RawJob>(`${JOB_SELECT} WHERE id = ?`).get(job.id as string)!
|
||||
);
|
||||
const created = this.db
|
||||
.prepare<[string], IndexingJobEntity>(`${JOB_SELECT} WHERE id = ?`)
|
||||
.get(job.id as string)!;
|
||||
return IndexingJobMapper.fromEntity(new IndexingJobEntity(created));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -142,7 +110,7 @@ export class JobQueue {
|
||||
}
|
||||
|
||||
const rawJob = this.db
|
||||
.prepare<[], RawJob>(
|
||||
.prepare<[], IndexingJobEntity>(
|
||||
`${JOB_SELECT}
|
||||
WHERE status = 'queued'
|
||||
ORDER BY created_at ASC LIMIT 1`
|
||||
@@ -151,7 +119,7 @@ export class JobQueue {
|
||||
|
||||
if (!rawJob) return;
|
||||
|
||||
const job = mapJob(rawJob);
|
||||
const job = IndexingJobMapper.fromEntity(new IndexingJobEntity(rawJob));
|
||||
this.isRunning = true;
|
||||
try {
|
||||
await this.pipeline.run(job);
|
||||
@@ -180,9 +148,9 @@ export class JobQueue {
|
||||
*/
|
||||
getJob(id: string): IndexingJob | null {
|
||||
const raw = this.db
|
||||
.prepare<[string], RawJob>(`${JOB_SELECT} WHERE id = ?`)
|
||||
.prepare<[string], IndexingJobEntity>(`${JOB_SELECT} WHERE id = ?`)
|
||||
.get(id);
|
||||
return raw ? mapJob(raw) : null;
|
||||
return raw ? IndexingJobMapper.fromEntity(new IndexingJobEntity(raw)) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -210,7 +178,9 @@ export class JobQueue {
|
||||
const sql = `${JOB_SELECT} ${where} ORDER BY created_at DESC LIMIT ?`;
|
||||
params.push(limit);
|
||||
|
||||
return (this.db.prepare<unknown[], RawJob>(sql).all(...params) as RawJob[]).map(mapJob);
|
||||
return (this.db.prepare<unknown[], IndexingJobEntity>(sql).all(...params) as IndexingJobEntity[]).map(
|
||||
(row) => IndexingJobMapper.fromEntity(new IndexingJobEntity(row))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user