Files
trueref/src/lib/server/services/repository.service.test.ts
Giancarmine Salucci 7630740403 feat(TRUEREF-0022): complete iteration 0 — worker-thread indexing, parallel jobs, SSE progress
- Move IndexingPipeline.run() into Worker Threads via WorkerPool
- Add dedicated embedding worker thread with single model instance
- Add stage/stageDetail columns to indexing_jobs schema
- Create ProgressBroadcaster for SSE channel management
- Add SSE endpoints: GET /api/v1/jobs/:id/stream, GET /api/v1/jobs/stream
- Replace UI polling with EventSource on repo detail and admin pages
- Add concurrency settings UI and API endpoint
- Build worker entries separately via esbuild
2026-03-30 17:08:23 +02:00

555 lines
18 KiB
TypeScript

/**
* Unit tests for RepositoryService.
*
* The service uses raw better-sqlite3 queries, so returned rows have
* snake_case column names matching the SQLite schema. Tests assert against
* those actual runtime keys.
*/
import { describe, it, expect, beforeEach } from 'vitest';
import Database from 'better-sqlite3';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { RepositoryService } from './repository.service';
import {
AlreadyExistsError,
InvalidInputError,
InvalidUrlError,
NotFoundError
} from '$lib/server/utils/validation';
// ---------------------------------------------------------------------------
// Test DB factory
// ---------------------------------------------------------------------------
function createTestDb(): Database.Database {
const client = new Database(':memory:');
client.pragma('foreign_keys = ON');
const migrationsFolder = join(import.meta.dirname, '../db/migrations');
for (const migration of [
'0000_large_master_chief.sql',
'0001_quick_nighthawk.sql',
'0002_silky_stellaris.sql',
'0003_multiversion_config.sql',
'0004_complete_sentry.sql'
]) {
const statements = readFileSync(join(migrationsFolder, migration), 'utf-8')
.split('--> statement-breakpoint')
.map((statement) => statement.trim())
.filter(Boolean);
for (const statement of statements) {
client.exec(statement);
}
}
return client;
}
// Raw row shape returned by better-sqlite3 SELECT * FROM repositories.
interface RawRepo {
id: string;
title: string;
description: string | null;
source: string;
source_url: string;
branch: string | null;
state: string;
total_snippets: number;
total_tokens: number;
trust_score: number;
benchmark_score: number;
stars: number | null;
github_token: string | null;
last_indexed_at: number | null;
created_at: number;
updated_at: number;
}
function makeService(client: Database.Database): RepositoryService {
return new RepositoryService(client);
}
// ---------------------------------------------------------------------------
// list() and count()
// ---------------------------------------------------------------------------
describe('RepositoryService.list()', () => {
let service: RepositoryService;
beforeEach(() => {
service = makeService(createTestDb());
});
it('returns empty array when no repositories exist', () => {
const result = service.list();
expect(result).toEqual([]);
});
it('returns all repositories after adding some', () => {
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/a' });
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/b' });
const result = service.list();
expect(result).toHaveLength(2);
});
it('filters by state', () => {
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/a' });
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/b' });
// All repos start as 'pending'.
const pending = service.list({ state: 'pending' });
expect(pending).toHaveLength(2);
const indexed = service.list({ state: 'indexed' });
expect(indexed).toHaveLength(0);
});
it('respects limit and offset', () => {
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/a' });
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/b' });
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/c' });
const page1 = service.list({ limit: 2, offset: 0 });
const page2 = service.list({ limit: 2, offset: 2 });
expect(page1).toHaveLength(2);
expect(page2).toHaveLength(1);
});
});
describe('RepositoryService.count()', () => {
let service: RepositoryService;
beforeEach(() => {
service = makeService(createTestDb());
});
it('returns 0 when empty', () => {
expect(service.count()).toBe(0);
});
it('returns correct count after adding repositories', () => {
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/a' });
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/b' });
expect(service.count()).toBe(2);
});
it('filters count by state', () => {
service.add({ source: 'github', sourceUrl: 'https://github.com/foo/a' });
expect(service.count('pending')).toBe(1);
expect(service.count('indexed')).toBe(0);
});
});
// ---------------------------------------------------------------------------
// get()
// ---------------------------------------------------------------------------
describe('RepositoryService.get()', () => {
let service: RepositoryService;
beforeEach(() => {
service = makeService(createTestDb());
});
it('returns null for a non-existent repository', () => {
expect(service.get('/not/found')).toBeNull();
});
it('returns the repository when it exists', () => {
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
const repo = service.get('/facebook/react') as unknown as RawRepo;
expect(repo).not.toBeNull();
expect(repo.id).toBe('/facebook/react');
expect(repo.source).toBe('github');
});
});
// ---------------------------------------------------------------------------
// add()
// ---------------------------------------------------------------------------
describe('RepositoryService.add()', () => {
let service: RepositoryService;
beforeEach(() => {
service = makeService(createTestDb());
});
it('adds a GitHub repository and derives the correct ID', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'https://github.com/facebook/react'
}) as unknown as RawRepo;
expect(repo.id).toBe('/facebook/react');
expect(repo.source).toBe('github');
expect(repo.state).toBe('pending');
});
it('accepts github.com/owner/repo URL format', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'github.com/facebook/react'
});
expect(repo.id).toBe('/facebook/react');
});
it('strips .git suffix from GitHub URLs', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'https://github.com/facebook/react.git'
});
expect(repo.id).toBe('/facebook/react');
});
it('uses override title when provided', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'https://github.com/facebook/react',
title: 'React Library'
});
expect(repo.title).toBe('React Library');
});
it('defaults title to repo name for GitHub repos', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'https://github.com/facebook/react'
});
expect(repo.title).toBe('react');
});
it('adds a local repository and derives the /local/ ID', () => {
const repo = service.add({
source: 'local',
sourceUrl: '/home/user/projects/my-sdk'
}) as unknown as RawRepo;
expect(repo.id).toBe('/local/my-sdk');
expect(repo.source).toBe('local');
});
it('resolves ID collisions for local repositories with -2, -3 suffix', () => {
service.add({ source: 'local', sourceUrl: '/home/user/a/my-sdk' });
const repo2 = service.add({ source: 'local', sourceUrl: '/home/user/b/my-sdk' });
expect(repo2.id).toBe('/local/my-sdk-2');
const repo3 = service.add({ source: 'local', sourceUrl: '/home/user/c/my-sdk' });
expect(repo3.id).toBe('/local/my-sdk-3');
});
it('throws AlreadyExistsError when a GitHub repo already exists', () => {
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
expect(() =>
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' })
).toThrow(AlreadyExistsError);
});
it('throws InvalidUrlError for a bad GitHub URL', () => {
expect(() =>
service.add({ source: 'github', sourceUrl: 'https://gitlab.com/foo/bar' })
).toThrow(InvalidUrlError);
});
it('throws InvalidInputError when sourceUrl is empty', () => {
expect(() => service.add({ source: 'github', sourceUrl: '' })).toThrow(InvalidInputError);
});
it('stores description and branch when provided', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'https://github.com/facebook/react',
description: 'A UI library',
branch: 'main'
}) as unknown as RawRepo;
expect(repo.description).toBe('A UI library');
expect(repo.branch).toBe('main');
});
it('initialises counters to zero', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'https://github.com/facebook/react'
});
expect(repo.totalSnippets).toBe(0);
expect(repo.totalTokens).toBe(0);
});
});
// ---------------------------------------------------------------------------
// update()
// ---------------------------------------------------------------------------
describe('RepositoryService.update()', () => {
let service: RepositoryService;
beforeEach(() => {
service = makeService(createTestDb());
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
});
it('updates the title', () => {
const updated = service.update('/facebook/react', { title: 'React' });
expect(updated.title).toBe('React');
});
it('updates the description', () => {
const updated = service.update('/facebook/react', { description: 'UI library' });
expect(updated.description).toBe('UI library');
});
it('updates the branch', () => {
const updated = service.update('/facebook/react', { branch: 'canary' }) as unknown as RawRepo;
expect(updated.branch).toBe('canary');
});
it('returns unchanged repository when input has no fields', () => {
const before = service.get('/facebook/react')!;
const updated = service.update('/facebook/react', {});
expect(updated.title).toBe(before.title);
});
it('throws NotFoundError for a non-existent repository', () => {
expect(() => service.update('/not/found', { title: 'New Title' })).toThrow(NotFoundError);
});
});
// ---------------------------------------------------------------------------
// remove()
// ---------------------------------------------------------------------------
describe('RepositoryService.remove()', () => {
let service: RepositoryService;
beforeEach(() => {
service = makeService(createTestDb());
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
});
it('removes an existing repository', () => {
service.remove('/facebook/react');
expect(service.get('/facebook/react')).toBeNull();
});
it('throws NotFoundError when the repository does not exist', () => {
expect(() => service.remove('/not/found')).toThrow(NotFoundError);
});
});
// ---------------------------------------------------------------------------
// getStats()
// ---------------------------------------------------------------------------
describe('RepositoryService.getStats()', () => {
let client: Database.Database;
let service: RepositoryService;
beforeEach(() => {
client = createTestDb();
service = makeService(client);
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
});
it('returns zero stats for a freshly added repository', () => {
const stats = service.getStats('/facebook/react');
expect(stats.totalSnippets).toBe(0);
expect(stats.totalTokens).toBe(0);
expect(stats.totalDocuments).toBe(0);
// lastIndexedAt may be undefined or null depending on the raw row value.
expect(stats.lastIndexedAt == null).toBe(true);
});
it('throws NotFoundError for a non-existent repository', () => {
expect(() => service.getStats('/not/found')).toThrow(NotFoundError);
});
it('counts documents correctly', () => {
const docId = crypto.randomUUID();
const now = Math.floor(Date.now() / 1000);
client
.prepare(
`INSERT INTO documents (id, repository_id, file_path, checksum, indexed_at)
VALUES (?, '/facebook/react', 'README.md', 'abc', ?)`
)
.run(docId, now);
const stats = service.getStats('/facebook/react');
expect(stats.totalDocuments).toBe(1);
});
});
// ---------------------------------------------------------------------------
// getVersions()
// ---------------------------------------------------------------------------
describe('RepositoryService.getVersions()', () => {
let client: Database.Database;
let service: RepositoryService;
beforeEach(() => {
client = createTestDb();
service = makeService(client);
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
});
it('returns an empty array when no versions exist', () => {
expect(service.getVersions('/facebook/react')).toEqual([]);
});
it('returns tags for all versions of a repository', () => {
const now = Math.floor(Date.now() / 1000);
client
.prepare(
`INSERT INTO repository_versions (id, repository_id, tag, created_at)
VALUES (?, '/facebook/react', ?, ?)`
)
.run('/facebook/react/v18.3.0', 'v18.3.0', now);
client
.prepare(
`INSERT INTO repository_versions (id, repository_id, tag, created_at)
VALUES (?, '/facebook/react', ?, ?)`
)
.run('/facebook/react/v17.0.2', 'v17.0.2', now - 1);
const versions = service.getVersions('/facebook/react');
expect(versions).toContain('v18.3.0');
expect(versions).toContain('v17.0.2');
});
});
// ---------------------------------------------------------------------------
// getIndexSummary()
// ---------------------------------------------------------------------------
describe('RepositoryService.getIndexSummary()', () => {
let client: Database.Database;
let service: RepositoryService;
beforeEach(() => {
client = createTestDb();
service = makeService(client);
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react', branch: 'main' });
});
it('returns embedding counts and indexed version labels', () => {
const now = Math.floor(Date.now() / 1000);
const docId = crypto.randomUUID();
const versionDocId = crypto.randomUUID();
const snippetId = crypto.randomUUID();
const versionSnippetId = crypto.randomUUID();
client
.prepare(
`INSERT INTO repository_versions (id, repository_id, tag, state, created_at)
VALUES (?, '/facebook/react', ?, 'indexed', ?)`
)
.run('/facebook/react/v18.3.0', 'v18.3.0', now);
client
.prepare(
`INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at)
VALUES (?, '/facebook/react', NULL, 'README.md', 'base', ?)`
)
.run(docId, now);
client
.prepare(
`INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at)
VALUES (?, '/facebook/react', ?, 'README.md', 'version', ?)`
)
.run(versionDocId, '/facebook/react/v18.3.0', now);
client
.prepare(
`INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at)
VALUES (?, ?, '/facebook/react', NULL, 'info', 'base snippet', ?)`
)
.run(snippetId, docId, now);
client
.prepare(
`INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at)
VALUES (?, ?, '/facebook/react', ?, 'info', 'version snippet', ?)`
)
.run(versionSnippetId, versionDocId, '/facebook/react/v18.3.0', now);
client
.prepare(
`INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at)
VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', 2, ?, ?)`
)
.run(snippetId, Buffer.from(Float32Array.from([1, 0]).buffer), now);
client
.prepare(
`INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at)
VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', 2, ?, ?)`
)
.run(versionSnippetId, Buffer.from(Float32Array.from([0, 1]).buffer), now);
expect(service.getIndexSummary('/facebook/react')).toEqual({
embeddingCount: 2,
indexedVersions: ['main', 'v18.3.0']
});
});
});
// ---------------------------------------------------------------------------
// createIndexingJob()
// ---------------------------------------------------------------------------
describe('RepositoryService.createIndexingJob()', () => {
let service: RepositoryService;
beforeEach(() => {
service = makeService(createTestDb());
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
});
it('creates a queued indexing job', () => {
const job = service.createIndexingJob('/facebook/react');
expect(job.id).toBeTruthy();
expect(job.repositoryId).toBe('/facebook/react');
expect(job.status).toBe('queued');
expect(job.progress).toBe(0);
});
it('returns the existing job when one is already queued', () => {
const job1 = service.createIndexingJob('/facebook/react');
const job2 = service.createIndexingJob('/facebook/react');
expect(job2.id).toBe(job1.id);
});
it('creates a new job once the previous job is done', () => {
const job1 = service.createIndexingJob('/facebook/react');
// Access the underlying db to simulate job completion.
const db = (service as unknown as { db: Database.Database }).db;
db.prepare(`UPDATE indexing_jobs SET status = 'done' WHERE id = ?`).run(job1.id);
const job2 = service.createIndexingJob('/facebook/react');
expect(job2.id).not.toBe(job1.id);
});
it('accepts an optional versionId', () => {
const job = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
expect(job.versionId).toBe('/facebook/react/v18.3.0');
});
it('allows separate jobs for the same repo but different versions', () => {
const defaultJob = service.createIndexingJob('/facebook/react');
const versionJob = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
expect(versionJob.id).not.toBe(defaultJob.id);
expect(defaultJob.versionId).toBeNull();
expect(versionJob.versionId).toBe('/facebook/react/v18.3.0');
});
it('returns the existing job when the same (repo, version) pair is already queued', () => {
const job1 = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
const job2 = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
expect(job2.id).toBe(job1.id);
});
it('returns the existing default-branch job when called again without a versionId', () => {
const job1 = service.createIndexingJob('/facebook/react');
const job2 = service.createIndexingJob('/facebook/react');
expect(job2.id).toBe(job1.id);
});
});