refactor: introduce domain model classes and mapper layer

Replace ad-hoc inline row casting (snake_case → camelCase) spread across
services, routes, and the indexing pipeline with explicit model classes
(Repository, IndexingJob, RepositoryVersion, Snippet, SearchResult) and
dedicated mapper classes that own the DB → domain conversion.

- Add src/lib/server/models/ with typed model classes for all domain entities
- Add src/lib/server/mappers/ with mapper classes per entity
- Remove duplicated RawRow interfaces and inline map functions from
  job-queue, repository.service, indexing.pipeline, and all API routes
- Add dtoJsonResponse helper to standardise JSON responses via SvelteKit json()
- Add api-contract.integration.test.ts as a regression baseline

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Giancarmine Salucci
2026-03-25 14:29:49 +01:00
parent 7994254e23
commit 215cadf070
39 changed files with 1339 additions and 562 deletions

View File

@@ -0,0 +1,5 @@
import { json } from '@sveltejs/kit';
export function dtoJsonResponse<T>(payload: T, init?: ResponseInit) {
return json(payload, init);
}

View File

@@ -12,16 +12,17 @@ import {
formatContextJson,
formatContextTxt
} from './formatters';
import type { LibrarySearchResult } from '$lib/server/search/search.service';
import type { SnippetSearchResult } from '$lib/server/search/search.service';
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result';
import { Repository } from '$lib/server/models/repository';
import { RepositoryVersion } from '$lib/server/models/repository-version';
import { Snippet } from '$lib/server/models/snippet';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function makeRepo(overrides: Partial<Repository> = {}): Repository {
return {
return new Repository({
id: '/facebook/react',
title: 'React',
description: 'A JavaScript library for building user interfaces',
@@ -39,11 +40,11 @@ function makeRepo(overrides: Partial<Repository> = {}): Repository {
createdAt: new Date('2024-01-01T00:00:00Z'),
updatedAt: new Date('2026-03-22T10:00:00Z'),
...overrides
};
});
}
function makeVersion(tag: string): RepositoryVersion {
return {
return new RepositoryVersion({
id: `/facebook/react/${tag}`,
repositoryId: '/facebook/react',
tag,
@@ -52,11 +53,11 @@ function makeVersion(tag: string): RepositoryVersion {
totalSnippets: 100,
indexedAt: new Date(),
createdAt: new Date()
};
});
}
function makeSnippet(overrides: Partial<Snippet> = {}): Snippet {
return {
return new Snippet({
id: 'snippet-1',
documentId: 'doc-1',
repositoryId: '/facebook/react',
@@ -69,15 +70,15 @@ function makeSnippet(overrides: Partial<Snippet> = {}): Snippet {
tokenCount: 45,
createdAt: new Date(),
...overrides
};
});
}
function makeSnippetResult(snippet: Snippet): SnippetSearchResult {
return {
return new SnippetSearchResult({
snippet,
score: -1.5,
repository: { id: snippet.repositoryId, title: 'React' }
};
});
}
// ---------------------------------------------------------------------------
@@ -109,11 +110,11 @@ describe('mapState', () => {
describe('formatLibrarySearchJson', () => {
it('returns results array with correct shape', () => {
const results: LibrarySearchResult[] = [
{
new LibrarySearchResult({
repository: makeRepo(),
versions: [makeVersion('v18.3.0'), makeVersion('v17.0.2')],
score: 150
}
})
];
const response = formatLibrarySearchJson(results);
@@ -138,7 +139,7 @@ describe('formatLibrarySearchJson', () => {
it('maps non-indexed state to initial', () => {
const results: LibrarySearchResult[] = [
{ repository: makeRepo({ state: 'pending' }), versions: [], score: 0 }
new LibrarySearchResult({ repository: makeRepo({ state: 'pending' }), versions: [], score: 0 })
];
const response = formatLibrarySearchJson(results);
expect(response.results[0].state).toBe('initial');
@@ -146,7 +147,7 @@ describe('formatLibrarySearchJson', () => {
it('handles null lastIndexedAt', () => {
const results: LibrarySearchResult[] = [
{ repository: makeRepo({ lastIndexedAt: null }), versions: [], score: 0 }
new LibrarySearchResult({ repository: makeRepo({ lastIndexedAt: null }), versions: [], score: 0 })
];
const response = formatLibrarySearchJson(results);
expect(response.results[0].lastUpdateDate).toBeNull();

View File

@@ -12,9 +12,23 @@
* error → error
*/
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
import type { LibrarySearchResult } from '$lib/server/search/search.service';
import type { SnippetSearchResult } from '$lib/server/search/search.service';
import { ContextResponseMapper } from '$lib/server/mappers/context-response.mapper.js';
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
import {
ContextJsonResponseDto,
LibrarySearchJsonResponseDto,
LibrarySearchJsonResultDto,
CodeSnippetJsonDto,
InfoSnippetJsonDto,
type SnippetJsonDto
} from '$lib/server/models/context-response.js';
export type LibrarySearchJsonResult = LibrarySearchJsonResultDto;
export type LibrarySearchJsonResponse = LibrarySearchJsonResponseDto;
export type CodeSnippetJson = CodeSnippetJsonDto;
export type InfoSnippetJson = InfoSnippetJsonDto;
export type SnippetJson = SnippetJsonDto;
export type ContextJsonResponse = ContextJsonResponseDto;
// ---------------------------------------------------------------------------
// State mapping
@@ -48,92 +62,11 @@ export const CORS_HEADERS = {
// /api/v1/libs/search — JSON response shape
// ---------------------------------------------------------------------------
export interface LibrarySearchJsonResult {
id: string;
title: string;
description: string | null;
branch: string | null;
lastUpdateDate: string | null;
state: Context7State;
totalTokens: number | null;
totalSnippets: number | null;
stars: number | null;
trustScore: number | null;
benchmarkScore: number | null;
versions: string[];
source: string;
}
export interface LibrarySearchJsonResponse {
results: LibrarySearchJsonResult[];
}
/**
* Convert internal LibrarySearchResult[] to the context7-compatible JSON body.
*/
export function formatLibrarySearchJson(results: LibrarySearchResult[]): LibrarySearchJsonResponse {
return {
results: results.map(({ repository, versions }) =>
formatSingleLibraryJson(repository, versions)
)
};
}
export function formatSingleLibraryJson(
repository: Repository,
versions: RepositoryVersion[]
): LibrarySearchJsonResult {
return {
id: repository.id,
title: repository.title,
description: repository.description ?? null,
branch: repository.branch ?? null,
lastUpdateDate: repository.lastIndexedAt ? repository.lastIndexedAt.toISOString() : null,
state: mapState(repository.state as TrueRefState),
totalTokens: repository.totalTokens ?? null,
totalSnippets: repository.totalSnippets ?? null,
stars: repository.stars ?? null,
trustScore: repository.trustScore ?? null,
benchmarkScore: repository.benchmarkScore ?? null,
versions: versions.map((v) => v.tag),
source: repository.sourceUrl
};
}
// ---------------------------------------------------------------------------
// /api/v1/context — JSON response shapes
// ---------------------------------------------------------------------------
export interface CodeListItem {
language: string;
code: string;
}
export interface CodeSnippetJson {
type: 'code';
title: string | null;
description: string | null;
language: string | null;
codeList: CodeListItem[];
id: string;
tokenCount: number | null;
pageTitle: string | null;
}
export interface InfoSnippetJson {
type: 'info';
text: string;
breadcrumb: string | null;
pageId: string;
tokenCount: number | null;
}
export type SnippetJson = CodeSnippetJson | InfoSnippetJson;
export interface ContextJsonResponse {
snippets: SnippetJson[];
rules: string[];
totalTokens: number;
export function formatLibrarySearchJson(results: LibrarySearchResult[]): LibrarySearchJsonResponseDto {
return ContextResponseMapper.toLibrarySearchJson(results);
}
/**
@@ -145,54 +78,8 @@ export interface ContextJsonResponse {
export function formatContextJson(
snippets: SnippetSearchResult[],
rules: string[]
): ContextJsonResponse {
const mapped: SnippetJson[] = snippets.map(({ snippet }) => {
if (snippet.type === 'code') {
const codeSnippet: CodeSnippetJson = {
type: 'code',
title: snippet.title ?? null,
description: snippet.breadcrumb ?? null,
language: snippet.language ?? null,
codeList: [
{
language: snippet.language ?? '',
code: snippet.content
}
],
id: snippet.id,
tokenCount: snippet.tokenCount ?? null,
pageTitle: extractPageTitle(snippet.breadcrumb)
};
return codeSnippet;
} else {
const infoSnippet: InfoSnippetJson = {
type: 'info',
text: snippet.content,
breadcrumb: snippet.breadcrumb ?? null,
pageId: snippet.id,
tokenCount: snippet.tokenCount ?? null
};
return infoSnippet;
}
});
const totalTokens = snippets.reduce((sum, { snippet }) => sum + (snippet.tokenCount ?? 0), 0);
return {
snippets: mapped,
rules,
totalTokens
};
}
/**
* Extract the top-level page title from a breadcrumb string.
* e.g. "Getting Started > Components" → "Getting Started"
*/
function extractPageTitle(breadcrumb: string | null | undefined): string | null {
if (!breadcrumb) return null;
const parts = breadcrumb.split('>');
return parts[0].trim() || null;
): ContextJsonResponseDto {
return ContextResponseMapper.toContextJson(snippets, rules);
}
// ---------------------------------------------------------------------------

View File

@@ -0,0 +1,71 @@
import {
CodeListItemDto,
CodeSnippetJsonDto,
ContextJsonResponseDto,
InfoSnippetJsonDto,
LibrarySearchJsonResponseDto,
LibrarySearchJsonResultDto,
type SnippetJsonDto
} from '$lib/server/models/context-response.js';
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
export class ContextResponseMapper {
static toLibrarySearchJson(results: LibrarySearchResult[]): LibrarySearchJsonResponseDto {
return new LibrarySearchJsonResponseDto(
results.map(
({ repository, versions }) =>
new LibrarySearchJsonResultDto({
id: repository.id,
title: repository.title,
description: repository.description ?? null,
branch: repository.branch ?? null,
lastUpdateDate: repository.lastIndexedAt
? repository.lastIndexedAt.toISOString()
: null,
state: repository.state === 'indexed' ? 'finalized' : repository.state === 'error' ? 'error' : 'initial',
totalTokens: repository.totalTokens ?? null,
totalSnippets: repository.totalSnippets ?? null,
stars: repository.stars ?? null,
trustScore: repository.trustScore ?? null,
benchmarkScore: repository.benchmarkScore ?? null,
versions: versions.map((version) => version.tag),
source: repository.sourceUrl
})
)
);
}
static toContextJson(snippets: SnippetSearchResult[], rules: string[]): ContextJsonResponseDto {
const mapped: SnippetJsonDto[] = snippets.map(({ snippet }) => {
if (snippet.type === 'code') {
return new CodeSnippetJsonDto({
title: snippet.title ?? null,
description: snippet.breadcrumb ?? null,
language: snippet.language ?? null,
codeList: [
new CodeListItemDto({
language: snippet.language ?? '',
code: snippet.content
})
],
id: snippet.id,
tokenCount: snippet.tokenCount ?? null,
pageTitle: snippet.breadcrumb ? snippet.breadcrumb.split('>')[0].trim() || null : null
});
}
return new InfoSnippetJsonDto({
text: snippet.content,
breadcrumb: snippet.breadcrumb ?? null,
pageId: snippet.id,
tokenCount: snippet.tokenCount ?? null
});
});
return new ContextJsonResponseDto({
snippets: mapped,
rules,
totalTokens: snippets.reduce((sum, result) => sum + (result.snippet.tokenCount ?? 0), 0)
});
}
}

View File

@@ -0,0 +1,36 @@
import { IndexingJob, IndexingJobDto, IndexingJobEntity } from '$lib/server/models/indexing-job.js';
export class IndexingJobMapper {
static fromEntity(entity: IndexingJobEntity): IndexingJob {
return new IndexingJob({
id: entity.id,
repositoryId: entity.repository_id,
versionId: entity.version_id,
status: entity.status,
progress: entity.progress,
totalFiles: entity.total_files,
processedFiles: entity.processed_files,
error: entity.error,
startedAt: entity.started_at != null ? new Date(entity.started_at * 1000) : null,
completedAt:
entity.completed_at != null ? new Date(entity.completed_at * 1000) : null,
createdAt: new Date(entity.created_at * 1000)
});
}
static toDto(domain: IndexingJob): IndexingJobDto {
return new IndexingJobDto({
id: domain.id,
repositoryId: domain.repositoryId,
versionId: domain.versionId,
status: domain.status,
progress: domain.progress,
totalFiles: domain.totalFiles,
processedFiles: domain.processedFiles,
error: domain.error,
startedAt: domain.startedAt,
completedAt: domain.completedAt,
createdAt: domain.createdAt
});
}
}

View File

@@ -0,0 +1,33 @@
import {
RepositoryVersion,
RepositoryVersionDto,
RepositoryVersionEntity
} from '$lib/server/models/repository-version.js';
export class RepositoryVersionMapper {
static fromEntity(entity: RepositoryVersionEntity): RepositoryVersion {
return new RepositoryVersion({
id: entity.id,
repositoryId: entity.repository_id,
tag: entity.tag,
title: entity.title,
state: entity.state,
totalSnippets: entity.total_snippets ?? 0,
indexedAt: entity.indexed_at != null ? new Date(entity.indexed_at * 1000) : null,
createdAt: new Date(entity.created_at * 1000)
});
}
static toDto(domain: RepositoryVersion): RepositoryVersionDto {
return new RepositoryVersionDto({
id: domain.id,
repositoryId: domain.repositoryId,
tag: domain.tag,
title: domain.title,
state: domain.state,
totalSnippets: domain.totalSnippets,
indexedAt: domain.indexedAt,
createdAt: domain.createdAt
});
}
}

View File

@@ -0,0 +1,67 @@
import { Repository, RepositoryDto, RepositoryEntity } from '$lib/server/models/repository.js';
export class RepositoryMapper {
static fromEntity(entity: RepositoryEntity): Repository {
return new Repository({
id: entity.id,
title: entity.title,
description: entity.description,
source: entity.source,
sourceUrl: entity.source_url,
branch: entity.branch,
state: entity.state,
totalSnippets: entity.total_snippets ?? 0,
totalTokens: entity.total_tokens ?? 0,
trustScore: entity.trust_score ?? 0,
benchmarkScore: entity.benchmark_score ?? 0,
stars: entity.stars,
githubToken: entity.github_token,
lastIndexedAt:
entity.last_indexed_at != null ? new Date(entity.last_indexed_at * 1000) : null,
createdAt: new Date(entity.created_at * 1000),
updatedAt: new Date(entity.updated_at * 1000)
});
}
static toEntity(domain: Repository): RepositoryEntity {
return new RepositoryEntity({
id: domain.id,
title: domain.title,
description: domain.description,
source: domain.source,
source_url: domain.sourceUrl,
branch: domain.branch,
state: domain.state,
total_snippets: domain.totalSnippets,
total_tokens: domain.totalTokens,
trust_score: domain.trustScore,
benchmark_score: domain.benchmarkScore,
stars: domain.stars,
github_token: domain.githubToken,
last_indexed_at:
domain.lastIndexedAt != null ? Math.floor(domain.lastIndexedAt.getTime() / 1000) : null,
created_at: Math.floor(domain.createdAt.getTime() / 1000),
updated_at: Math.floor(domain.updatedAt.getTime() / 1000)
});
}
static toDto(domain: Repository): RepositoryDto {
return new RepositoryDto({
id: domain.id,
title: domain.title,
description: domain.description,
source: domain.source,
sourceUrl: domain.sourceUrl,
branch: domain.branch,
state: domain.state,
totalSnippets: domain.totalSnippets,
totalTokens: domain.totalTokens,
trustScore: domain.trustScore,
benchmarkScore: domain.benchmarkScore,
stars: domain.stars,
lastIndexedAt: domain.lastIndexedAt,
createdAt: domain.createdAt,
updatedAt: domain.updatedAt
});
}
}

View File

@@ -0,0 +1,35 @@
import { LibrarySearchResult, SnippetRepositoryRef, SnippetSearchResult } from '$lib/server/models/search-result.js';
import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js';
import { RepositoryVersionMapper } from '$lib/server/mappers/repository-version.mapper.js';
import { SnippetMapper } from '$lib/server/mappers/snippet.mapper.js';
import { RepositoryEntity } from '$lib/server/models/repository.js';
import { RepositoryVersionEntity } from '$lib/server/models/repository-version.js';
import { SnippetEntity } from '$lib/server/models/snippet.js';
export class SearchResultMapper {
static snippetFromEntity(
entity: SnippetEntity,
repository: { id: string; title: string },
score: number
): SnippetSearchResult {
return new SnippetSearchResult({
snippet: SnippetMapper.fromEntity(entity),
score,
repository: new SnippetRepositoryRef(repository)
});
}
static libraryFromEntity(
repositoryEntity: RepositoryEntity,
versionEntities: RepositoryVersionEntity[],
score: number
): LibrarySearchResult {
return new LibrarySearchResult({
repository: RepositoryMapper.fromEntity(repositoryEntity),
versions: versionEntities.map((version) =>
RepositoryVersionMapper.fromEntity(version)
),
score
});
}
}

View File

@@ -0,0 +1,19 @@
import { Snippet, SnippetEntity } from '$lib/server/models/snippet.js';
export class SnippetMapper {
static fromEntity(entity: SnippetEntity): Snippet {
return new Snippet({
id: entity.id,
documentId: entity.document_id,
repositoryId: entity.repository_id,
versionId: entity.version_id,
type: entity.type,
title: entity.title,
content: entity.content,
language: entity.language,
breadcrumb: entity.breadcrumb,
tokenCount: entity.token_count,
createdAt: new Date(entity.created_at * 1000)
});
}
}

View File

@@ -0,0 +1,99 @@
export class LibrarySearchJsonResultDto {
id: string;
title: string;
description: string | null;
branch: string | null;
lastUpdateDate: string | null;
state: 'initial' | 'finalized' | 'error';
totalTokens: number | null;
totalSnippets: number | null;
stars: number | null;
trustScore: number | null;
benchmarkScore: number | null;
versions: string[];
source: string;
constructor(props: LibrarySearchJsonResultDto) {
this.id = props.id;
this.title = props.title;
this.description = props.description;
this.branch = props.branch;
this.lastUpdateDate = props.lastUpdateDate;
this.state = props.state;
this.totalTokens = props.totalTokens;
this.totalSnippets = props.totalSnippets;
this.stars = props.stars;
this.trustScore = props.trustScore;
this.benchmarkScore = props.benchmarkScore;
this.versions = props.versions;
this.source = props.source;
}
}
export class LibrarySearchJsonResponseDto {
results: LibrarySearchJsonResultDto[];
constructor(results: LibrarySearchJsonResultDto[]) {
this.results = results;
}
}
export class CodeListItemDto {
language: string;
code: string;
constructor(props: CodeListItemDto) {
this.language = props.language;
this.code = props.code;
}
}
export class CodeSnippetJsonDto {
type: 'code' = 'code';
title: string | null;
description: string | null;
language: string | null;
codeList: CodeListItemDto[];
id: string;
tokenCount: number | null;
pageTitle: string | null;
constructor(props: Omit<CodeSnippetJsonDto, 'type'>) {
this.title = props.title;
this.description = props.description;
this.language = props.language;
this.codeList = props.codeList;
this.id = props.id;
this.tokenCount = props.tokenCount;
this.pageTitle = props.pageTitle;
}
}
export class InfoSnippetJsonDto {
type: 'info' = 'info';
text: string;
breadcrumb: string | null;
pageId: string;
tokenCount: number | null;
constructor(props: Omit<InfoSnippetJsonDto, 'type'>) {
this.text = props.text;
this.breadcrumb = props.breadcrumb;
this.pageId = props.pageId;
this.tokenCount = props.tokenCount;
}
}
export type SnippetJsonDto = CodeSnippetJsonDto | InfoSnippetJsonDto;
export class ContextJsonResponseDto {
snippets: SnippetJsonDto[];
rules: string[];
totalTokens: number;
constructor(props: ContextJsonResponseDto) {
this.snippets = props.snippets;
this.rules = props.rules;
this.totalTokens = props.totalTokens;
}
}

View File

@@ -0,0 +1,125 @@
export interface IndexingJobEntityProps {
id: string;
repository_id: string;
version_id: string | null;
status: 'queued' | 'running' | 'done' | 'failed';
progress: number;
total_files: number;
processed_files: number;
error: string | null;
started_at: number | null;
completed_at: number | null;
created_at: number;
}
export class IndexingJobEntity {
id: string;
repository_id: string;
version_id: string | null;
status: 'queued' | 'running' | 'done' | 'failed';
progress: number;
total_files: number;
processed_files: number;
error: string | null;
started_at: number | null;
completed_at: number | null;
created_at: number;
constructor(props: IndexingJobEntityProps) {
this.id = props.id;
this.repository_id = props.repository_id;
this.version_id = props.version_id;
this.status = props.status;
this.progress = props.progress;
this.total_files = props.total_files;
this.processed_files = props.processed_files;
this.error = props.error;
this.started_at = props.started_at;
this.completed_at = props.completed_at;
this.created_at = props.created_at;
}
}
export interface IndexingJobProps {
id: string;
repositoryId: string;
versionId: string | null;
status: 'queued' | 'running' | 'done' | 'failed';
progress: number;
totalFiles: number;
processedFiles: number;
error: string | null;
startedAt: Date | null;
completedAt: Date | null;
createdAt: Date;
}
export class IndexingJob {
id: string;
repositoryId: string;
versionId: string | null;
status: 'queued' | 'running' | 'done' | 'failed';
progress: number;
totalFiles: number;
processedFiles: number;
error: string | null;
startedAt: Date | null;
completedAt: Date | null;
createdAt: Date;
constructor(props: IndexingJobProps) {
this.id = props.id;
this.repositoryId = props.repositoryId;
this.versionId = props.versionId;
this.status = props.status;
this.progress = props.progress;
this.totalFiles = props.totalFiles;
this.processedFiles = props.processedFiles;
this.error = props.error;
this.startedAt = props.startedAt;
this.completedAt = props.completedAt;
this.createdAt = props.createdAt;
}
}
export interface IndexingJobDtoProps {
id: string;
repositoryId: string;
versionId: string | null;
status: 'queued' | 'running' | 'done' | 'failed';
progress: number;
totalFiles: number;
processedFiles: number;
error: string | null;
startedAt: Date | null;
completedAt: Date | null;
createdAt: Date;
}
export class IndexingJobDto {
id: string;
repositoryId: string;
versionId: string | null;
status: 'queued' | 'running' | 'done' | 'failed';
progress: number;
totalFiles: number;
processedFiles: number;
error: string | null;
startedAt: Date | null;
completedAt: Date | null;
createdAt: Date;
constructor(props: IndexingJobDtoProps) {
this.id = props.id;
this.repositoryId = props.repositoryId;
this.versionId = props.versionId;
this.status = props.status;
this.progress = props.progress;
this.totalFiles = props.totalFiles;
this.processedFiles = props.processedFiles;
this.error = props.error;
this.startedAt = props.startedAt;
this.completedAt = props.completedAt;
this.createdAt = props.createdAt;
}
}

View File

@@ -0,0 +1,98 @@
export interface RepositoryVersionEntityProps {
id: string;
repository_id: string;
tag: string;
title: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
total_snippets: number | null;
indexed_at: number | null;
created_at: number;
}
export class RepositoryVersionEntity {
id: string;
repository_id: string;
tag: string;
title: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
total_snippets: number | null;
indexed_at: number | null;
created_at: number;
constructor(props: RepositoryVersionEntityProps) {
this.id = props.id;
this.repository_id = props.repository_id;
this.tag = props.tag;
this.title = props.title;
this.state = props.state;
this.total_snippets = props.total_snippets;
this.indexed_at = props.indexed_at;
this.created_at = props.created_at;
}
}
export interface RepositoryVersionProps {
id: string;
repositoryId: string;
tag: string;
title: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
indexedAt: Date | null;
createdAt: Date;
}
export class RepositoryVersion {
id: string;
repositoryId: string;
tag: string;
title: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
indexedAt: Date | null;
createdAt: Date;
constructor(props: RepositoryVersionProps) {
this.id = props.id;
this.repositoryId = props.repositoryId;
this.tag = props.tag;
this.title = props.title;
this.state = props.state;
this.totalSnippets = props.totalSnippets;
this.indexedAt = props.indexedAt;
this.createdAt = props.createdAt;
}
}
export interface RepositoryVersionDtoProps {
id: string;
repositoryId: string;
tag: string;
title: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
indexedAt: Date | null;
createdAt: Date;
}
export class RepositoryVersionDto {
id: string;
repositoryId: string;
tag: string;
title: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
indexedAt: Date | null;
createdAt: Date;
constructor(props: RepositoryVersionDtoProps) {
this.id = props.id;
this.repositoryId = props.repositoryId;
this.tag = props.tag;
this.title = props.title;
this.state = props.state;
this.totalSnippets = props.totalSnippets;
this.indexedAt = props.indexedAt;
this.createdAt = props.createdAt;
}
}

View File

@@ -0,0 +1,167 @@
export interface RepositoryEntityProps {
id: string;
title: string;
description: string | null;
source: 'github' | 'local';
source_url: string;
branch: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
total_snippets: number | null;
total_tokens: number | null;
trust_score: number | null;
benchmark_score: number | null;
stars: number | null;
github_token: string | null;
last_indexed_at: number | null;
created_at: number;
updated_at: number;
}
export class RepositoryEntity {
id: string;
title: string;
description: string | null;
source: 'github' | 'local';
source_url: string;
branch: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
total_snippets: number | null;
total_tokens: number | null;
trust_score: number | null;
benchmark_score: number | null;
stars: number | null;
github_token: string | null;
last_indexed_at: number | null;
created_at: number;
updated_at: number;
constructor(props: RepositoryEntityProps) {
this.id = props.id;
this.title = props.title;
this.description = props.description;
this.source = props.source;
this.source_url = props.source_url;
this.branch = props.branch;
this.state = props.state;
this.total_snippets = props.total_snippets;
this.total_tokens = props.total_tokens;
this.trust_score = props.trust_score;
this.benchmark_score = props.benchmark_score;
this.stars = props.stars;
this.github_token = props.github_token;
this.last_indexed_at = props.last_indexed_at;
this.created_at = props.created_at;
this.updated_at = props.updated_at;
}
}
export interface RepositoryProps {
id: string;
title: string;
description: string | null;
source: 'github' | 'local';
sourceUrl: string;
branch: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
totalTokens: number;
trustScore: number;
benchmarkScore: number;
stars: number | null;
githubToken: string | null;
lastIndexedAt: Date | null;
createdAt: Date;
updatedAt: Date;
}
export class Repository {
id: string;
title: string;
description: string | null;
source: 'github' | 'local';
sourceUrl: string;
branch: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
totalTokens: number;
trustScore: number;
benchmarkScore: number;
stars: number | null;
githubToken: string | null;
lastIndexedAt: Date | null;
createdAt: Date;
updatedAt: Date;
constructor(props: RepositoryProps) {
this.id = props.id;
this.title = props.title;
this.description = props.description;
this.source = props.source;
this.sourceUrl = props.sourceUrl;
this.branch = props.branch;
this.state = props.state;
this.totalSnippets = props.totalSnippets;
this.totalTokens = props.totalTokens;
this.trustScore = props.trustScore;
this.benchmarkScore = props.benchmarkScore;
this.stars = props.stars;
this.githubToken = props.githubToken;
this.lastIndexedAt = props.lastIndexedAt;
this.createdAt = props.createdAt;
this.updatedAt = props.updatedAt;
}
}
export interface RepositoryDtoProps {
id: string;
title: string;
description: string | null;
source: 'github' | 'local';
sourceUrl: string;
branch: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
totalTokens: number;
trustScore: number;
benchmarkScore: number;
stars: number | null;
lastIndexedAt: Date | null;
createdAt: Date;
updatedAt: Date;
}
export class RepositoryDto {
id: string;
title: string;
description: string | null;
source: 'github' | 'local';
sourceUrl: string;
branch: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
totalTokens: number;
trustScore: number;
benchmarkScore: number;
stars: number | null;
lastIndexedAt: Date | null;
createdAt: Date;
updatedAt: Date;
constructor(props: RepositoryDtoProps) {
this.id = props.id;
this.title = props.title;
this.description = props.description;
this.source = props.source;
this.sourceUrl = props.sourceUrl;
this.branch = props.branch;
this.state = props.state;
this.totalSnippets = props.totalSnippets;
this.totalTokens = props.totalTokens;
this.trustScore = props.trustScore;
this.benchmarkScore = props.benchmarkScore;
this.stars = props.stars;
this.lastIndexedAt = props.lastIndexedAt;
this.createdAt = props.createdAt;
this.updatedAt = props.updatedAt;
}
}

View File

@@ -0,0 +1,54 @@
import { Repository } from '$lib/server/models/repository.js';
import { RepositoryVersion } from '$lib/server/models/repository-version.js';
import { Snippet } from '$lib/server/models/snippet.js';
export interface SnippetRepositoryRefProps {
id: string;
title: string;
}
export class SnippetRepositoryRef {
id: string;
title: string;
constructor(props: SnippetRepositoryRefProps) {
this.id = props.id;
this.title = props.title;
}
}
export interface SnippetSearchResultProps {
snippet: Snippet;
score: number;
repository: SnippetRepositoryRef;
}
export class SnippetSearchResult {
snippet: Snippet;
score: number;
repository: SnippetRepositoryRef;
constructor(props: SnippetSearchResultProps) {
this.snippet = props.snippet;
this.score = props.score;
this.repository = props.repository;
}
}
export interface LibrarySearchResultProps {
repository: Repository;
versions: RepositoryVersion[];
score: number;
}
export class LibrarySearchResult {
repository: Repository;
versions: RepositoryVersion[];
score: number;
constructor(props: LibrarySearchResultProps) {
this.repository = props.repository;
this.versions = props.versions;
this.score = props.score;
}
}

View File

@@ -0,0 +1,83 @@
export interface SnippetEntityProps {
id: string;
document_id: string;
repository_id: string;
version_id: string | null;
type: 'code' | 'info';
title: string | null;
content: string;
language: string | null;
breadcrumb: string | null;
token_count: number | null;
created_at: number;
}
export class SnippetEntity {
id: string;
document_id: string;
repository_id: string;
version_id: string | null;
type: 'code' | 'info';
title: string | null;
content: string;
language: string | null;
breadcrumb: string | null;
token_count: number | null;
created_at: number;
constructor(props: SnippetEntityProps) {
this.id = props.id;
this.document_id = props.document_id;
this.repository_id = props.repository_id;
this.version_id = props.version_id;
this.type = props.type;
this.title = props.title;
this.content = props.content;
this.language = props.language;
this.breadcrumb = props.breadcrumb;
this.token_count = props.token_count;
this.created_at = props.created_at;
}
}
export interface SnippetProps {
id: string;
documentId: string;
repositoryId: string;
versionId: string | null;
type: 'code' | 'info';
title: string | null;
content: string;
language: string | null;
breadcrumb: string | null;
tokenCount: number | null;
createdAt: Date;
}
export class Snippet {
id: string;
documentId: string;
repositoryId: string;
versionId: string | null;
type: 'code' | 'info';
title: string | null;
content: string;
language: string | null;
breadcrumb: string | null;
tokenCount: number | null;
createdAt: Date;
constructor(props: SnippetProps) {
this.id = props.id;
this.documentId = props.documentId;
this.repositoryId = props.repositoryId;
this.versionId = props.versionId;
this.type = props.type;
this.title = props.title;
this.content = props.content;
this.language = props.language;
this.breadcrumb = props.breadcrumb;
this.tokenCount = props.tokenCount;
this.createdAt = props.createdAt;
}
}

View File

@@ -458,6 +458,31 @@ describe('IndexingPipeline', () => {
expect(updated.progress).toBe(100);
});
it('uses the repository source_url when crawling local repositories', async () => {
const crawl = vi.fn().mockResolvedValue({
files: [],
totalFiles: 0,
skippedFiles: 0,
branch: 'local',
commitSha: 'abc'
});
const pipeline = new IndexingPipeline(
db,
vi.fn() as never,
{ crawl } as never,
null
);
const job = makeJob();
await pipeline.run(job as never);
expect(crawl).toHaveBeenCalledWith({
rootPath: '/tmp/test-repo',
ref: undefined
});
});
it('integration: handles unchanged, modified, added, and deleted files in one run', async () => {
// ---- First run: index three files -----------------------------------
const firstFiles = [

View File

@@ -15,10 +15,13 @@
import { createHash } from 'node:crypto';
import type Database from 'better-sqlite3';
import type { Document, IndexingJob, NewDocument, NewSnippet, Repository } from '$lib/types';
import type { Document, NewDocument, NewSnippet } from '$lib/types';
import type { crawl as GithubCrawlFn } from '$lib/server/crawler/github.crawler.js';
import type { LocalCrawler } from '$lib/server/crawler/local.crawler.js';
import type { EmbeddingService } from '$lib/server/embeddings/embedding.service.js';
import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js';
import { IndexingJob } from '$lib/server/models/indexing-job.js';
import { Repository, RepositoryEntity } from '$lib/server/models/repository.js';
import { parseFile } from '$lib/server/parser/index.js';
import { computeTrustScore } from '$lib/server/search/trust-score.js';
import { computeDiff } from './diff.js';
@@ -399,11 +402,10 @@ export class IndexingPipeline {
}
private getRepository(id: string): Repository | null {
return (
(this.db
.prepare<[string], Repository>(`SELECT * FROM repositories WHERE id = ?`)
.get(id) as Repository | undefined) ?? null
);
const raw = this.db
.prepare<[string], RepositoryEntity>(`SELECT * FROM repositories WHERE id = ?`)
.get(id);
return raw ? RepositoryMapper.fromEntity(new RepositoryEntity(raw)) : null;
}
private updateJob(id: string, fields: Record<string, unknown>): void {

View File

@@ -7,48 +7,15 @@
*/
import type Database from 'better-sqlite3';
import type { IndexingJob, NewIndexingJob } from '$lib/types';
import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js';
import { IndexingJob, IndexingJobEntity } from '$lib/server/models/indexing-job.js';
import type { IndexingPipeline } from './indexing.pipeline.js';
// ---------------------------------------------------------------------------
// SQL projection + row mapper (mirrors repository.service.ts pattern)
// ---------------------------------------------------------------------------
const JOB_SELECT = `
SELECT id,
repository_id AS repositoryId,
version_id AS versionId,
status, progress,
total_files AS totalFiles,
processed_files AS processedFiles,
error,
started_at AS startedAt,
completed_at AS completedAt,
created_at AS createdAt
FROM indexing_jobs`;
interface RawJob {
id: string;
repositoryId: string;
versionId: string | null;
status: 'queued' | 'running' | 'done' | 'failed';
progress: number;
totalFiles: number;
processedFiles: number;
error: string | null;
startedAt: number | null;
completedAt: number | null;
createdAt: number;
}
function mapJob(raw: RawJob): IndexingJob {
return {
...raw,
startedAt: raw.startedAt != null ? new Date(raw.startedAt * 1000) : null,
completedAt: raw.completedAt != null ? new Date(raw.completedAt * 1000) : null,
createdAt: new Date(raw.createdAt * 1000)
};
}
const JOB_SELECT = `SELECT * FROM indexing_jobs`;
export class JobQueue {
private isRunning = false;
@@ -71,7 +38,7 @@ export class JobQueue {
enqueue(repositoryId: string, versionId?: string): IndexingJob {
// Return early if there's already an active job for this repo.
const activeRaw = this.db
.prepare<[string], RawJob>(
.prepare<[string], IndexingJobEntity>(
`${JOB_SELECT}
WHERE repository_id = ? AND status IN ('queued', 'running')
ORDER BY created_at DESC LIMIT 1`
@@ -81,11 +48,11 @@ export class JobQueue {
if (activeRaw) {
// Ensure the queue is draining even if enqueue was called concurrently.
if (!this.isRunning) setImmediate(() => this.processNext());
return mapJob(activeRaw);
return IndexingJobMapper.fromEntity(new IndexingJobEntity(activeRaw));
}
const now = Math.floor(Date.now() / 1000);
const job: NewIndexingJob = {
const job = new IndexingJob({
id: crypto.randomUUID(),
repositoryId,
versionId: versionId ?? null,
@@ -97,7 +64,7 @@ export class JobQueue {
startedAt: null,
completedAt: null,
createdAt: new Date(now * 1000)
};
});
this.db
.prepare(
@@ -125,9 +92,10 @@ export class JobQueue {
setImmediate(() => this.processNext());
}
return mapJob(
this.db.prepare<[string], RawJob>(`${JOB_SELECT} WHERE id = ?`).get(job.id as string)!
);
const created = this.db
.prepare<[string], IndexingJobEntity>(`${JOB_SELECT} WHERE id = ?`)
.get(job.id as string)!;
return IndexingJobMapper.fromEntity(new IndexingJobEntity(created));
}
/**
@@ -142,7 +110,7 @@ export class JobQueue {
}
const rawJob = this.db
.prepare<[], RawJob>(
.prepare<[], IndexingJobEntity>(
`${JOB_SELECT}
WHERE status = 'queued'
ORDER BY created_at ASC LIMIT 1`
@@ -151,7 +119,7 @@ export class JobQueue {
if (!rawJob) return;
const job = mapJob(rawJob);
const job = IndexingJobMapper.fromEntity(new IndexingJobEntity(rawJob));
this.isRunning = true;
try {
await this.pipeline.run(job);
@@ -180,9 +148,9 @@ export class JobQueue {
*/
getJob(id: string): IndexingJob | null {
const raw = this.db
.prepare<[string], RawJob>(`${JOB_SELECT} WHERE id = ?`)
.prepare<[string], IndexingJobEntity>(`${JOB_SELECT} WHERE id = ?`)
.get(id);
return raw ? mapJob(raw) : null;
return raw ? IndexingJobMapper.fromEntity(new IndexingJobEntity(raw)) : null;
}
/**
@@ -210,7 +178,9 @@ export class JobQueue {
const sql = `${JOB_SELECT} ${where} ORDER BY created_at DESC LIMIT ?`;
params.push(limit);
return (this.db.prepare<unknown[], RawJob>(sql).all(...params) as RawJob[]).map(mapJob);
return (this.db.prepare<unknown[], IndexingJobEntity>(sql).all(...params) as IndexingJobEntity[]).map(
(row) => IndexingJobMapper.fromEntity(new IndexingJobEntity(row))
);
}
/**

View File

@@ -5,7 +5,7 @@
* responses and MCP tool outputs.
*/
import type { LibrarySearchResult, SnippetSearchResult } from './search.service';
import type { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
// ---------------------------------------------------------------------------
// Library search formatter (`resolve-library-id`)
@@ -22,7 +22,7 @@ export function formatLibraryResults(results: LibrarySearchResult[]): string {
return results
.map((r, i) => {
const repo = r.repository;
const versions = r.versions.map((v) => v.tag).join(', ') || 'default branch';
const versions = r.versions.map((version) => version.tag).join(', ') || 'default branch';
return [
`${i + 1}. ${repo.title}`,
` Library ID: ${repo.id}`,

View File

@@ -13,11 +13,12 @@
import type Database from 'better-sqlite3';
import type { EmbeddingProvider } from '../embeddings/provider.js';
import type { SnippetSearchResult } from './search.service.js';
import { SnippetSearchResult, SnippetRepositoryRef } from '$lib/server/models/search-result.js';
import { SnippetEntity } from '$lib/server/models/snippet.js';
import { SearchResultMapper } from '$lib/server/mappers/search-result.mapper.js';
import { SearchService } from './search.service.js';
import { VectorSearch } from './vector.search.js';
import { reciprocalRankFusion } from './rrf.js';
import type { Snippet } from '$lib/types';
// ---------------------------------------------------------------------------
// Public interfaces
@@ -54,18 +55,7 @@ export interface SearchConfig {
// Raw DB row used when re-fetching snippets by ID
// ---------------------------------------------------------------------------
interface RawSnippetById {
id: string;
document_id: string;
repository_id: string;
version_id: string | null;
type: 'code' | 'info';
title: string | null;
content: string;
language: string | null;
breadcrumb: string | null;
token_count: number | null;
created_at: number;
interface RawSnippetById extends SnippetEntity {
repo_id: string;
repo_title: string;
}
@@ -200,25 +190,17 @@ export class HybridSearchService {
const row = byId.get(id);
if (!row) continue;
const snippet: Snippet = {
id: row.id,
documentId: row.document_id,
repositoryId: row.repository_id,
versionId: row.version_id,
type: row.type,
title: row.title,
content: row.content,
language: row.language,
breadcrumb: row.breadcrumb,
tokenCount: row.token_count,
createdAt: new Date(row.created_at * 1000)
};
results.push({
snippet,
score: 0, // RRF score not mapped to BM25 scale; consumers use rank position.
repository: { id: row.repo_id, title: row.repo_title }
});
results.push(
new SnippetSearchResult({
snippet: SearchResultMapper.snippetFromEntity(
new SnippetEntity(row),
new SnippetRepositoryRef({ id: row.repo_id, title: row.repo_title }),
0
).snippet,
score: 0,
repository: new SnippetRepositoryRef({ id: row.repo_id, title: row.repo_title })
})
);
}
return results;

View File

@@ -8,9 +8,15 @@
*/
import type Database from 'better-sqlite3';
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
import { RepositoryEntity } from '$lib/server/models/repository.js';
import { RepositoryVersionEntity } from '$lib/server/models/repository-version.js';
import { SnippetEntity } from '$lib/server/models/snippet.js';
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
import { SearchResultMapper } from '$lib/server/mappers/search-result.mapper.js';
import { preprocessQuery } from './query-preprocessor';
export { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
// ---------------------------------------------------------------------------
// Public interface types
// ---------------------------------------------------------------------------
@@ -25,13 +31,6 @@ export interface SnippetSearchOptions {
offset?: number;
}
export interface SnippetSearchResult {
snippet: Snippet;
/** BM25 rank — negative value; lower (more negative) = more relevant. */
score: number;
repository: Pick<Repository, 'id' | 'title'>;
}
export interface LibrarySearchOptions {
libraryName: string;
/** Semantic relevance hint (reserved for future hybrid use). */
@@ -40,53 +39,19 @@ export interface LibrarySearchOptions {
limit?: number;
}
export interface LibrarySearchResult {
repository: Repository;
versions: RepositoryVersion[];
/** Composite relevance score. Higher = more relevant. */
score: number;
}
// ---------------------------------------------------------------------------
// Raw DB row types
// ---------------------------------------------------------------------------
/** Raw row returned by the snippet FTS query (snake_case column names). */
interface RawSnippetRow {
id: string;
document_id: string;
repository_id: string;
version_id: string | null;
type: 'code' | 'info';
title: string | null;
content: string;
language: string | null;
breadcrumb: string | null;
token_count: number | null;
created_at: number;
interface RawSnippetRow extends SnippetEntity {
repo_id: string;
repo_title: string;
score: number;
}
/** Raw row returned by the library search query. */
interface RawRepoRow {
id: string;
title: string;
description: string | null;
source: 'github' | 'local';
source_url: string;
branch: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
total_snippets: number | null;
total_tokens: number | null;
trust_score: number | null;
benchmark_score: number | null;
stars: number | null;
github_token: string | null;
last_indexed_at: number | null;
created_at: number;
updated_at: number;
interface RawRepoRow extends RepositoryEntity {
exact_match: number;
prefix_match: number;
desc_match: number;
@@ -95,70 +60,7 @@ interface RawRepoRow {
}
/** Raw row returned by the version query. */
interface RawVersionRow {
id: string;
repository_id: string;
tag: string;
title: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
total_snippets: number | null;
indexed_at: number | null;
created_at: number;
}
// ---------------------------------------------------------------------------
// Mappers: raw DB rows → domain types
// ---------------------------------------------------------------------------
function mapSnippet(row: RawSnippetRow): Snippet {
return {
id: row.id,
documentId: row.document_id,
repositoryId: row.repository_id,
versionId: row.version_id,
type: row.type,
title: row.title,
content: row.content,
language: row.language,
breadcrumb: row.breadcrumb,
tokenCount: row.token_count,
createdAt: new Date(row.created_at * 1000)
};
}
function mapRepository(row: RawRepoRow): Repository {
return {
id: row.id,
title: row.title,
description: row.description,
source: row.source,
sourceUrl: row.source_url,
branch: row.branch,
state: row.state,
totalSnippets: row.total_snippets,
totalTokens: row.total_tokens,
trustScore: row.trust_score,
benchmarkScore: row.benchmark_score,
stars: row.stars,
githubToken: row.github_token,
lastIndexedAt: row.last_indexed_at ? new Date(row.last_indexed_at * 1000) : null,
createdAt: new Date(row.created_at * 1000),
updatedAt: new Date(row.updated_at * 1000)
};
}
function mapVersion(row: RawVersionRow): RepositoryVersion {
return {
id: row.id,
repositoryId: row.repository_id,
tag: row.tag,
title: row.title,
state: row.state,
totalSnippets: row.total_snippets,
indexedAt: row.indexed_at ? new Date(row.indexed_at * 1000) : null,
createdAt: new Date(row.created_at * 1000)
};
}
type RawVersionRow = RepositoryVersionEntity;
// ---------------------------------------------------------------------------
// SearchService
@@ -229,11 +131,12 @@ export class SearchService {
const rows = this.db.prepare(sql).all(...params) as RawSnippetRow[];
return rows.map((row) => ({
snippet: mapSnippet(row),
score: row.score,
repository: { id: row.repo_id, title: row.repo_title }
}));
return rows.map((row) =>
SearchResultMapper.snippetFromEntity(new SnippetEntity(row), {
id: row.repo_id,
title: row.repo_title
}, row.score)
);
}
// -------------------------------------------------------------------------
@@ -284,14 +187,13 @@ export class SearchService {
) as RawRepoRow[];
return rows.map((row) => {
const repository = mapRepository(row);
const compositeScore =
row.exact_match + row.prefix_match + row.desc_match + row.snippet_score + row.trust_component;
return {
repository,
versions: this.getVersions(row.id),
score: compositeScore
};
return SearchResultMapper.libraryFromEntity(
new RepositoryEntity(row),
this.getVersionEntities(row.id),
compositeScore
);
});
}
@@ -299,12 +201,11 @@ export class SearchService {
// Private helpers
// -------------------------------------------------------------------------
private getVersions(repositoryId: string): RepositoryVersion[] {
const rows = this.db
private getVersionEntities(repositoryId: string): RepositoryVersionEntity[] {
return this.db
.prepare(
`SELECT * FROM repository_versions WHERE repository_id = ? ORDER BY created_at DESC`
)
.all(repositoryId) as RawVersionRow[];
return rows.map(mapVersion);
}
}

View File

@@ -281,9 +281,9 @@ describe('RepositoryService.add()', () => {
const repo = service.add({
source: 'github',
sourceUrl: 'https://github.com/facebook/react'
}) as unknown as RawRepo;
expect(repo.total_snippets).toBe(0);
expect(repo.total_tokens).toBe(0);
});
expect(repo.totalSnippets).toBe(0);
expect(repo.totalTokens).toBe(0);
});
});
@@ -443,9 +443,9 @@ describe('RepositoryService.createIndexingJob()', () => {
});
it('creates a queued indexing job', () => {
const job = service.createIndexingJob('/facebook/react') as unknown as RawJob;
const job = service.createIndexingJob('/facebook/react');
expect(job.id).toBeTruthy();
expect(job.repository_id).toBe('/facebook/react');
expect(job.repositoryId).toBe('/facebook/react');
expect(job.status).toBe('queued');
expect(job.progress).toBe(0);
});
@@ -467,10 +467,7 @@ describe('RepositoryService.createIndexingJob()', () => {
});
it('accepts an optional versionId', () => {
const job = service.createIndexingJob(
'/facebook/react',
'/facebook/react/v18.3.0'
) as unknown as RawJob;
expect(job.version_id).toBe('/facebook/react/v18.3.0');
const job = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
expect(job.versionId).toBe('/facebook/react/v18.3.0');
});
});

View File

@@ -4,7 +4,10 @@
*/
import type Database from 'better-sqlite3';
import type { Repository, NewRepository, IndexingJob, NewIndexingJob } from '$lib/types';
import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js';
import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js';
import { Repository, RepositoryEntity } from '$lib/server/models/repository.js';
import { IndexingJob, IndexingJobEntity } from '$lib/server/models/indexing-job.js';
import { resolveGitHubId, resolveLocalId } from '$lib/server/utils/id-resolver';
import {
AlreadyExistsError,
@@ -47,16 +50,18 @@ export class RepositoryService {
const offset = options?.offset ?? 0;
if (options?.state) {
return this.db
const rows = this.db
.prepare(
`SELECT * FROM repositories WHERE state = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`
)
.all(options.state, limit, offset) as Repository[];
.all(options.state, limit, offset) as RepositoryEntity[];
return rows.map((row) => RepositoryMapper.fromEntity(new RepositoryEntity(row)));
}
return this.db
const rows = this.db
.prepare(`SELECT * FROM repositories ORDER BY created_at DESC LIMIT ? OFFSET ?`)
.all(limit, offset) as Repository[];
.all(limit, offset) as RepositoryEntity[];
return rows.map((row) => RepositoryMapper.fromEntity(new RepositoryEntity(row)));
}
/**
@@ -79,10 +84,10 @@ export class RepositoryService {
* Get a single repository by ID.
*/
get(id: string): Repository | null {
return (
(this.db.prepare(`SELECT * FROM repositories WHERE id = ?`).get(id) as Repository | undefined) ??
null
);
const row = this.db.prepare(`SELECT * FROM repositories WHERE id = ?`).get(id) as
| RepositoryEntity
| undefined;
return row ? RepositoryMapper.fromEntity(new RepositoryEntity(row)) : null;
}
/**
@@ -126,24 +131,25 @@ export class RepositoryService {
}
const now = Math.floor(Date.now() / 1000);
const repo: Record<string, unknown> = {
const repo = new Repository({
id,
title,
description: input.description ?? null,
source: input.source,
source_url: input.sourceUrl,
sourceUrl: input.sourceUrl,
branch: input.branch ?? 'main',
state: 'pending',
total_snippets: 0,
total_tokens: 0,
trust_score: 0,
benchmark_score: 0,
totalSnippets: 0,
totalTokens: 0,
trustScore: 0,
benchmarkScore: 0,
stars: null,
github_token: input.githubToken ?? null,
last_indexed_at: null,
created_at: now,
updated_at: now
};
githubToken: input.githubToken ?? null,
lastIndexedAt: null,
createdAt: new Date(now * 1000),
updatedAt: new Date(now * 1000)
});
const entity = RepositoryMapper.toEntity(repo);
this.db
.prepare(
@@ -154,22 +160,22 @@ export class RepositoryService {
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
repo.id,
repo.title,
repo.description,
repo.source,
repo.source_url,
repo.branch,
repo.state,
repo.total_snippets,
repo.total_tokens,
repo.trust_score,
repo.benchmark_score,
repo.stars,
repo.github_token,
repo.last_indexed_at,
repo.created_at,
repo.updated_at
entity.id,
entity.title,
entity.description,
entity.source,
entity.source_url,
entity.branch,
entity.state,
entity.total_snippets,
entity.total_tokens,
entity.trust_score,
entity.benchmark_score,
entity.stars,
entity.github_token,
entity.last_indexed_at,
entity.created_at,
entity.updated_at
);
return this.get(id)!;
@@ -274,24 +280,37 @@ export class RepositoryService {
WHERE repository_id = ? AND status IN ('queued', 'running')
ORDER BY created_at DESC LIMIT 1`
)
.get(repositoryId) as IndexingJob | undefined;
.get(repositoryId) as IndexingJobEntity | undefined;
if (runningJob) return runningJob;
if (runningJob) return IndexingJobMapper.fromEntity(new IndexingJobEntity(runningJob));
const now = Math.floor(Date.now() / 1000);
const job: Record<string, unknown> = {
const job = new IndexingJob({
id: crypto.randomUUID(),
repository_id: repositoryId,
version_id: versionId ?? null,
repositoryId,
versionId: versionId ?? null,
status: 'queued',
progress: 0,
total_files: 0,
processed_files: 0,
totalFiles: 0,
processedFiles: 0,
error: null,
startedAt: null,
completedAt: null,
createdAt: new Date(now * 1000)
});
const entity = new IndexingJobEntity({
id: job.id,
repository_id: job.repositoryId,
version_id: job.versionId,
status: job.status,
progress: job.progress,
total_files: job.totalFiles,
processed_files: job.processedFiles,
error: job.error,
started_at: null,
completed_at: null,
created_at: now
};
});
this.db
.prepare(
@@ -301,21 +320,22 @@ export class RepositoryService {
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
)
.run(
job.id,
job.repository_id,
job.version_id,
job.status,
job.progress,
job.total_files,
job.processed_files,
job.error,
job.started_at,
job.completed_at,
job.created_at
entity.id,
entity.repository_id,
entity.version_id,
entity.status,
entity.progress,
entity.total_files,
entity.processed_files,
entity.error,
entity.started_at,
entity.completed_at,
entity.created_at
);
return this.db
const created = this.db
.prepare(`SELECT * FROM indexing_jobs WHERE id = ?`)
.get(job.id) as IndexingJob;
.get(job.id) as IndexingJobEntity;
return IndexingJobMapper.fromEntity(new IndexingJobEntity(created));
}
}

View File

@@ -115,26 +115,19 @@ describe('VersionService.list()', () => {
describe('VersionService.add()', () => {
it('creates a version with the correct ID format', () => {
const { versionService } = setup();
const version = versionService.add(
'/facebook/react',
'v18.3.0',
'React v18.3.0'
) as unknown as RawVersion;
const version = versionService.add('/facebook/react', 'v18.3.0', 'React v18.3.0');
expect(version.id).toBe('/facebook/react/v18.3.0');
expect(version.repository_id).toBe('/facebook/react');
expect(version.repositoryId).toBe('/facebook/react');
expect(version.tag).toBe('v18.3.0');
expect(version.title).toBe('React v18.3.0');
expect(version.state).toBe('pending');
expect(version.total_snippets).toBe(0);
expect(version.indexed_at).toBeNull();
expect(version.totalSnippets).toBe(0);
expect(version.indexedAt).toBeNull();
});
it('creates a version without a title', () => {
const { versionService } = setup();
const version = versionService.add(
'/facebook/react',
'v18.3.0'
) as unknown as RawVersion;
const version = versionService.add('/facebook/react', 'v18.3.0');
expect(version.title).toBeNull();
});
@@ -154,7 +147,7 @@ describe('VersionService.add()', () => {
// Use a repo name without dots so resolveGitHubId produces a predictable ID.
repoService.add({ source: 'github', sourceUrl: 'https://github.com/vercel/nextjs' });
versionService.add('/facebook/react', 'v18.3.0');
const v = versionService.add('/vercel/nextjs', 'v18.3.0') as unknown as RawVersion;
const v = versionService.add('/vercel/nextjs', 'v18.3.0');
expect(v.id).toBe('/vercel/nextjs/v18.3.0');
});
});
@@ -211,13 +204,11 @@ describe('VersionService.getByTag()', () => {
it('returns the version record when it exists', () => {
const { versionService } = setup();
versionService.add('/facebook/react', 'v18.3.0', 'React v18.3.0');
const version = versionService.getByTag(
'/facebook/react',
'v18.3.0'
) as unknown as RawVersion;
const version = versionService.getByTag('/facebook/react', 'v18.3.0');
expect(version).not.toBeNull();
if (!version) throw new Error('Expected version to exist');
expect(version.tag).toBe('v18.3.0');
expect(version.repository_id).toBe('/facebook/react');
expect(version?.repositoryId).toBe('/facebook/react');
});
});
@@ -266,10 +257,8 @@ describe('VersionService.registerFromConfig()', () => {
versionService.registerFromConfig('/facebook/react', [
{ tag: 'v18.3.0', title: 'React v18.3.0' }
]);
const version = versionService.getByTag(
'/facebook/react',
'v18.3.0'
) as unknown as RawVersion;
const version = versionService.getByTag('/facebook/react', 'v18.3.0');
if (!version) throw new Error('Expected version to exist');
expect(version.state).toBe('pending');
});
});

View File

@@ -6,7 +6,11 @@
*/
import type Database from 'better-sqlite3';
import type { RepositoryVersion } from '$lib/types';
import { RepositoryVersionMapper } from '$lib/server/mappers/repository-version.mapper.js';
import {
RepositoryVersion,
RepositoryVersionEntity
} from '$lib/server/models/repository-version.js';
import { AlreadyExistsError, NotFoundError } from '$lib/server/utils/validation';
export class VersionService {
@@ -16,13 +20,14 @@ export class VersionService {
* List all versions for a repository, newest first.
*/
list(repositoryId: string): RepositoryVersion[] {
return this.db
const rows = this.db
.prepare(
`SELECT * FROM repository_versions
WHERE repository_id = ?
ORDER BY created_at DESC`
)
.all(repositoryId) as RepositoryVersion[];
.all(repositoryId) as RepositoryVersionEntity[];
return rows.map((row) => RepositoryVersionMapper.fromEntity(new RepositoryVersionEntity(row)));
}
/**
@@ -60,9 +65,10 @@ export class VersionService {
)
.run(id, repositoryId, tag, title ?? null, now);
return this.db
const row = this.db
.prepare(`SELECT * FROM repository_versions WHERE id = ?`)
.get(id) as RepositoryVersion;
.get(id) as RepositoryVersionEntity;
return RepositoryVersionMapper.fromEntity(new RepositoryVersionEntity(row));
}
/**
@@ -86,13 +92,12 @@ export class VersionService {
* Returns `null` when not found.
*/
getByTag(repositoryId: string, tag: string): RepositoryVersion | null {
return (
(this.db
.prepare(
`SELECT * FROM repository_versions WHERE repository_id = ? AND tag = ?`
)
.get(repositoryId, tag) as RepositoryVersion | undefined) ?? null
);
const row = this.db
.prepare(
`SELECT * FROM repository_versions WHERE repository_id = ? AND tag = ?`
)
.get(repositoryId, tag) as RepositoryVersionEntity | undefined;
return row ? RepositoryVersionMapper.fromEntity(new RepositoryVersionEntity(row)) : null;
}
/**