refactor: introduce domain model classes and mapper layer
Replace ad-hoc inline row casting (snake_case → camelCase) spread across services, routes, and the indexing pipeline with explicit model classes (Repository, IndexingJob, RepositoryVersion, Snippet, SearchResult) and dedicated mapper classes that own the DB → domain conversion. - Add src/lib/server/models/ with typed model classes for all domain entities - Add src/lib/server/mappers/ with mapper classes per entity - Remove duplicated RawRow interfaces and inline map functions from job-queue, repository.service, indexing.pipeline, and all API routes - Add dtoJsonResponse helper to standardise JSON responses via SvelteKit json() - Add api-contract.integration.test.ts as a regression baseline Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
5
src/lib/server/api/dto-response.ts
Normal file
5
src/lib/server/api/dto-response.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
import { json } from '@sveltejs/kit';
|
||||
|
||||
export function dtoJsonResponse<T>(payload: T, init?: ResponseInit) {
|
||||
return json(payload, init);
|
||||
}
|
||||
@@ -12,16 +12,17 @@ import {
|
||||
formatContextJson,
|
||||
formatContextTxt
|
||||
} from './formatters';
|
||||
import type { LibrarySearchResult } from '$lib/server/search/search.service';
|
||||
import type { SnippetSearchResult } from '$lib/server/search/search.service';
|
||||
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
|
||||
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result';
|
||||
import { Repository } from '$lib/server/models/repository';
|
||||
import { RepositoryVersion } from '$lib/server/models/repository-version';
|
||||
import { Snippet } from '$lib/server/models/snippet';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeRepo(overrides: Partial<Repository> = {}): Repository {
|
||||
return {
|
||||
return new Repository({
|
||||
id: '/facebook/react',
|
||||
title: 'React',
|
||||
description: 'A JavaScript library for building user interfaces',
|
||||
@@ -39,11 +40,11 @@ function makeRepo(overrides: Partial<Repository> = {}): Repository {
|
||||
createdAt: new Date('2024-01-01T00:00:00Z'),
|
||||
updatedAt: new Date('2026-03-22T10:00:00Z'),
|
||||
...overrides
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function makeVersion(tag: string): RepositoryVersion {
|
||||
return {
|
||||
return new RepositoryVersion({
|
||||
id: `/facebook/react/${tag}`,
|
||||
repositoryId: '/facebook/react',
|
||||
tag,
|
||||
@@ -52,11 +53,11 @@ function makeVersion(tag: string): RepositoryVersion {
|
||||
totalSnippets: 100,
|
||||
indexedAt: new Date(),
|
||||
createdAt: new Date()
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function makeSnippet(overrides: Partial<Snippet> = {}): Snippet {
|
||||
return {
|
||||
return new Snippet({
|
||||
id: 'snippet-1',
|
||||
documentId: 'doc-1',
|
||||
repositoryId: '/facebook/react',
|
||||
@@ -69,15 +70,15 @@ function makeSnippet(overrides: Partial<Snippet> = {}): Snippet {
|
||||
tokenCount: 45,
|
||||
createdAt: new Date(),
|
||||
...overrides
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function makeSnippetResult(snippet: Snippet): SnippetSearchResult {
|
||||
return {
|
||||
return new SnippetSearchResult({
|
||||
snippet,
|
||||
score: -1.5,
|
||||
repository: { id: snippet.repositoryId, title: 'React' }
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -109,11 +110,11 @@ describe('mapState', () => {
|
||||
describe('formatLibrarySearchJson', () => {
|
||||
it('returns results array with correct shape', () => {
|
||||
const results: LibrarySearchResult[] = [
|
||||
{
|
||||
new LibrarySearchResult({
|
||||
repository: makeRepo(),
|
||||
versions: [makeVersion('v18.3.0'), makeVersion('v17.0.2')],
|
||||
score: 150
|
||||
}
|
||||
})
|
||||
];
|
||||
|
||||
const response = formatLibrarySearchJson(results);
|
||||
@@ -138,7 +139,7 @@ describe('formatLibrarySearchJson', () => {
|
||||
|
||||
it('maps non-indexed state to initial', () => {
|
||||
const results: LibrarySearchResult[] = [
|
||||
{ repository: makeRepo({ state: 'pending' }), versions: [], score: 0 }
|
||||
new LibrarySearchResult({ repository: makeRepo({ state: 'pending' }), versions: [], score: 0 })
|
||||
];
|
||||
const response = formatLibrarySearchJson(results);
|
||||
expect(response.results[0].state).toBe('initial');
|
||||
@@ -146,7 +147,7 @@ describe('formatLibrarySearchJson', () => {
|
||||
|
||||
it('handles null lastIndexedAt', () => {
|
||||
const results: LibrarySearchResult[] = [
|
||||
{ repository: makeRepo({ lastIndexedAt: null }), versions: [], score: 0 }
|
||||
new LibrarySearchResult({ repository: makeRepo({ lastIndexedAt: null }), versions: [], score: 0 })
|
||||
];
|
||||
const response = formatLibrarySearchJson(results);
|
||||
expect(response.results[0].lastUpdateDate).toBeNull();
|
||||
|
||||
@@ -12,9 +12,23 @@
|
||||
* error → error
|
||||
*/
|
||||
|
||||
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
|
||||
import type { LibrarySearchResult } from '$lib/server/search/search.service';
|
||||
import type { SnippetSearchResult } from '$lib/server/search/search.service';
|
||||
import { ContextResponseMapper } from '$lib/server/mappers/context-response.mapper.js';
|
||||
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
|
||||
import {
|
||||
ContextJsonResponseDto,
|
||||
LibrarySearchJsonResponseDto,
|
||||
LibrarySearchJsonResultDto,
|
||||
CodeSnippetJsonDto,
|
||||
InfoSnippetJsonDto,
|
||||
type SnippetJsonDto
|
||||
} from '$lib/server/models/context-response.js';
|
||||
|
||||
export type LibrarySearchJsonResult = LibrarySearchJsonResultDto;
|
||||
export type LibrarySearchJsonResponse = LibrarySearchJsonResponseDto;
|
||||
export type CodeSnippetJson = CodeSnippetJsonDto;
|
||||
export type InfoSnippetJson = InfoSnippetJsonDto;
|
||||
export type SnippetJson = SnippetJsonDto;
|
||||
export type ContextJsonResponse = ContextJsonResponseDto;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// State mapping
|
||||
@@ -48,92 +62,11 @@ export const CORS_HEADERS = {
|
||||
// /api/v1/libs/search — JSON response shape
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface LibrarySearchJsonResult {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
branch: string | null;
|
||||
lastUpdateDate: string | null;
|
||||
state: Context7State;
|
||||
totalTokens: number | null;
|
||||
totalSnippets: number | null;
|
||||
stars: number | null;
|
||||
trustScore: number | null;
|
||||
benchmarkScore: number | null;
|
||||
versions: string[];
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface LibrarySearchJsonResponse {
|
||||
results: LibrarySearchJsonResult[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert internal LibrarySearchResult[] to the context7-compatible JSON body.
|
||||
*/
|
||||
export function formatLibrarySearchJson(results: LibrarySearchResult[]): LibrarySearchJsonResponse {
|
||||
return {
|
||||
results: results.map(({ repository, versions }) =>
|
||||
formatSingleLibraryJson(repository, versions)
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
export function formatSingleLibraryJson(
|
||||
repository: Repository,
|
||||
versions: RepositoryVersion[]
|
||||
): LibrarySearchJsonResult {
|
||||
return {
|
||||
id: repository.id,
|
||||
title: repository.title,
|
||||
description: repository.description ?? null,
|
||||
branch: repository.branch ?? null,
|
||||
lastUpdateDate: repository.lastIndexedAt ? repository.lastIndexedAt.toISOString() : null,
|
||||
state: mapState(repository.state as TrueRefState),
|
||||
totalTokens: repository.totalTokens ?? null,
|
||||
totalSnippets: repository.totalSnippets ?? null,
|
||||
stars: repository.stars ?? null,
|
||||
trustScore: repository.trustScore ?? null,
|
||||
benchmarkScore: repository.benchmarkScore ?? null,
|
||||
versions: versions.map((v) => v.tag),
|
||||
source: repository.sourceUrl
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /api/v1/context — JSON response shapes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface CodeListItem {
|
||||
language: string;
|
||||
code: string;
|
||||
}
|
||||
|
||||
export interface CodeSnippetJson {
|
||||
type: 'code';
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
language: string | null;
|
||||
codeList: CodeListItem[];
|
||||
id: string;
|
||||
tokenCount: number | null;
|
||||
pageTitle: string | null;
|
||||
}
|
||||
|
||||
export interface InfoSnippetJson {
|
||||
type: 'info';
|
||||
text: string;
|
||||
breadcrumb: string | null;
|
||||
pageId: string;
|
||||
tokenCount: number | null;
|
||||
}
|
||||
|
||||
export type SnippetJson = CodeSnippetJson | InfoSnippetJson;
|
||||
|
||||
export interface ContextJsonResponse {
|
||||
snippets: SnippetJson[];
|
||||
rules: string[];
|
||||
totalTokens: number;
|
||||
export function formatLibrarySearchJson(results: LibrarySearchResult[]): LibrarySearchJsonResponseDto {
|
||||
return ContextResponseMapper.toLibrarySearchJson(results);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,54 +78,8 @@ export interface ContextJsonResponse {
|
||||
export function formatContextJson(
|
||||
snippets: SnippetSearchResult[],
|
||||
rules: string[]
|
||||
): ContextJsonResponse {
|
||||
const mapped: SnippetJson[] = snippets.map(({ snippet }) => {
|
||||
if (snippet.type === 'code') {
|
||||
const codeSnippet: CodeSnippetJson = {
|
||||
type: 'code',
|
||||
title: snippet.title ?? null,
|
||||
description: snippet.breadcrumb ?? null,
|
||||
language: snippet.language ?? null,
|
||||
codeList: [
|
||||
{
|
||||
language: snippet.language ?? '',
|
||||
code: snippet.content
|
||||
}
|
||||
],
|
||||
id: snippet.id,
|
||||
tokenCount: snippet.tokenCount ?? null,
|
||||
pageTitle: extractPageTitle(snippet.breadcrumb)
|
||||
};
|
||||
return codeSnippet;
|
||||
} else {
|
||||
const infoSnippet: InfoSnippetJson = {
|
||||
type: 'info',
|
||||
text: snippet.content,
|
||||
breadcrumb: snippet.breadcrumb ?? null,
|
||||
pageId: snippet.id,
|
||||
tokenCount: snippet.tokenCount ?? null
|
||||
};
|
||||
return infoSnippet;
|
||||
}
|
||||
});
|
||||
|
||||
const totalTokens = snippets.reduce((sum, { snippet }) => sum + (snippet.tokenCount ?? 0), 0);
|
||||
|
||||
return {
|
||||
snippets: mapped,
|
||||
rules,
|
||||
totalTokens
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the top-level page title from a breadcrumb string.
|
||||
* e.g. "Getting Started > Components" → "Getting Started"
|
||||
*/
|
||||
function extractPageTitle(breadcrumb: string | null | undefined): string | null {
|
||||
if (!breadcrumb) return null;
|
||||
const parts = breadcrumb.split('>');
|
||||
return parts[0].trim() || null;
|
||||
): ContextJsonResponseDto {
|
||||
return ContextResponseMapper.toContextJson(snippets, rules);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
71
src/lib/server/mappers/context-response.mapper.ts
Normal file
71
src/lib/server/mappers/context-response.mapper.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import {
|
||||
CodeListItemDto,
|
||||
CodeSnippetJsonDto,
|
||||
ContextJsonResponseDto,
|
||||
InfoSnippetJsonDto,
|
||||
LibrarySearchJsonResponseDto,
|
||||
LibrarySearchJsonResultDto,
|
||||
type SnippetJsonDto
|
||||
} from '$lib/server/models/context-response.js';
|
||||
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
|
||||
|
||||
export class ContextResponseMapper {
|
||||
static toLibrarySearchJson(results: LibrarySearchResult[]): LibrarySearchJsonResponseDto {
|
||||
return new LibrarySearchJsonResponseDto(
|
||||
results.map(
|
||||
({ repository, versions }) =>
|
||||
new LibrarySearchJsonResultDto({
|
||||
id: repository.id,
|
||||
title: repository.title,
|
||||
description: repository.description ?? null,
|
||||
branch: repository.branch ?? null,
|
||||
lastUpdateDate: repository.lastIndexedAt
|
||||
? repository.lastIndexedAt.toISOString()
|
||||
: null,
|
||||
state: repository.state === 'indexed' ? 'finalized' : repository.state === 'error' ? 'error' : 'initial',
|
||||
totalTokens: repository.totalTokens ?? null,
|
||||
totalSnippets: repository.totalSnippets ?? null,
|
||||
stars: repository.stars ?? null,
|
||||
trustScore: repository.trustScore ?? null,
|
||||
benchmarkScore: repository.benchmarkScore ?? null,
|
||||
versions: versions.map((version) => version.tag),
|
||||
source: repository.sourceUrl
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
static toContextJson(snippets: SnippetSearchResult[], rules: string[]): ContextJsonResponseDto {
|
||||
const mapped: SnippetJsonDto[] = snippets.map(({ snippet }) => {
|
||||
if (snippet.type === 'code') {
|
||||
return new CodeSnippetJsonDto({
|
||||
title: snippet.title ?? null,
|
||||
description: snippet.breadcrumb ?? null,
|
||||
language: snippet.language ?? null,
|
||||
codeList: [
|
||||
new CodeListItemDto({
|
||||
language: snippet.language ?? '',
|
||||
code: snippet.content
|
||||
})
|
||||
],
|
||||
id: snippet.id,
|
||||
tokenCount: snippet.tokenCount ?? null,
|
||||
pageTitle: snippet.breadcrumb ? snippet.breadcrumb.split('>')[0].trim() || null : null
|
||||
});
|
||||
}
|
||||
|
||||
return new InfoSnippetJsonDto({
|
||||
text: snippet.content,
|
||||
breadcrumb: snippet.breadcrumb ?? null,
|
||||
pageId: snippet.id,
|
||||
tokenCount: snippet.tokenCount ?? null
|
||||
});
|
||||
});
|
||||
|
||||
return new ContextJsonResponseDto({
|
||||
snippets: mapped,
|
||||
rules,
|
||||
totalTokens: snippets.reduce((sum, result) => sum + (result.snippet.tokenCount ?? 0), 0)
|
||||
});
|
||||
}
|
||||
}
|
||||
36
src/lib/server/mappers/indexing-job.mapper.ts
Normal file
36
src/lib/server/mappers/indexing-job.mapper.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { IndexingJob, IndexingJobDto, IndexingJobEntity } from '$lib/server/models/indexing-job.js';
|
||||
|
||||
export class IndexingJobMapper {
|
||||
static fromEntity(entity: IndexingJobEntity): IndexingJob {
|
||||
return new IndexingJob({
|
||||
id: entity.id,
|
||||
repositoryId: entity.repository_id,
|
||||
versionId: entity.version_id,
|
||||
status: entity.status,
|
||||
progress: entity.progress,
|
||||
totalFiles: entity.total_files,
|
||||
processedFiles: entity.processed_files,
|
||||
error: entity.error,
|
||||
startedAt: entity.started_at != null ? new Date(entity.started_at * 1000) : null,
|
||||
completedAt:
|
||||
entity.completed_at != null ? new Date(entity.completed_at * 1000) : null,
|
||||
createdAt: new Date(entity.created_at * 1000)
|
||||
});
|
||||
}
|
||||
|
||||
static toDto(domain: IndexingJob): IndexingJobDto {
|
||||
return new IndexingJobDto({
|
||||
id: domain.id,
|
||||
repositoryId: domain.repositoryId,
|
||||
versionId: domain.versionId,
|
||||
status: domain.status,
|
||||
progress: domain.progress,
|
||||
totalFiles: domain.totalFiles,
|
||||
processedFiles: domain.processedFiles,
|
||||
error: domain.error,
|
||||
startedAt: domain.startedAt,
|
||||
completedAt: domain.completedAt,
|
||||
createdAt: domain.createdAt
|
||||
});
|
||||
}
|
||||
}
|
||||
33
src/lib/server/mappers/repository-version.mapper.ts
Normal file
33
src/lib/server/mappers/repository-version.mapper.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import {
|
||||
RepositoryVersion,
|
||||
RepositoryVersionDto,
|
||||
RepositoryVersionEntity
|
||||
} from '$lib/server/models/repository-version.js';
|
||||
|
||||
export class RepositoryVersionMapper {
|
||||
static fromEntity(entity: RepositoryVersionEntity): RepositoryVersion {
|
||||
return new RepositoryVersion({
|
||||
id: entity.id,
|
||||
repositoryId: entity.repository_id,
|
||||
tag: entity.tag,
|
||||
title: entity.title,
|
||||
state: entity.state,
|
||||
totalSnippets: entity.total_snippets ?? 0,
|
||||
indexedAt: entity.indexed_at != null ? new Date(entity.indexed_at * 1000) : null,
|
||||
createdAt: new Date(entity.created_at * 1000)
|
||||
});
|
||||
}
|
||||
|
||||
static toDto(domain: RepositoryVersion): RepositoryVersionDto {
|
||||
return new RepositoryVersionDto({
|
||||
id: domain.id,
|
||||
repositoryId: domain.repositoryId,
|
||||
tag: domain.tag,
|
||||
title: domain.title,
|
||||
state: domain.state,
|
||||
totalSnippets: domain.totalSnippets,
|
||||
indexedAt: domain.indexedAt,
|
||||
createdAt: domain.createdAt
|
||||
});
|
||||
}
|
||||
}
|
||||
67
src/lib/server/mappers/repository.mapper.ts
Normal file
67
src/lib/server/mappers/repository.mapper.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import { Repository, RepositoryDto, RepositoryEntity } from '$lib/server/models/repository.js';
|
||||
|
||||
export class RepositoryMapper {
|
||||
static fromEntity(entity: RepositoryEntity): Repository {
|
||||
return new Repository({
|
||||
id: entity.id,
|
||||
title: entity.title,
|
||||
description: entity.description,
|
||||
source: entity.source,
|
||||
sourceUrl: entity.source_url,
|
||||
branch: entity.branch,
|
||||
state: entity.state,
|
||||
totalSnippets: entity.total_snippets ?? 0,
|
||||
totalTokens: entity.total_tokens ?? 0,
|
||||
trustScore: entity.trust_score ?? 0,
|
||||
benchmarkScore: entity.benchmark_score ?? 0,
|
||||
stars: entity.stars,
|
||||
githubToken: entity.github_token,
|
||||
lastIndexedAt:
|
||||
entity.last_indexed_at != null ? new Date(entity.last_indexed_at * 1000) : null,
|
||||
createdAt: new Date(entity.created_at * 1000),
|
||||
updatedAt: new Date(entity.updated_at * 1000)
|
||||
});
|
||||
}
|
||||
|
||||
static toEntity(domain: Repository): RepositoryEntity {
|
||||
return new RepositoryEntity({
|
||||
id: domain.id,
|
||||
title: domain.title,
|
||||
description: domain.description,
|
||||
source: domain.source,
|
||||
source_url: domain.sourceUrl,
|
||||
branch: domain.branch,
|
||||
state: domain.state,
|
||||
total_snippets: domain.totalSnippets,
|
||||
total_tokens: domain.totalTokens,
|
||||
trust_score: domain.trustScore,
|
||||
benchmark_score: domain.benchmarkScore,
|
||||
stars: domain.stars,
|
||||
github_token: domain.githubToken,
|
||||
last_indexed_at:
|
||||
domain.lastIndexedAt != null ? Math.floor(domain.lastIndexedAt.getTime() / 1000) : null,
|
||||
created_at: Math.floor(domain.createdAt.getTime() / 1000),
|
||||
updated_at: Math.floor(domain.updatedAt.getTime() / 1000)
|
||||
});
|
||||
}
|
||||
|
||||
static toDto(domain: Repository): RepositoryDto {
|
||||
return new RepositoryDto({
|
||||
id: domain.id,
|
||||
title: domain.title,
|
||||
description: domain.description,
|
||||
source: domain.source,
|
||||
sourceUrl: domain.sourceUrl,
|
||||
branch: domain.branch,
|
||||
state: domain.state,
|
||||
totalSnippets: domain.totalSnippets,
|
||||
totalTokens: domain.totalTokens,
|
||||
trustScore: domain.trustScore,
|
||||
benchmarkScore: domain.benchmarkScore,
|
||||
stars: domain.stars,
|
||||
lastIndexedAt: domain.lastIndexedAt,
|
||||
createdAt: domain.createdAt,
|
||||
updatedAt: domain.updatedAt
|
||||
});
|
||||
}
|
||||
}
|
||||
35
src/lib/server/mappers/search-result.mapper.ts
Normal file
35
src/lib/server/mappers/search-result.mapper.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { LibrarySearchResult, SnippetRepositoryRef, SnippetSearchResult } from '$lib/server/models/search-result.js';
|
||||
import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js';
|
||||
import { RepositoryVersionMapper } from '$lib/server/mappers/repository-version.mapper.js';
|
||||
import { SnippetMapper } from '$lib/server/mappers/snippet.mapper.js';
|
||||
import { RepositoryEntity } from '$lib/server/models/repository.js';
|
||||
import { RepositoryVersionEntity } from '$lib/server/models/repository-version.js';
|
||||
import { SnippetEntity } from '$lib/server/models/snippet.js';
|
||||
|
||||
export class SearchResultMapper {
|
||||
static snippetFromEntity(
|
||||
entity: SnippetEntity,
|
||||
repository: { id: string; title: string },
|
||||
score: number
|
||||
): SnippetSearchResult {
|
||||
return new SnippetSearchResult({
|
||||
snippet: SnippetMapper.fromEntity(entity),
|
||||
score,
|
||||
repository: new SnippetRepositoryRef(repository)
|
||||
});
|
||||
}
|
||||
|
||||
static libraryFromEntity(
|
||||
repositoryEntity: RepositoryEntity,
|
||||
versionEntities: RepositoryVersionEntity[],
|
||||
score: number
|
||||
): LibrarySearchResult {
|
||||
return new LibrarySearchResult({
|
||||
repository: RepositoryMapper.fromEntity(repositoryEntity),
|
||||
versions: versionEntities.map((version) =>
|
||||
RepositoryVersionMapper.fromEntity(version)
|
||||
),
|
||||
score
|
||||
});
|
||||
}
|
||||
}
|
||||
19
src/lib/server/mappers/snippet.mapper.ts
Normal file
19
src/lib/server/mappers/snippet.mapper.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { Snippet, SnippetEntity } from '$lib/server/models/snippet.js';
|
||||
|
||||
export class SnippetMapper {
|
||||
static fromEntity(entity: SnippetEntity): Snippet {
|
||||
return new Snippet({
|
||||
id: entity.id,
|
||||
documentId: entity.document_id,
|
||||
repositoryId: entity.repository_id,
|
||||
versionId: entity.version_id,
|
||||
type: entity.type,
|
||||
title: entity.title,
|
||||
content: entity.content,
|
||||
language: entity.language,
|
||||
breadcrumb: entity.breadcrumb,
|
||||
tokenCount: entity.token_count,
|
||||
createdAt: new Date(entity.created_at * 1000)
|
||||
});
|
||||
}
|
||||
}
|
||||
99
src/lib/server/models/context-response.ts
Normal file
99
src/lib/server/models/context-response.ts
Normal file
@@ -0,0 +1,99 @@
|
||||
export class LibrarySearchJsonResultDto {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
branch: string | null;
|
||||
lastUpdateDate: string | null;
|
||||
state: 'initial' | 'finalized' | 'error';
|
||||
totalTokens: number | null;
|
||||
totalSnippets: number | null;
|
||||
stars: number | null;
|
||||
trustScore: number | null;
|
||||
benchmarkScore: number | null;
|
||||
versions: string[];
|
||||
source: string;
|
||||
|
||||
constructor(props: LibrarySearchJsonResultDto) {
|
||||
this.id = props.id;
|
||||
this.title = props.title;
|
||||
this.description = props.description;
|
||||
this.branch = props.branch;
|
||||
this.lastUpdateDate = props.lastUpdateDate;
|
||||
this.state = props.state;
|
||||
this.totalTokens = props.totalTokens;
|
||||
this.totalSnippets = props.totalSnippets;
|
||||
this.stars = props.stars;
|
||||
this.trustScore = props.trustScore;
|
||||
this.benchmarkScore = props.benchmarkScore;
|
||||
this.versions = props.versions;
|
||||
this.source = props.source;
|
||||
}
|
||||
}
|
||||
|
||||
export class LibrarySearchJsonResponseDto {
|
||||
results: LibrarySearchJsonResultDto[];
|
||||
|
||||
constructor(results: LibrarySearchJsonResultDto[]) {
|
||||
this.results = results;
|
||||
}
|
||||
}
|
||||
|
||||
export class CodeListItemDto {
|
||||
language: string;
|
||||
code: string;
|
||||
|
||||
constructor(props: CodeListItemDto) {
|
||||
this.language = props.language;
|
||||
this.code = props.code;
|
||||
}
|
||||
}
|
||||
|
||||
export class CodeSnippetJsonDto {
|
||||
type: 'code' = 'code';
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
language: string | null;
|
||||
codeList: CodeListItemDto[];
|
||||
id: string;
|
||||
tokenCount: number | null;
|
||||
pageTitle: string | null;
|
||||
|
||||
constructor(props: Omit<CodeSnippetJsonDto, 'type'>) {
|
||||
this.title = props.title;
|
||||
this.description = props.description;
|
||||
this.language = props.language;
|
||||
this.codeList = props.codeList;
|
||||
this.id = props.id;
|
||||
this.tokenCount = props.tokenCount;
|
||||
this.pageTitle = props.pageTitle;
|
||||
}
|
||||
}
|
||||
|
||||
export class InfoSnippetJsonDto {
|
||||
type: 'info' = 'info';
|
||||
text: string;
|
||||
breadcrumb: string | null;
|
||||
pageId: string;
|
||||
tokenCount: number | null;
|
||||
|
||||
constructor(props: Omit<InfoSnippetJsonDto, 'type'>) {
|
||||
this.text = props.text;
|
||||
this.breadcrumb = props.breadcrumb;
|
||||
this.pageId = props.pageId;
|
||||
this.tokenCount = props.tokenCount;
|
||||
}
|
||||
}
|
||||
|
||||
export type SnippetJsonDto = CodeSnippetJsonDto | InfoSnippetJsonDto;
|
||||
|
||||
export class ContextJsonResponseDto {
|
||||
snippets: SnippetJsonDto[];
|
||||
rules: string[];
|
||||
totalTokens: number;
|
||||
|
||||
constructor(props: ContextJsonResponseDto) {
|
||||
this.snippets = props.snippets;
|
||||
this.rules = props.rules;
|
||||
this.totalTokens = props.totalTokens;
|
||||
}
|
||||
}
|
||||
125
src/lib/server/models/indexing-job.ts
Normal file
125
src/lib/server/models/indexing-job.ts
Normal file
@@ -0,0 +1,125 @@
|
||||
export interface IndexingJobEntityProps {
|
||||
id: string;
|
||||
repository_id: string;
|
||||
version_id: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
total_files: number;
|
||||
processed_files: number;
|
||||
error: string | null;
|
||||
started_at: number | null;
|
||||
completed_at: number | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export class IndexingJobEntity {
|
||||
id: string;
|
||||
repository_id: string;
|
||||
version_id: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
total_files: number;
|
||||
processed_files: number;
|
||||
error: string | null;
|
||||
started_at: number | null;
|
||||
completed_at: number | null;
|
||||
created_at: number;
|
||||
|
||||
constructor(props: IndexingJobEntityProps) {
|
||||
this.id = props.id;
|
||||
this.repository_id = props.repository_id;
|
||||
this.version_id = props.version_id;
|
||||
this.status = props.status;
|
||||
this.progress = props.progress;
|
||||
this.total_files = props.total_files;
|
||||
this.processed_files = props.processed_files;
|
||||
this.error = props.error;
|
||||
this.started_at = props.started_at;
|
||||
this.completed_at = props.completed_at;
|
||||
this.created_at = props.created_at;
|
||||
}
|
||||
}
|
||||
|
||||
export interface IndexingJobProps {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
totalFiles: number;
|
||||
processedFiles: number;
|
||||
error: string | null;
|
||||
startedAt: Date | null;
|
||||
completedAt: Date | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export class IndexingJob {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
totalFiles: number;
|
||||
processedFiles: number;
|
||||
error: string | null;
|
||||
startedAt: Date | null;
|
||||
completedAt: Date | null;
|
||||
createdAt: Date;
|
||||
|
||||
constructor(props: IndexingJobProps) {
|
||||
this.id = props.id;
|
||||
this.repositoryId = props.repositoryId;
|
||||
this.versionId = props.versionId;
|
||||
this.status = props.status;
|
||||
this.progress = props.progress;
|
||||
this.totalFiles = props.totalFiles;
|
||||
this.processedFiles = props.processedFiles;
|
||||
this.error = props.error;
|
||||
this.startedAt = props.startedAt;
|
||||
this.completedAt = props.completedAt;
|
||||
this.createdAt = props.createdAt;
|
||||
}
|
||||
}
|
||||
|
||||
export interface IndexingJobDtoProps {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
totalFiles: number;
|
||||
processedFiles: number;
|
||||
error: string | null;
|
||||
startedAt: Date | null;
|
||||
completedAt: Date | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export class IndexingJobDto {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
totalFiles: number;
|
||||
processedFiles: number;
|
||||
error: string | null;
|
||||
startedAt: Date | null;
|
||||
completedAt: Date | null;
|
||||
createdAt: Date;
|
||||
|
||||
constructor(props: IndexingJobDtoProps) {
|
||||
this.id = props.id;
|
||||
this.repositoryId = props.repositoryId;
|
||||
this.versionId = props.versionId;
|
||||
this.status = props.status;
|
||||
this.progress = props.progress;
|
||||
this.totalFiles = props.totalFiles;
|
||||
this.processedFiles = props.processedFiles;
|
||||
this.error = props.error;
|
||||
this.startedAt = props.startedAt;
|
||||
this.completedAt = props.completedAt;
|
||||
this.createdAt = props.createdAt;
|
||||
}
|
||||
}
|
||||
98
src/lib/server/models/repository-version.ts
Normal file
98
src/lib/server/models/repository-version.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
export interface RepositoryVersionEntityProps {
|
||||
id: string;
|
||||
repository_id: string;
|
||||
tag: string;
|
||||
title: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
total_snippets: number | null;
|
||||
indexed_at: number | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export class RepositoryVersionEntity {
|
||||
id: string;
|
||||
repository_id: string;
|
||||
tag: string;
|
||||
title: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
total_snippets: number | null;
|
||||
indexed_at: number | null;
|
||||
created_at: number;
|
||||
|
||||
constructor(props: RepositoryVersionEntityProps) {
|
||||
this.id = props.id;
|
||||
this.repository_id = props.repository_id;
|
||||
this.tag = props.tag;
|
||||
this.title = props.title;
|
||||
this.state = props.state;
|
||||
this.total_snippets = props.total_snippets;
|
||||
this.indexed_at = props.indexed_at;
|
||||
this.created_at = props.created_at;
|
||||
}
|
||||
}
|
||||
|
||||
export interface RepositoryVersionProps {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
tag: string;
|
||||
title: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
indexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export class RepositoryVersion {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
tag: string;
|
||||
title: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
indexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
|
||||
constructor(props: RepositoryVersionProps) {
|
||||
this.id = props.id;
|
||||
this.repositoryId = props.repositoryId;
|
||||
this.tag = props.tag;
|
||||
this.title = props.title;
|
||||
this.state = props.state;
|
||||
this.totalSnippets = props.totalSnippets;
|
||||
this.indexedAt = props.indexedAt;
|
||||
this.createdAt = props.createdAt;
|
||||
}
|
||||
}
|
||||
|
||||
export interface RepositoryVersionDtoProps {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
tag: string;
|
||||
title: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
indexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export class RepositoryVersionDto {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
tag: string;
|
||||
title: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
indexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
|
||||
constructor(props: RepositoryVersionDtoProps) {
|
||||
this.id = props.id;
|
||||
this.repositoryId = props.repositoryId;
|
||||
this.tag = props.tag;
|
||||
this.title = props.title;
|
||||
this.state = props.state;
|
||||
this.totalSnippets = props.totalSnippets;
|
||||
this.indexedAt = props.indexedAt;
|
||||
this.createdAt = props.createdAt;
|
||||
}
|
||||
}
|
||||
167
src/lib/server/models/repository.ts
Normal file
167
src/lib/server/models/repository.ts
Normal file
@@ -0,0 +1,167 @@
|
||||
export interface RepositoryEntityProps {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
source: 'github' | 'local';
|
||||
source_url: string;
|
||||
branch: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
total_snippets: number | null;
|
||||
total_tokens: number | null;
|
||||
trust_score: number | null;
|
||||
benchmark_score: number | null;
|
||||
stars: number | null;
|
||||
github_token: string | null;
|
||||
last_indexed_at: number | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export class RepositoryEntity {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
source: 'github' | 'local';
|
||||
source_url: string;
|
||||
branch: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
total_snippets: number | null;
|
||||
total_tokens: number | null;
|
||||
trust_score: number | null;
|
||||
benchmark_score: number | null;
|
||||
stars: number | null;
|
||||
github_token: string | null;
|
||||
last_indexed_at: number | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
|
||||
constructor(props: RepositoryEntityProps) {
|
||||
this.id = props.id;
|
||||
this.title = props.title;
|
||||
this.description = props.description;
|
||||
this.source = props.source;
|
||||
this.source_url = props.source_url;
|
||||
this.branch = props.branch;
|
||||
this.state = props.state;
|
||||
this.total_snippets = props.total_snippets;
|
||||
this.total_tokens = props.total_tokens;
|
||||
this.trust_score = props.trust_score;
|
||||
this.benchmark_score = props.benchmark_score;
|
||||
this.stars = props.stars;
|
||||
this.github_token = props.github_token;
|
||||
this.last_indexed_at = props.last_indexed_at;
|
||||
this.created_at = props.created_at;
|
||||
this.updated_at = props.updated_at;
|
||||
}
|
||||
}
|
||||
|
||||
export interface RepositoryProps {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
source: 'github' | 'local';
|
||||
sourceUrl: string;
|
||||
branch: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
totalTokens: number;
|
||||
trustScore: number;
|
||||
benchmarkScore: number;
|
||||
stars: number | null;
|
||||
githubToken: string | null;
|
||||
lastIndexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export class Repository {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
source: 'github' | 'local';
|
||||
sourceUrl: string;
|
||||
branch: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
totalTokens: number;
|
||||
trustScore: number;
|
||||
benchmarkScore: number;
|
||||
stars: number | null;
|
||||
githubToken: string | null;
|
||||
lastIndexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
|
||||
constructor(props: RepositoryProps) {
|
||||
this.id = props.id;
|
||||
this.title = props.title;
|
||||
this.description = props.description;
|
||||
this.source = props.source;
|
||||
this.sourceUrl = props.sourceUrl;
|
||||
this.branch = props.branch;
|
||||
this.state = props.state;
|
||||
this.totalSnippets = props.totalSnippets;
|
||||
this.totalTokens = props.totalTokens;
|
||||
this.trustScore = props.trustScore;
|
||||
this.benchmarkScore = props.benchmarkScore;
|
||||
this.stars = props.stars;
|
||||
this.githubToken = props.githubToken;
|
||||
this.lastIndexedAt = props.lastIndexedAt;
|
||||
this.createdAt = props.createdAt;
|
||||
this.updatedAt = props.updatedAt;
|
||||
}
|
||||
}
|
||||
|
||||
export interface RepositoryDtoProps {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
source: 'github' | 'local';
|
||||
sourceUrl: string;
|
||||
branch: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
totalTokens: number;
|
||||
trustScore: number;
|
||||
benchmarkScore: number;
|
||||
stars: number | null;
|
||||
lastIndexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
export class RepositoryDto {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
source: 'github' | 'local';
|
||||
sourceUrl: string;
|
||||
branch: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
totalSnippets: number;
|
||||
totalTokens: number;
|
||||
trustScore: number;
|
||||
benchmarkScore: number;
|
||||
stars: number | null;
|
||||
lastIndexedAt: Date | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
|
||||
constructor(props: RepositoryDtoProps) {
|
||||
this.id = props.id;
|
||||
this.title = props.title;
|
||||
this.description = props.description;
|
||||
this.source = props.source;
|
||||
this.sourceUrl = props.sourceUrl;
|
||||
this.branch = props.branch;
|
||||
this.state = props.state;
|
||||
this.totalSnippets = props.totalSnippets;
|
||||
this.totalTokens = props.totalTokens;
|
||||
this.trustScore = props.trustScore;
|
||||
this.benchmarkScore = props.benchmarkScore;
|
||||
this.stars = props.stars;
|
||||
this.lastIndexedAt = props.lastIndexedAt;
|
||||
this.createdAt = props.createdAt;
|
||||
this.updatedAt = props.updatedAt;
|
||||
}
|
||||
}
|
||||
54
src/lib/server/models/search-result.ts
Normal file
54
src/lib/server/models/search-result.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { Repository } from '$lib/server/models/repository.js';
|
||||
import { RepositoryVersion } from '$lib/server/models/repository-version.js';
|
||||
import { Snippet } from '$lib/server/models/snippet.js';
|
||||
|
||||
export interface SnippetRepositoryRefProps {
|
||||
id: string;
|
||||
title: string;
|
||||
}
|
||||
|
||||
export class SnippetRepositoryRef {
|
||||
id: string;
|
||||
title: string;
|
||||
|
||||
constructor(props: SnippetRepositoryRefProps) {
|
||||
this.id = props.id;
|
||||
this.title = props.title;
|
||||
}
|
||||
}
|
||||
|
||||
export interface SnippetSearchResultProps {
|
||||
snippet: Snippet;
|
||||
score: number;
|
||||
repository: SnippetRepositoryRef;
|
||||
}
|
||||
|
||||
export class SnippetSearchResult {
|
||||
snippet: Snippet;
|
||||
score: number;
|
||||
repository: SnippetRepositoryRef;
|
||||
|
||||
constructor(props: SnippetSearchResultProps) {
|
||||
this.snippet = props.snippet;
|
||||
this.score = props.score;
|
||||
this.repository = props.repository;
|
||||
}
|
||||
}
|
||||
|
||||
export interface LibrarySearchResultProps {
|
||||
repository: Repository;
|
||||
versions: RepositoryVersion[];
|
||||
score: number;
|
||||
}
|
||||
|
||||
export class LibrarySearchResult {
|
||||
repository: Repository;
|
||||
versions: RepositoryVersion[];
|
||||
score: number;
|
||||
|
||||
constructor(props: LibrarySearchResultProps) {
|
||||
this.repository = props.repository;
|
||||
this.versions = props.versions;
|
||||
this.score = props.score;
|
||||
}
|
||||
}
|
||||
83
src/lib/server/models/snippet.ts
Normal file
83
src/lib/server/models/snippet.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
export interface SnippetEntityProps {
|
||||
id: string;
|
||||
document_id: string;
|
||||
repository_id: string;
|
||||
version_id: string | null;
|
||||
type: 'code' | 'info';
|
||||
title: string | null;
|
||||
content: string;
|
||||
language: string | null;
|
||||
breadcrumb: string | null;
|
||||
token_count: number | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export class SnippetEntity {
|
||||
id: string;
|
||||
document_id: string;
|
||||
repository_id: string;
|
||||
version_id: string | null;
|
||||
type: 'code' | 'info';
|
||||
title: string | null;
|
||||
content: string;
|
||||
language: string | null;
|
||||
breadcrumb: string | null;
|
||||
token_count: number | null;
|
||||
created_at: number;
|
||||
|
||||
constructor(props: SnippetEntityProps) {
|
||||
this.id = props.id;
|
||||
this.document_id = props.document_id;
|
||||
this.repository_id = props.repository_id;
|
||||
this.version_id = props.version_id;
|
||||
this.type = props.type;
|
||||
this.title = props.title;
|
||||
this.content = props.content;
|
||||
this.language = props.language;
|
||||
this.breadcrumb = props.breadcrumb;
|
||||
this.token_count = props.token_count;
|
||||
this.created_at = props.created_at;
|
||||
}
|
||||
}
|
||||
|
||||
export interface SnippetProps {
|
||||
id: string;
|
||||
documentId: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
type: 'code' | 'info';
|
||||
title: string | null;
|
||||
content: string;
|
||||
language: string | null;
|
||||
breadcrumb: string | null;
|
||||
tokenCount: number | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export class Snippet {
|
||||
id: string;
|
||||
documentId: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
type: 'code' | 'info';
|
||||
title: string | null;
|
||||
content: string;
|
||||
language: string | null;
|
||||
breadcrumb: string | null;
|
||||
tokenCount: number | null;
|
||||
createdAt: Date;
|
||||
|
||||
constructor(props: SnippetProps) {
|
||||
this.id = props.id;
|
||||
this.documentId = props.documentId;
|
||||
this.repositoryId = props.repositoryId;
|
||||
this.versionId = props.versionId;
|
||||
this.type = props.type;
|
||||
this.title = props.title;
|
||||
this.content = props.content;
|
||||
this.language = props.language;
|
||||
this.breadcrumb = props.breadcrumb;
|
||||
this.tokenCount = props.tokenCount;
|
||||
this.createdAt = props.createdAt;
|
||||
}
|
||||
}
|
||||
@@ -458,6 +458,31 @@ describe('IndexingPipeline', () => {
|
||||
expect(updated.progress).toBe(100);
|
||||
});
|
||||
|
||||
it('uses the repository source_url when crawling local repositories', async () => {
|
||||
const crawl = vi.fn().mockResolvedValue({
|
||||
files: [],
|
||||
totalFiles: 0,
|
||||
skippedFiles: 0,
|
||||
branch: 'local',
|
||||
commitSha: 'abc'
|
||||
});
|
||||
|
||||
const pipeline = new IndexingPipeline(
|
||||
db,
|
||||
vi.fn() as never,
|
||||
{ crawl } as never,
|
||||
null
|
||||
);
|
||||
|
||||
const job = makeJob();
|
||||
await pipeline.run(job as never);
|
||||
|
||||
expect(crawl).toHaveBeenCalledWith({
|
||||
rootPath: '/tmp/test-repo',
|
||||
ref: undefined
|
||||
});
|
||||
});
|
||||
|
||||
it('integration: handles unchanged, modified, added, and deleted files in one run', async () => {
|
||||
// ---- First run: index three files -----------------------------------
|
||||
const firstFiles = [
|
||||
|
||||
@@ -15,10 +15,13 @@
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { Document, IndexingJob, NewDocument, NewSnippet, Repository } from '$lib/types';
|
||||
import type { Document, NewDocument, NewSnippet } from '$lib/types';
|
||||
import type { crawl as GithubCrawlFn } from '$lib/server/crawler/github.crawler.js';
|
||||
import type { LocalCrawler } from '$lib/server/crawler/local.crawler.js';
|
||||
import type { EmbeddingService } from '$lib/server/embeddings/embedding.service.js';
|
||||
import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js';
|
||||
import { IndexingJob } from '$lib/server/models/indexing-job.js';
|
||||
import { Repository, RepositoryEntity } from '$lib/server/models/repository.js';
|
||||
import { parseFile } from '$lib/server/parser/index.js';
|
||||
import { computeTrustScore } from '$lib/server/search/trust-score.js';
|
||||
import { computeDiff } from './diff.js';
|
||||
@@ -399,11 +402,10 @@ export class IndexingPipeline {
|
||||
}
|
||||
|
||||
private getRepository(id: string): Repository | null {
|
||||
return (
|
||||
(this.db
|
||||
.prepare<[string], Repository>(`SELECT * FROM repositories WHERE id = ?`)
|
||||
.get(id) as Repository | undefined) ?? null
|
||||
);
|
||||
const raw = this.db
|
||||
.prepare<[string], RepositoryEntity>(`SELECT * FROM repositories WHERE id = ?`)
|
||||
.get(id);
|
||||
return raw ? RepositoryMapper.fromEntity(new RepositoryEntity(raw)) : null;
|
||||
}
|
||||
|
||||
private updateJob(id: string, fields: Record<string, unknown>): void {
|
||||
|
||||
@@ -7,48 +7,15 @@
|
||||
*/
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { IndexingJob, NewIndexingJob } from '$lib/types';
|
||||
import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js';
|
||||
import { IndexingJob, IndexingJobEntity } from '$lib/server/models/indexing-job.js';
|
||||
import type { IndexingPipeline } from './indexing.pipeline.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SQL projection + row mapper (mirrors repository.service.ts pattern)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const JOB_SELECT = `
|
||||
SELECT id,
|
||||
repository_id AS repositoryId,
|
||||
version_id AS versionId,
|
||||
status, progress,
|
||||
total_files AS totalFiles,
|
||||
processed_files AS processedFiles,
|
||||
error,
|
||||
started_at AS startedAt,
|
||||
completed_at AS completedAt,
|
||||
created_at AS createdAt
|
||||
FROM indexing_jobs`;
|
||||
|
||||
interface RawJob {
|
||||
id: string;
|
||||
repositoryId: string;
|
||||
versionId: string | null;
|
||||
status: 'queued' | 'running' | 'done' | 'failed';
|
||||
progress: number;
|
||||
totalFiles: number;
|
||||
processedFiles: number;
|
||||
error: string | null;
|
||||
startedAt: number | null;
|
||||
completedAt: number | null;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
function mapJob(raw: RawJob): IndexingJob {
|
||||
return {
|
||||
...raw,
|
||||
startedAt: raw.startedAt != null ? new Date(raw.startedAt * 1000) : null,
|
||||
completedAt: raw.completedAt != null ? new Date(raw.completedAt * 1000) : null,
|
||||
createdAt: new Date(raw.createdAt * 1000)
|
||||
};
|
||||
}
|
||||
const JOB_SELECT = `SELECT * FROM indexing_jobs`;
|
||||
|
||||
export class JobQueue {
|
||||
private isRunning = false;
|
||||
@@ -71,7 +38,7 @@ export class JobQueue {
|
||||
enqueue(repositoryId: string, versionId?: string): IndexingJob {
|
||||
// Return early if there's already an active job for this repo.
|
||||
const activeRaw = this.db
|
||||
.prepare<[string], RawJob>(
|
||||
.prepare<[string], IndexingJobEntity>(
|
||||
`${JOB_SELECT}
|
||||
WHERE repository_id = ? AND status IN ('queued', 'running')
|
||||
ORDER BY created_at DESC LIMIT 1`
|
||||
@@ -81,11 +48,11 @@ export class JobQueue {
|
||||
if (activeRaw) {
|
||||
// Ensure the queue is draining even if enqueue was called concurrently.
|
||||
if (!this.isRunning) setImmediate(() => this.processNext());
|
||||
return mapJob(activeRaw);
|
||||
return IndexingJobMapper.fromEntity(new IndexingJobEntity(activeRaw));
|
||||
}
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const job: NewIndexingJob = {
|
||||
const job = new IndexingJob({
|
||||
id: crypto.randomUUID(),
|
||||
repositoryId,
|
||||
versionId: versionId ?? null,
|
||||
@@ -97,7 +64,7 @@ export class JobQueue {
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
createdAt: new Date(now * 1000)
|
||||
};
|
||||
});
|
||||
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -125,9 +92,10 @@ export class JobQueue {
|
||||
setImmediate(() => this.processNext());
|
||||
}
|
||||
|
||||
return mapJob(
|
||||
this.db.prepare<[string], RawJob>(`${JOB_SELECT} WHERE id = ?`).get(job.id as string)!
|
||||
);
|
||||
const created = this.db
|
||||
.prepare<[string], IndexingJobEntity>(`${JOB_SELECT} WHERE id = ?`)
|
||||
.get(job.id as string)!;
|
||||
return IndexingJobMapper.fromEntity(new IndexingJobEntity(created));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -142,7 +110,7 @@ export class JobQueue {
|
||||
}
|
||||
|
||||
const rawJob = this.db
|
||||
.prepare<[], RawJob>(
|
||||
.prepare<[], IndexingJobEntity>(
|
||||
`${JOB_SELECT}
|
||||
WHERE status = 'queued'
|
||||
ORDER BY created_at ASC LIMIT 1`
|
||||
@@ -151,7 +119,7 @@ export class JobQueue {
|
||||
|
||||
if (!rawJob) return;
|
||||
|
||||
const job = mapJob(rawJob);
|
||||
const job = IndexingJobMapper.fromEntity(new IndexingJobEntity(rawJob));
|
||||
this.isRunning = true;
|
||||
try {
|
||||
await this.pipeline.run(job);
|
||||
@@ -180,9 +148,9 @@ export class JobQueue {
|
||||
*/
|
||||
getJob(id: string): IndexingJob | null {
|
||||
const raw = this.db
|
||||
.prepare<[string], RawJob>(`${JOB_SELECT} WHERE id = ?`)
|
||||
.prepare<[string], IndexingJobEntity>(`${JOB_SELECT} WHERE id = ?`)
|
||||
.get(id);
|
||||
return raw ? mapJob(raw) : null;
|
||||
return raw ? IndexingJobMapper.fromEntity(new IndexingJobEntity(raw)) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -210,7 +178,9 @@ export class JobQueue {
|
||||
const sql = `${JOB_SELECT} ${where} ORDER BY created_at DESC LIMIT ?`;
|
||||
params.push(limit);
|
||||
|
||||
return (this.db.prepare<unknown[], RawJob>(sql).all(...params) as RawJob[]).map(mapJob);
|
||||
return (this.db.prepare<unknown[], IndexingJobEntity>(sql).all(...params) as IndexingJobEntity[]).map(
|
||||
(row) => IndexingJobMapper.fromEntity(new IndexingJobEntity(row))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* responses and MCP tool outputs.
|
||||
*/
|
||||
|
||||
import type { LibrarySearchResult, SnippetSearchResult } from './search.service';
|
||||
import type { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Library search formatter (`resolve-library-id`)
|
||||
@@ -22,7 +22,7 @@ export function formatLibraryResults(results: LibrarySearchResult[]): string {
|
||||
return results
|
||||
.map((r, i) => {
|
||||
const repo = r.repository;
|
||||
const versions = r.versions.map((v) => v.tag).join(', ') || 'default branch';
|
||||
const versions = r.versions.map((version) => version.tag).join(', ') || 'default branch';
|
||||
return [
|
||||
`${i + 1}. ${repo.title}`,
|
||||
` Library ID: ${repo.id}`,
|
||||
|
||||
@@ -13,11 +13,12 @@
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { EmbeddingProvider } from '../embeddings/provider.js';
|
||||
import type { SnippetSearchResult } from './search.service.js';
|
||||
import { SnippetSearchResult, SnippetRepositoryRef } from '$lib/server/models/search-result.js';
|
||||
import { SnippetEntity } from '$lib/server/models/snippet.js';
|
||||
import { SearchResultMapper } from '$lib/server/mappers/search-result.mapper.js';
|
||||
import { SearchService } from './search.service.js';
|
||||
import { VectorSearch } from './vector.search.js';
|
||||
import { reciprocalRankFusion } from './rrf.js';
|
||||
import type { Snippet } from '$lib/types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public interfaces
|
||||
@@ -54,18 +55,7 @@ export interface SearchConfig {
|
||||
// Raw DB row used when re-fetching snippets by ID
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface RawSnippetById {
|
||||
id: string;
|
||||
document_id: string;
|
||||
repository_id: string;
|
||||
version_id: string | null;
|
||||
type: 'code' | 'info';
|
||||
title: string | null;
|
||||
content: string;
|
||||
language: string | null;
|
||||
breadcrumb: string | null;
|
||||
token_count: number | null;
|
||||
created_at: number;
|
||||
interface RawSnippetById extends SnippetEntity {
|
||||
repo_id: string;
|
||||
repo_title: string;
|
||||
}
|
||||
@@ -200,25 +190,17 @@ export class HybridSearchService {
|
||||
const row = byId.get(id);
|
||||
if (!row) continue;
|
||||
|
||||
const snippet: Snippet = {
|
||||
id: row.id,
|
||||
documentId: row.document_id,
|
||||
repositoryId: row.repository_id,
|
||||
versionId: row.version_id,
|
||||
type: row.type,
|
||||
title: row.title,
|
||||
content: row.content,
|
||||
language: row.language,
|
||||
breadcrumb: row.breadcrumb,
|
||||
tokenCount: row.token_count,
|
||||
createdAt: new Date(row.created_at * 1000)
|
||||
};
|
||||
|
||||
results.push({
|
||||
snippet,
|
||||
score: 0, // RRF score not mapped to BM25 scale; consumers use rank position.
|
||||
repository: { id: row.repo_id, title: row.repo_title }
|
||||
});
|
||||
results.push(
|
||||
new SnippetSearchResult({
|
||||
snippet: SearchResultMapper.snippetFromEntity(
|
||||
new SnippetEntity(row),
|
||||
new SnippetRepositoryRef({ id: row.repo_id, title: row.repo_title }),
|
||||
0
|
||||
).snippet,
|
||||
score: 0,
|
||||
repository: new SnippetRepositoryRef({ id: row.repo_id, title: row.repo_title })
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
return results;
|
||||
|
||||
@@ -8,9 +8,15 @@
|
||||
*/
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
|
||||
import { RepositoryEntity } from '$lib/server/models/repository.js';
|
||||
import { RepositoryVersionEntity } from '$lib/server/models/repository-version.js';
|
||||
import { SnippetEntity } from '$lib/server/models/snippet.js';
|
||||
import { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
|
||||
import { SearchResultMapper } from '$lib/server/mappers/search-result.mapper.js';
|
||||
import { preprocessQuery } from './query-preprocessor';
|
||||
|
||||
export { LibrarySearchResult, SnippetSearchResult } from '$lib/server/models/search-result.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public interface types
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -25,13 +31,6 @@ export interface SnippetSearchOptions {
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export interface SnippetSearchResult {
|
||||
snippet: Snippet;
|
||||
/** BM25 rank — negative value; lower (more negative) = more relevant. */
|
||||
score: number;
|
||||
repository: Pick<Repository, 'id' | 'title'>;
|
||||
}
|
||||
|
||||
export interface LibrarySearchOptions {
|
||||
libraryName: string;
|
||||
/** Semantic relevance hint (reserved for future hybrid use). */
|
||||
@@ -40,53 +39,19 @@ export interface LibrarySearchOptions {
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface LibrarySearchResult {
|
||||
repository: Repository;
|
||||
versions: RepositoryVersion[];
|
||||
/** Composite relevance score. Higher = more relevant. */
|
||||
score: number;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Raw DB row types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Raw row returned by the snippet FTS query (snake_case column names). */
|
||||
interface RawSnippetRow {
|
||||
id: string;
|
||||
document_id: string;
|
||||
repository_id: string;
|
||||
version_id: string | null;
|
||||
type: 'code' | 'info';
|
||||
title: string | null;
|
||||
content: string;
|
||||
language: string | null;
|
||||
breadcrumb: string | null;
|
||||
token_count: number | null;
|
||||
created_at: number;
|
||||
interface RawSnippetRow extends SnippetEntity {
|
||||
repo_id: string;
|
||||
repo_title: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
/** Raw row returned by the library search query. */
|
||||
interface RawRepoRow {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
source: 'github' | 'local';
|
||||
source_url: string;
|
||||
branch: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
total_snippets: number | null;
|
||||
total_tokens: number | null;
|
||||
trust_score: number | null;
|
||||
benchmark_score: number | null;
|
||||
stars: number | null;
|
||||
github_token: string | null;
|
||||
last_indexed_at: number | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
interface RawRepoRow extends RepositoryEntity {
|
||||
exact_match: number;
|
||||
prefix_match: number;
|
||||
desc_match: number;
|
||||
@@ -95,70 +60,7 @@ interface RawRepoRow {
|
||||
}
|
||||
|
||||
/** Raw row returned by the version query. */
|
||||
interface RawVersionRow {
|
||||
id: string;
|
||||
repository_id: string;
|
||||
tag: string;
|
||||
title: string | null;
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
total_snippets: number | null;
|
||||
indexed_at: number | null;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mappers: raw DB rows → domain types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function mapSnippet(row: RawSnippetRow): Snippet {
|
||||
return {
|
||||
id: row.id,
|
||||
documentId: row.document_id,
|
||||
repositoryId: row.repository_id,
|
||||
versionId: row.version_id,
|
||||
type: row.type,
|
||||
title: row.title,
|
||||
content: row.content,
|
||||
language: row.language,
|
||||
breadcrumb: row.breadcrumb,
|
||||
tokenCount: row.token_count,
|
||||
createdAt: new Date(row.created_at * 1000)
|
||||
};
|
||||
}
|
||||
|
||||
function mapRepository(row: RawRepoRow): Repository {
|
||||
return {
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
description: row.description,
|
||||
source: row.source,
|
||||
sourceUrl: row.source_url,
|
||||
branch: row.branch,
|
||||
state: row.state,
|
||||
totalSnippets: row.total_snippets,
|
||||
totalTokens: row.total_tokens,
|
||||
trustScore: row.trust_score,
|
||||
benchmarkScore: row.benchmark_score,
|
||||
stars: row.stars,
|
||||
githubToken: row.github_token,
|
||||
lastIndexedAt: row.last_indexed_at ? new Date(row.last_indexed_at * 1000) : null,
|
||||
createdAt: new Date(row.created_at * 1000),
|
||||
updatedAt: new Date(row.updated_at * 1000)
|
||||
};
|
||||
}
|
||||
|
||||
function mapVersion(row: RawVersionRow): RepositoryVersion {
|
||||
return {
|
||||
id: row.id,
|
||||
repositoryId: row.repository_id,
|
||||
tag: row.tag,
|
||||
title: row.title,
|
||||
state: row.state,
|
||||
totalSnippets: row.total_snippets,
|
||||
indexedAt: row.indexed_at ? new Date(row.indexed_at * 1000) : null,
|
||||
createdAt: new Date(row.created_at * 1000)
|
||||
};
|
||||
}
|
||||
type RawVersionRow = RepositoryVersionEntity;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SearchService
|
||||
@@ -229,11 +131,12 @@ export class SearchService {
|
||||
|
||||
const rows = this.db.prepare(sql).all(...params) as RawSnippetRow[];
|
||||
|
||||
return rows.map((row) => ({
|
||||
snippet: mapSnippet(row),
|
||||
score: row.score,
|
||||
repository: { id: row.repo_id, title: row.repo_title }
|
||||
}));
|
||||
return rows.map((row) =>
|
||||
SearchResultMapper.snippetFromEntity(new SnippetEntity(row), {
|
||||
id: row.repo_id,
|
||||
title: row.repo_title
|
||||
}, row.score)
|
||||
);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -284,14 +187,13 @@ export class SearchService {
|
||||
) as RawRepoRow[];
|
||||
|
||||
return rows.map((row) => {
|
||||
const repository = mapRepository(row);
|
||||
const compositeScore =
|
||||
row.exact_match + row.prefix_match + row.desc_match + row.snippet_score + row.trust_component;
|
||||
return {
|
||||
repository,
|
||||
versions: this.getVersions(row.id),
|
||||
score: compositeScore
|
||||
};
|
||||
return SearchResultMapper.libraryFromEntity(
|
||||
new RepositoryEntity(row),
|
||||
this.getVersionEntities(row.id),
|
||||
compositeScore
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -299,12 +201,11 @@ export class SearchService {
|
||||
// Private helpers
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private getVersions(repositoryId: string): RepositoryVersion[] {
|
||||
const rows = this.db
|
||||
private getVersionEntities(repositoryId: string): RepositoryVersionEntity[] {
|
||||
return this.db
|
||||
.prepare(
|
||||
`SELECT * FROM repository_versions WHERE repository_id = ? ORDER BY created_at DESC`
|
||||
)
|
||||
.all(repositoryId) as RawVersionRow[];
|
||||
return rows.map(mapVersion);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -281,9 +281,9 @@ describe('RepositoryService.add()', () => {
|
||||
const repo = service.add({
|
||||
source: 'github',
|
||||
sourceUrl: 'https://github.com/facebook/react'
|
||||
}) as unknown as RawRepo;
|
||||
expect(repo.total_snippets).toBe(0);
|
||||
expect(repo.total_tokens).toBe(0);
|
||||
});
|
||||
expect(repo.totalSnippets).toBe(0);
|
||||
expect(repo.totalTokens).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -443,9 +443,9 @@ describe('RepositoryService.createIndexingJob()', () => {
|
||||
});
|
||||
|
||||
it('creates a queued indexing job', () => {
|
||||
const job = service.createIndexingJob('/facebook/react') as unknown as RawJob;
|
||||
const job = service.createIndexingJob('/facebook/react');
|
||||
expect(job.id).toBeTruthy();
|
||||
expect(job.repository_id).toBe('/facebook/react');
|
||||
expect(job.repositoryId).toBe('/facebook/react');
|
||||
expect(job.status).toBe('queued');
|
||||
expect(job.progress).toBe(0);
|
||||
});
|
||||
@@ -467,10 +467,7 @@ describe('RepositoryService.createIndexingJob()', () => {
|
||||
});
|
||||
|
||||
it('accepts an optional versionId', () => {
|
||||
const job = service.createIndexingJob(
|
||||
'/facebook/react',
|
||||
'/facebook/react/v18.3.0'
|
||||
) as unknown as RawJob;
|
||||
expect(job.version_id).toBe('/facebook/react/v18.3.0');
|
||||
const job = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
|
||||
expect(job.versionId).toBe('/facebook/react/v18.3.0');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,7 +4,10 @@
|
||||
*/
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { Repository, NewRepository, IndexingJob, NewIndexingJob } from '$lib/types';
|
||||
import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js';
|
||||
import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js';
|
||||
import { Repository, RepositoryEntity } from '$lib/server/models/repository.js';
|
||||
import { IndexingJob, IndexingJobEntity } from '$lib/server/models/indexing-job.js';
|
||||
import { resolveGitHubId, resolveLocalId } from '$lib/server/utils/id-resolver';
|
||||
import {
|
||||
AlreadyExistsError,
|
||||
@@ -47,16 +50,18 @@ export class RepositoryService {
|
||||
const offset = options?.offset ?? 0;
|
||||
|
||||
if (options?.state) {
|
||||
return this.db
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT * FROM repositories WHERE state = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`
|
||||
)
|
||||
.all(options.state, limit, offset) as Repository[];
|
||||
.all(options.state, limit, offset) as RepositoryEntity[];
|
||||
return rows.map((row) => RepositoryMapper.fromEntity(new RepositoryEntity(row)));
|
||||
}
|
||||
|
||||
return this.db
|
||||
const rows = this.db
|
||||
.prepare(`SELECT * FROM repositories ORDER BY created_at DESC LIMIT ? OFFSET ?`)
|
||||
.all(limit, offset) as Repository[];
|
||||
.all(limit, offset) as RepositoryEntity[];
|
||||
return rows.map((row) => RepositoryMapper.fromEntity(new RepositoryEntity(row)));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -79,10 +84,10 @@ export class RepositoryService {
|
||||
* Get a single repository by ID.
|
||||
*/
|
||||
get(id: string): Repository | null {
|
||||
return (
|
||||
(this.db.prepare(`SELECT * FROM repositories WHERE id = ?`).get(id) as Repository | undefined) ??
|
||||
null
|
||||
);
|
||||
const row = this.db.prepare(`SELECT * FROM repositories WHERE id = ?`).get(id) as
|
||||
| RepositoryEntity
|
||||
| undefined;
|
||||
return row ? RepositoryMapper.fromEntity(new RepositoryEntity(row)) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -126,24 +131,25 @@ export class RepositoryService {
|
||||
}
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const repo: Record<string, unknown> = {
|
||||
const repo = new Repository({
|
||||
id,
|
||||
title,
|
||||
description: input.description ?? null,
|
||||
source: input.source,
|
||||
source_url: input.sourceUrl,
|
||||
sourceUrl: input.sourceUrl,
|
||||
branch: input.branch ?? 'main',
|
||||
state: 'pending',
|
||||
total_snippets: 0,
|
||||
total_tokens: 0,
|
||||
trust_score: 0,
|
||||
benchmark_score: 0,
|
||||
totalSnippets: 0,
|
||||
totalTokens: 0,
|
||||
trustScore: 0,
|
||||
benchmarkScore: 0,
|
||||
stars: null,
|
||||
github_token: input.githubToken ?? null,
|
||||
last_indexed_at: null,
|
||||
created_at: now,
|
||||
updated_at: now
|
||||
};
|
||||
githubToken: input.githubToken ?? null,
|
||||
lastIndexedAt: null,
|
||||
createdAt: new Date(now * 1000),
|
||||
updatedAt: new Date(now * 1000)
|
||||
});
|
||||
const entity = RepositoryMapper.toEntity(repo);
|
||||
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -154,22 +160,22 @@ export class RepositoryService {
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(
|
||||
repo.id,
|
||||
repo.title,
|
||||
repo.description,
|
||||
repo.source,
|
||||
repo.source_url,
|
||||
repo.branch,
|
||||
repo.state,
|
||||
repo.total_snippets,
|
||||
repo.total_tokens,
|
||||
repo.trust_score,
|
||||
repo.benchmark_score,
|
||||
repo.stars,
|
||||
repo.github_token,
|
||||
repo.last_indexed_at,
|
||||
repo.created_at,
|
||||
repo.updated_at
|
||||
entity.id,
|
||||
entity.title,
|
||||
entity.description,
|
||||
entity.source,
|
||||
entity.source_url,
|
||||
entity.branch,
|
||||
entity.state,
|
||||
entity.total_snippets,
|
||||
entity.total_tokens,
|
||||
entity.trust_score,
|
||||
entity.benchmark_score,
|
||||
entity.stars,
|
||||
entity.github_token,
|
||||
entity.last_indexed_at,
|
||||
entity.created_at,
|
||||
entity.updated_at
|
||||
);
|
||||
|
||||
return this.get(id)!;
|
||||
@@ -274,24 +280,37 @@ export class RepositoryService {
|
||||
WHERE repository_id = ? AND status IN ('queued', 'running')
|
||||
ORDER BY created_at DESC LIMIT 1`
|
||||
)
|
||||
.get(repositoryId) as IndexingJob | undefined;
|
||||
.get(repositoryId) as IndexingJobEntity | undefined;
|
||||
|
||||
if (runningJob) return runningJob;
|
||||
if (runningJob) return IndexingJobMapper.fromEntity(new IndexingJobEntity(runningJob));
|
||||
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const job: Record<string, unknown> = {
|
||||
const job = new IndexingJob({
|
||||
id: crypto.randomUUID(),
|
||||
repository_id: repositoryId,
|
||||
version_id: versionId ?? null,
|
||||
repositoryId,
|
||||
versionId: versionId ?? null,
|
||||
status: 'queued',
|
||||
progress: 0,
|
||||
total_files: 0,
|
||||
processed_files: 0,
|
||||
totalFiles: 0,
|
||||
processedFiles: 0,
|
||||
error: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
createdAt: new Date(now * 1000)
|
||||
});
|
||||
const entity = new IndexingJobEntity({
|
||||
id: job.id,
|
||||
repository_id: job.repositoryId,
|
||||
version_id: job.versionId,
|
||||
status: job.status,
|
||||
progress: job.progress,
|
||||
total_files: job.totalFiles,
|
||||
processed_files: job.processedFiles,
|
||||
error: job.error,
|
||||
started_at: null,
|
||||
completed_at: null,
|
||||
created_at: now
|
||||
};
|
||||
});
|
||||
|
||||
this.db
|
||||
.prepare(
|
||||
@@ -301,21 +320,22 @@ export class RepositoryService {
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(
|
||||
job.id,
|
||||
job.repository_id,
|
||||
job.version_id,
|
||||
job.status,
|
||||
job.progress,
|
||||
job.total_files,
|
||||
job.processed_files,
|
||||
job.error,
|
||||
job.started_at,
|
||||
job.completed_at,
|
||||
job.created_at
|
||||
entity.id,
|
||||
entity.repository_id,
|
||||
entity.version_id,
|
||||
entity.status,
|
||||
entity.progress,
|
||||
entity.total_files,
|
||||
entity.processed_files,
|
||||
entity.error,
|
||||
entity.started_at,
|
||||
entity.completed_at,
|
||||
entity.created_at
|
||||
);
|
||||
|
||||
return this.db
|
||||
const created = this.db
|
||||
.prepare(`SELECT * FROM indexing_jobs WHERE id = ?`)
|
||||
.get(job.id) as IndexingJob;
|
||||
.get(job.id) as IndexingJobEntity;
|
||||
return IndexingJobMapper.fromEntity(new IndexingJobEntity(created));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,26 +115,19 @@ describe('VersionService.list()', () => {
|
||||
describe('VersionService.add()', () => {
|
||||
it('creates a version with the correct ID format', () => {
|
||||
const { versionService } = setup();
|
||||
const version = versionService.add(
|
||||
'/facebook/react',
|
||||
'v18.3.0',
|
||||
'React v18.3.0'
|
||||
) as unknown as RawVersion;
|
||||
const version = versionService.add('/facebook/react', 'v18.3.0', 'React v18.3.0');
|
||||
expect(version.id).toBe('/facebook/react/v18.3.0');
|
||||
expect(version.repository_id).toBe('/facebook/react');
|
||||
expect(version.repositoryId).toBe('/facebook/react');
|
||||
expect(version.tag).toBe('v18.3.0');
|
||||
expect(version.title).toBe('React v18.3.0');
|
||||
expect(version.state).toBe('pending');
|
||||
expect(version.total_snippets).toBe(0);
|
||||
expect(version.indexed_at).toBeNull();
|
||||
expect(version.totalSnippets).toBe(0);
|
||||
expect(version.indexedAt).toBeNull();
|
||||
});
|
||||
|
||||
it('creates a version without a title', () => {
|
||||
const { versionService } = setup();
|
||||
const version = versionService.add(
|
||||
'/facebook/react',
|
||||
'v18.3.0'
|
||||
) as unknown as RawVersion;
|
||||
const version = versionService.add('/facebook/react', 'v18.3.0');
|
||||
expect(version.title).toBeNull();
|
||||
});
|
||||
|
||||
@@ -154,7 +147,7 @@ describe('VersionService.add()', () => {
|
||||
// Use a repo name without dots so resolveGitHubId produces a predictable ID.
|
||||
repoService.add({ source: 'github', sourceUrl: 'https://github.com/vercel/nextjs' });
|
||||
versionService.add('/facebook/react', 'v18.3.0');
|
||||
const v = versionService.add('/vercel/nextjs', 'v18.3.0') as unknown as RawVersion;
|
||||
const v = versionService.add('/vercel/nextjs', 'v18.3.0');
|
||||
expect(v.id).toBe('/vercel/nextjs/v18.3.0');
|
||||
});
|
||||
});
|
||||
@@ -211,13 +204,11 @@ describe('VersionService.getByTag()', () => {
|
||||
it('returns the version record when it exists', () => {
|
||||
const { versionService } = setup();
|
||||
versionService.add('/facebook/react', 'v18.3.0', 'React v18.3.0');
|
||||
const version = versionService.getByTag(
|
||||
'/facebook/react',
|
||||
'v18.3.0'
|
||||
) as unknown as RawVersion;
|
||||
const version = versionService.getByTag('/facebook/react', 'v18.3.0');
|
||||
expect(version).not.toBeNull();
|
||||
if (!version) throw new Error('Expected version to exist');
|
||||
expect(version.tag).toBe('v18.3.0');
|
||||
expect(version.repository_id).toBe('/facebook/react');
|
||||
expect(version?.repositoryId).toBe('/facebook/react');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -266,10 +257,8 @@ describe('VersionService.registerFromConfig()', () => {
|
||||
versionService.registerFromConfig('/facebook/react', [
|
||||
{ tag: 'v18.3.0', title: 'React v18.3.0' }
|
||||
]);
|
||||
const version = versionService.getByTag(
|
||||
'/facebook/react',
|
||||
'v18.3.0'
|
||||
) as unknown as RawVersion;
|
||||
const version = versionService.getByTag('/facebook/react', 'v18.3.0');
|
||||
if (!version) throw new Error('Expected version to exist');
|
||||
expect(version.state).toBe('pending');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,7 +6,11 @@
|
||||
*/
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { RepositoryVersion } from '$lib/types';
|
||||
import { RepositoryVersionMapper } from '$lib/server/mappers/repository-version.mapper.js';
|
||||
import {
|
||||
RepositoryVersion,
|
||||
RepositoryVersionEntity
|
||||
} from '$lib/server/models/repository-version.js';
|
||||
import { AlreadyExistsError, NotFoundError } from '$lib/server/utils/validation';
|
||||
|
||||
export class VersionService {
|
||||
@@ -16,13 +20,14 @@ export class VersionService {
|
||||
* List all versions for a repository, newest first.
|
||||
*/
|
||||
list(repositoryId: string): RepositoryVersion[] {
|
||||
return this.db
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT * FROM repository_versions
|
||||
WHERE repository_id = ?
|
||||
ORDER BY created_at DESC`
|
||||
)
|
||||
.all(repositoryId) as RepositoryVersion[];
|
||||
.all(repositoryId) as RepositoryVersionEntity[];
|
||||
return rows.map((row) => RepositoryVersionMapper.fromEntity(new RepositoryVersionEntity(row)));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -60,9 +65,10 @@ export class VersionService {
|
||||
)
|
||||
.run(id, repositoryId, tag, title ?? null, now);
|
||||
|
||||
return this.db
|
||||
const row = this.db
|
||||
.prepare(`SELECT * FROM repository_versions WHERE id = ?`)
|
||||
.get(id) as RepositoryVersion;
|
||||
.get(id) as RepositoryVersionEntity;
|
||||
return RepositoryVersionMapper.fromEntity(new RepositoryVersionEntity(row));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -86,13 +92,12 @@ export class VersionService {
|
||||
* Returns `null` when not found.
|
||||
*/
|
||||
getByTag(repositoryId: string, tag: string): RepositoryVersion | null {
|
||||
return (
|
||||
(this.db
|
||||
.prepare(
|
||||
`SELECT * FROM repository_versions WHERE repository_id = ? AND tag = ?`
|
||||
)
|
||||
.get(repositoryId, tag) as RepositoryVersion | undefined) ?? null
|
||||
);
|
||||
const row = this.db
|
||||
.prepare(
|
||||
`SELECT * FROM repository_versions WHERE repository_id = ? AND tag = ?`
|
||||
)
|
||||
.get(repositoryId, tag) as RepositoryVersionEntity | undefined;
|
||||
return row ? RepositoryVersionMapper.fromEntity(new RepositoryVersionEntity(row)) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user