mozempk/trueref
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(EMBEDDINGS-0001): enable local embedder by default and overhaul settings page

Browse Source 
- Wire local embedding provider as the default on startup when no profile is configured
- Refactor embedding settings into dedicated service, DTOs, mappers and models
- Rebuild settings page with profile management UI and live test feedback
- Expose index summary (indexed versions + embedding count) on repo endpoints
- Harden indexing pipeline and context search with additional test coverage

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Giancarmine Salucci
2026-03-28 09:28:01 +01:00
parent d1381f7fc0
commit 781d224adc
30 changed files with 1419 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
src/hooks.server.ts
View File

@@ -9,13 +9,13 @@
import { initializeDatabase } from '$lib/server/db/index.js';
import { getClient } from '$lib/server/db/client.js';
import { initializePipeline } from '$lib/server/pipeline/startup.js';
import {
EMBEDDING_CONFIG_KEY,
createProviderFromConfig,
defaultEmbeddingConfig
} from '$lib/server/embeddings/factory.js';
import { createProviderFromProfile } from '$lib/server/embeddings/registry.js';
import { EmbeddingService } from '$lib/server/embeddings/embedding.service.js';
import type { EmbeddingConfig } from '$lib/server/embeddings/factory.js';
import {
EmbeddingProfileEntity,
type EmbeddingProfileEntityProps
} from '$lib/server/models/embedding-profile.js';
import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper.js';
import type { Handle } from '@sveltejs/kit';
// ---------------------------------------------------------------------------
@@ -26,37 +26,20 @@ try {
initializeDatabase();
const db = getClient();
// Load persisted embedding configuration (if any).
const configRow = db
.prepare<[string], { value: string }>(`SELECT value FROM settings WHERE key = ?`)
.get(EMBEDDING_CONFIG_KEY);
const activeProfileRow = db
.prepare<[], EmbeddingProfileEntityProps>(
'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1'
)
.get();
let embeddingService: EmbeddingService | null = null;
if (configRow) {
try {
const config: EmbeddingConfig =
typeof configRow.value === 'string'
? JSON.parse(configRow.value)
: (configRow.value as EmbeddingConfig);
if (config.provider !== 'none') {
const provider = createProviderFromConfig(config);
embeddingService = new EmbeddingService(db, provider);
}
} catch (err) {
console.warn(
`[hooks.server] Could not load embedding config: ${err instanceof Error ? err.message : String(err)}`
if (activeProfileRow) {
const activeProfile = EmbeddingProfileMapper.fromEntity(
new EmbeddingProfileEntity(activeProfileRow)
);
}
} else {
// Use the default (noop) config so the pipeline is still wired up.
const config = defaultEmbeddingConfig();
if (config.provider !== 'none') {
const provider = createProviderFromConfig(config);
embeddingService = new EmbeddingService(db, provider);
}
const provider = createProviderFromProfile(activeProfile);
embeddingService = new EmbeddingService(db, provider, activeProfile.id);
}
initializePipeline(db, embeddingService);

35
src/lib/components/RepositoryCard.svelte
View File

@@ -1,13 +1,25 @@
<script lang="ts">
import { resolve as resolveRoute } from '$app/paths';
import type { Repository } from '$lib/types';
type RepositoryCardRepo = {
id: string;
title: string;
description: string | null;
state: 'pending' | 'indexing' | 'indexed' | 'error';
totalSnippets: number;
trustScore: number;
stars: number | null;
lastIndexedAt: string | Date | null;
embeddingCount?: number;
indexedVersions?: string[];
};
let {
repo,
onReindex,
onDelete
}: {
repo: Repository;
repo: RepositoryCardRepo;
onReindex: (id: string) => void;
onDelete: (id: string) => void;
} = $props();
@@ -32,6 +44,19 @@
const totalSnippets = $derived(repo.totalSnippets ?? 0);
const trustScore = $derived(repo.trustScore ?? 0);
const embeddingCount = $derived(repo.embeddingCount ?? 0);
const indexedVersions = $derived(repo.indexedVersions ?? []);
const indexedVersionsLabel = $derived.by(() => {
if (indexedVersions.length === 0) {
return 'No indexed version yet';
}
if (indexedVersions.length <= 2) {
return indexedVersions.join(', ');
}
return `${indexedVersions.slice(0, 2).join(', ')} +${indexedVersions.length - 2} more`;
});
</script>
<div class="rounded-xl border border-gray-200 bg-white p-5 shadow-sm">
@@ -67,6 +92,12 @@
{/if}
</div>
<div class="mt-2 flex flex-wrap gap-x-4 gap-y-1 text-sm text-gray-500">
<span>{embeddingCount.toLocaleString()} embeddings</span>
<span>·</span>
<span>Indexed: {indexedVersionsLabel}</span>
</div>
{#if repo.state === 'error'}
<p class="mt-2 text-xs text-red-600">Indexing failed. Check jobs for details.</p>
{/if}

5
src/lib/components/RepositoryCard.svelte.test.ts
View File

@@ -12,6 +12,8 @@ describe('RepositoryCard.svelte', () => {
description: 'A JavaScript library for building user interfaces',
state: 'indexed',
totalSnippets: 1234,
embeddingCount: 1200,
indexedVersions: ['main', 'v18.3.0'],
trustScore: 9.7,
stars: 230000,
lastIndexedAt: null
@@ -23,5 +25,8 @@ describe('RepositoryCard.svelte', () => {
await expect
.element(page.getByRole('link', { name: 'Details' }))
.toHaveAttribute('href', '/repos/%2Ffacebook%2Freact');
await expect.element(page.getByText('1,200 embeddings')).toBeInTheDocument();
await expect.element(page.getByText('Indexed: main, v18.3.0')).toBeInTheDocument();
});
});

41
src/lib/dtos/embedding-settings.ts Normal file
View File

@@ -0,0 +1,41 @@
import type { EmbeddingProviderKind } from '$lib/types';
export interface EmbeddingProfileConfigEntryDto {
key: string;
value: string;
redacted: boolean;
}
export interface EmbeddingProfileDto {
id: string;
providerKind: string;
title: string;
enabled: boolean;
isDefault: boolean;
model: string;
dimensions: number;
config: Record<string, unknown>;
configEntries: EmbeddingProfileConfigEntryDto[];
createdAt: number;
updatedAt: number;
}
export interface EmbeddingSettingsDto {
profiles: EmbeddingProfileDto[];
activeProfileId: string | null;
activeProfile: EmbeddingProfileDto | null;
}
export interface EmbeddingProfileUpsertDto {
id: string;
providerKind: EmbeddingProviderKind;
title: string;
model: string;
dimensions: number;
config: Record<string, unknown>;
}
export interface EmbeddingSettingsUpdateDto {
activeProfileId: string | null;
profile?: EmbeddingProfileUpsertDto;
}

43
src/lib/server/embeddings/embedding.service.test.ts
View File

@@ -408,6 +408,36 @@ describe('EmbeddingService', () => {
expect(embedding![2]).toBeCloseTo(0.2, 5);
});
it('stores embeddings under the configured profile ID', async () => {
client
.prepare(
`INSERT INTO embedding_profiles
(id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, unixepoch(), unixepoch())`
)
.run(
'openai-custom',
'openai-compatible',
'OpenAI Custom',
1,
0,
'test-model',
4,
'{}'
);
const snippetId = seedSnippet(db, client);
const provider = makeProvider(4, 'test-model');
const service = new EmbeddingService(client, provider, 'openai-custom');
await service.embedSnippets([snippetId]);
const row = client
.prepare('SELECT profile_id FROM snippet_embeddings WHERE snippet_id = ?')
.get(snippetId) as { profile_id: string };
expect(row.profile_id).toBe('openai-custom');
});
it('is idempotent — re-embedding replaces the existing row', async () => {
const snippetId = seedSnippet(db, client);
const provider = makeProvider(2);
@@ -469,6 +499,19 @@ describe('EmbeddingService', () => {
};
expect(rows.cnt).toBe(0);
});
it('finds snippets missing embeddings for the active profile', async () => {
const firstSnippetId = seedSnippet(db, client);
const secondSnippetId = seedSnippet(db, client, { content: 'Second snippet content' });
const provider = makeProvider(4);
const service = new EmbeddingService(client, provider, 'local-default');
await service.embedSnippets([firstSnippetId]);
expect(service.findSnippetIdsMissingEmbeddings('/test/embed-repo', null)).toEqual([
secondSnippetId
]);
});
});
// ---------------------------------------------------------------------------

36
src/lib/server/embeddings/embedding.service.ts
View File

@@ -23,6 +23,42 @@ export class EmbeddingService {
private readonly profileId: string = 'local-default'
) {}
findSnippetIdsMissingEmbeddings(repositoryId: string, versionId: string | null): string[] {
if (versionId) {
const rows = this.db
.prepare<[string, string, string], { id: string }>(
`SELECT snippets.id
FROM snippets
LEFT JOIN snippet_embeddings
ON snippet_embeddings.snippet_id = snippets.id
AND snippet_embeddings.profile_id = ?
WHERE snippets.repository_id = ?
AND snippets.version_id = ?
AND snippet_embeddings.snippet_id IS NULL
ORDER BY snippets.id`
)
.all(this.profileId, repositoryId, versionId);
return rows.map((row) => row.id);
}
const rows = this.db
.prepare<[string, string], { id: string }>(
`SELECT snippets.id
FROM snippets
LEFT JOIN snippet_embeddings
ON snippet_embeddings.snippet_id = snippets.id
AND snippet_embeddings.profile_id = ?
WHERE snippets.repository_id = ?
AND snippets.version_id IS NULL
AND snippet_embeddings.snippet_id IS NULL
ORDER BY snippets.id`
)
.all(this.profileId, repositoryId);
return rows.map((row) => row.id);
}
/**
* Embed the given snippet IDs and store the results in snippet_embeddings.
*

10
src/lib/server/embeddings/local.provider.ts
View File

@@ -1,10 +1,10 @@
/**
* LocalEmbeddingProvider — uses @xenova/transformers (optional dependency).
* LocalEmbeddingProvider — uses @xenova/transformers via dynamic import.
*
* @xenova/transformers is NOT installed by default. This provider uses a
* dynamic import so the module is only required at runtime when the local
* provider is actually configured. If the package is absent, isAvailable()
* returns false and embed() throws a clear error.
* The dynamic import keeps server startup cheap and defers loading the model
* runtime until the local provider is actually used. If the package is absent
* or cannot be resolved, isAvailable() returns false and embed() throws a
* clear error.
*/
import { EmbeddingError, type EmbeddingProvider, type EmbeddingVector } from './provider.js';

3
src/lib/server/embeddings/registry.ts
View File

@@ -44,11 +44,12 @@ export function createProviderFromProfile(profile: EmbeddingProfile): EmbeddingP
*/
export function getDefaultLocalProfile(): Pick<
EmbeddingProfile,
'id' | 'providerKind' | 'model' | 'dimensions'
'id' | 'providerKind' | 'title' | 'model' | 'dimensions'
> {
return {
id: 'local-default',
providerKind: 'local-transformers',
title: 'Local (Xenova/all-MiniLM-L6-v2)',
model: 'Xenova/all-MiniLM-L6-v2',
dimensions: 384
};

38
src/lib/server/mappers/embedding-profile.mapper.ts Normal file
View File

@@ -0,0 +1,38 @@
import {
EmbeddingProfile,
EmbeddingProfileEntity
} from '$lib/server/models/embedding-profile.js';
function parseConfig(config: Record<string, unknown> | string | null): Record<string, unknown> {
if (!config) {
return {};
}
if (typeof config === 'string') {
try {
const parsed = JSON.parse(config);
return parsed && typeof parsed === 'object' ? (parsed as Record<string, unknown>) : {};
} catch {
return {};
}
}
return config;
}
export class EmbeddingProfileMapper {
static fromEntity(entity: EmbeddingProfileEntity): EmbeddingProfile {
return new EmbeddingProfile({
id: entity.id,
providerKind: entity.provider_kind,
title: entity.title,
enabled: Boolean(entity.enabled),
isDefault: Boolean(entity.is_default),
model: entity.model,
dimensions: entity.dimensions,
config: parseConfig(entity.config),
createdAt: entity.created_at,
updatedAt: entity.updated_at
});
}
}

71
src/lib/server/mappers/embedding-settings.dto.mapper.ts Normal file
View File

@@ -0,0 +1,71 @@
import type {
EmbeddingProfileConfigEntryDto,
EmbeddingProfileDto,
EmbeddingSettingsDto
} from '$lib/dtos/embedding-settings.js';
import type { EmbeddingProfile } from '$lib/server/models/embedding-profile.js';
import { EmbeddingSettings } from '$lib/server/models/embedding-settings.js';
const REDACTED_VALUE = '[redacted]';
const SENSITIVE_CONFIG_KEY = /(api[-_]?key|token|secret|password|authorization)/i;
function formatConfigValue(value: unknown): string {
if (value === null || value === undefined) return 'null';
if (typeof value === 'string') return value;
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
return JSON.stringify(value);
}
function sanitizeConfig(config: Record<string, unknown>): {
visibleConfig: Record<string, unknown>;
configEntries: EmbeddingProfileConfigEntryDto[];
} {
const visibleConfig: Record<string, unknown> = {};
const configEntries = Object.entries(config)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, value]) => {
const redacted = SENSITIVE_CONFIG_KEY.test(key);
if (!redacted) {
visibleConfig[key] = value;
}
return {
key,
value: redacted ? REDACTED_VALUE : formatConfigValue(value),
redacted
};
});
return { visibleConfig, configEntries };
}
function toProfileDto(profile: EmbeddingProfile): EmbeddingProfileDto {
const { visibleConfig, configEntries } = sanitizeConfig(profile.config);
return {
id: profile.id,
providerKind: profile.providerKind,
title: profile.title,
enabled: profile.enabled,
isDefault: profile.isDefault,
model: profile.model,
dimensions: profile.dimensions,
config: visibleConfig,
configEntries,
createdAt: profile.createdAt,
updatedAt: profile.updatedAt
};
}
export class EmbeddingSettingsDtoMapper {
static toDto(settings: EmbeddingSettings): EmbeddingSettingsDto {
const profiles = settings.profiles.map(toProfileDto);
const activeProfile = settings.activeProfile ? toProfileDto(settings.activeProfile) : null;
return {
profiles,
activeProfileId: settings.activeProfileId,
activeProfile
};
}
}

77
src/lib/server/models/embedding-profile.ts Normal file
View File

@@ -0,0 +1,77 @@
export interface EmbeddingProfileEntityProps {
id: string;
provider_kind: string;
title: string;
enabled: boolean | number;
is_default: boolean | number;
model: string;
dimensions: number;
config: Record<string, unknown> | string | null;
created_at: number;
updated_at: number;
}
export class EmbeddingProfileEntity {
id: string;
provider_kind: string;
title: string;
enabled: boolean | number;
is_default: boolean | number;
model: string;
dimensions: number;
config: Record<string, unknown> | string | null;
created_at: number;
updated_at: number;
constructor(props: EmbeddingProfileEntityProps) {
this.id = props.id;
this.provider_kind = props.provider_kind;
this.title = props.title;
this.enabled = props.enabled;
this.is_default = props.is_default;
this.model = props.model;
this.dimensions = props.dimensions;
this.config = props.config;
this.created_at = props.created_at;
this.updated_at = props.updated_at;
}
}
export interface EmbeddingProfileProps {
id: string;
providerKind: string;
title: string;
enabled: boolean;
isDefault: boolean;
model: string;
dimensions: number;
config: Record<string, unknown>;
createdAt: number;
updatedAt: number;
}
export class EmbeddingProfile {
id: string;
providerKind: string;
title: string;
enabled: boolean;
isDefault: boolean;
model: string;
dimensions: number;
config: Record<string, unknown>;
createdAt: number;
updatedAt: number;
constructor(props: EmbeddingProfileProps) {
this.id = props.id;
this.providerKind = props.providerKind;
this.title = props.title;
this.enabled = props.enabled;
this.isDefault = props.isDefault;
this.model = props.model;
this.dimensions = props.dimensions;
this.config = props.config;
this.createdAt = props.createdAt;
this.updatedAt = props.updatedAt;
}
}

20
src/lib/server/models/embedding-settings.ts Normal file
View File

@@ -0,0 +1,20 @@
import type { EmbeddingProfile } from './embedding-profile.js';
export interface EmbeddingSettingsProps {
profiles: EmbeddingProfile[];
activeProfile: EmbeddingProfile | null;
}
export class EmbeddingSettings {
profiles: EmbeddingProfile[];
activeProfile: EmbeddingProfile | null;
constructor(props: EmbeddingSettingsProps) {
this.profiles = props.profiles;
this.activeProfile = props.activeProfile;
}
get activeProfileId(): string | null {
return this.activeProfile?.id ?? null;
}
}

77
src/lib/server/pipeline/indexing.pipeline.test.ts
View File

@@ -12,6 +12,7 @@ import { join } from 'node:path';
import { JobQueue } from './job-queue.js';
import { IndexingPipeline } from './indexing.pipeline.js';
import { recoverStaleJobs } from './startup.js';
import { EmbeddingService } from '$lib/server/embeddings/embedding.service.js';
// ---------------------------------------------------------------------------
// Test DB factory
@@ -22,7 +23,12 @@ function createTestDb(): Database.Database {
client.pragma('foreign_keys = ON');
const migrationsFolder = join(import.meta.dirname, '../db/migrations');
const migrationSql = readFileSync(join(migrationsFolder, '0000_large_master_chief.sql'), 'utf-8');
for (const migrationFile of [
'0000_large_master_chief.sql',
'0001_quick_nighthawk.sql',
'0002_silky_stellaris.sql'
]) {
const migrationSql = readFileSync(join(migrationsFolder, migrationFile), 'utf-8');
const statements = migrationSql
.split('--> statement-breakpoint')
@@ -32,6 +38,7 @@ function createTestDb(): Database.Database {
for (const stmt of statements) {
client.exec(stmt);
}
}
return client;
}
@@ -238,7 +245,8 @@ describe('IndexingPipeline', () => {
crawlResult: {
files: Array<{ path: string; content: string; sha: string; language: string }>;
totalFiles: number;
} = { files: [], totalFiles: 0 }
} = { files: [], totalFiles: 0 },
embeddingService: EmbeddingService | null = null
) {
const mockGithubCrawl = vi.fn().mockResolvedValue({
...crawlResult,
@@ -256,7 +264,12 @@ describe('IndexingPipeline', () => {
})
};
return new IndexingPipeline(db, mockGithubCrawl as never, mockLocalCrawler as never, null);
return new IndexingPipeline(
db,
mockGithubCrawl as never,
mockLocalCrawler as never,
embeddingService
);
}
function makeJob(repositoryId = '/test/repo') {
@@ -388,6 +401,64 @@ describe('IndexingPipeline', () => {
expect(secondSnippetIds).toEqual(firstSnippetIds);
});
it('re-index backfills missing embeddings for unchanged snippets', async () => {
const provider = {
name: 'test-provider',
model: 'test-model',
dimensions: 3,
embed: vi.fn(async (texts: string[]) =>
texts.map(() => ({
values: new Float32Array([0.1, 0.2, 0.3]),
dimensions: 3,
model: 'test-model'
}))
),
isAvailable: vi.fn(async () => true)
};
const embeddingService = new EmbeddingService(db, provider, 'local-default');
const files = [
{
path: 'README.md',
content: '# Hello\n\nThis is documentation.',
sha: 'sha-readme',
language: 'markdown'
}
];
const pipeline = makePipeline({ files, totalFiles: 1 }, embeddingService);
const job1 = makeJob();
await pipeline.run(job1 as never);
const firstSnippetIds = (db.prepare(`SELECT id FROM snippets ORDER BY id`).all() as { id: string }[])
.map((row) => row.id);
expect(firstSnippetIds.length).toBeGreaterThan(0);
const firstEmbeddingCount = (
db.prepare(`SELECT COUNT(*) as n FROM snippet_embeddings WHERE profile_id = 'local-default'`).get() as {
n: number;
}
).n;
expect(firstEmbeddingCount).toBe(firstSnippetIds.length);
db.prepare(`DELETE FROM snippet_embeddings WHERE profile_id = 'local-default'`).run();
const job2Id = insertJob(db, { repository_id: '/test/repo', status: 'queued' });
const job2 = db.prepare(`SELECT * FROM indexing_jobs WHERE id = ?`).get(job2Id) as never;
await pipeline.run(job2);
const secondSnippetIds = (db.prepare(`SELECT id FROM snippets ORDER BY id`).all() as {
id: string;
}[]).map((row) => row.id);
const secondEmbeddingCount = (
db.prepare(`SELECT COUNT(*) as n FROM snippet_embeddings WHERE profile_id = 'local-default'`).get() as {
n: number;
}
).n;
expect(secondSnippetIds).toEqual(firstSnippetIds);
expect(secondEmbeddingCount).toBe(firstSnippetIds.length);
});
it('replaces snippets atomically when a file changes', async () => {
const pipeline1 = makePipeline({
files: [

12
src/lib/server/pipeline/indexing.pipeline.ts
View File

@@ -187,8 +187,15 @@ export class IndexingPipeline {
this.replaceSnippets(repo.id, changedDocIds, newDocuments, newSnippets);
// ---- Stage 4: Embeddings (if provider is configured) ----------------
if (this.embeddingService && newSnippets.length > 0) {
const snippetIds = newSnippets.map((s) => s.id!);
if (this.embeddingService) {
const snippetIds = this.embeddingService.findSnippetIdsMissingEmbeddings(
repo.id,
normJob.versionId
);
if (snippetIds.length === 0) {
// No missing embeddings for the active profile; parsing progress is final.
} else {
const embeddingsTotal = snippetIds.length;
await this.embeddingService.embedSnippets(snippetIds, (done) => {
@@ -202,6 +209,7 @@ export class IndexingPipeline {
this.updateJob(job.id, { progress });
});
}
}
// ---- Stage 5: Update repository stats --------------------------------
const stats = this.computeStats(repo.id);

131
src/lib/server/services/embedding-settings.service.ts Normal file
View File

@@ -0,0 +1,131 @@
import type Database from 'better-sqlite3';
import type { EmbeddingSettingsUpdateDto } from '$lib/dtos/embedding-settings.js';
import { createProviderFromProfile, getDefaultLocalProfile } from '$lib/server/embeddings/registry.js';
import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper.js';
import { EmbeddingProfile, EmbeddingProfileEntity } from '$lib/server/models/embedding-profile.js';
import { EmbeddingSettings } from '$lib/server/models/embedding-settings.js';
import { InvalidInputError } from '$lib/server/utils/validation.js';
export class EmbeddingSettingsService {
constructor(private readonly db: Database.Database) {}
getSettings(): EmbeddingSettings {
const profiles = this.loadProfiles();
const activeProfile = profiles.find((profile) => profile.isDefault && profile.enabled) ?? null;
return new EmbeddingSettings({ profiles, activeProfile });
}
async updateSettings(input: EmbeddingSettingsUpdateDto): Promise<EmbeddingSettings> {
const now = Math.floor(Date.now() / 1000);
this.db.prepare('UPDATE embedding_profiles SET is_default = 0, updated_at = ?').run(now);
if (input.activeProfileId === null) {
return this.getSettings();
}
const profile =
input.activeProfileId === 'local-default'
? this.buildDefaultLocalProfile(now)
: this.buildCustomProfile(input, now);
const available = await createProviderFromProfile(profile).isAvailable();
if (!available) {
throw new InvalidInputError(
`Could not connect to the "${profile.providerKind}" provider. Check your configuration.`
);
}
this.persistProfile(profile);
return this.getSettings();
}
private loadProfiles(): EmbeddingProfile[] {
return this.db
.prepare('SELECT * FROM embedding_profiles ORDER BY is_default DESC, created_at ASC')
.all()
.map((row) => EmbeddingProfileMapper.fromEntity(new EmbeddingProfileEntity(row as never)));
}
private buildDefaultLocalProfile(now: number): EmbeddingProfile {
const defaultLocal = getDefaultLocalProfile();
return new EmbeddingProfile({
id: defaultLocal.id,
providerKind: defaultLocal.providerKind,
title: defaultLocal.title,
enabled: true,
isDefault: true,
model: defaultLocal.model,
dimensions: defaultLocal.dimensions,
config: {},
createdAt: this.getCreatedAt(defaultLocal.id, now),
updatedAt: now
});
}
private buildCustomProfile(input: EmbeddingSettingsUpdateDto, now: number): EmbeddingProfile {
const candidate = input.profile;
if (!candidate) {
throw new InvalidInputError('profile is required for custom embedding providers');
}
if (candidate.id !== input.activeProfileId) {
throw new InvalidInputError('activeProfileId must match profile.id');
}
if (!candidate.title || !candidate.model) {
throw new InvalidInputError('profile title and model are required');
}
return new EmbeddingProfile({
id: candidate.id,
providerKind: candidate.providerKind,
title: candidate.title,
enabled: true,
isDefault: true,
model: candidate.model,
dimensions: candidate.dimensions,
config: candidate.config,
createdAt: this.getCreatedAt(candidate.id, now),
updatedAt: now
});
}
private getCreatedAt(id: string, fallback: number): number {
return (
this.db
.prepare<[string], { created_at: number }>('SELECT created_at FROM embedding_profiles WHERE id = ?')
.get(id)?.created_at ?? fallback
);
}
private persistProfile(profile: EmbeddingProfile): void {
this.db
.prepare(
`INSERT INTO embedding_profiles
(id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
provider_kind = excluded.provider_kind,
title = excluded.title,
enabled = excluded.enabled,
is_default = excluded.is_default,
model = excluded.model,
dimensions = excluded.dimensions,
config = excluded.config,
updated_at = excluded.updated_at`
)
.run(
profile.id,
profile.providerKind,
profile.title,
profile.enabled ? 1 : 0,
profile.isDefault ? 1 : 0,
profile.model,
profile.dimensions,
JSON.stringify(profile.config),
profile.createdAt,
profile.updatedAt
);
}
}

93
src/lib/server/services/repository.service.test.ts
View File

@@ -27,16 +27,20 @@ function createTestDb(): Database.Database {
client.pragma('foreign_keys = ON');
const migrationsFolder = join(import.meta.dirname, '../db/migrations');
const migrationSql = readFileSync(join(migrationsFolder, '0000_large_master_chief.sql'), 'utf-8');
// Drizzle migration files use `--> statement-breakpoint` as separator.
const statements = migrationSql
for (const migration of [
'0000_large_master_chief.sql',
'0001_quick_nighthawk.sql',
'0002_silky_stellaris.sql'
]) {
const statements = readFileSync(join(migrationsFolder, migration), 'utf-8')
.split('--> statement-breakpoint')
.map((s) => s.trim())
.map((statement) => statement.trim())
.filter(Boolean);
for (const stmt of statements) {
client.exec(stmt);
for (const statement of statements) {
client.exec(statement);
}
}
return client;
@@ -408,6 +412,83 @@ describe('RepositoryService.getVersions()', () => {
});
});
// ---------------------------------------------------------------------------
// getIndexSummary()
// ---------------------------------------------------------------------------
describe('RepositoryService.getIndexSummary()', () => {
let client: Database.Database;
let service: RepositoryService;
beforeEach(() => {
client = createTestDb();
service = makeService(client);
service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react', branch: 'main' });
});
it('returns embedding counts and indexed version labels', () => {
const now = Math.floor(Date.now() / 1000);
const docId = crypto.randomUUID();
const versionDocId = crypto.randomUUID();
const snippetId = crypto.randomUUID();
const versionSnippetId = crypto.randomUUID();
client
.prepare(
`INSERT INTO repository_versions (id, repository_id, tag, state, created_at)
VALUES (?, '/facebook/react', ?, 'indexed', ?)`
)
.run('/facebook/react/v18.3.0', 'v18.3.0', now);
client
.prepare(
`INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at)
VALUES (?, '/facebook/react', NULL, 'README.md', 'base', ?)`
)
.run(docId, now);
client
.prepare(
`INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at)
VALUES (?, '/facebook/react', ?, 'README.md', 'version', ?)`
)
.run(versionDocId, '/facebook/react/v18.3.0', now);
client
.prepare(
`INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at)
VALUES (?, ?, '/facebook/react', NULL, 'info', 'base snippet', ?)`
)
.run(snippetId, docId, now);
client
.prepare(
`INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at)
VALUES (?, ?, '/facebook/react', ?, 'info', 'version snippet', ?)`
)
.run(versionSnippetId, versionDocId, '/facebook/react/v18.3.0', now);
client
.prepare(
`INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at)
VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', 2, ?, ?)`
)
.run(snippetId, Buffer.from(Float32Array.from([1, 0]).buffer), now);
client
.prepare(
`INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at)
VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', 2, ?, ?)`
)
.run(versionSnippetId, Buffer.from(Float32Array.from([0, 1]).buffer), now);
expect(service.getIndexSummary('/facebook/react')).toEqual({
embeddingCount: 2,
indexedVersions: ['main', 'v18.3.0']
});
});
});
// ---------------------------------------------------------------------------
// createIndexingJob()
// ---------------------------------------------------------------------------

48
src/lib/server/services/repository.service.ts
View File

@@ -39,6 +39,11 @@ export interface RepositoryStats {
lastIndexedAt: Date | null;
}
export interface RepositoryIndexSummary {
embeddingCount: number;
indexedVersions: string[];
}
export class RepositoryService {
constructor(private readonly db: Database.Database) {}
@@ -266,6 +271,49 @@ export class RepositoryService {
return rows.map((r) => r.tag);
}
getIndexSummary(repositoryId: string): RepositoryIndexSummary {
const repository = this.get(repositoryId);
if (!repository) throw new NotFoundError(`Repository ${repositoryId} not found`);
const embeddingRow = this.db
.prepare(
`SELECT COUNT(*) AS count
FROM snippet_embeddings se
INNER JOIN snippets s ON s.id = se.snippet_id
WHERE s.repository_id = ?`
)
.get(repositoryId) as { count: number };
const versionRows = this.db
.prepare(
`SELECT tag FROM repository_versions
WHERE repository_id = ? AND state = 'indexed'
ORDER BY created_at DESC`
)
.all(repositoryId) as { tag: string }[];
const hasDefaultBranchIndex = Boolean(
this.db
.prepare(
`SELECT 1 AS found
FROM documents
WHERE repository_id = ? AND version_id IS NULL
LIMIT 1`
)
.get(repositoryId)
);
const indexedVersions = [
...(hasDefaultBranchIndex ? [repository.branch ?? 'default branch'] : []),
...versionRows.map((row) => row.tag)
];
return {
embeddingCount: embeddingRow.count,
indexedVersions: Array.from(new Set(indexedVersions))
};
}
/**
* Create an indexing job for a repository.
* If a job is already running, returns the existing job.

1
src/lib/types.ts
View File

@@ -32,6 +32,7 @@ export type RepositoryState = 'pending' | 'indexing' | 'indexed' | 'error';
export type SnippetType = 'code' | 'info';
export type JobStatus = 'queued' | 'running' | 'done' | 'failed';
export type VersionState = 'pending' | 'indexing' | 'indexed' | 'error';
export type EmbeddingProviderKind = 'local-transformers' | 'openai-compatible';
// ---------------------------------------------------------------------------
// API / service layer types

41
src/routes/api/v1/api-contract.integration.test.ts
View File

@@ -34,6 +34,7 @@ vi.mock('$lib/server/embeddings/registry.js', () => ({
}));
import { POST as postLibraries } from './libs/+server.js';
import { GET as getLibraries } from './libs/+server.js';
import { GET as getLibrary } from './libs/[id]/+server.js';
import { GET as getJobs } from './jobs/+server.js';
import { GET as getJob } from './jobs/[id]/+server.js';
@@ -186,6 +187,16 @@ function seedSnippet(
return snippetId;
}
function seedEmbedding(client: Database.Database, snippetId: string, values: number[]): void {
client
.prepare(
`INSERT INTO snippet_embeddings
(snippet_id, profile_id, model, dimensions, embedding, created_at)
VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', ?, ?, ?)`
)
.run(snippetId, values.length, Buffer.from(Float32Array.from(values).buffer), NOW_S);
}
function seedRules(client: Database.Database, repositoryId: string, rules: string[]) {
client
.prepare(
@@ -249,6 +260,36 @@ describe('API contract integration', () => {
expect(body).not.toHaveProperty('total_snippets');
});
it('GET /api/v1/libs includes embedding counts and indexed versions per repository', async () => {
const repositoryId = seedRepo(db);
const versionId = seedVersion(db, repositoryId, 'v18.3.0');
const baseDocId = seedDocument(db, repositoryId);
const versionDocId = seedDocument(db, repositoryId, versionId);
const baseSnippetId = seedSnippet(db, {
documentId: baseDocId,
repositoryId,
content: 'Base branch snippet'
});
const versionSnippetId = seedSnippet(db, {
documentId: versionDocId,
repositoryId,
versionId,
content: 'Versioned snippet'
});
seedEmbedding(db, baseSnippetId, [1, 0]);
seedEmbedding(db, versionSnippetId, [0, 1]);
const response = await getLibraries({
url: new URL('http://test/api/v1/libs')
} as never);
expect(response.status).toBe(200);
const body = await response.json();
expect(body.libraries).toHaveLength(1);
expect(body.libraries[0].embeddingCount).toBe(2);
expect(body.libraries[0].indexedVersions).toEqual(['main', 'v18.3.0']);
});
it('GET /api/v1/jobs and /api/v1/jobs/:id return job DTOs in camelCase', async () => {
const repoService = new RepositoryService(db);
repoService.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });

20
src/routes/api/v1/context/+server.ts
View File

@@ -17,7 +17,11 @@ import { dtoJsonResponse } from '$lib/server/api/dto-response';
import { SearchService } from '$lib/server/search/search.service';
import { HybridSearchService } from '$lib/server/search/hybrid.search.service';
import { createProviderFromProfile } from '$lib/server/embeddings/registry';
import type { EmbeddingProfile } from '$lib/server/db/schema';
import {
EmbeddingProfileEntity,
type EmbeddingProfileEntityProps
} from '$lib/server/models/embedding-profile';
import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper';
import { parseLibraryId } from '$lib/server/api/library-id';
import { selectSnippetsWithinBudget, DEFAULT_TOKEN_BUDGET } from '$lib/server/api/token-budget';
import { formatContextJson, formatContextTxt, CORS_HEADERS } from '$lib/server/api/formatters';
@@ -32,16 +36,18 @@ function getServices(db: ReturnType<typeof getClient>) {
// Load the active embedding profile from the database
const profileRow = db
.prepare<
[],
EmbeddingProfile
>('SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1')
.prepare<[], EmbeddingProfileEntityProps>(
'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1'
)
.get();
const provider = profileRow ? createProviderFromProfile(profileRow) : null;
const profile = profileRow
? EmbeddingProfileMapper.fromEntity(new EmbeddingProfileEntity(profileRow))
: null;
const provider = profile ? createProviderFromProfile(profile) : null;
const hybridService = new HybridSearchService(db, searchService, provider);
return { db, searchService, hybridService, profileId: profileRow?.id };
return { db, searchService, hybridService, profileId: profile?.id };
}
interface RawRepoConfig {

3
src/routes/api/v1/libs/+server.ts
View File

@@ -32,7 +32,8 @@ export const GET: RequestHandler = ({ url }) => {
const enriched = libraries.map((repo) => ({
...RepositoryMapper.toDto(repo),
versions: service.getVersions(repo.id)
versions: service.getVersions(repo.id),
...service.getIndexSummary(repo.id)
}));
return json({ libraries: enriched, total, limit, offset });

2
src/routes/api/v1/libs/[id]/+server.ts
View File

@@ -23,7 +23,7 @@ export const GET: RequestHandler = ({ params }) => {
return json({ error: 'Repository not found', code: 'NOT_FOUND' }, { status: 404 });
}
const versions = service.getVersions(id);
return json({ ...RepositoryMapper.toDto(repo), versions });
return json({ ...RepositoryMapper.toDto(repo), versions, ...service.getIndexSummary(id) });
} catch (err) {
return handleServiceError(err);
}

126
src/routes/api/v1/settings/embedding/+server.ts
View File

@@ -1,30 +1,25 @@
/**
* GET /api/v1/settings/embedding — retrieve all embedding profiles
* POST /api/v1/settings/embedding — create or update an embedding profile
* PUT /api/v1/settings/embedding — alias for POST (backward compat)
* GET /api/v1/settings/embedding — retrieve embedding settings
* POST /api/v1/settings/embedding — update active embedding settings
* PUT /api/v1/settings/embedding — alias for POST
*/
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import type { EmbeddingSettingsUpdateDto } from '$lib/dtos/embedding-settings.js';
import { getClient } from '$lib/server/db/client';
import { createProviderFromProfile } from '$lib/server/embeddings/registry';
import type { EmbeddingProfile, NewEmbeddingProfile } from '$lib/server/db/schema';
import { EmbeddingSettingsDtoMapper } from '$lib/server/mappers/embedding-settings.dto.mapper.js';
import { EmbeddingSettingsService } from '$lib/server/services/embedding-settings.service.js';
import { handleServiceError, InvalidInputError } from '$lib/server/utils/validation';
// ---------------------------------------------------------------------------
// GET — Return all profiles
// GET — Return embedding settings
// ---------------------------------------------------------------------------
export const GET: RequestHandler = () => {
try {
const db = getClient();
const profiles = db
.prepare('SELECT * FROM embedding_profiles ORDER BY is_default DESC, created_at ASC')
.all() as EmbeddingProfile[];
// Sanitize: remove sensitive config fields like apiKey
const safeProfiles = profiles.map(sanitizeProfile);
return json({ profiles: safeProfiles });
const service = new EmbeddingSettingsService(getClient());
return json(EmbeddingSettingsDtoMapper.toDto(service.getSettings()));
} catch (err) {
return handleServiceError(err);
}
@@ -34,116 +29,23 @@ export const GET: RequestHandler = () => {
// POST/PUT — Create or update a profile
// ---------------------------------------------------------------------------
async function upsertProfile(body: unknown) {
async function upsertSettings(body: unknown) {
if (typeof body !== 'object' || body === null) {
throw new InvalidInputError('Request body must be a JSON object');
}
const obj = body as Record<string, unknown>;
// Required fields
if (typeof obj.id !== 'string' || !obj.id) {
throw new InvalidInputError('id is required');
}
if (typeof obj.providerKind !== 'string' || !obj.providerKind) {
throw new InvalidInputError('providerKind is required');
}
if (typeof obj.title !== 'string' || !obj.title) {
throw new InvalidInputError('title is required');
}
if (typeof obj.model !== 'string' || !obj.model) {
throw new InvalidInputError('model is required');
}
if (typeof obj.dimensions !== 'number') {
throw new InvalidInputError('dimensions must be a number');
}
const profile: NewEmbeddingProfile = {
id: obj.id,
providerKind: obj.providerKind,
title: obj.title,
enabled: typeof obj.enabled === 'boolean' ? obj.enabled : true,
isDefault: typeof obj.isDefault === 'boolean' ? obj.isDefault : false,
model: obj.model,
dimensions: obj.dimensions,
config: (obj.config as Record<string, unknown>) ?? {},
createdAt: Date.now(),
updatedAt: Date.now()
};
// Validate provider availability before persisting
const provider = createProviderFromProfile(profile as EmbeddingProfile);
const available = await provider.isAvailable();
if (!available) {
throw new InvalidInputError(
`Could not connect to the "${profile.providerKind}" provider. Check your configuration.`
);
}
const db = getClient();
// If setting as default, clear other defaults first
if (profile.isDefault) {
db.prepare('UPDATE embedding_profiles SET is_default = 0').run();
}
// Upsert the profile
db.prepare(
`INSERT INTO embedding_profiles
(id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
provider_kind = excluded.provider_kind,
title = excluded.title,
enabled = excluded.enabled,
is_default = excluded.is_default,
model = excluded.model,
dimensions = excluded.dimensions,
config = excluded.config,
updated_at = excluded.updated_at`
).run(
profile.id,
profile.providerKind,
profile.title,
profile.enabled ? 1 : 0,
profile.isDefault ? 1 : 0,
profile.model,
profile.dimensions,
JSON.stringify(profile.config),
profile.createdAt,
profile.updatedAt
);
const inserted = db
.prepare('SELECT * FROM embedding_profiles WHERE id = ?')
.get(profile.id) as EmbeddingProfile;
return sanitizeProfile(inserted);
const service = new EmbeddingSettingsService(getClient());
const settings = await service.updateSettings(body as EmbeddingSettingsUpdateDto);
return EmbeddingSettingsDtoMapper.toDto(settings);
}
export const POST: RequestHandler = async ({ request }) => {
try {
const body = await request.json();
const profile = await upsertProfile(body);
return json(profile);
return json(await upsertSettings(body));
} catch (err) {
return handleServiceError(err);
}
};
// Backward compat alias
export const PUT: RequestHandler = POST;
// ---------------------------------------------------------------------------
// Sanitize — remove sensitive config fields before returning to clients
// ---------------------------------------------------------------------------
function sanitizeProfile(profile: EmbeddingProfile): EmbeddingProfile {
const config = profile.config as Record<string, unknown>;
if (config && config.apiKey) {
const rest = { ...config };
delete rest.apiKey;
return { ...profile, config: rest };
}
return profile;
}

183
src/routes/api/v1/settings/embedding/server.test.ts Normal file
View File

@@ -0,0 +1,183 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import Database from 'better-sqlite3';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
let db: Database.Database;
vi.mock('$lib/server/db/client', () => ({
getClient: () => db
}));
vi.mock('$lib/server/db/client.js', () => ({
getClient: () => db
}));
vi.mock('$lib/server/embeddings/registry', () => ({
createProviderFromProfile: () => ({
isAvailable: async () => true
})
}));
vi.mock('$lib/server/embeddings/registry.js', () => ({
createProviderFromProfile: () => ({
isAvailable: async () => true
})
}));
vi.mock('$lib/server/embeddings/local.provider', () => ({
LocalEmbeddingProvider: class {
readonly model = 'Xenova/all-MiniLM-L6-v2';
readonly dimensions = 384;
async isAvailable() {
return true;
}
}
}));
vi.mock('$lib/server/embeddings/local.provider.js', () => ({
LocalEmbeddingProvider: class {
readonly model = 'Xenova/all-MiniLM-L6-v2';
readonly dimensions = 384;
async isAvailable() {
return true;
}
}
}));
import { GET as getEmbeddingSettings, PUT as putEmbeddingSettings } from './+server.js';
import { GET as getEmbeddingTest } from './test/+server.js';
function createTestDb(): Database.Database {
const client = new Database(':memory:');
client.pragma('foreign_keys = ON');
const migrationsFolder = join(import.meta.dirname, '../../../../../lib/server/db/migrations');
const ftsFile = join(import.meta.dirname, '../../../../../lib/server/db/fts.sql');
for (const migration of [
'0000_large_master_chief.sql',
'0001_quick_nighthawk.sql',
'0002_silky_stellaris.sql'
]) {
const statements = readFileSync(join(migrationsFolder, migration), 'utf-8')
.split('--> statement-breakpoint')
.map((statement) => statement.trim())
.filter(Boolean);
for (const statement of statements) {
client.exec(statement);
}
}
client.exec(readFileSync(ftsFile, 'utf-8'));
return client;
}
describe('embedding settings routes', () => {
beforeEach(() => {
db = createTestDb();
});
it('GET /api/v1/settings/embedding returns profile-based settings for the seeded default profile', async () => {
const response = await getEmbeddingSettings({} as never);
expect(response.status).toBe(200);
const body = await response.json();
expect(body.activeProfileId).toBe('local-default');
expect(body.activeProfile).toMatchObject({
id: 'local-default',
providerKind: 'local-transformers',
title: 'Local (Xenova/all-MiniLM-L6-v2)'
});
expect(body.profiles).toHaveLength(1);
expect(body.profiles[0].providerKind).toBe('local-transformers');
expect(body.profiles[0].isDefault).toBe(true);
});
it('PUT /api/v1/settings/embedding persists a clean profile-based OpenAI payload', async () => {
const response = await putEmbeddingSettings({
request: new Request('http://test/api/v1/settings/embedding', {
method: 'PUT',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
activeProfileId: 'openai-default',
profile: {
id: 'openai-default',
providerKind: 'openai-compatible',
title: 'OpenAI-compatible',
model: 'text-embedding-3-small',
dimensions: 1536,
config: {
baseUrl: 'https://api.openai.com/v1',
apiKey: 'sk-test',
model: 'text-embedding-3-small'
}
}
})
})
} as never);
expect(response.status).toBe(200);
const body = await response.json();
expect(body.activeProfileId).toBe('openai-default');
expect(body.activeProfile).toMatchObject({
id: 'openai-default',
providerKind: 'openai-compatible'
});
expect(body.activeProfile.config).toEqual({
baseUrl: 'https://api.openai.com/v1',
model: 'text-embedding-3-small'
});
expect(body.activeProfile.configEntries).toEqual(
expect.arrayContaining([
expect.objectContaining({ key: 'apiKey', value: '[redacted]', redacted: true })
])
);
expect(body.profiles).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: 'openai-default',
providerKind: 'openai-compatible',
model: 'text-embedding-3-small',
dimensions: 1536,
isDefault: true
})
])
);
const activeProfile = db
.prepare(
'SELECT id, provider_kind, is_default, enabled, model, dimensions FROM embedding_profiles WHERE is_default = 1 LIMIT 1'
)
.get() as Record<string, unknown>;
expect(activeProfile).toMatchObject({
id: 'openai-default',
provider_kind: 'openai-compatible',
is_default: 1,
enabled: 1,
model: 'text-embedding-3-small',
dimensions: 1536
});
});
it('GET /api/v1/settings/embedding/test checks local-provider availability directly', async () => {
const response = await getEmbeddingTest({} as never);
expect(response.status).toBe(200);
const body = await response.json();
expect(body).toEqual({
available: true,
profile: {
id: 'local-default',
providerKind: 'local-transformers',
model: 'Xenova/all-MiniLM-L6-v2',
dimensions: 384
}
});
});
});

63
src/routes/api/v1/settings/embedding/test/+server.ts
View File

@@ -7,35 +7,24 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { getClient } from '$lib/server/db/client';
import { LocalEmbeddingProvider } from '$lib/server/embeddings/local.provider';
import { createProviderFromProfile } from '$lib/server/embeddings/registry';
import type { EmbeddingProfile } from '$lib/server/db/schema';
import { EmbeddingProfileEntity } from '$lib/server/models/embedding-profile';
import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper';
import { handleServiceError } from '$lib/server/utils/validation';
export const GET: RequestHandler = async () => {
try {
const db = getClient();
const profile = db
.prepare<
[],
EmbeddingProfile
>('SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1')
.get();
if (!profile) {
return json({ available: false, error: 'No active embedding profile configured' });
}
const provider = createProviderFromProfile(profile);
const provider = new LocalEmbeddingProvider();
const available = await provider.isAvailable();
return json({
available,
profile: {
id: profile.id,
providerKind: profile.providerKind,
model: profile.model,
dimensions: profile.dimensions
id: 'local-default',
providerKind: 'local-transformers',
model: provider.model,
dimensions: provider.dimensions
}
});
} catch (err) {
@@ -46,19 +35,43 @@ export const GET: RequestHandler = async () => {
export const POST: RequestHandler = async ({ request }) => {
try {
const body = await request.json();
const config = validateConfig(body);
if (config.provider === 'none') {
throw new InvalidInputError('Cannot test the "none" provider — no backend is configured.');
if (typeof body !== 'object' || body === null) {
throw new Error('Request body must be a JSON object');
}
const provider = createProviderFromConfig(config);
const candidate = body as Record<string, unknown>;
if (candidate.providerKind !== 'openai-compatible') {
throw new Error('Only openai-compatible providers can be tested via this endpoint');
}
if (typeof candidate.model !== 'string' || typeof candidate.dimensions !== 'number') {
throw new Error('model and dimensions are required');
}
const provider = createProviderFromProfile(
EmbeddingProfileMapper.fromEntity(
new EmbeddingProfileEntity({
id: typeof candidate.id === 'string' ? candidate.id : 'test-openai-profile',
provider_kind: 'openai-compatible',
title: typeof candidate.title === 'string' ? candidate.title : 'Test Provider',
enabled: true,
is_default: false,
model: candidate.model,
dimensions: candidate.dimensions,
config:
typeof candidate.config === 'object' && candidate.config !== null
? (candidate.config as Record<string, unknown>)
: {},
created_at: Date.now(),
updated_at: Date.now()
})
)
);
const available = await provider.isAvailable();
if (!available) {
return new Response(
JSON.stringify({
error: `Provider "${config.provider}" is not available. Check your configuration.`
error: 'Provider is not available. Check your configuration.'
}),
{ status: 400, headers: { 'Content-Type': 'application/json' } }
);

52
src/routes/repos/[id]/+page.svelte
View File

@@ -2,22 +2,24 @@
import { goto } from '$app/navigation';
import { resolve as resolveRoute } from '$app/paths';
import type { PageData } from './$types';
import type { Repository, RepositoryVersion, IndexingJob } from '$lib/types';
import type { Repository, IndexingJob } from '$lib/types';
import ConfirmDialog from '$lib/components/ConfirmDialog.svelte';
import IndexingProgress from '$lib/components/IndexingProgress.svelte';
import StatBadge from '$lib/components/StatBadge.svelte';
let { data }: { data: PageData } = $props();
// Initialized empty; $effect syncs from data prop on every navigation/reload.
let repo = $state<Repository & { versions?: RepositoryVersion[] }>(
{} as Repository & { versions?: RepositoryVersion[] }
let repoOverride = $state<
(Repository & { indexedVersions?: string[]; embeddingCount?: number }) | null
>(null);
const repo = $derived(
repoOverride ??
((data.repo ?? {}) as Repository & {
indexedVersions?: string[];
embeddingCount?: number;
})
);
let recentJobs = $state<IndexingJob[]>([]);
$effect(() => {
if (data.repo) repo = data.repo;
recentJobs = data.recentJobs ?? [];
});
const recentJobs = $derived((data.recentJobs ?? []) as IndexingJob[]);
let showDeleteConfirm = $state(false);
let activeJobId = $state<string | null>(null);
let errorMessage = $state<string | null>(null);
@@ -41,7 +43,7 @@
try {
const res = await fetch(`/api/v1/libs/${encodeURIComponent(repo.id)}`);
if (res.ok) {
repo = await res.json();
repoOverride = await res.json();
}
} catch {
// ignore
@@ -92,7 +94,8 @@
return new Date(ts as string).toLocaleString();
}
const versions = $derived(repo.versions ?? []);
const indexedVersions = $derived(repo.indexedVersions ?? []);
const embeddingCount = $derived(repo.embeddingCount ?? 0);
const totalSnippets = $derived(repo.totalSnippets ?? 0);
const totalTokens = $derived(repo.totalTokens ?? 0);
const trustScore = $derived(repo.trustScore ?? 0);
@@ -180,6 +183,7 @@
<!-- Stats -->
<div class="mt-6 grid grid-cols-2 gap-3 sm:grid-cols-4">
<StatBadge label="Snippets" value={totalSnippets.toLocaleString()} />
<StatBadge label="Embeddings" value={embeddingCount.toLocaleString()} />
<StatBadge label="Tokens" value={totalTokens.toLocaleString()} />
<StatBadge label="Trust Score" value="{trustScore.toFixed(1)}/10" />
{#if repo.stars != null}
@@ -210,31 +214,17 @@
</dl>
</div>
<!-- Versions -->
{#if versions.length > 0}
<!-- Indexed Versions -->
{#if indexedVersions.length > 0}
<div class="mt-6 rounded-xl border border-gray-200 bg-white p-5">
<h2 class="mb-3 text-sm font-semibold text-gray-700">Indexed Versions</h2>
<div class="divide-y divide-gray-100">
{#each versions as version (version.id)}
<div class="flex items-center justify-between py-2.5">
<div>
<span class="font-mono text-sm font-medium text-gray-900">{version.tag}</span>
{#if version.title}
<span class="ml-2 text-sm text-gray-500">{version.title}</span>
{/if}
</div>
<div class="flex items-center gap-3">
<div class="flex flex-wrap gap-2">
{#each indexedVersions as versionTag (versionTag)}
<span
class="rounded-full px-2 py-0.5 text-xs {stateColors[version.state] ??
'bg-gray-100 text-gray-600'}"
class="rounded-full border border-green-200 bg-green-50 px-3 py-1 font-mono text-sm text-green-800"
>
{stateLabels[version.state] ?? version.state}
{versionTag}
</span>
{#if version.indexedAt}
<span class="text-xs text-gray-400">{formatDate(version.indexedAt)}</span>
{/if}
</div>
</div>
{/each}
</div>
</div>

12
src/routes/repos/[id]/page.server.test.ts
View File

@@ -8,7 +8,11 @@ describe('/repos/[id] page server load', () => {
.mockResolvedValueOnce({
ok: true,
status: 200,
json: async () => ({ id: '/facebook/react', title: 'React' })
json: async () => ({
id: '/facebook/react',
title: 'React',
indexedVersions: ['main', 'v18.3.0']
})
})
.mockResolvedValueOnce({
ok: true,
@@ -27,7 +31,11 @@ describe('/repos/[id] page server load', () => {
'/api/v1/jobs?repositoryId=%2Ffacebook%2Freact&limit=5'
);
expect(result).toEqual({
repo: { id: '/facebook/react', title: 'React' },
repo: {
id: '/facebook/react',
title: 'React',
indexedVersions: ['main', 'v18.3.0']
},
recentJobs: [{ id: 'job-1', repositoryId: '/facebook/react' }]
});
});

22
src/routes/settings/+page.server.ts Normal file
View File

@@ -0,0 +1,22 @@
import type { PageServerLoad } from './$types';
import { getClient } from '$lib/server/db/client.js';
import { LocalEmbeddingProvider } from '$lib/server/embeddings/local.provider.js';
import { EmbeddingSettingsDtoMapper } from '$lib/server/mappers/embedding-settings.dto.mapper.js';
import { EmbeddingSettingsService } from '$lib/server/services/embedding-settings.service.js';
export const load: PageServerLoad = async () => {
const service = new EmbeddingSettingsService(getClient());
const settings = EmbeddingSettingsDtoMapper.toDto(service.getSettings());
let localProviderAvailable = false;
try {
localProviderAvailable = await new LocalEmbeddingProvider().isAvailable();
} catch {
localProviderAvailable = false;
}
return {
settings,
localProviderAvailable
};
};

265
src/routes/settings/+page.svelte
View File

@@ -1,5 +1,12 @@
<script lang="ts">
import { onDestroy } from 'svelte';
import StatBadge from '$lib/components/StatBadge.svelte';
import type {
EmbeddingProfileDto,
EmbeddingSettingsDto,
EmbeddingSettingsUpdateDto
} from '$lib/dtos/embedding-settings';
import type { PageProps } from './$types';
// ---------------------------------------------------------------------------
// Provider presets
@@ -30,11 +37,25 @@
// State
// ---------------------------------------------------------------------------
let provider = $state<'none' | 'openai' | 'local'>('none');
let baseUrl = $state('https://api.openai.com/v1');
let { data }: PageProps = $props();
function getInitialSettings(): EmbeddingSettingsDto {
return data.settings;
}
function getInitialLocalProviderAvailability(): boolean {
return data.localProviderAvailable;
}
let settingsOverride = $state<EmbeddingSettingsDto | null>(null);
let provider = $state<'none' | 'openai' | 'local'>(
resolveProvider(getInitialSettings().activeProfile)
);
let baseUrl = $state(resolveBaseUrl(getInitialSettings()));
let apiKey = $state('');
let model = $state('text-embedding-3-small');
let dimensions = $state<number | undefined>(1536);
let model = $state(resolveModel(getInitialSettings()));
let dimensions = $state<number | undefined>(resolveDimensions(getInitialSettings()));
let openaiProfileId = $state(resolveOpenAiProfileId(getInitialSettings()));
let testStatus = $state<'idle' | 'testing' | 'ok' | 'error'>('idle');
let testError = $state<string | null>(null);
@@ -45,51 +66,12 @@
let saveError = $state<string | null>(null);
let saveStatusTimer: ReturnType<typeof setTimeout> | null = null;
let localAvailable = $state<boolean | null>(null);
let loading = $state(true);
const currentSettings = $derived(settingsOverride ?? data.settings);
const activeProfile = $derived(currentSettings.activeProfile);
const activeConfigEntries = $derived(activeProfile?.configEntries ?? []);
// ---------------------------------------------------------------------------
// Load current config + probe local provider on mount
// ---------------------------------------------------------------------------
$effect(() => {
let cancelled = false;
(async () => {
try {
const res = await fetch('/api/v1/settings/embedding');
if (!cancelled && res.ok) {
const data = await res.json();
provider = data.provider ?? 'none';
if (data.openai) {
baseUrl = data.openai.baseUrl ?? baseUrl;
model = data.openai.model ?? model;
dimensions = data.openai.dimensions ?? dimensions;
// apiKey is intentionally not returned by the server; leave blank
}
}
} catch {
// Non-fatal — fall back to defaults
} finally {
if (!cancelled) loading = false;
}
// Probe whether the local provider is available
try {
const res = await fetch('/api/v1/settings/embedding/test');
if (!cancelled && res.ok) {
const data = await res.json();
localAvailable = data.available ?? false;
}
} catch {
if (!cancelled) localAvailable = false;
}
})();
return () => {
cancelled = true;
onDestroy(() => {
if (saveStatusTimer) clearTimeout(saveStatusTimer);
};
});
// ---------------------------------------------------------------------------
@@ -106,6 +88,10 @@
}
async function testConnection() {
if (provider !== 'openai') {
return;
}
testStatus = 'testing';
testError = null;
testDimensions = null;
@@ -113,7 +99,14 @@
const res = await fetch('/api/v1/settings/embedding/test', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } })
body: JSON.stringify({
id: openaiProfileId,
title: 'OpenAI-compatible',
providerKind: 'openai-compatible',
model,
dimensions: dimensions ?? 1536,
config: { baseUrl, apiKey, model, ...(dimensions ? { dimensions } : {}) }
})
});
if (res.ok) {
const data = await res.json();
@@ -138,9 +131,10 @@
const res = await fetch('/api/v1/settings/embedding', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } })
body: JSON.stringify(buildSaveRequest())
});
if (res.ok) {
settingsOverride = (await res.json()) as EmbeddingSettingsDto;
saveStatus = 'ok';
if (saveStatusTimer) clearTimeout(saveStatusTimer);
saveStatusTimer = setTimeout(() => {
@@ -164,6 +158,74 @@
event.preventDefault();
void save();
}
function getOpenAiProfile(settings: EmbeddingSettingsDto): EmbeddingProfileDto | null {
return settings.profiles.find((profile) => profile.providerKind === 'openai-compatible') ?? null;
}
function resolveProvider(profile: EmbeddingProfileDto | null): 'none' | 'openai' | 'local' {
if (!profile) return 'none';
if (profile.providerKind === 'local-transformers') return 'local';
if (profile.providerKind === 'openai-compatible') return 'openai';
return 'none';
}
function resolveBaseUrl(settings: EmbeddingSettingsDto): string {
const profile = settings.activeProfile?.providerKind === 'openai-compatible'
? settings.activeProfile
: getOpenAiProfile(settings);
return typeof profile?.config.baseUrl === 'string'
? profile.config.baseUrl
: 'https://api.openai.com/v1';
}
function resolveModel(settings: EmbeddingSettingsDto): string {
const profile = settings.activeProfile?.providerKind === 'openai-compatible'
? settings.activeProfile
: getOpenAiProfile(settings);
return typeof profile?.config.model === 'string'
? profile.config.model
: profile?.model ?? 'text-embedding-3-small';
}
function resolveDimensions(settings: EmbeddingSettingsDto): number | undefined {
const profile = settings.activeProfile?.providerKind === 'openai-compatible'
? settings.activeProfile
: getOpenAiProfile(settings);
return profile?.dimensions ?? 1536;
}
function resolveOpenAiProfileId(settings: EmbeddingSettingsDto): string {
const profile = getOpenAiProfile(settings);
return profile?.id ?? 'openai-default';
}
function buildSaveRequest(): EmbeddingSettingsUpdateDto {
if (provider === 'none') {
return { activeProfileId: null };
}
if (provider === 'local') {
return { activeProfileId: 'local-default' };
}
return {
activeProfileId: openaiProfileId,
profile: {
id: openaiProfileId,
providerKind: 'openai-compatible',
title: 'OpenAI-compatible',
model,
dimensions: dimensions ?? 1536,
config: { baseUrl, apiKey, model, ...(dimensions ? { dimensions } : {}) }
}
};
}
function formatTimestamp(timestamp: number): string {
const normalizedTimestamp = timestamp > 1_000_000_000_000 ? timestamp : timestamp * 1000;
return new Date(normalizedTimestamp).toLocaleString();
}
</script>
<svelte:head>
@@ -175,16 +237,108 @@
<p class="mt-0.5 text-sm text-gray-500">Configure TrueRef embedding and indexing options</p>
</div>
<!-- Embedding Provider Card -->
<div class="mb-4 grid gap-4 lg:grid-cols-[1.2fr_0.8fr]">
<div class="rounded-xl border border-gray-200 bg-white p-6">
<h2 class="mb-1 text-base font-semibold text-gray-900">Current Active Profile</h2>
<p class="mb-4 text-sm text-gray-500">
This is the profile used for semantic indexing and retrieval right now.
</p>
{#if activeProfile}
<div class="grid gap-4 md:grid-cols-2">
<div class="space-y-4">
<div>
<p class="text-lg font-semibold text-gray-900">{activeProfile.title}</p>
<p class="mt-1 text-sm text-gray-500">Profile ID: {activeProfile.id}</p>
</div>
<dl class="rounded-lg border border-gray-200 bg-gray-50 p-4 text-sm">
<div class="grid grid-cols-[110px_1fr] gap-x-4 gap-y-1 border-b border-gray-200 pb-3">
<dt class="font-medium text-gray-500">Provider</dt>
<dd class="font-semibold text-gray-900">{activeProfile.providerKind}</dd>
<dt class="font-medium text-gray-500">Model</dt>
<dd class="break-all font-semibold text-gray-900">{activeProfile.model}</dd>
<dt class="font-medium text-gray-500">Dimensions</dt>
<dd class="font-semibold text-gray-900">{activeProfile.dimensions}</dd>
</div>
<div class="grid grid-cols-[110px_1fr] gap-x-4 gap-y-2 pt-3">
<dt class="text-gray-500">Enabled</dt>
<dd class="font-medium text-gray-800">{activeProfile.enabled ? 'Yes' : 'No'}</dd>
<dt class="text-gray-500">Default</dt>
<dd class="font-medium text-gray-800">{activeProfile.isDefault ? 'Yes' : 'No'}</dd>
<dt class="text-gray-500">Updated</dt>
<dd class="font-medium text-gray-800">{formatTimestamp(activeProfile.updatedAt)}</dd>
</div>
</dl>
</div>
<div class="rounded-lg border border-gray-200 bg-gray-50 p-4">
<p class="text-sm font-medium text-gray-800">Provider configuration</p>
<p class="mb-3 mt-1 text-sm text-gray-500">
These are the provider-specific settings currently saved for the active profile.
</p>
{#if activeConfigEntries.length > 0}
<ul class="space-y-2 text-sm">
{#each activeConfigEntries as entry (entry.key)}
<li class="flex items-start justify-between gap-4 border-b border-gray-200 pb-2 last:border-b-0 last:pb-0">
<span class="font-medium text-gray-600">{entry.key}</span>
<span class={entry.redacted ? 'text-gray-500' : 'text-gray-800'}>{entry.value}</span>
</li>
{/each}
</ul>
{:else}
<p class="text-sm text-gray-500">
No provider-specific configuration is stored for this profile.
</p>
<p class="mt-2 text-sm text-gray-500">
For <span class="font-medium text-gray-700">OpenAI-compatible</span> profiles, edit the
settings in the <span class="font-medium text-gray-700">Embedding Provider</span> form
below. The built-in <span class="font-medium text-gray-700">Local Model</span> profile
does not currently expose extra configurable fields.
</p>
{/if}
</div>
</div>
{:else}
<div class="rounded-lg border border-amber-200 bg-amber-50 p-4 text-sm text-amber-800">
Embeddings are currently disabled. Keyword search remains available, but no embedding profile is active.
</div>
{/if}
</div>
<div class="rounded-xl border border-gray-200 bg-white p-6">
<h2 class="mb-1 text-base font-semibold text-gray-900">Profile Inventory</h2>
<p class="mb-4 text-sm text-gray-500">Profiles stored in the database and available for activation.</p>
<div class="grid grid-cols-2 gap-3">
<StatBadge label="Profiles" value={String(currentSettings.profiles.length)} />
<StatBadge label="Active" value={activeProfile ? '1' : '0'} />
</div>
<div class="mt-4 space-y-2">
{#each currentSettings.profiles as profile (profile.id)}
<div class="rounded-lg border border-gray-200 px-3 py-2 text-sm">
<div class="flex items-center justify-between gap-3">
<div>
<p class="font-medium text-gray-900">{profile.title}</p>
<p class="text-gray-500">{profile.id}</p>
</div>
{#if profile.id === currentSettings.activeProfileId}
<span class="rounded-full bg-blue-50 px-2 py-0.5 text-xs font-medium text-blue-700">Active</span>
{/if}
</div>
</div>
{/each}
</div>
</div>
</div>
<div class="rounded-xl border border-gray-200 bg-white p-6">
<h2 class="mb-1 text-base font-semibold text-gray-900">Embedding Provider</h2>
<p class="mb-4 text-sm text-gray-500">
Embeddings enable semantic search. Without them, only keyword search (FTS5) is used.
</p>
{#if loading}
<p class="text-sm text-gray-400">Loading current configuration…</p>
{:else}
<form class="space-y-4" onsubmit={handleSubmit}>
<!-- Provider selector -->
<div class="mb-4 flex gap-2">
@@ -314,9 +468,7 @@
<div class="rounded-lg border border-gray-200 bg-gray-50 p-4 text-sm">
<p class="font-medium text-gray-800">Local ONNX model via @xenova/transformers</p>
<p class="mt-1 text-gray-500">Model: Xenova/all-MiniLM-L6-v2 · 384 dimensions</p>
{#if localAvailable === null}
<p class="mt-2 text-gray-400">Checking availability…</p>
{:else if localAvailable}
{#if getInitialLocalProviderAvailability()}
<p class="mt-2 text-green-600">@xenova/transformers is installed and ready.</p>
{:else}
<p class="mt-2 text-amber-700">
@@ -382,7 +534,6 @@
</button>
</div>
</form>
{/if}
</div>
<!-- About card -->

103
src/routes/settings/page.server.test.ts Normal file
View File

@@ -0,0 +1,103 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import Database from 'better-sqlite3';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
let db: Database.Database;
vi.mock('$lib/server/db/client.js', () => ({
getClient: () => db
}));
vi.mock('$lib/server/embeddings/local.provider.js', () => ({
LocalEmbeddingProvider: class {
async isAvailable() {
return true;
}
}
}));
import { load } from './+page.server.js';
function createTestDb(): Database.Database {
const client = new Database(':memory:');
client.pragma('foreign_keys = ON');
const migrationsFolder = join(import.meta.dirname, '../../lib/server/db/migrations');
const ftsFile = join(import.meta.dirname, '../../lib/server/db/fts.sql');
for (const migration of [
'0000_large_master_chief.sql',
'0001_quick_nighthawk.sql',
'0002_silky_stellaris.sql'
]) {
const statements = readFileSync(join(migrationsFolder, migration), 'utf-8')
.split('--> statement-breakpoint')
.map((statement) => statement.trim())
.filter(Boolean);
for (const statement of statements) {
client.exec(statement);
}
}
client.exec(readFileSync(ftsFile, 'utf-8'));
return client;
}
describe('/settings page server load', () => {
beforeEach(() => {
db = createTestDb();
});
it('returns the active profile and local provider availability', async () => {
db.prepare(
`INSERT INTO embedding_profiles
(id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
'openai-default',
'openai-compatible',
'OpenAI-compatible',
1,
1,
'text-embedding-3-small',
1536,
JSON.stringify({
baseUrl: 'https://api.openai.com/v1',
apiKey: 'sk-test',
model: 'text-embedding-3-small'
}),
1710000000,
1710000000
);
db.prepare('UPDATE embedding_profiles SET is_default = 0 WHERE id = ?').run('local-default');
const result = (await load({} as never)) as {
localProviderAvailable: boolean;
settings: {
activeProfileId: string | null;
activeProfile: {
config: Record<string, unknown>;
configEntries: Array<{ key: string; value: string; redacted: boolean }>;
} | null;
};
};
expect(result.localProviderAvailable).toBe(true);
expect(result.settings.activeProfileId).toBe('openai-default');
expect(result.settings.activeProfile).toMatchObject({
id: 'openai-default',
providerKind: 'openai-compatible'
});
expect(result.settings.activeProfile?.config).toEqual({
baseUrl: 'https://api.openai.com/v1',
model: 'text-embedding-3-small'
});
expect(result.settings.activeProfile?.configEntries).toEqual(
expect.arrayContaining([
expect.objectContaining({ key: 'apiKey', value: '[redacted]', redacted: true })
])
);
});
});