mozempk/trueref
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(TRUEREF-0020): add embedding profiles, default local embeddings, and version-scoped semantic retrieval

Browse Source 
- Add embedding_profiles table with provider registry pattern
- Install @xenova/transformers as runtime dependency
- Update snippet_embeddings with composite PK (snippet_id, profile_id)
- Seed default local profile using Xenova/all-MiniLM-L6-v2
- Add provider registry (local-transformers, openai-compatible)
- Update EmbeddingService to persist and retrieve by profileId
- Add version-scoped VectorSearch with optional versionId filtering
- Add searchMode (auto|keyword|semantic|hybrid) to HybridSearchService
- Update API /context route to load active profile, support searchMode/alpha params
- Extend MCP query-docs tool with searchMode and alpha parameters
- Update settings API to work with embedding_profiles table
- Add comprehensive test coverage for profiles, registry, version scoping

Status: 445/451 tests passing, core feature complete
This commit is contained in:
Giancarmine Salucci
2026-03-25 19:16:37 +01:00
parent fef6f66930
commit 169df4d984
19 changed files with 2668 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/routes/api/v1/context/+server.ts
View File

@@ -16,6 +16,8 @@ import { getClient } from '$lib/server/db/client';
import { dtoJsonResponse } from '$lib/server/api/dto-response';
import { SearchService } from '$lib/server/search/search.service';
import { HybridSearchService } from '$lib/server/search/hybrid.search.service';
import { createProviderFromProfile } from '$lib/server/embeddings/registry';
import type { EmbeddingProfile } from '$lib/server/db/schema';
import { parseLibraryId } from '$lib/server/api/library-id';
import { selectSnippetsWithinBudget, DEFAULT_TOKEN_BUDGET } from '$lib/server/api/token-budget';
import {
@@ -28,12 +30,20 @@ import {
// Helpers
// ---------------------------------------------------------------------------
function getServices() {
const db = getClient();
function getServices(db: ReturnType<typeof getClient>) {
const searchService = new SearchService(db);
// No embedding provider — pure FTS5 mode (alpha=0 equivalent).
const hybridService = new HybridSearchService(db, searchService, null);
return { db, searchService, hybridService };
// Load the active embedding profile from the database
const profileRow = db
.prepare<[], EmbeddingProfile>(
'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1'
)
.get();
const provider = profileRow ? createProviderFromProfile(profileRow) : null;
const hybridService = new HybridSearchService(db, searchService, provider);
return { db, searchService, hybridService, profileId: profileRow?.id };
}
interface RawRepoConfig {
@@ -93,6 +103,14 @@ export const GET: RequestHandler = async ({ url }) => {
const tokensRaw = parseInt(url.searchParams.get('tokens') ?? String(DEFAULT_TOKEN_BUDGET), 10);
const maxTokens = isNaN(tokensRaw) || tokensRaw < 1 ? DEFAULT_TOKEN_BUDGET : tokensRaw;
// Parse searchMode and alpha
const rawMode = url.searchParams.get('searchMode') ?? 'auto';
const searchMode = ['auto', 'keyword', 'semantic', 'hybrid'].includes(rawMode)
? (rawMode as 'auto' | 'keyword' | 'semantic' | 'hybrid')
: 'auto';
const alphaRaw = parseFloat(url.searchParams.get('alpha') ?? '0.5');
const alpha = isNaN(alphaRaw) ? 0.5 : Math.max(0, Math.min(1, alphaRaw));
// Parse the libraryId
let parsed: ReturnType<typeof parseLibraryId>;
try {
@@ -108,7 +126,8 @@ export const GET: RequestHandler = async ({ url }) => {
}
try {
const { db, hybridService } = getServices();
const db = getClient();
const { hybridService, profileId } = getServices(db);
// Verify the repository exists and check its state.
const repo = db
@@ -158,7 +177,10 @@ export const GET: RequestHandler = async ({ url }) => {
const searchResults = await hybridService.search(query, {
repositoryId: parsed.repositoryId,
versionId,
limit: 50 // fetch more than needed; token budget will trim
limit: 50, // fetch more than needed; token budget will trim
searchMode,
alpha,
profileId
});
// Apply token budget.

216
src/routes/api/v1/settings/embedding/+server.ts
View File

@@ -1,147 +1,149 @@
/**
* GET /api/v1/settings/embedding — retrieve current embedding configuration
* PUT /api/v1/settings/embedding — update embedding configuration
* GET /api/v1/settings/embedding — retrieve all embedding profiles
* POST /api/v1/settings/embedding — create or update an embedding profile
* PUT /api/v1/settings/embedding — alias for POST (backward compat)
*/
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { getClient } from '$lib/server/db/client';
import {
EMBEDDING_CONFIG_KEY,
createProviderFromConfig,
defaultEmbeddingConfig,
type EmbeddingConfig
} from '$lib/server/embeddings/factory';
import { createProviderFromProfile } from '$lib/server/embeddings/registry';
import type { EmbeddingProfile, NewEmbeddingProfile } from '$lib/server/db/schema';
import { handleServiceError, InvalidInputError } from '$lib/server/utils/validation';
// ---------------------------------------------------------------------------
// Helpers
// GET — Return all profiles
// ---------------------------------------------------------------------------
function readConfig(db: ReturnType<typeof getClient>): EmbeddingConfig {
const row = db
.prepare(`SELECT value FROM settings WHERE key = ?`)
.get(EMBEDDING_CONFIG_KEY) as { value: string } | undefined;
if (!row) return defaultEmbeddingConfig();
export const GET: RequestHandler = () => {
try {
return JSON.parse(row.value) as EmbeddingConfig;
} catch {
return defaultEmbeddingConfig();
}
}
const db = getClient();
const profiles = db
.prepare('SELECT * FROM embedding_profiles ORDER BY is_default DESC, created_at ASC')
.all() as EmbeddingProfile[];
function validateConfig(body: unknown): EmbeddingConfig {
// Sanitize: remove sensitive config fields like apiKey
const safeProfiles = profiles.map(sanitizeProfile);
return json({ profiles: safeProfiles });
} catch (err) {
return handleServiceError(err);
}
};
// ---------------------------------------------------------------------------
// POST/PUT — Create or update a profile
// ---------------------------------------------------------------------------
async function upsertProfile(body: unknown) {
if (typeof body !== 'object' || body === null) {
throw new InvalidInputError('Request body must be a JSON object');
}
const obj = body as Record<string, unknown>;
const provider = obj.provider;
if (provider !== 'openai' && provider !== 'local' && provider !== 'none') {
// Required fields
if (typeof obj.id !== 'string' || !obj.id) {
throw new InvalidInputError('id is required');
}
if (typeof obj.providerKind !== 'string' || !obj.providerKind) {
throw new InvalidInputError('providerKind is required');
}
if (typeof obj.title !== 'string' || !obj.title) {
throw new InvalidInputError('title is required');
}
if (typeof obj.model !== 'string' || !obj.model) {
throw new InvalidInputError('model is required');
}
if (typeof obj.dimensions !== 'number') {
throw new InvalidInputError('dimensions must be a number');
}
const profile: NewEmbeddingProfile = {
id: obj.id,
providerKind: obj.providerKind,
title: obj.title,
enabled: typeof obj.enabled === 'boolean' ? obj.enabled : true,
isDefault: typeof obj.isDefault === 'boolean' ? obj.isDefault : false,
model: obj.model,
dimensions: obj.dimensions,
config: (obj.config as Record<string, unknown>) ?? {},
createdAt: Date.now(),
updatedAt: Date.now()
};
// Validate provider availability before persisting
const provider = createProviderFromProfile(profile as EmbeddingProfile);
const available = await provider.isAvailable();
if (!available) {
throw new InvalidInputError(
`Invalid provider "${String(provider)}". Must be one of: openai, local, none.`
`Could not connect to the "${profile.providerKind}" provider. Check your configuration.`
);
}
if (provider === 'openai') {
const openai = obj.openai as Record<string, unknown> | undefined;
if (!openai || typeof openai !== 'object') {
throw new InvalidInputError('openai config object is required when provider is "openai"');
}
if (typeof openai.baseUrl !== 'string' || !openai.baseUrl) {
throw new InvalidInputError('openai.baseUrl must be a non-empty string');
}
if (typeof openai.apiKey !== 'string' || !openai.apiKey) {
throw new InvalidInputError('openai.apiKey must be a non-empty string');
}
if (typeof openai.model !== 'string' || !openai.model) {
throw new InvalidInputError('openai.model must be a non-empty string');
}
const db = getClient();
const config: EmbeddingConfig = {
provider: 'openai',
openai: {
baseUrl: openai.baseUrl as string,
apiKey: openai.apiKey as string,
model: openai.model as string,
dimensions:
typeof openai.dimensions === 'number' ? (openai.dimensions as number) : undefined,
maxBatchSize:
typeof openai.maxBatchSize === 'number'
? (openai.maxBatchSize as number)
: undefined
}
};
return config;
// If setting as default, clear other defaults first
if (profile.isDefault) {
db.prepare('UPDATE embedding_profiles SET is_default = 0').run();
}
return { provider: provider as 'local' | 'none' };
// Upsert the profile
db.prepare(
`INSERT INTO embedding_profiles
(id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
provider_kind = excluded.provider_kind,
title = excluded.title,
enabled = excluded.enabled,
is_default = excluded.is_default,
model = excluded.model,
dimensions = excluded.dimensions,
config = excluded.config,
updated_at = excluded.updated_at`
).run(
profile.id,
profile.providerKind,
profile.title,
profile.enabled ? 1 : 0,
profile.isDefault ? 1 : 0,
profile.model,
profile.dimensions,
JSON.stringify(profile.config),
profile.createdAt,
profile.updatedAt
);
const inserted = db
.prepare('SELECT * FROM embedding_profiles WHERE id = ?')
.get(profile.id) as EmbeddingProfile;
return sanitizeProfile(inserted);
}
// ---------------------------------------------------------------------------
// GET
// ---------------------------------------------------------------------------
export const GET: RequestHandler = () => {
try {
const db = getClient();
const config = readConfig(db);
// Strip the apiKey from the response for security.
const safeConfig = sanitizeForResponse(config);
return json(safeConfig);
} catch (err) {
return handleServiceError(err);
}
};
// ---------------------------------------------------------------------------
// PUT
// ---------------------------------------------------------------------------
export const PUT: RequestHandler = async ({ request }) => {
export const POST: RequestHandler = async ({ request }) => {
try {
const body = await request.json();
const config = validateConfig(body);
// Verify provider connectivity before persisting (skip for noop).
if (config.provider !== 'none') {
const provider = createProviderFromConfig(config);
const available = await provider.isAvailable();
if (!available) {
throw new InvalidInputError(
`Could not connect to the "${config.provider}" embedding provider. Check your configuration.`
);
}
}
const db = getClient();
db.prepare(
`INSERT INTO settings (key, value, updated_at)
VALUES (?, ?, unixepoch())
ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`
).run(EMBEDDING_CONFIG_KEY, JSON.stringify(config));
const safeConfig = sanitizeForResponse(config);
return json(safeConfig);
const profile = await upsertProfile(body);
return json(profile);
} catch (err) {
return handleServiceError(err);
}
};
// Backward compat alias
export const PUT: RequestHandler = POST;
// ---------------------------------------------------------------------------
// Sanitize — remove sensitive fields before returning to clients
// Sanitize — remove sensitive config fields before returning to clients
// ---------------------------------------------------------------------------
function sanitizeForResponse(config: EmbeddingConfig): Omit<EmbeddingConfig, 'openai'> & {
openai?: Omit<NonNullable<EmbeddingConfig['openai']>, 'apiKey'>;
} {
if (config.provider === 'openai' && config.openai) {
const { apiKey: _apiKey, ...rest } = config.openai;
return { ...config, openai: rest };
function sanitizeProfile(profile: EmbeddingProfile): EmbeddingProfile {
const config = profile.config as Record<string, unknown>;
if (config && config.apiKey) {
const { apiKey: _apiKey, ...rest } = config;
return { ...profile, config: rest };
}
return config;
return profile;
}

93
src/routes/api/v1/settings/embedding/test/+server.ts
View File

@@ -1,82 +1,47 @@
/**
* POST /api/v1/settings/embedding/test
* GET /api/v1/settings/embedding/test
*
* Validates an embedding provider configuration by creating a provider
* instance and calling embed(['test']). Returns success with dimensions
* or a descriptive error without persisting any changes.
* Tests the active default embedding profile by creating a provider instance
* and checking availability. Returns success with profile metadata or error.
*/
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import {
createProviderFromConfig,
type EmbeddingConfig
} from '$lib/server/embeddings/factory';
import { handleServiceError, InvalidInputError } from '$lib/server/utils/validation';
import { getClient } from '$lib/server/db/client';
import { createProviderFromProfile } from '$lib/server/embeddings/registry';
import type { EmbeddingProfile } from '$lib/server/db/schema';
import { handleServiceError } from '$lib/server/utils/validation';
export const GET: RequestHandler = async () => {
try {
const provider = createProviderFromConfig({ provider: 'local' });
const db = getClient();
const profile = db
.prepare<[], EmbeddingProfile>(
'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1'
)
.get();
if (!profile) {
return json({ available: false, error: 'No active embedding profile configured' });
}
const provider = createProviderFromProfile(profile);
const available = await provider.isAvailable();
return json({ available });
return json({
available,
profile: {
id: profile.id,
providerKind: profile.providerKind,
model: profile.model,
dimensions: profile.dimensions
}
});
} catch (err) {
return handleServiceError(err);
}
};
// ---------------------------------------------------------------------------
// Validate — reuse the same shape accepted by PUT /settings/embedding
// ---------------------------------------------------------------------------
function validateConfig(body: unknown): EmbeddingConfig {
if (typeof body !== 'object' || body === null) {
throw new InvalidInputError('Request body must be a JSON object');
}
const obj = body as Record<string, unknown>;
const provider = obj.provider;
if (provider !== 'openai' && provider !== 'local' && provider !== 'none') {
throw new InvalidInputError(
`Invalid provider "${String(provider)}". Must be one of: openai, local, none.`
);
}
if (provider === 'openai') {
const openai = obj.openai as Record<string, unknown> | undefined;
if (!openai || typeof openai !== 'object') {
throw new InvalidInputError('openai config object is required when provider is "openai"');
}
if (typeof openai.baseUrl !== 'string' || !openai.baseUrl) {
throw new InvalidInputError('openai.baseUrl must be a non-empty string');
}
if (typeof openai.apiKey !== 'string' || !openai.apiKey) {
throw new InvalidInputError('openai.apiKey must be a non-empty string');
}
if (typeof openai.model !== 'string' || !openai.model) {
throw new InvalidInputError('openai.model must be a non-empty string');
}
return {
provider: 'openai',
openai: {
baseUrl: openai.baseUrl as string,
apiKey: openai.apiKey as string,
model: openai.model as string,
dimensions:
typeof openai.dimensions === 'number' ? (openai.dimensions as number) : undefined,
maxBatchSize:
typeof openai.maxBatchSize === 'number' ? (openai.maxBatchSize as number) : undefined
}
};
}
return { provider: provider as 'local' | 'none' };
}
// ---------------------------------------------------------------------------
// POST
// ---------------------------------------------------------------------------
export const POST: RequestHandler = async ({ request }) => {
try {