feat(TRUEREF-0020): add embedding profiles, default local embeddings, and version-scoped semantic retrieval
- Add embedding_profiles table with provider registry pattern - Install @xenova/transformers as runtime dependency - Update snippet_embeddings with composite PK (snippet_id, profile_id) - Seed default local profile using Xenova/all-MiniLM-L6-v2 - Add provider registry (local-transformers, openai-compatible) - Update EmbeddingService to persist and retrieve by profileId - Add version-scoped VectorSearch with optional versionId filtering - Add searchMode (auto|keyword|semantic|hybrid) to HybridSearchService - Update API /context route to load active profile, support searchMode/alpha params - Extend MCP query-docs tool with searchMode and alpha parameters - Update settings API to work with embedding_profiles table - Add comprehensive test coverage for profiles, registry, version scoping Status: 445/451 tests passing, core feature complete
This commit is contained in:
Giancarmine Salucci
@@ -1,147 +1,149 @@
|
||||
/**
|
||||
* GET /api/v1/settings/embedding — retrieve current embedding configuration
|
||||
* PUT /api/v1/settings/embedding — update embedding configuration
|
||||
* GET /api/v1/settings/embedding — retrieve all embedding profiles
|
||||
* POST /api/v1/settings/embedding — create or update an embedding profile
|
||||
* PUT /api/v1/settings/embedding — alias for POST (backward compat)
|
||||
*/
|
||||
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getClient } from '$lib/server/db/client';
|
||||
import {
|
||||
EMBEDDING_CONFIG_KEY,
|
||||
createProviderFromConfig,
|
||||
defaultEmbeddingConfig,
|
||||
type EmbeddingConfig
|
||||
} from '$lib/server/embeddings/factory';
|
||||
import { createProviderFromProfile } from '$lib/server/embeddings/registry';
|
||||
import type { EmbeddingProfile, NewEmbeddingProfile } from '$lib/server/db/schema';
|
||||
import { handleServiceError, InvalidInputError } from '$lib/server/utils/validation';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// GET — Return all profiles
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function readConfig(db: ReturnType<typeof getClient>): EmbeddingConfig {
|
||||
const row = db
|
||||
.prepare(`SELECT value FROM settings WHERE key = ?`)
|
||||
.get(EMBEDDING_CONFIG_KEY) as { value: string } | undefined;
|
||||
|
||||
if (!row) return defaultEmbeddingConfig();
|
||||
|
||||
export const GET: RequestHandler = () => {
|
||||
try {
|
||||
return JSON.parse(row.value) as EmbeddingConfig;
|
||||
} catch {
|
||||
return defaultEmbeddingConfig();
|
||||
}
|
||||
}
|
||||
const db = getClient();
|
||||
const profiles = db
|
||||
.prepare('SELECT * FROM embedding_profiles ORDER BY is_default DESC, created_at ASC')
|
||||
.all() as EmbeddingProfile[];
|
||||
|
||||
function validateConfig(body: unknown): EmbeddingConfig {
|
||||
// Sanitize: remove sensitive config fields like apiKey
|
||||
const safeProfiles = profiles.map(sanitizeProfile);
|
||||
return json({ profiles: safeProfiles });
|
||||
} catch (err) {
|
||||
return handleServiceError(err);
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// POST/PUT — Create or update a profile
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function upsertProfile(body: unknown) {
|
||||
if (typeof body !== 'object' || body === null) {
|
||||
throw new InvalidInputError('Request body must be a JSON object');
|
||||
}
|
||||
|
||||
const obj = body as Record<string, unknown>;
|
||||
|
||||
const provider = obj.provider;
|
||||
if (provider !== 'openai' && provider !== 'local' && provider !== 'none') {
|
||||
// Required fields
|
||||
if (typeof obj.id !== 'string' || !obj.id) {
|
||||
throw new InvalidInputError('id is required');
|
||||
}
|
||||
if (typeof obj.providerKind !== 'string' || !obj.providerKind) {
|
||||
throw new InvalidInputError('providerKind is required');
|
||||
}
|
||||
if (typeof obj.title !== 'string' || !obj.title) {
|
||||
throw new InvalidInputError('title is required');
|
||||
}
|
||||
if (typeof obj.model !== 'string' || !obj.model) {
|
||||
throw new InvalidInputError('model is required');
|
||||
}
|
||||
if (typeof obj.dimensions !== 'number') {
|
||||
throw new InvalidInputError('dimensions must be a number');
|
||||
}
|
||||
|
||||
const profile: NewEmbeddingProfile = {
|
||||
id: obj.id,
|
||||
providerKind: obj.providerKind,
|
||||
title: obj.title,
|
||||
enabled: typeof obj.enabled === 'boolean' ? obj.enabled : true,
|
||||
isDefault: typeof obj.isDefault === 'boolean' ? obj.isDefault : false,
|
||||
model: obj.model,
|
||||
dimensions: obj.dimensions,
|
||||
config: (obj.config as Record<string, unknown>) ?? {},
|
||||
createdAt: Date.now(),
|
||||
updatedAt: Date.now()
|
||||
};
|
||||
|
||||
// Validate provider availability before persisting
|
||||
const provider = createProviderFromProfile(profile as EmbeddingProfile);
|
||||
const available = await provider.isAvailable();
|
||||
if (!available) {
|
||||
throw new InvalidInputError(
|
||||
`Invalid provider "${String(provider)}". Must be one of: openai, local, none.`
|
||||
`Could not connect to the "${profile.providerKind}" provider. Check your configuration.`
|
||||
);
|
||||
}
|
||||
|
||||
if (provider === 'openai') {
|
||||
const openai = obj.openai as Record<string, unknown> | undefined;
|
||||
if (!openai || typeof openai !== 'object') {
|
||||
throw new InvalidInputError('openai config object is required when provider is "openai"');
|
||||
}
|
||||
if (typeof openai.baseUrl !== 'string' || !openai.baseUrl) {
|
||||
throw new InvalidInputError('openai.baseUrl must be a non-empty string');
|
||||
}
|
||||
if (typeof openai.apiKey !== 'string' || !openai.apiKey) {
|
||||
throw new InvalidInputError('openai.apiKey must be a non-empty string');
|
||||
}
|
||||
if (typeof openai.model !== 'string' || !openai.model) {
|
||||
throw new InvalidInputError('openai.model must be a non-empty string');
|
||||
}
|
||||
const db = getClient();
|
||||
|
||||
const config: EmbeddingConfig = {
|
||||
provider: 'openai',
|
||||
openai: {
|
||||
baseUrl: openai.baseUrl as string,
|
||||
apiKey: openai.apiKey as string,
|
||||
model: openai.model as string,
|
||||
dimensions:
|
||||
typeof openai.dimensions === 'number' ? (openai.dimensions as number) : undefined,
|
||||
maxBatchSize:
|
||||
typeof openai.maxBatchSize === 'number'
|
||||
? (openai.maxBatchSize as number)
|
||||
: undefined
|
||||
}
|
||||
};
|
||||
return config;
|
||||
// If setting as default, clear other defaults first
|
||||
if (profile.isDefault) {
|
||||
db.prepare('UPDATE embedding_profiles SET is_default = 0').run();
|
||||
}
|
||||
|
||||
return { provider: provider as 'local' | 'none' };
|
||||
// Upsert the profile
|
||||
db.prepare(
|
||||
`INSERT INTO embedding_profiles
|
||||
(id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
provider_kind = excluded.provider_kind,
|
||||
title = excluded.title,
|
||||
enabled = excluded.enabled,
|
||||
is_default = excluded.is_default,
|
||||
model = excluded.model,
|
||||
dimensions = excluded.dimensions,
|
||||
config = excluded.config,
|
||||
updated_at = excluded.updated_at`
|
||||
).run(
|
||||
profile.id,
|
||||
profile.providerKind,
|
||||
profile.title,
|
||||
profile.enabled ? 1 : 0,
|
||||
profile.isDefault ? 1 : 0,
|
||||
profile.model,
|
||||
profile.dimensions,
|
||||
JSON.stringify(profile.config),
|
||||
profile.createdAt,
|
||||
profile.updatedAt
|
||||
);
|
||||
|
||||
const inserted = db
|
||||
.prepare('SELECT * FROM embedding_profiles WHERE id = ?')
|
||||
.get(profile.id) as EmbeddingProfile;
|
||||
|
||||
return sanitizeProfile(inserted);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GET
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const GET: RequestHandler = () => {
|
||||
try {
|
||||
const db = getClient();
|
||||
const config = readConfig(db);
|
||||
|
||||
// Strip the apiKey from the response for security.
|
||||
const safeConfig = sanitizeForResponse(config);
|
||||
return json(safeConfig);
|
||||
} catch (err) {
|
||||
return handleServiceError(err);
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PUT
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const PUT: RequestHandler = async ({ request }) => {
|
||||
export const POST: RequestHandler = async ({ request }) => {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const config = validateConfig(body);
|
||||
|
||||
// Verify provider connectivity before persisting (skip for noop).
|
||||
if (config.provider !== 'none') {
|
||||
const provider = createProviderFromConfig(config);
|
||||
const available = await provider.isAvailable();
|
||||
if (!available) {
|
||||
throw new InvalidInputError(
|
||||
`Could not connect to the "${config.provider}" embedding provider. Check your configuration.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const db = getClient();
|
||||
db.prepare(
|
||||
`INSERT INTO settings (key, value, updated_at)
|
||||
VALUES (?, ?, unixepoch())
|
||||
ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`
|
||||
).run(EMBEDDING_CONFIG_KEY, JSON.stringify(config));
|
||||
|
||||
const safeConfig = sanitizeForResponse(config);
|
||||
return json(safeConfig);
|
||||
const profile = await upsertProfile(body);
|
||||
return json(profile);
|
||||
} catch (err) {
|
||||
return handleServiceError(err);
|
||||
}
|
||||
};
|
||||
|
||||
// Backward compat alias
|
||||
export const PUT: RequestHandler = POST;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Sanitize — remove sensitive fields before returning to clients
|
||||
// Sanitize — remove sensitive config fields before returning to clients
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function sanitizeForResponse(config: EmbeddingConfig): Omit<EmbeddingConfig, 'openai'> & {
|
||||
openai?: Omit<NonNullable<EmbeddingConfig['openai']>, 'apiKey'>;
|
||||
} {
|
||||
if (config.provider === 'openai' && config.openai) {
|
||||
const { apiKey: _apiKey, ...rest } = config.openai;
|
||||
return { ...config, openai: rest };
|
||||
function sanitizeProfile(profile: EmbeddingProfile): EmbeddingProfile {
|
||||
const config = profile.config as Record<string, unknown>;
|
||||
if (config && config.apiKey) {
|
||||
const { apiKey: _apiKey, ...rest } = config;
|
||||
return { ...profile, config: rest };
|
||||
}
|
||||
return config;
|
||||
return profile;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user