feat(TRUEREF-0020): add embedding profiles, default local embeddings, and version-scoped semantic retrieval
- Add embedding_profiles table with provider registry pattern - Install @xenova/transformers as runtime dependency - Update snippet_embeddings with composite PK (snippet_id, profile_id) - Seed default local profile using Xenova/all-MiniLM-L6-v2 - Add provider registry (local-transformers, openai-compatible) - Update EmbeddingService to persist and retrieve by profileId - Add version-scoped VectorSearch with optional versionId filtering - Add searchMode (auto|keyword|semantic|hybrid) to HybridSearchService - Update API /context route to load active profile, support searchMode/alpha params - Extend MCP query-docs tool with searchMode and alpha parameters - Update settings API to work with embedding_profiles table - Add comprehensive test coverage for profiles, registry, version scoping Status: 445/451 tests passing, core feature complete
This commit is contained in:
Giancarmine Salucci
@@ -25,16 +25,18 @@ function createTestDb(): Database.Database {
|
||||
client.pragma('foreign_keys = ON');
|
||||
|
||||
const migrationsFolder = join(import.meta.dirname, '../db/migrations');
|
||||
const migrationSql = readFileSync(
|
||||
join(migrationsFolder, '0000_large_master_chief.sql'),
|
||||
'utf-8'
|
||||
);
|
||||
const statements = migrationSql
|
||||
.split('--> statement-breakpoint')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
for (const stmt of statements) {
|
||||
client.exec(stmt);
|
||||
|
||||
// Run all migrations in order
|
||||
const migrations = ['0000_large_master_chief.sql', '0001_quick_nighthawk.sql', '0002_silky_stellaris.sql'];
|
||||
for (const migrationFile of migrations) {
|
||||
const migrationSql = readFileSync(join(migrationsFolder, migrationFile), 'utf-8');
|
||||
const statements = migrationSql
|
||||
.split('--> statement-breakpoint')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
for (const stmt of statements) {
|
||||
client.exec(stmt);
|
||||
}
|
||||
}
|
||||
|
||||
const ftsSql = readFileSync(join(import.meta.dirname, '../db/fts.sql'), 'utf-8');
|
||||
@@ -104,16 +106,17 @@ function seedEmbedding(
|
||||
client: Database.Database,
|
||||
snippetId: string,
|
||||
values: number[],
|
||||
profileId = 'local-default',
|
||||
model = 'test-model'
|
||||
): void {
|
||||
const f32 = new Float32Array(values);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT OR REPLACE INTO snippet_embeddings
|
||||
(snippet_id, model, dimensions, embedding, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)`
|
||||
(snippet_id, profile_id, model, dimensions, embedding, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(snippetId, model, values.length, Buffer.from(f32.buffer), NOW_S);
|
||||
.run(snippetId, profileId, model, values.length, Buffer.from(f32.buffer), NOW_S);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -621,4 +624,203 @@ describe('HybridSearchService', () => {
|
||||
const results = await svc.search('default alpha hybrid', { repositoryId: repoId });
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
it('filters by versionId — excludes snippets from other versions', async () => {
|
||||
const client = createTestDb();
|
||||
const repoId = seedRepo(client);
|
||||
const docId = seedDocument(client, repoId);
|
||||
|
||||
// Create two versions
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO repository_versions (id, repository_id, tag, state, total_snippets, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run('/test/repo/v1.0', repoId, 'v1.0', 'indexed', 0, NOW_S);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO repository_versions (id, repository_id, tag, state, total_snippets, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run('/test/repo/v2.0', repoId, 'v2.0', 'indexed', 0, NOW_S);
|
||||
|
||||
// Create embedding profile
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO embedding_profiles (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run('test-profile', 'local-transformers', 'Test', 1, 1, 'test-model', 4, '{}', NOW_S, NOW_S);
|
||||
|
||||
// Snippet A in version 1.0
|
||||
const snippetA = seedSnippet(client, {
|
||||
repositoryId: repoId,
|
||||
documentId: docId,
|
||||
content: 'version 1 text'
|
||||
});
|
||||
client
|
||||
.prepare('UPDATE snippets SET version_id = ? WHERE id = ?')
|
||||
.run('/test/repo/v1.0', snippetA);
|
||||
|
||||
// Seed embedding for snippetA
|
||||
const embedA = [0.1, 0.2, 0.3, 0.4];
|
||||
const f32A = new Float32Array(embedA);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(snippetA, 'test-profile', 'test-model', 4, Buffer.from(f32A.buffer), NOW_S);
|
||||
|
||||
// Snippet B in version 2.0
|
||||
const snippetB = seedSnippet(client, {
|
||||
repositoryId: repoId,
|
||||
documentId: docId,
|
||||
content: 'version 2 text'
|
||||
});
|
||||
client
|
||||
.prepare('UPDATE snippets SET version_id = ? WHERE id = ?')
|
||||
.run('/test/repo/v2.0', snippetB);
|
||||
|
||||
// Seed embedding for snippetB
|
||||
const embedB = [0.2, 0.3, 0.4, 0.5];
|
||||
const f32B = new Float32Array(embedB);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(snippetB, 'test-profile', 'test-model', 4, Buffer.from(f32B.buffer), NOW_S);
|
||||
|
||||
const vs = new VectorSearch(client);
|
||||
const query = new Float32Array([0.1, 0.2, 0.3, 0.4]);
|
||||
|
||||
// Query with versionId v1.0 should only return snippetA
|
||||
const resultsV1 = vs.vectorSearch(query, {
|
||||
repositoryId: repoId,
|
||||
versionId: '/test/repo/v1.0',
|
||||
profileId: 'test-profile'
|
||||
});
|
||||
expect(resultsV1.map((r) => r.snippetId)).toContain(snippetA);
|
||||
expect(resultsV1.map((r) => r.snippetId)).not.toContain(snippetB);
|
||||
|
||||
// Query with versionId v2.0 should only return snippetB
|
||||
const resultsV2 = vs.vectorSearch(query, {
|
||||
repositoryId: repoId,
|
||||
versionId: '/test/repo/v2.0',
|
||||
profileId: 'test-profile'
|
||||
});
|
||||
expect(resultsV2.map((r) => r.snippetId)).not.toContain(snippetA);
|
||||
expect(resultsV2.map((r) => r.snippetId)).toContain(snippetB);
|
||||
|
||||
// Query without versionId should return both
|
||||
const resultsAll = vs.vectorSearch(query, {
|
||||
repositoryId: repoId,
|
||||
profileId: 'test-profile'
|
||||
});
|
||||
expect(resultsAll.map((r) => r.snippetId)).toContain(snippetA);
|
||||
expect(resultsAll.map((r) => r.snippetId)).toContain(snippetB);
|
||||
});
|
||||
|
||||
it('searchMode=keyword never calls provider.embed()', async () => {
|
||||
const client = createTestDb();
|
||||
const repoId = seedRepo(client);
|
||||
const docId = seedDocument(client, repoId);
|
||||
|
||||
const snippetId = seedSnippet(client, {
|
||||
repositoryId: repoId,
|
||||
documentId: docId,
|
||||
content: 'keyword only test'
|
||||
});
|
||||
|
||||
client.exec(
|
||||
`INSERT INTO snippets_fts (id, repository_id, version_id, title, breadcrumb, content)
|
||||
VALUES ('${snippetId}', '${repoId}', NULL, NULL, NULL, 'keyword only test')`
|
||||
);
|
||||
|
||||
let embedCalled = false;
|
||||
const mockProvider: EmbeddingProvider = {
|
||||
name: 'mock',
|
||||
dimensions: 4,
|
||||
model: 'test-model',
|
||||
async embed() {
|
||||
embedCalled = true;
|
||||
return [];
|
||||
},
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
const searchService = new SearchService(client);
|
||||
const hybridService = new HybridSearchService(client, searchService, mockProvider);
|
||||
|
||||
const results = await hybridService.search('keyword', {
|
||||
repositoryId: repoId,
|
||||
searchMode: 'keyword'
|
||||
});
|
||||
|
||||
expect(embedCalled).toBe(false);
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('searchMode=semantic uses only vector search', async () => {
|
||||
const client = createTestDb();
|
||||
const repoId = seedRepo(client);
|
||||
const docId = seedDocument(client, repoId);
|
||||
|
||||
// Create profile
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO embedding_profiles (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run('test-profile', 'local-transformers', 'Test', 1, 1, 'test-model', 4, '{}', NOW_S, NOW_S);
|
||||
|
||||
const snippetId = seedSnippet(client, {
|
||||
repositoryId: repoId,
|
||||
documentId: docId,
|
||||
content: 'semantic test'
|
||||
});
|
||||
|
||||
// Seed embedding
|
||||
const embed = [0.5, 0.5, 0.5, 0.5];
|
||||
const f32 = new Float32Array(embed);
|
||||
client
|
||||
.prepare(
|
||||
`INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.run(snippetId, 'test-profile', 'test-model', 4, Buffer.from(f32.buffer), NOW_S);
|
||||
|
||||
const mockProvider: EmbeddingProvider = {
|
||||
name: 'mock',
|
||||
dimensions: 4,
|
||||
model: 'test-model',
|
||||
async embed() {
|
||||
return [
|
||||
{
|
||||
values: new Float32Array([0.5, 0.5, 0.5, 0.5]),
|
||||
dimensions: 4,
|
||||
model: 'test-model'
|
||||
}
|
||||
];
|
||||
},
|
||||
async isAvailable() {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
const searchService = new SearchService(client);
|
||||
const hybridService = new HybridSearchService(client, searchService, mockProvider);
|
||||
|
||||
const results = await hybridService.search('semantic', {
|
||||
repositoryId: repoId,
|
||||
searchMode: 'semantic',
|
||||
profileId: 'test-profile'
|
||||
});
|
||||
|
||||
// Should return results (alpha=1 pure vector mode)
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user