TRUEREF-0023 rewrite indexing pipeline - parallel reads - serialized writes

This commit is contained in:
Giancarmine Salucci
2026-04-02 09:49:38 +02:00
parent 9525c58e9a
commit f86be4106b
68 changed files with 5042 additions and 3131 deletions

View File

@@ -12,11 +12,7 @@ import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import * as schema from '../db/schema.js';
import {
loadSqliteVec,
sqliteVecRowidTableName,
sqliteVecTableName
} from '../db/sqlite-vec.js';
import { loadSqliteVec, sqliteVecRowidTableName, sqliteVecTableName } from '../db/sqlite-vec.js';
import { SqliteVecStore } from '../search/sqlite-vec.store.js';
import { NoopEmbeddingProvider, EmbeddingError, type EmbeddingVector } from './provider.js';
@@ -424,6 +420,25 @@ describe('EmbeddingService', () => {
expect(embedding![2]).toBeCloseTo(0.2, 5);
});
it('can delegate embedding persistence to an injected writer', async () => {
const snippetId = seedSnippet(db, client);
const provider = makeProvider(4);
const persistEmbeddings = vi.fn().mockResolvedValue(undefined);
const service = new EmbeddingService(client, provider, 'local-default', {
persistEmbeddings
});
await service.embedSnippets([snippetId]);
expect(persistEmbeddings).toHaveBeenCalledTimes(1);
const rows = client
.prepare(
'SELECT COUNT(*) AS cnt FROM snippet_embeddings WHERE snippet_id = ? AND profile_id = ?'
)
.get(snippetId, 'local-default') as { cnt: number };
expect(rows.cnt).toBe(0);
});
it('stores embeddings under the configured profile ID', async () => {
client
.prepare(
@@ -431,16 +446,7 @@ describe('EmbeddingService', () => {
(id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, unixepoch(), unixepoch())`
)
.run(
'openai-custom',
'openai-compatible',
'OpenAI Custom',
1,
0,
'test-model',
4,
'{}'
);
.run('openai-custom', 'openai-compatible', 'OpenAI Custom', 1, 0, 'test-model', 4, '{}');
const snippetId = seedSnippet(db, client);
const provider = makeProvider(4, 'test-model');