feat(TRUEREF-0023): add sqlite-vec search pipeline
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
|
||||
import type Database from 'better-sqlite3';
|
||||
import type { EmbeddingProvider } from './provider.js';
|
||||
import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js';
|
||||
|
||||
interface SnippetRow {
|
||||
id: string;
|
||||
@@ -17,11 +18,15 @@ const BATCH_SIZE = 50;
|
||||
const TEXT_MAX_CHARS = 2048;
|
||||
|
||||
export class EmbeddingService {
|
||||
private readonly sqliteVecStore: SqliteVecStore;
|
||||
|
||||
constructor(
|
||||
private readonly db: Database.Database,
|
||||
private readonly provider: EmbeddingProvider,
|
||||
private readonly profileId: string = 'local-default'
|
||||
) {}
|
||||
) {
|
||||
this.sqliteVecStore = new SqliteVecStore(db);
|
||||
}
|
||||
|
||||
findSnippetIdsMissingEmbeddings(repositoryId: string, versionId: string | null): string[] {
|
||||
if (versionId) {
|
||||
@@ -104,13 +109,19 @@ export class EmbeddingService {
|
||||
for (let j = 0; j < batchSnippets.length; j++) {
|
||||
const snippet = batchSnippets[j];
|
||||
const embedding = embeddings[j];
|
||||
|
||||
insert.run(
|
||||
snippet.id,
|
||||
this.profileId,
|
||||
embedding.model,
|
||||
embedding.dimensions,
|
||||
Buffer.from(embedding.values.buffer)
|
||||
Buffer.from(
|
||||
embedding.values.buffer,
|
||||
embedding.values.byteOffset,
|
||||
embedding.values.byteLength
|
||||
)
|
||||
);
|
||||
this.sqliteVecStore.upsertEmbedding(this.profileId, snippet.id, embedding.values);
|
||||
}
|
||||
});
|
||||
insertMany();
|
||||
|
||||
Reference in New Issue
Block a user