521 lines
14 KiB
TypeScript
521 lines
14 KiB
TypeScript
import { describe, it, expect, beforeEach } from 'vitest';
|
|
import Database from 'better-sqlite3';
|
|
import { drizzle } from 'drizzle-orm/better-sqlite3';
|
|
import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
|
|
import { readFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { eq } from 'drizzle-orm';
|
|
import * as schema from './schema';
|
|
import { loadSqliteVec, sqliteVecRowidTableName, sqliteVecTableName } from './sqlite-vec';
|
|
import {
|
|
repositories,
|
|
repositoryVersions,
|
|
documents,
|
|
snippets,
|
|
snippetEmbeddings,
|
|
indexingJobs,
|
|
repositoryConfigs,
|
|
settings
|
|
} from './schema';
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function createTestDb() {
|
|
const client = new Database(':memory:');
|
|
client.pragma('foreign_keys = ON');
|
|
loadSqliteVec(client);
|
|
|
|
const db = drizzle(client, { schema });
|
|
|
|
// Run migrations from the generated migration folder.
|
|
const migrationsFolder = join(import.meta.dirname, 'migrations');
|
|
migrate(db, { migrationsFolder });
|
|
|
|
// Apply FTS5 DDL using exec() which handles multi-statement SQL with comments.
|
|
const ftsSql = readFileSync(join(import.meta.dirname, 'fts.sql'), 'utf-8');
|
|
client.exec(ftsSql);
|
|
|
|
return { db, client };
|
|
}
|
|
|
|
const now = new Date();
|
|
const nowTimestamp = Math.floor(now.getTime() / 1000);
|
|
|
|
function makeRepo(overrides: Partial<schema.NewRepository> = {}): schema.NewRepository {
|
|
return {
|
|
id: '/test/repo',
|
|
title: 'Test Repo',
|
|
source: 'github',
|
|
sourceUrl: 'https://github.com/test/repo',
|
|
createdAt: now,
|
|
updatedAt: now,
|
|
...overrides
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
describe('repositories table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
|
|
beforeEach(() => {
|
|
({ db } = createTestDb());
|
|
});
|
|
|
|
it('inserts and retrieves a repository', () => {
|
|
const repo = makeRepo();
|
|
db.insert(repositories).values(repo).run();
|
|
|
|
const result = db.select().from(repositories).all();
|
|
expect(result).toHaveLength(1);
|
|
expect(result[0].id).toBe('/test/repo');
|
|
expect(result[0].title).toBe('Test Repo');
|
|
expect(result[0].source).toBe('github');
|
|
expect(result[0].state).toBe('pending');
|
|
expect(result[0].totalSnippets).toBe(0);
|
|
expect(result[0].totalTokens).toBe(0);
|
|
expect(result[0].trustScore).toBe(0);
|
|
expect(result[0].benchmarkScore).toBe(0);
|
|
});
|
|
|
|
it('allows nullable optional fields', () => {
|
|
const repo = makeRepo({ description: null, stars: null, githubToken: null });
|
|
db.insert(repositories).values(repo).run();
|
|
|
|
const result = db.select().from(repositories).all();
|
|
expect(result[0].description).toBeNull();
|
|
expect(result[0].stars).toBeNull();
|
|
expect(result[0].githubToken).toBeNull();
|
|
});
|
|
|
|
it('supports all state enum values', () => {
|
|
const states = ['pending', 'indexing', 'indexed', 'error'] as const;
|
|
for (const state of states) {
|
|
db.insert(repositories)
|
|
.values(makeRepo({ id: `/test/${state}`, state }))
|
|
.run();
|
|
}
|
|
const results = db.select().from(repositories).all();
|
|
const resultStates = results.map((r) => r.state).sort();
|
|
expect(resultStates).toEqual([...states].sort());
|
|
});
|
|
});
|
|
|
|
describe('repository_versions table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
|
|
beforeEach(() => {
|
|
({ db } = createTestDb());
|
|
db.insert(repositories).values(makeRepo()).run();
|
|
});
|
|
|
|
it('inserts a version linked to a repository', () => {
|
|
db.insert(repositoryVersions)
|
|
.values({
|
|
id: '/test/repo/v1.0.0',
|
|
repositoryId: '/test/repo',
|
|
tag: 'v1.0.0',
|
|
title: 'Version 1.0.0',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
|
|
const result = db.select().from(repositoryVersions).all();
|
|
expect(result).toHaveLength(1);
|
|
expect(result[0].tag).toBe('v1.0.0');
|
|
expect(result[0].repositoryId).toBe('/test/repo');
|
|
});
|
|
|
|
it('cascades delete when parent repository is deleted', () => {
|
|
db.insert(repositoryVersions)
|
|
.values({
|
|
id: '/test/repo/v1.0.0',
|
|
repositoryId: '/test/repo',
|
|
tag: 'v1.0.0',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
|
|
db.delete(repositories).where(eq(repositories.id, '/test/repo')).run();
|
|
|
|
const result = db.select().from(repositoryVersions).all();
|
|
expect(result).toHaveLength(0);
|
|
});
|
|
});
|
|
|
|
describe('documents table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
|
|
beforeEach(() => {
|
|
({ db } = createTestDb());
|
|
db.insert(repositories).values(makeRepo()).run();
|
|
});
|
|
|
|
it('inserts a document', () => {
|
|
db.insert(documents)
|
|
.values({
|
|
id: crypto.randomUUID(),
|
|
repositoryId: '/test/repo',
|
|
filePath: 'README.md',
|
|
checksum: 'abc123',
|
|
indexedAt: now
|
|
})
|
|
.run();
|
|
|
|
const result = db.select().from(documents).all();
|
|
expect(result).toHaveLength(1);
|
|
expect(result[0].filePath).toBe('README.md');
|
|
expect(result[0].checksum).toBe('abc123');
|
|
});
|
|
|
|
it('cascades delete when repository is deleted', () => {
|
|
db.insert(documents)
|
|
.values({
|
|
id: crypto.randomUUID(),
|
|
repositoryId: '/test/repo',
|
|
filePath: 'README.md',
|
|
checksum: 'abc123',
|
|
indexedAt: now
|
|
})
|
|
.run();
|
|
|
|
db.delete(repositories).where(eq(repositories.id, '/test/repo')).run();
|
|
|
|
const result = db.select().from(documents).all();
|
|
expect(result).toHaveLength(0);
|
|
});
|
|
});
|
|
|
|
describe('snippets table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
let docId: string;
|
|
|
|
beforeEach(() => {
|
|
({ db } = createTestDb());
|
|
db.insert(repositories).values(makeRepo()).run();
|
|
docId = crypto.randomUUID();
|
|
db.insert(documents)
|
|
.values({
|
|
id: docId,
|
|
repositoryId: '/test/repo',
|
|
filePath: 'README.md',
|
|
checksum: 'abc123',
|
|
indexedAt: now
|
|
})
|
|
.run();
|
|
});
|
|
|
|
it('inserts a code snippet', () => {
|
|
const snippetId = crypto.randomUUID();
|
|
db.insert(snippets)
|
|
.values({
|
|
id: snippetId,
|
|
documentId: docId,
|
|
repositoryId: '/test/repo',
|
|
type: 'code',
|
|
content: 'console.log("hello")',
|
|
language: 'javascript',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
|
|
const result = db.select().from(snippets).all();
|
|
expect(result).toHaveLength(1);
|
|
expect(result[0].type).toBe('code');
|
|
expect(result[0].language).toBe('javascript');
|
|
});
|
|
|
|
it('inserts an info snippet', () => {
|
|
const snippetId = crypto.randomUUID();
|
|
db.insert(snippets)
|
|
.values({
|
|
id: snippetId,
|
|
documentId: docId,
|
|
repositoryId: '/test/repo',
|
|
type: 'info',
|
|
content: 'This is documentation text.',
|
|
breadcrumb: 'Intro > Overview',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
|
|
const result = db.select().from(snippets).all();
|
|
expect(result[0].breadcrumb).toBe('Intro > Overview');
|
|
});
|
|
|
|
it('cascades delete when document is deleted', () => {
|
|
db.insert(snippets)
|
|
.values({
|
|
id: crypto.randomUUID(),
|
|
documentId: docId,
|
|
repositoryId: '/test/repo',
|
|
type: 'info',
|
|
content: 'Some content.',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
|
|
db.delete(documents).where(eq(documents.id, docId)).run();
|
|
|
|
const result = db.select().from(snippets).all();
|
|
expect(result).toHaveLength(0);
|
|
});
|
|
});
|
|
|
|
describe('snippet_embeddings table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
let client: Database.Database;
|
|
let snippetId: string;
|
|
|
|
beforeEach(() => {
|
|
({ db, client } = createTestDb());
|
|
db.insert(repositories).values(makeRepo()).run();
|
|
const docId = crypto.randomUUID();
|
|
db.insert(documents)
|
|
.values({
|
|
id: docId,
|
|
repositoryId: '/test/repo',
|
|
filePath: 'README.md',
|
|
checksum: 'abc123',
|
|
indexedAt: now
|
|
})
|
|
.run();
|
|
snippetId = crypto.randomUUID();
|
|
db.insert(snippets)
|
|
.values({
|
|
id: snippetId,
|
|
documentId: docId,
|
|
repositoryId: '/test/repo',
|
|
type: 'info',
|
|
content: 'hello world',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
});
|
|
|
|
it('stores a Float32Array embedding as blob', () => {
|
|
const vec = new Float32Array([0.1, 0.2, 0.3, 0.4]);
|
|
const buf = Buffer.from(vec.buffer);
|
|
|
|
db.insert(snippetEmbeddings)
|
|
.values({
|
|
snippetId,
|
|
profileId: 'local-default',
|
|
model: 'text-embedding-3-small',
|
|
dimensions: 4,
|
|
embedding: buf,
|
|
createdAt: nowTimestamp
|
|
})
|
|
.run();
|
|
|
|
const result = db.select().from(snippetEmbeddings).all();
|
|
expect(result).toHaveLength(1);
|
|
expect(result[0].model).toBe('text-embedding-3-small');
|
|
expect(result[0].dimensions).toBe(4);
|
|
|
|
const retrieved = new Float32Array(
|
|
(result[0].embedding as Buffer).buffer,
|
|
(result[0].embedding as Buffer).byteOffset,
|
|
(result[0].embedding as Buffer).byteLength / 4
|
|
);
|
|
// Float32Array has ~7 decimal digits of precision; use toBeCloseTo.
|
|
expect(retrieved[0]).toBeCloseTo(0.1, 5);
|
|
expect(retrieved[1]).toBeCloseTo(0.2, 5);
|
|
expect(retrieved[2]).toBeCloseTo(0.3, 5);
|
|
expect(retrieved[3]).toBeCloseTo(0.4, 5);
|
|
});
|
|
|
|
it('cascades delete when snippet is deleted', () => {
|
|
const vec = new Float32Array([1, 2]);
|
|
db.insert(snippetEmbeddings)
|
|
.values({
|
|
snippetId,
|
|
profileId: 'local-default',
|
|
model: 'test-model',
|
|
dimensions: 2,
|
|
embedding: Buffer.from(vec.buffer),
|
|
createdAt: nowTimestamp
|
|
})
|
|
.run();
|
|
|
|
db.delete(snippets).where(eq(snippets.id, snippetId)).run();
|
|
|
|
const result = db.select().from(snippetEmbeddings).all();
|
|
expect(result).toHaveLength(0);
|
|
});
|
|
|
|
it('keeps the relational schema free of vec_embedding and retains the profile index', () => {
|
|
const columns = client
|
|
.prepare("PRAGMA table_info('snippet_embeddings')")
|
|
.all() as Array<{ name: string }>;
|
|
expect(columns.map((column) => column.name)).not.toContain('vec_embedding');
|
|
|
|
const indexes = client
|
|
.prepare("PRAGMA index_list('snippet_embeddings')")
|
|
.all() as Array<{ name: string }>;
|
|
expect(indexes.map((index) => index.name)).toContain('idx_embeddings_profile');
|
|
});
|
|
|
|
it('loads sqlite-vec idempotently and derives deterministic per-profile table names', () => {
|
|
expect(() => loadSqliteVec(client)).not.toThrow();
|
|
const tableName = sqliteVecTableName('local-default');
|
|
const rowidTableName = sqliteVecRowidTableName('local-default');
|
|
|
|
expect(tableName).toMatch(/^snippet_embeddings_vec_local_default_[0-9a-f]{8}$/);
|
|
expect(rowidTableName).toMatch(/^snippet_embeddings_vec_rowids_local_default_[0-9a-f]{8}$/);
|
|
expect(sqliteVecTableName('local-default')).toBe(tableName);
|
|
expect(sqliteVecRowidTableName('local-default')).toBe(rowidTableName);
|
|
expect(sqliteVecTableName('local-default')).not.toBe(sqliteVecTableName('openai/custom'));
|
|
});
|
|
});
|
|
|
|
describe('indexing_jobs table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
|
|
beforeEach(() => {
|
|
({ db } = createTestDb());
|
|
db.insert(repositories).values(makeRepo()).run();
|
|
});
|
|
|
|
it('creates a job with default queued status', () => {
|
|
db.insert(indexingJobs)
|
|
.values({
|
|
id: crypto.randomUUID(),
|
|
repositoryId: '/test/repo',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
|
|
const result = db.select().from(indexingJobs).all();
|
|
expect(result[0].status).toBe('queued');
|
|
expect(result[0].progress).toBe(0);
|
|
expect(result[0].totalFiles).toBe(0);
|
|
expect(result[0].processedFiles).toBe(0);
|
|
});
|
|
|
|
it('supports all status enum values', () => {
|
|
const statuses = ['queued', 'running', 'done', 'failed'] as const;
|
|
for (const status of statuses) {
|
|
db.insert(indexingJobs)
|
|
.values({
|
|
id: crypto.randomUUID(),
|
|
repositoryId: '/test/repo',
|
|
status,
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
}
|
|
const results = db.select().from(indexingJobs).all();
|
|
expect(results.map((r) => r.status).sort()).toEqual([...statuses].sort());
|
|
});
|
|
});
|
|
|
|
describe('repository_configs table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
|
|
beforeEach(() => {
|
|
({ db } = createTestDb());
|
|
db.insert(repositories).values(makeRepo()).run();
|
|
});
|
|
|
|
it('stores JSON array fields correctly', () => {
|
|
db.insert(repositoryConfigs)
|
|
.values({
|
|
repositoryId: '/test/repo',
|
|
projectTitle: 'My SDK',
|
|
folders: ['docs', 'src'],
|
|
excludeFolders: ['node_modules', '.git'],
|
|
excludeFiles: ['*.test.ts'],
|
|
rules: ['Always use TypeScript'],
|
|
previousVersions: [{ tag: 'v1.0.0', title: 'Version 1' }],
|
|
updatedAt: now
|
|
})
|
|
.run();
|
|
|
|
const result = db.select().from(repositoryConfigs).all();
|
|
expect(result).toHaveLength(1);
|
|
expect(result[0].folders).toEqual(['docs', 'src']);
|
|
expect(result[0].excludeFolders).toEqual(['node_modules', '.git']);
|
|
expect(result[0].rules).toEqual(['Always use TypeScript']);
|
|
expect(result[0].previousVersions).toEqual([{ tag: 'v1.0.0', title: 'Version 1' }]);
|
|
});
|
|
});
|
|
|
|
describe('settings table', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
|
|
beforeEach(() => {
|
|
({ db } = createTestDb());
|
|
});
|
|
|
|
it('stores and retrieves key-value settings', () => {
|
|
db.insert(settings)
|
|
.values({ key: 'embeddingProvider', value: { provider: 'openai' }, updatedAt: now })
|
|
.run();
|
|
|
|
const result = db.select().from(settings).all();
|
|
expect(result).toHaveLength(1);
|
|
expect(result[0].key).toBe('embeddingProvider');
|
|
expect(result[0].value).toEqual({ provider: 'openai' });
|
|
});
|
|
});
|
|
|
|
describe('FTS5 virtual table (snippets_fts)', () => {
|
|
let db: ReturnType<typeof createTestDb>['db'];
|
|
let client: Database.Database;
|
|
|
|
beforeEach(() => {
|
|
({ db, client } = createTestDb());
|
|
db.insert(repositories).values(makeRepo()).run();
|
|
const docId = crypto.randomUUID();
|
|
db.insert(documents)
|
|
.values({
|
|
id: docId,
|
|
repositoryId: '/test/repo',
|
|
filePath: 'README.md',
|
|
checksum: 'abc',
|
|
indexedAt: now
|
|
})
|
|
.run();
|
|
db.insert(snippets)
|
|
.values({
|
|
id: crypto.randomUUID(),
|
|
documentId: docId,
|
|
repositoryId: '/test/repo',
|
|
type: 'info',
|
|
content: 'The quick brown fox jumps over the lazy dog',
|
|
title: 'Fox story',
|
|
breadcrumb: 'Animals > Foxes',
|
|
createdAt: now
|
|
})
|
|
.run();
|
|
});
|
|
|
|
it('FTS table exists and is queryable', () => {
|
|
const result = client
|
|
.prepare(`SELECT rowid FROM snippets_fts WHERE snippets_fts MATCH 'fox'`)
|
|
.all();
|
|
expect(result.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('insert trigger keeps FTS in sync', () => {
|
|
const result = client
|
|
.prepare(`SELECT rowid FROM snippets_fts WHERE snippets_fts MATCH 'quick'`)
|
|
.all();
|
|
expect(result.length).toBe(1);
|
|
});
|
|
|
|
it('delete trigger removes entry from FTS', () => {
|
|
db.delete(snippets).run();
|
|
|
|
const result = client
|
|
.prepare(`SELECT rowid FROM snippets_fts WHERE snippets_fts MATCH 'quick'`)
|
|
.all();
|
|
expect(result.length).toBe(0);
|
|
});
|
|
});
|