From d1381f7fc052fa35909239ecb11bac6a1cbf5c3f Mon Sep 17 00:00:00 2001 From: Giancarmine Salucci Date: Fri, 27 Mar 2026 19:01:47 +0100 Subject: [PATCH 1/7] fix(ROUTING-0001): repair repo routing and isolate MCP filtering --- docs/FINDINGS.md | 23 + scripts/build.mjs | 79 +++ src/lib/components/RepositoryCard.svelte | 6 +- .../components/RepositoryCard.svelte.test.ts | 27 + .../api/v1/api-contract.integration.test.ts | 35 + src/routes/api/v1/context/+server.ts | 19 +- src/routes/repos/[id]/+page.server.ts | 8 +- src/routes/repos/[id]/page.server.test.ts | 34 + src/routes/route-file-conventions.test.ts | 33 + test-output.txt | 632 ------------------ 10 files changed, 252 insertions(+), 644 deletions(-) create mode 100644 scripts/build.mjs create mode 100644 src/lib/components/RepositoryCard.svelte.test.ts create mode 100644 src/routes/repos/[id]/page.server.test.ts create mode 100644 src/routes/route-file-conventions.test.ts delete mode 100644 test-output.txt diff --git a/docs/FINDINGS.md b/docs/FINDINGS.md index 13cf45f..13892fd 100644 --- a/docs/FINDINGS.md +++ b/docs/FINDINGS.md @@ -165,3 +165,26 @@ Add subsequent research below this section. - Risks / follow-ups: - Base-aware navigation fixes must preserve internal app routing semantics and should not replace intentional external navigation, because SvelteKit `goto(...)` no longer accepts external URLs. - Settings and search page lifecycle changes must avoid reintroducing SSR-triggered fetches or self-triggered URL loops; client-only bootstrap logic should remain mounted once and URL-sync effects must stay idempotent. + +### 2026-03-27 — ROUTING-0001 planning research + +- Task: Plan the repository-detail routing fix for slash-bearing repository IDs causing homepage SSR failures and invalid `/repos/[id]` navigation. +- Files inspected: + - `package.json` + - `src/lib/components/RepositoryCard.svelte` + - `src/routes/+page.svelte` + - `src/routes/+page.server.ts` + - `src/routes/repos/[id]/+page.server.ts` + - `src/routes/repos/[id]/+page.svelte` + - `src/routes/api/v1/api-contract.integration.test.ts` + - `src/lib/types.ts` +- Findings: + - The app is on SvelteKit `^2.50.2` and uses `$app/paths.resolve(...)` for internal navigation, including `resolveRoute('/repos/[id]', { id: repo.id })` in `RepositoryCard.svelte`. + - SvelteKit’s `[id]` route is a single-segment dynamic parameter. Context7 routing docs show slash-containing values belong to rest parameters like `[...param]`, so raw repository IDs containing `/` are invalid inputs for `resolveRoute('/repos/[id]', ...)`. + - The repository model intentionally stores slash-bearing IDs such as `/facebook/react`, and the existing API surface consistently treats those IDs as percent-encoded path segments. The integration contract already passes `params.id = encodeURIComponent('/facebook/react')` for `/api/v1/libs/[id]` handlers, which then call `decodeURIComponent(params.id)`. + - The homepage SSR failure is therefore rooted in UI link generation, not repository listing fetches: rendering `RepositoryCard.svelte` with a raw slash-bearing `repo.id` can throw before page load completes, which explains repeated `500` responses on `/`. + - The repo detail page currently forwards `params.id` directly into `encodeURIComponent(...)` for downstream API requests. Once detail links are generated as encoded single segments, the page loader and client-side refresh/delete/reindex flows need one normalization step so API calls continue targeting the stored repository ID instead of a doubly encoded value. + - No existing browser-facing test covers homepage card navigation or `/repos/[id]` loader behavior; the closest current evidence is the API contract test file, which already exercises encoded repository IDs on HTTP endpoints and provides reusable fixtures for slash-bearing IDs. +- Risks / follow-ups: + - The fix should preserve the existing `/repos/[id]` route shape instead of redesigning it to a rest route unless a broader navigation contract change is explicitly requested. + - Any normalization helper introduced for the repo detail page should be reused consistently across server load and client event handlers to avoid mixed encoded and decoded repository IDs during navigation and fetches. diff --git a/scripts/build.mjs b/scripts/build.mjs new file mode 100644 index 0000000..5531a15 --- /dev/null +++ b/scripts/build.mjs @@ -0,0 +1,79 @@ +import { mkdir, readdir, rename } from 'node:fs/promises'; +import { basename, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawn } from 'node:child_process'; + +const rootDir = dirname(dirname(fileURLToPath(import.meta.url))); +const routesDir = join(rootDir, 'src', 'routes'); +const renamedRouteTestFiles = []; + +async function collectReservedRouteTestFiles(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + const files = []; + + for (const entry of entries) { + const entryPath = join(directory, entry.name); + + if (entry.isDirectory()) { + files.push(...(await collectReservedRouteTestFiles(entryPath))); + continue; + } + + if (!entry.name.startsWith('+')) { + continue; + } + + if (!entry.name.includes('.test.') && !entry.name.includes('.spec.')) { + continue; + } + + files.push(entryPath); + } + + return files; +} + +async function renameReservedRouteTests() { + const reservedRouteTestFiles = await collectReservedRouteTestFiles(routesDir); + + for (const sourcePath of reservedRouteTestFiles) { + const targetPath = join(dirname(sourcePath), basename(sourcePath).slice(1)); + await rename(sourcePath, targetPath); + renamedRouteTestFiles.push({ sourcePath, targetPath }); + } +} + +async function restoreReservedRouteTests() { + for (const { sourcePath, targetPath } of renamedRouteTestFiles.reverse()) { + await mkdir(dirname(sourcePath), { recursive: true }); + await rename(targetPath, sourcePath); + } +} + +function runViteBuild() { + const viteBinPath = join(rootDir, 'node_modules', 'vite', 'bin', 'vite.js'); + + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [viteBinPath, 'build'], { + cwd: rootDir, + stdio: 'inherit' + }); + + child.once('error', reject); + child.once('exit', (code) => { + if (code === 0) { + resolve(); + return; + } + + reject(new Error(`vite build exited with code ${code ?? 'unknown'}`)); + }); + }); +} + +try { + await renameReservedRouteTests(); + await runViteBuild(); +} finally { + await restoreReservedRouteTests(); +} \ No newline at end of file diff --git a/src/lib/components/RepositoryCard.svelte b/src/lib/components/RepositoryCard.svelte index 18d6d07..dcc959c 100644 --- a/src/lib/components/RepositoryCard.svelte +++ b/src/lib/components/RepositoryCard.svelte @@ -26,6 +26,10 @@ error: 'Error' }; + const detailsHref = $derived( + resolveRoute('/repos/[id]', { id: encodeURIComponent(repo.id) }) + ); + const totalSnippets = $derived(repo.totalSnippets ?? 0); const trustScore = $derived(repo.trustScore ?? 0); @@ -77,7 +81,7 @@ {repo.state === 'indexing' ? 'Indexing...' : 'Re-index'} Details diff --git a/src/lib/components/RepositoryCard.svelte.test.ts b/src/lib/components/RepositoryCard.svelte.test.ts new file mode 100644 index 0000000..ee2be10 --- /dev/null +++ b/src/lib/components/RepositoryCard.svelte.test.ts @@ -0,0 +1,27 @@ +import { page } from 'vitest/browser'; +import { describe, expect, it, vi } from 'vitest'; +import { render } from 'vitest-browser-svelte'; +import RepositoryCard from './RepositoryCard.svelte'; + +describe('RepositoryCard.svelte', () => { + it('encodes slash-bearing repository ids in the details href', async () => { + render(RepositoryCard, { + repo: { + id: '/facebook/react', + title: 'React', + description: 'A JavaScript library for building user interfaces', + state: 'indexed', + totalSnippets: 1234, + trustScore: 9.7, + stars: 230000, + lastIndexedAt: null + } as never, + onReindex: vi.fn(), + onDelete: vi.fn() + }); + + await expect + .element(page.getByRole('link', { name: 'Details' })) + .toHaveAttribute('href', '/repos/%2Ffacebook%2Freact'); + }); +}); \ No newline at end of file diff --git a/src/routes/api/v1/api-contract.integration.test.ts b/src/routes/api/v1/api-contract.integration.test.ts index 19c97f7..58c7e09 100644 --- a/src/routes/api/v1/api-contract.integration.test.ts +++ b/src/routes/api/v1/api-contract.integration.test.ts @@ -39,6 +39,7 @@ import { GET as getJobs } from './jobs/+server.js'; import { GET as getJob } from './jobs/[id]/+server.js'; import { GET as getVersions, POST as postVersions } from './libs/[id]/versions/+server.js'; import { GET as getContext } from './context/+server.js'; +import { DEFAULT_TOKEN_BUDGET } from '$lib/server/api/token-budget.js'; const NOW_S = Math.floor(Date.now() / 1000); @@ -325,6 +326,40 @@ describe('API contract integration', () => { expect(body).toContain('Result count: 0'); }); + it('GET /api/v1/context does not token-filter default JSON responses for the UI', async () => { + const repositoryId = seedRepo(db); + const documentId = seedDocument(db, repositoryId); + + seedSnippet(db, { + documentId, + repositoryId, + type: 'info', + title: 'Large result', + content: 'Large result body', + tokenCount: DEFAULT_TOKEN_BUDGET + 1 + }); + seedSnippet(db, { + documentId, + repositoryId, + type: 'info', + title: 'Small result', + content: 'Small result body', + tokenCount: 5 + }); + + const response = await getContext({ + url: new URL( + `http://test/api/v1/context?libraryId=${encodeURIComponent(repositoryId)}&query=${encodeURIComponent('result')}` + ) + } as never); + + expect(response.status).toBe(200); + const body = await response.json(); + + expect(body.snippets).toHaveLength(2); + expect(body.resultCount).toBe(2); + }); + it('GET /api/v1/context returns additive repository and version metadata for versioned results', async () => { const repositoryId = seedRepo(db); const versionId = seedVersion(db, repositoryId, 'v18.3.0'); diff --git a/src/routes/api/v1/context/+server.ts b/src/routes/api/v1/context/+server.ts index bb2b5fd..784182c 100644 --- a/src/routes/api/v1/context/+server.ts +++ b/src/routes/api/v1/context/+server.ts @@ -124,6 +124,7 @@ export const GET: RequestHandler = async ({ url }) => { } const responseType = url.searchParams.get('type') ?? 'json'; + const applyTokenBudget = responseType === 'txt' || url.searchParams.has('tokens'); const tokensRaw = parseInt(url.searchParams.get('tokens') ?? String(DEFAULT_TOKEN_BUDGET), 10); const maxTokens = isNaN(tokensRaw) || tokensRaw < 1 ? DEFAULT_TOKEN_BUDGET : tokensRaw; @@ -212,15 +213,17 @@ export const GET: RequestHandler = async ({ url }) => { profileId }); - // Apply token budget. - const snippets = searchResults.map((r) => r.snippet); - const selected = selectSnippetsWithinBudget(snippets, maxTokens); + const selectedResults = applyTokenBudget + ? (() => { + const snippets = searchResults.map((r) => r.snippet); + const selected = selectSnippetsWithinBudget(snippets, maxTokens); - // Re-wrap selected snippets as SnippetSearchResult for formatters. - const selectedResults = selected.map((snippet) => { - const found = searchResults.find((r) => r.snippet.id === snippet.id)!; - return found; - }); + return selected.map((snippet) => { + const found = searchResults.find((r) => r.snippet.id === snippet.id)!; + return found; + }); + })() + : searchResults; const snippetVersionIds = Array.from( new Set( diff --git a/src/routes/repos/[id]/+page.server.ts b/src/routes/repos/[id]/+page.server.ts index 2517398..0f8e36c 100644 --- a/src/routes/repos/[id]/+page.server.ts +++ b/src/routes/repos/[id]/+page.server.ts @@ -2,8 +2,8 @@ import type { PageServerLoad } from './$types'; import { error } from '@sveltejs/kit'; export const load: PageServerLoad = async ({ fetch, params }) => { - const id = params.id; - const res = await fetch(`/api/v1/libs/${encodeURIComponent(id)}`); + const repositoryId = decodeURIComponent(params.id); + const res = await fetch(`/api/v1/libs/${encodeURIComponent(repositoryId)}`); if (res.status === 404) { error(404, 'Repository not found'); @@ -16,7 +16,9 @@ export const load: PageServerLoad = async ({ fetch, params }) => { const repo = await res.json(); // Fetch recent jobs - const jobsRes = await fetch(`/api/v1/jobs?repositoryId=${encodeURIComponent(id)}&limit=5`); + const jobsRes = await fetch( + `/api/v1/jobs?repositoryId=${encodeURIComponent(repositoryId)}&limit=5` + ); const jobsData = jobsRes.ok ? await jobsRes.json() : { jobs: [] }; return { diff --git a/src/routes/repos/[id]/page.server.test.ts b/src/routes/repos/[id]/page.server.test.ts new file mode 100644 index 0000000..db1db3e --- /dev/null +++ b/src/routes/repos/[id]/page.server.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it, vi } from 'vitest'; +import { load } from './+page.server'; + +describe('/repos/[id] page server load', () => { + it('decodes the route param once before calling downstream APIs', async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ id: '/facebook/react', title: 'React' }) + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ jobs: [{ id: 'job-1', repositoryId: '/facebook/react' }] }) + }); + + const result = await load({ + fetch, + params: { id: encodeURIComponent('/facebook/react') } + } as never); + + expect(fetch).toHaveBeenNthCalledWith(1, '/api/v1/libs/%2Ffacebook%2Freact'); + expect(fetch).toHaveBeenNthCalledWith( + 2, + '/api/v1/jobs?repositoryId=%2Ffacebook%2Freact&limit=5' + ); + expect(result).toEqual({ + repo: { id: '/facebook/react', title: 'React' }, + recentJobs: [{ id: 'job-1', repositoryId: '/facebook/react' }] + }); + }); +}); \ No newline at end of file diff --git a/src/routes/route-file-conventions.test.ts b/src/routes/route-file-conventions.test.ts new file mode 100644 index 0000000..40c460e --- /dev/null +++ b/src/routes/route-file-conventions.test.ts @@ -0,0 +1,33 @@ +import { readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +function collectReservedRouteTestFiles(directory: string): string[] { + const entries = readdirSync(directory, { withFileTypes: true }); + const reservedTestFiles: string[] = []; + + for (const entry of entries) { + const entryPath = join(directory, entry.name); + + if (entry.isDirectory()) { + reservedTestFiles.push(...collectReservedRouteTestFiles(entryPath)); + continue; + } + + if (!entry.name.startsWith('+')) continue; + if (!entry.name.includes('.test.') && !entry.name.includes('.spec.')) continue; + + reservedTestFiles.push(entryPath); + } + + return reservedTestFiles; +} + +describe('SvelteKit route file conventions', () => { + it('does not place test files in reserved +prefixed route filenames', () => { + const routeDirectory = import.meta.dirname; + const reservedTestFiles = collectReservedRouteTestFiles(routeDirectory); + + expect(reservedTestFiles).toEqual([]); + }); +}); \ No newline at end of file diff --git a/test-output.txt b/test-output.txt deleted file mode 100644 index 852450b..0000000 --- a/test-output.txt +++ /dev/null @@ -1,632 +0,0 @@ - -> trueref@0.0.1 test:unit -> vitest - - - DEV  v4.1.0 /home/moze/Sources/trueref - -19:10:26 [vite] (client) Re-optimizing dependencies because lockfile has changed - ❯  server  src/lib/server/embeddings/embedding.service.test.ts (0 test) - ✓  server  src/lib/server/parser/code.parser.test.ts (20 tests) 22ms - ✓  server  src/lib/server/services/version.service.test.ts (19 tests) 37ms - ✓  server  src/lib/server/services/repository.service.test.ts (37 tests) 57ms -stderr | src/lib/server/crawler/local.crawler.test.ts > LocalCrawler.crawl() — config file detection > gracefully handles a malformed config file -[LocalCrawler] Failed to parse config file: /tmp/trueref-test-ptITIP/trueref.json - - ✓  server  src/lib/server/config/config-parser.test.ts (50 tests) 21ms -stderr | src/lib/server/pipeline/indexing.pipeline.test.ts > IndexingPipeline > marks job as failed and repo as error when pipeline throws -[IndexingPipeline] Job c44d7e22-6127-49e7-82b7-eb724726c888 failed: crawl failed - -stderr | src/lib/server/pipeline/indexing.pipeline.test.ts -[JobQueue] No pipeline configured — cannot process jobs. - -stderr | src/lib/server/pipeline/indexing.pipeline.test.ts -[JobQueue] No pipeline configured — cannot process jobs. - -stderr | src/lib/server/pipeline/indexing.pipeline.test.ts -[JobQueue] No pipeline configured — cannot process jobs. - - ✓  server  src/lib/server/search/search.service.test.ts (43 tests) 43ms - ✓  server  src/lib/server/pipeline/indexing.pipeline.test.ts (20 tests) 42ms - ✓  server  src/lib/server/crawler/gitignore-parser.test.ts (29 tests) 11ms - ✓  server  src/lib/server/crawler/github-tags.test.ts (10 tests) 9ms - ✓  server  src/routes/api/v1/api-contract.integration.test.ts (4 tests) 48ms - ❯  server  src/lib/server/db/schema.test.ts (19 tests | 19 failed) 50ms - × inserts and retrieves a repository 12ms - × allows nullable optional fields 3ms - × supports all state enum values 2ms - × inserts a version linked to a repository 4ms - × cascades delete when parent repository is deleted 2ms - × inserts a document 1ms - × cascades delete when repository is deleted 2ms - × inserts a code snippet 2ms - × inserts an info snippet 2ms - × cascades delete when document is deleted 2ms - × stores a Float32Array embedding as blob 2ms - × cascades delete when snippet is deleted 2ms - × creates a job with default queued status 2ms - × supports all status enum values 2ms - × stores JSON array fields correctly 2ms - × stores and retrieves key-value settings 2ms - × FTS table exists and is queryable 1ms - × insert trigger keeps FTS in sync 2ms - × delete trigger removes entry from FTS 2ms - ❯  server  src/lib/server/search/hybrid.search.service.test.ts (33 tests | 16 failed) 52ms - ✓ returns 1.0 for identical vectors 2ms - ✓ returns 0.0 for orthogonal vectors 0ms - ✓ returns -1.0 for opposite vectors 0ms - ✓ returns 0 for zero-magnitude vector 0ms - ✓ throws when dimensions do not match 1ms - ✓ computes correct similarity for non-trivial vectors 0ms - ✓ returns empty array for empty inputs 1ms - ✓ fuses a single list preserving order 1ms - ✓ deduplicates items appearing in multiple lists 0ms - ✓ boosts items appearing in multiple lists 0ms - ✓ assigns higher rrfScore to higher-ranked items 0ms - ✓ handles three lists correctly 0ms - ✓ produces positive rrfScores 0ms - × returns empty array when no embeddings exist 10ms - × returns results sorted by descending cosine similarity 2ms - × respects the limit parameter 4ms - × only returns snippets from the specified repository 2ms - × handles embeddings with negative values 1ms - ✓ returns FTS5 results when embeddingProvider is null 2ms - ✓ returns FTS5 results when alpha = 0 1ms - ✓ returns empty array when FTS5 query is blank and no provider 1ms - ✓ falls back to FTS5 when noop provider returns empty embeddings 2ms - × returns results when hybrid mode is active (alpha = 0.5) 1ms - × deduplicates snippets appearing in both FTS5 and vector results 1ms - × respects the limit option 1ms - × returns vector-ranked results when alpha = 1 1ms - × results include snippet and repository metadata 1ms - × all results belong to the requested repository 1ms - × filters by snippet type when provided 1ms - × uses alpha = 0.5 when not specified 1ms - × filters by versionId — excludes snippets from other versions 3ms - × searchMode=keyword never calls provider.embed() 3ms - × searchMode=semantic uses only vector search 2ms - ✓  server  src/lib/server/api/formatters.test.ts (20 tests) 9ms - ✓  server  src/lib/server/pipeline/diff.test.ts (9 tests) 8ms - ✓  server  src/lib/server/api/library-id.test.ts (8 tests) 6ms - ✓  server  src/lib/server/api/token-budget.test.ts (7 tests) 6ms - ✓  server  src/lib/server/parser/markdown.parser.test.ts (14 tests) 9ms - ✓  server  src/lib/vitest-examples/greet.spec.ts (1 test) 3ms - ✓  server  src/lib/server/crawler/local.crawler.test.ts (50 tests) 658ms - ✓  server  src/mcp/index.test.ts (7 tests) 985ms - ✓  client (chromium)  src/lib/vitest-examples/Welcome.svelte.spec.ts (1 test) 9ms -stderr | src/lib/server/crawler/github.crawler.test.ts > crawl() > skips files that fail to download without throwing -[GitHubCrawler] Could not download: src/index.ts — skipping. - - ✓  server  src/lib/server/crawler/github.crawler.test.ts (50 tests) 6082ms - ✓ retries on failure and returns eventual success  3003ms - ✓ throws after exhausting all attempts  3003ms - -⎯⎯⎯⎯⎯⎯ Failed Suites 1 ⎯⎯⎯⎯⎯⎯⎯ - - FAIL   server  src/lib/server/embeddings/embedding.service.test.ts [ src/lib/server/embeddings/embedding.service.test.ts ] -Error: Transform failed with 1 error: -/home/moze/Sources/trueref/src/lib/server/embeddings/embedding.service.test.ts:408:2: ERROR: "await" can only be used inside an "async" function - Plugin: vite:esbuild - File: /home/moze/Sources/trueref/src/lib/server/embeddings/embedding.service.test.ts:408:2 - - "await" can only be used inside an "async" function - 406 | }); - 407 | - 408 | await service.embedSnippets([snippetId]); - | ^ - 409 | - 410 | const retrieved = service.getEmbedding(snippetId); -  - ❯ failureErrorWithLog node_modules/vite/node_modules/esbuild/lib/main.js:1748:15 - ❯ node_modules/vite/node_modules/esbuild/lib/main.js:1017:50 - ❯ responseCallbacks. node_modules/vite/node_modules/esbuild/lib/main.js:884:9 - ❯ handleIncomingPacket node_modules/vite/node_modules/esbuild/lib/main.js:939:12 - ❯ Socket.readFromStdout node_modules/vite/node_modules/esbuild/lib/main.js:862:7 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/36]⎯ - - -⎯⎯⎯⎯⎯⎯ Failed Tests 35 ⎯⎯⎯⎯⎯⎯⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > repositories table > inserts and retrieves a repository - FAIL   server  src/lib/server/db/schema.test.ts > repositories table > allows nullable optional fields - FAIL   server  src/lib/server/db/schema.test.ts > repositories table > supports all state enum values -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:63:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:63:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > repository_versions table > inserts a version linked to a repository - FAIL   server  src/lib/server/db/schema.test.ts > repository_versions table > cascades delete when parent repository is deleted -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:109:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:109:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > documents table > inserts a document - FAIL   server  src/lib/server/db/schema.test.ts > documents table > cascades delete when repository is deleted -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:151:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:151:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > snippets table > inserts a code snippet - FAIL   server  src/lib/server/db/schema.test.ts > snippets table > inserts an info snippet - FAIL   server  src/lib/server/db/schema.test.ts > snippets table > cascades delete when document is deleted -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:195:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:195:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > snippet_embeddings table > stores a Float32Array embedding as blob - FAIL   server  src/lib/server/db/schema.test.ts > snippet_embeddings table > cascades delete when snippet is deleted -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:271:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:271:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[6/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > indexing_jobs table > creates a job with default queued status - FAIL   server  src/lib/server/db/schema.test.ts > indexing_jobs table > supports all status enum values -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:350:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:350:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[7/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > repository_configs table > stores JSON array fields correctly -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:391:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:391:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[8/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > settings table > stores and retrieves key-value settings -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:422:13 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:422:13 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[9/36]⎯ - - FAIL   server  src/lib/server/db/schema.test.ts > FTS5 virtual table (snippets_fts) > FTS table exists and is queryable - FAIL   server  src/lib/server/db/schema.test.ts > FTS5 virtual table (snippets_fts) > insert trigger keeps FTS in sync - FAIL   server  src/lib/server/db/schema.test.ts > FTS5 virtual table (snippets_fts) > delete trigger removes entry from FTS -DrizzleError: Failed to run the query ' -INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' - ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 - ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 - ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 -  30| // Run migrations from the generated migration folder. -  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); -  32| migrate(db, { migrationsFolder }); -  | ^ -  33| -  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… - ❯ src/lib/server/db/schema.test.ts:442:21 - -Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 - ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 - ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 - ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 - ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 - ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 - ❯ src/lib/server/db/schema.test.ts:442:21 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ -Serialized Error: { code: 'SQLITE_ERROR' } -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[10/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > returns empty array when no embeddings exist -SqliteError: no such column: se.profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ VectorSearch.vectorSearch src/lib/server/search/vector.search.ts:100:24 -  98| } -  99| - 100| const rows = this.db.prepare(sql).all(..… -  | ^ - 101| - 102| const scored: VectorSearchResult[] = rows.map((row) => { - ❯ src/lib/server/search/hybrid.search.service.test.ts:289:22 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[11/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > returns results sorted by descending cosine similarity -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:302:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[12/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > respects the limit parameter -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:321:4 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[13/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > only returns snippets from the specified repository -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:340:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[14/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > handles embeddings with negative values -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:352:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[15/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > returns results when hybrid mode is active (alpha = 0.5) -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:430:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[16/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > deduplicates snippets appearing in both FTS5 and vector results -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:449:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[17/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > respects the limit option -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:471:4 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[18/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > returns vector-ranked results when alpha = 1 -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:503:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[19/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > results include snippet and repository metadata -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:528:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[20/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > all results belong to the requested repository -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:556:4 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[21/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > filters by snippet type when provided -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:591:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[22/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > uses alpha = 0.5 when not specified -SqliteError: table snippet_embeddings has no column named profile_id - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 - 110| const f32 = new Float32Array(values); - 111| client - 112| .prepare( -  | ^ - 113| `INSERT OR REPLACE INTO snippet_embeddings - 114| (snippet_id, profile_id, model, dimensions, embedding, create… - ❯ src/lib/server/search/hybrid.search.service.test.ts:616:3 - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[23/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > filters by versionId — excludes snippets from other versions -SqliteError: no such table: embedding_profiles - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ src/lib/server/search/hybrid.search.service.test.ts:647:5 - 645| // Create embedding profile - 646| client - 647| .prepare( -  | ^ - 648| `INSERT INTO embedding_profiles (id, provider_kind, title, enabled… - 649| VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[24/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > searchMode=keyword never calls provider.embed() -SqliteError: table snippets_fts has no column named id - ❯ Database.exec node_modules/better-sqlite3/lib/methods/wrappers.js:9:14 - ❯ src/lib/server/search/hybrid.search.service.test.ts:734:10 - 732| }); - 733| - 734| client.exec( -  | ^ - 735| `INSERT INTO snippets_fts (id, repository_id, version_id, title, br… - 736| VALUES ('${snippetId}', '${repoId}', NULL, NULL, NULL, 'keyword… - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[25/36]⎯ - - FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > searchMode=semantic uses only vector search -SqliteError: no such table: embedding_profiles - ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 - ❯ src/lib/server/search/hybrid.search.service.test.ts:772:5 - 770| // Create profile - 771| client - 772| .prepare( -  | ^ - 773| `INSERT INTO embedding_profiles (id, provider_kind, title, enabled… - 774| VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[26/36]⎯ - - - Test Files  3 failed | 19 passed (22) - Tests  35 failed | 416 passed (451) - Start at  19:10:26 - Duration  6.93s (transform 7.37s, setup 0ms, import 9.29s, tests 8.17s, environment 11ms) - - FAIL  Tests failed. Watching for file changes... - press h to show help, press q to quit -Cancelling test run. Press CTRL+c again to exit forcefully. - From 781d224adca13bd5c76d3a851d2c120441437454 Mon Sep 17 00:00:00 2001 From: Giancarmine Salucci Date: Sat, 28 Mar 2026 09:28:01 +0100 Subject: [PATCH 2/7] feat(EMBEDDINGS-0001): enable local embedder by default and overhaul settings page - Wire local embedding provider as the default on startup when no profile is configured - Refactor embedding settings into dedicated service, DTOs, mappers and models - Rebuild settings page with profile management UI and live test feedback - Expose index summary (indexed versions + embedding count) on repo endpoints - Harden indexing pipeline and context search with additional test coverage Co-Authored-By: Claude Sonnet 4.6 --- src/hooks.server.ts | 51 ++-- src/lib/components/RepositoryCard.svelte | 35 ++- .../components/RepositoryCard.svelte.test.ts | 5 + src/lib/dtos/embedding-settings.ts | 41 +++ .../embeddings/embedding.service.test.ts | 43 +++ .../server/embeddings/embedding.service.ts | 36 +++ src/lib/server/embeddings/local.provider.ts | 10 +- src/lib/server/embeddings/registry.ts | 3 +- .../mappers/embedding-profile.mapper.ts | 38 +++ .../mappers/embedding-settings.dto.mapper.ts | 71 +++++ src/lib/server/models/embedding-profile.ts | 77 +++++ src/lib/server/models/embedding-settings.ts | 20 ++ .../server/pipeline/indexing.pipeline.test.ts | 89 +++++- src/lib/server/pipeline/indexing.pipeline.ts | 32 ++- .../services/embedding-settings.service.ts | 131 +++++++++ .../services/repository.service.test.ts | 97 ++++++- src/lib/server/services/repository.service.ts | 48 ++++ src/lib/types.ts | 1 + .../api/v1/api-contract.integration.test.ts | 41 +++ src/routes/api/v1/context/+server.ts | 20 +- src/routes/api/v1/libs/+server.ts | 3 +- src/routes/api/v1/libs/[id]/+server.ts | 2 +- .../api/v1/settings/embedding/+server.ts | 126 +------- .../api/v1/settings/embedding/server.test.ts | 183 ++++++++++++ .../api/v1/settings/embedding/test/+server.ts | 63 ++-- src/routes/repos/[id]/+page.svelte | 58 ++-- src/routes/repos/[id]/page.server.test.ts | 12 +- src/routes/settings/+page.server.ts | 22 ++ src/routes/settings/+page.svelte | 271 ++++++++++++++---- src/routes/settings/page.server.test.ts | 103 +++++++ 30 files changed, 1419 insertions(+), 313 deletions(-) create mode 100644 src/lib/dtos/embedding-settings.ts create mode 100644 src/lib/server/mappers/embedding-profile.mapper.ts create mode 100644 src/lib/server/mappers/embedding-settings.dto.mapper.ts create mode 100644 src/lib/server/models/embedding-profile.ts create mode 100644 src/lib/server/models/embedding-settings.ts create mode 100644 src/lib/server/services/embedding-settings.service.ts create mode 100644 src/routes/api/v1/settings/embedding/server.test.ts create mode 100644 src/routes/settings/+page.server.ts create mode 100644 src/routes/settings/page.server.test.ts diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 56fb88d..d66593b 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -9,13 +9,13 @@ import { initializeDatabase } from '$lib/server/db/index.js'; import { getClient } from '$lib/server/db/client.js'; import { initializePipeline } from '$lib/server/pipeline/startup.js'; -import { - EMBEDDING_CONFIG_KEY, - createProviderFromConfig, - defaultEmbeddingConfig -} from '$lib/server/embeddings/factory.js'; +import { createProviderFromProfile } from '$lib/server/embeddings/registry.js'; import { EmbeddingService } from '$lib/server/embeddings/embedding.service.js'; -import type { EmbeddingConfig } from '$lib/server/embeddings/factory.js'; +import { + EmbeddingProfileEntity, + type EmbeddingProfileEntityProps +} from '$lib/server/models/embedding-profile.js'; +import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper.js'; import type { Handle } from '@sveltejs/kit'; // --------------------------------------------------------------------------- @@ -26,37 +26,20 @@ try { initializeDatabase(); const db = getClient(); - - // Load persisted embedding configuration (if any). - const configRow = db - .prepare<[string], { value: string }>(`SELECT value FROM settings WHERE key = ?`) - .get(EMBEDDING_CONFIG_KEY); + const activeProfileRow = db + .prepare<[], EmbeddingProfileEntityProps>( + 'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1' + ) + .get(); let embeddingService: EmbeddingService | null = null; - if (configRow) { - try { - const config: EmbeddingConfig = - typeof configRow.value === 'string' - ? JSON.parse(configRow.value) - : (configRow.value as EmbeddingConfig); - - if (config.provider !== 'none') { - const provider = createProviderFromConfig(config); - embeddingService = new EmbeddingService(db, provider); - } - } catch (err) { - console.warn( - `[hooks.server] Could not load embedding config: ${err instanceof Error ? err.message : String(err)}` - ); - } - } else { - // Use the default (noop) config so the pipeline is still wired up. - const config = defaultEmbeddingConfig(); - if (config.provider !== 'none') { - const provider = createProviderFromConfig(config); - embeddingService = new EmbeddingService(db, provider); - } + if (activeProfileRow) { + const activeProfile = EmbeddingProfileMapper.fromEntity( + new EmbeddingProfileEntity(activeProfileRow) + ); + const provider = createProviderFromProfile(activeProfile); + embeddingService = new EmbeddingService(db, provider, activeProfile.id); } initializePipeline(db, embeddingService); diff --git a/src/lib/components/RepositoryCard.svelte b/src/lib/components/RepositoryCard.svelte index dcc959c..8cbdb84 100644 --- a/src/lib/components/RepositoryCard.svelte +++ b/src/lib/components/RepositoryCard.svelte @@ -1,13 +1,25 @@
@@ -67,6 +92,12 @@ {/if}
+
+ {embeddingCount.toLocaleString()} embeddings + · + Indexed: {indexedVersionsLabel} +
+ {#if repo.state === 'error'}

Indexing failed. Check jobs for details.

{/if} diff --git a/src/lib/components/RepositoryCard.svelte.test.ts b/src/lib/components/RepositoryCard.svelte.test.ts index ee2be10..34a5511 100644 --- a/src/lib/components/RepositoryCard.svelte.test.ts +++ b/src/lib/components/RepositoryCard.svelte.test.ts @@ -12,6 +12,8 @@ describe('RepositoryCard.svelte', () => { description: 'A JavaScript library for building user interfaces', state: 'indexed', totalSnippets: 1234, + embeddingCount: 1200, + indexedVersions: ['main', 'v18.3.0'], trustScore: 9.7, stars: 230000, lastIndexedAt: null @@ -23,5 +25,8 @@ describe('RepositoryCard.svelte', () => { await expect .element(page.getByRole('link', { name: 'Details' })) .toHaveAttribute('href', '/repos/%2Ffacebook%2Freact'); + + await expect.element(page.getByText('1,200 embeddings')).toBeInTheDocument(); + await expect.element(page.getByText('Indexed: main, v18.3.0')).toBeInTheDocument(); }); }); \ No newline at end of file diff --git a/src/lib/dtos/embedding-settings.ts b/src/lib/dtos/embedding-settings.ts new file mode 100644 index 0000000..f35c85f --- /dev/null +++ b/src/lib/dtos/embedding-settings.ts @@ -0,0 +1,41 @@ +import type { EmbeddingProviderKind } from '$lib/types'; + +export interface EmbeddingProfileConfigEntryDto { + key: string; + value: string; + redacted: boolean; +} + +export interface EmbeddingProfileDto { + id: string; + providerKind: string; + title: string; + enabled: boolean; + isDefault: boolean; + model: string; + dimensions: number; + config: Record; + configEntries: EmbeddingProfileConfigEntryDto[]; + createdAt: number; + updatedAt: number; +} + +export interface EmbeddingSettingsDto { + profiles: EmbeddingProfileDto[]; + activeProfileId: string | null; + activeProfile: EmbeddingProfileDto | null; +} + +export interface EmbeddingProfileUpsertDto { + id: string; + providerKind: EmbeddingProviderKind; + title: string; + model: string; + dimensions: number; + config: Record; +} + +export interface EmbeddingSettingsUpdateDto { + activeProfileId: string | null; + profile?: EmbeddingProfileUpsertDto; +} \ No newline at end of file diff --git a/src/lib/server/embeddings/embedding.service.test.ts b/src/lib/server/embeddings/embedding.service.test.ts index 7100789..4d49b7c 100644 --- a/src/lib/server/embeddings/embedding.service.test.ts +++ b/src/lib/server/embeddings/embedding.service.test.ts @@ -408,6 +408,36 @@ describe('EmbeddingService', () => { expect(embedding![2]).toBeCloseTo(0.2, 5); }); + it('stores embeddings under the configured profile ID', async () => { + client + .prepare( + `INSERT INTO embedding_profiles + (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, unixepoch(), unixepoch())` + ) + .run( + 'openai-custom', + 'openai-compatible', + 'OpenAI Custom', + 1, + 0, + 'test-model', + 4, + '{}' + ); + + const snippetId = seedSnippet(db, client); + const provider = makeProvider(4, 'test-model'); + const service = new EmbeddingService(client, provider, 'openai-custom'); + + await service.embedSnippets([snippetId]); + + const row = client + .prepare('SELECT profile_id FROM snippet_embeddings WHERE snippet_id = ?') + .get(snippetId) as { profile_id: string }; + expect(row.profile_id).toBe('openai-custom'); + }); + it('is idempotent — re-embedding replaces the existing row', async () => { const snippetId = seedSnippet(db, client); const provider = makeProvider(2); @@ -469,6 +499,19 @@ describe('EmbeddingService', () => { }; expect(rows.cnt).toBe(0); }); + + it('finds snippets missing embeddings for the active profile', async () => { + const firstSnippetId = seedSnippet(db, client); + const secondSnippetId = seedSnippet(db, client, { content: 'Second snippet content' }); + const provider = makeProvider(4); + const service = new EmbeddingService(client, provider, 'local-default'); + + await service.embedSnippets([firstSnippetId]); + + expect(service.findSnippetIdsMissingEmbeddings('/test/embed-repo', null)).toEqual([ + secondSnippetId + ]); + }); }); // --------------------------------------------------------------------------- diff --git a/src/lib/server/embeddings/embedding.service.ts b/src/lib/server/embeddings/embedding.service.ts index 8a60589..1dc18d5 100644 --- a/src/lib/server/embeddings/embedding.service.ts +++ b/src/lib/server/embeddings/embedding.service.ts @@ -23,6 +23,42 @@ export class EmbeddingService { private readonly profileId: string = 'local-default' ) {} + findSnippetIdsMissingEmbeddings(repositoryId: string, versionId: string | null): string[] { + if (versionId) { + const rows = this.db + .prepare<[string, string, string], { id: string }>( + `SELECT snippets.id + FROM snippets + LEFT JOIN snippet_embeddings + ON snippet_embeddings.snippet_id = snippets.id + AND snippet_embeddings.profile_id = ? + WHERE snippets.repository_id = ? + AND snippets.version_id = ? + AND snippet_embeddings.snippet_id IS NULL + ORDER BY snippets.id` + ) + .all(this.profileId, repositoryId, versionId); + + return rows.map((row) => row.id); + } + + const rows = this.db + .prepare<[string, string], { id: string }>( + `SELECT snippets.id + FROM snippets + LEFT JOIN snippet_embeddings + ON snippet_embeddings.snippet_id = snippets.id + AND snippet_embeddings.profile_id = ? + WHERE snippets.repository_id = ? + AND snippets.version_id IS NULL + AND snippet_embeddings.snippet_id IS NULL + ORDER BY snippets.id` + ) + .all(this.profileId, repositoryId); + + return rows.map((row) => row.id); + } + /** * Embed the given snippet IDs and store the results in snippet_embeddings. * diff --git a/src/lib/server/embeddings/local.provider.ts b/src/lib/server/embeddings/local.provider.ts index af638e2..eac0618 100644 --- a/src/lib/server/embeddings/local.provider.ts +++ b/src/lib/server/embeddings/local.provider.ts @@ -1,10 +1,10 @@ /** - * LocalEmbeddingProvider — uses @xenova/transformers (optional dependency). + * LocalEmbeddingProvider — uses @xenova/transformers via dynamic import. * - * @xenova/transformers is NOT installed by default. This provider uses a - * dynamic import so the module is only required at runtime when the local - * provider is actually configured. If the package is absent, isAvailable() - * returns false and embed() throws a clear error. + * The dynamic import keeps server startup cheap and defers loading the model + * runtime until the local provider is actually used. If the package is absent + * or cannot be resolved, isAvailable() returns false and embed() throws a + * clear error. */ import { EmbeddingError, type EmbeddingProvider, type EmbeddingVector } from './provider.js'; diff --git a/src/lib/server/embeddings/registry.ts b/src/lib/server/embeddings/registry.ts index d9b9bef..e5c699e 100644 --- a/src/lib/server/embeddings/registry.ts +++ b/src/lib/server/embeddings/registry.ts @@ -44,11 +44,12 @@ export function createProviderFromProfile(profile: EmbeddingProfile): EmbeddingP */ export function getDefaultLocalProfile(): Pick< EmbeddingProfile, - 'id' | 'providerKind' | 'model' | 'dimensions' + 'id' | 'providerKind' | 'title' | 'model' | 'dimensions' > { return { id: 'local-default', providerKind: 'local-transformers', + title: 'Local (Xenova/all-MiniLM-L6-v2)', model: 'Xenova/all-MiniLM-L6-v2', dimensions: 384 }; diff --git a/src/lib/server/mappers/embedding-profile.mapper.ts b/src/lib/server/mappers/embedding-profile.mapper.ts new file mode 100644 index 0000000..8126083 --- /dev/null +++ b/src/lib/server/mappers/embedding-profile.mapper.ts @@ -0,0 +1,38 @@ +import { + EmbeddingProfile, + EmbeddingProfileEntity +} from '$lib/server/models/embedding-profile.js'; + +function parseConfig(config: Record | string | null): Record { + if (!config) { + return {}; + } + + if (typeof config === 'string') { + try { + const parsed = JSON.parse(config); + return parsed && typeof parsed === 'object' ? (parsed as Record) : {}; + } catch { + return {}; + } + } + + return config; +} + +export class EmbeddingProfileMapper { + static fromEntity(entity: EmbeddingProfileEntity): EmbeddingProfile { + return new EmbeddingProfile({ + id: entity.id, + providerKind: entity.provider_kind, + title: entity.title, + enabled: Boolean(entity.enabled), + isDefault: Boolean(entity.is_default), + model: entity.model, + dimensions: entity.dimensions, + config: parseConfig(entity.config), + createdAt: entity.created_at, + updatedAt: entity.updated_at + }); + } +} \ No newline at end of file diff --git a/src/lib/server/mappers/embedding-settings.dto.mapper.ts b/src/lib/server/mappers/embedding-settings.dto.mapper.ts new file mode 100644 index 0000000..e1f12a8 --- /dev/null +++ b/src/lib/server/mappers/embedding-settings.dto.mapper.ts @@ -0,0 +1,71 @@ +import type { + EmbeddingProfileConfigEntryDto, + EmbeddingProfileDto, + EmbeddingSettingsDto +} from '$lib/dtos/embedding-settings.js'; +import type { EmbeddingProfile } from '$lib/server/models/embedding-profile.js'; +import { EmbeddingSettings } from '$lib/server/models/embedding-settings.js'; + +const REDACTED_VALUE = '[redacted]'; +const SENSITIVE_CONFIG_KEY = /(api[-_]?key|token|secret|password|authorization)/i; + +function formatConfigValue(value: unknown): string { + if (value === null || value === undefined) return 'null'; + if (typeof value === 'string') return value; + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + return JSON.stringify(value); +} + +function sanitizeConfig(config: Record): { + visibleConfig: Record; + configEntries: EmbeddingProfileConfigEntryDto[]; +} { + const visibleConfig: Record = {}; + const configEntries = Object.entries(config) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => { + const redacted = SENSITIVE_CONFIG_KEY.test(key); + if (!redacted) { + visibleConfig[key] = value; + } + + return { + key, + value: redacted ? REDACTED_VALUE : formatConfigValue(value), + redacted + }; + }); + + return { visibleConfig, configEntries }; +} + +function toProfileDto(profile: EmbeddingProfile): EmbeddingProfileDto { + const { visibleConfig, configEntries } = sanitizeConfig(profile.config); + + return { + id: profile.id, + providerKind: profile.providerKind, + title: profile.title, + enabled: profile.enabled, + isDefault: profile.isDefault, + model: profile.model, + dimensions: profile.dimensions, + config: visibleConfig, + configEntries, + createdAt: profile.createdAt, + updatedAt: profile.updatedAt + }; +} + +export class EmbeddingSettingsDtoMapper { + static toDto(settings: EmbeddingSettings): EmbeddingSettingsDto { + const profiles = settings.profiles.map(toProfileDto); + const activeProfile = settings.activeProfile ? toProfileDto(settings.activeProfile) : null; + + return { + profiles, + activeProfileId: settings.activeProfileId, + activeProfile + }; + } +} \ No newline at end of file diff --git a/src/lib/server/models/embedding-profile.ts b/src/lib/server/models/embedding-profile.ts new file mode 100644 index 0000000..47c7a65 --- /dev/null +++ b/src/lib/server/models/embedding-profile.ts @@ -0,0 +1,77 @@ +export interface EmbeddingProfileEntityProps { + id: string; + provider_kind: string; + title: string; + enabled: boolean | number; + is_default: boolean | number; + model: string; + dimensions: number; + config: Record | string | null; + created_at: number; + updated_at: number; +} + +export class EmbeddingProfileEntity { + id: string; + provider_kind: string; + title: string; + enabled: boolean | number; + is_default: boolean | number; + model: string; + dimensions: number; + config: Record | string | null; + created_at: number; + updated_at: number; + + constructor(props: EmbeddingProfileEntityProps) { + this.id = props.id; + this.provider_kind = props.provider_kind; + this.title = props.title; + this.enabled = props.enabled; + this.is_default = props.is_default; + this.model = props.model; + this.dimensions = props.dimensions; + this.config = props.config; + this.created_at = props.created_at; + this.updated_at = props.updated_at; + } +} + +export interface EmbeddingProfileProps { + id: string; + providerKind: string; + title: string; + enabled: boolean; + isDefault: boolean; + model: string; + dimensions: number; + config: Record; + createdAt: number; + updatedAt: number; +} + +export class EmbeddingProfile { + id: string; + providerKind: string; + title: string; + enabled: boolean; + isDefault: boolean; + model: string; + dimensions: number; + config: Record; + createdAt: number; + updatedAt: number; + + constructor(props: EmbeddingProfileProps) { + this.id = props.id; + this.providerKind = props.providerKind; + this.title = props.title; + this.enabled = props.enabled; + this.isDefault = props.isDefault; + this.model = props.model; + this.dimensions = props.dimensions; + this.config = props.config; + this.createdAt = props.createdAt; + this.updatedAt = props.updatedAt; + } +} \ No newline at end of file diff --git a/src/lib/server/models/embedding-settings.ts b/src/lib/server/models/embedding-settings.ts new file mode 100644 index 0000000..9fe86be --- /dev/null +++ b/src/lib/server/models/embedding-settings.ts @@ -0,0 +1,20 @@ +import type { EmbeddingProfile } from './embedding-profile.js'; + +export interface EmbeddingSettingsProps { + profiles: EmbeddingProfile[]; + activeProfile: EmbeddingProfile | null; +} + +export class EmbeddingSettings { + profiles: EmbeddingProfile[]; + activeProfile: EmbeddingProfile | null; + + constructor(props: EmbeddingSettingsProps) { + this.profiles = props.profiles; + this.activeProfile = props.activeProfile; + } + + get activeProfileId(): string | null { + return this.activeProfile?.id ?? null; + } +} \ No newline at end of file diff --git a/src/lib/server/pipeline/indexing.pipeline.test.ts b/src/lib/server/pipeline/indexing.pipeline.test.ts index c5faa54..5d3405c 100644 --- a/src/lib/server/pipeline/indexing.pipeline.test.ts +++ b/src/lib/server/pipeline/indexing.pipeline.test.ts @@ -12,6 +12,7 @@ import { join } from 'node:path'; import { JobQueue } from './job-queue.js'; import { IndexingPipeline } from './indexing.pipeline.js'; import { recoverStaleJobs } from './startup.js'; +import { EmbeddingService } from '$lib/server/embeddings/embedding.service.js'; // --------------------------------------------------------------------------- // Test DB factory @@ -22,15 +23,21 @@ function createTestDb(): Database.Database { client.pragma('foreign_keys = ON'); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); - const migrationSql = readFileSync(join(migrationsFolder, '0000_large_master_chief.sql'), 'utf-8'); + for (const migrationFile of [ + '0000_large_master_chief.sql', + '0001_quick_nighthawk.sql', + '0002_silky_stellaris.sql' + ]) { + const migrationSql = readFileSync(join(migrationsFolder, migrationFile), 'utf-8'); - const statements = migrationSql - .split('--> statement-breakpoint') - .map((s) => s.trim()) - .filter(Boolean); + const statements = migrationSql + .split('--> statement-breakpoint') + .map((s) => s.trim()) + .filter(Boolean); - for (const stmt of statements) { - client.exec(stmt); + for (const stmt of statements) { + client.exec(stmt); + } } return client; @@ -238,7 +245,8 @@ describe('IndexingPipeline', () => { crawlResult: { files: Array<{ path: string; content: string; sha: string; language: string }>; totalFiles: number; - } = { files: [], totalFiles: 0 } + } = { files: [], totalFiles: 0 }, + embeddingService: EmbeddingService | null = null ) { const mockGithubCrawl = vi.fn().mockResolvedValue({ ...crawlResult, @@ -256,7 +264,12 @@ describe('IndexingPipeline', () => { }) }; - return new IndexingPipeline(db, mockGithubCrawl as never, mockLocalCrawler as never, null); + return new IndexingPipeline( + db, + mockGithubCrawl as never, + mockLocalCrawler as never, + embeddingService + ); } function makeJob(repositoryId = '/test/repo') { @@ -388,6 +401,64 @@ describe('IndexingPipeline', () => { expect(secondSnippetIds).toEqual(firstSnippetIds); }); + it('re-index backfills missing embeddings for unchanged snippets', async () => { + const provider = { + name: 'test-provider', + model: 'test-model', + dimensions: 3, + embed: vi.fn(async (texts: string[]) => + texts.map(() => ({ + values: new Float32Array([0.1, 0.2, 0.3]), + dimensions: 3, + model: 'test-model' + })) + ), + isAvailable: vi.fn(async () => true) + }; + const embeddingService = new EmbeddingService(db, provider, 'local-default'); + const files = [ + { + path: 'README.md', + content: '# Hello\n\nThis is documentation.', + sha: 'sha-readme', + language: 'markdown' + } + ]; + + const pipeline = makePipeline({ files, totalFiles: 1 }, embeddingService); + const job1 = makeJob(); + await pipeline.run(job1 as never); + + const firstSnippetIds = (db.prepare(`SELECT id FROM snippets ORDER BY id`).all() as { id: string }[]) + .map((row) => row.id); + expect(firstSnippetIds.length).toBeGreaterThan(0); + + const firstEmbeddingCount = ( + db.prepare(`SELECT COUNT(*) as n FROM snippet_embeddings WHERE profile_id = 'local-default'`).get() as { + n: number; + } + ).n; + expect(firstEmbeddingCount).toBe(firstSnippetIds.length); + + db.prepare(`DELETE FROM snippet_embeddings WHERE profile_id = 'local-default'`).run(); + + const job2Id = insertJob(db, { repository_id: '/test/repo', status: 'queued' }); + const job2 = db.prepare(`SELECT * FROM indexing_jobs WHERE id = ?`).get(job2Id) as never; + await pipeline.run(job2); + + const secondSnippetIds = (db.prepare(`SELECT id FROM snippets ORDER BY id`).all() as { + id: string; + }[]).map((row) => row.id); + const secondEmbeddingCount = ( + db.prepare(`SELECT COUNT(*) as n FROM snippet_embeddings WHERE profile_id = 'local-default'`).get() as { + n: number; + } + ).n; + + expect(secondSnippetIds).toEqual(firstSnippetIds); + expect(secondEmbeddingCount).toBe(firstSnippetIds.length); + }); + it('replaces snippets atomically when a file changes', async () => { const pipeline1 = makePipeline({ files: [ diff --git a/src/lib/server/pipeline/indexing.pipeline.ts b/src/lib/server/pipeline/indexing.pipeline.ts index 0bbc1ed..8e8c677 100644 --- a/src/lib/server/pipeline/indexing.pipeline.ts +++ b/src/lib/server/pipeline/indexing.pipeline.ts @@ -187,20 +187,28 @@ export class IndexingPipeline { this.replaceSnippets(repo.id, changedDocIds, newDocuments, newSnippets); // ---- Stage 4: Embeddings (if provider is configured) ---------------- - if (this.embeddingService && newSnippets.length > 0) { - const snippetIds = newSnippets.map((s) => s.id!); + if (this.embeddingService) { + const snippetIds = this.embeddingService.findSnippetIdsMissingEmbeddings( + repo.id, + normJob.versionId + ); + + if (snippetIds.length === 0) { + // No missing embeddings for the active profile; parsing progress is final. + } else { const embeddingsTotal = snippetIds.length; - await this.embeddingService.embedSnippets(snippetIds, (done) => { - const progress = calculateProgress( - processedFiles, - totalFiles, - done, - embeddingsTotal, - true - ); - this.updateJob(job.id, { progress }); - }); + await this.embeddingService.embedSnippets(snippetIds, (done) => { + const progress = calculateProgress( + processedFiles, + totalFiles, + done, + embeddingsTotal, + true + ); + this.updateJob(job.id, { progress }); + }); + } } // ---- Stage 5: Update repository stats -------------------------------- diff --git a/src/lib/server/services/embedding-settings.service.ts b/src/lib/server/services/embedding-settings.service.ts new file mode 100644 index 0000000..e0ffb10 --- /dev/null +++ b/src/lib/server/services/embedding-settings.service.ts @@ -0,0 +1,131 @@ +import type Database from 'better-sqlite3'; +import type { EmbeddingSettingsUpdateDto } from '$lib/dtos/embedding-settings.js'; +import { createProviderFromProfile, getDefaultLocalProfile } from '$lib/server/embeddings/registry.js'; +import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper.js'; +import { EmbeddingProfile, EmbeddingProfileEntity } from '$lib/server/models/embedding-profile.js'; +import { EmbeddingSettings } from '$lib/server/models/embedding-settings.js'; +import { InvalidInputError } from '$lib/server/utils/validation.js'; + +export class EmbeddingSettingsService { + constructor(private readonly db: Database.Database) {} + + getSettings(): EmbeddingSettings { + const profiles = this.loadProfiles(); + const activeProfile = profiles.find((profile) => profile.isDefault && profile.enabled) ?? null; + + return new EmbeddingSettings({ profiles, activeProfile }); + } + + async updateSettings(input: EmbeddingSettingsUpdateDto): Promise { + const now = Math.floor(Date.now() / 1000); + + this.db.prepare('UPDATE embedding_profiles SET is_default = 0, updated_at = ?').run(now); + + if (input.activeProfileId === null) { + return this.getSettings(); + } + + const profile = + input.activeProfileId === 'local-default' + ? this.buildDefaultLocalProfile(now) + : this.buildCustomProfile(input, now); + + const available = await createProviderFromProfile(profile).isAvailable(); + if (!available) { + throw new InvalidInputError( + `Could not connect to the "${profile.providerKind}" provider. Check your configuration.` + ); + } + + this.persistProfile(profile); + return this.getSettings(); + } + + private loadProfiles(): EmbeddingProfile[] { + return this.db + .prepare('SELECT * FROM embedding_profiles ORDER BY is_default DESC, created_at ASC') + .all() + .map((row) => EmbeddingProfileMapper.fromEntity(new EmbeddingProfileEntity(row as never))); + } + + private buildDefaultLocalProfile(now: number): EmbeddingProfile { + const defaultLocal = getDefaultLocalProfile(); + + return new EmbeddingProfile({ + id: defaultLocal.id, + providerKind: defaultLocal.providerKind, + title: defaultLocal.title, + enabled: true, + isDefault: true, + model: defaultLocal.model, + dimensions: defaultLocal.dimensions, + config: {}, + createdAt: this.getCreatedAt(defaultLocal.id, now), + updatedAt: now + }); + } + + private buildCustomProfile(input: EmbeddingSettingsUpdateDto, now: number): EmbeddingProfile { + const candidate = input.profile; + if (!candidate) { + throw new InvalidInputError('profile is required for custom embedding providers'); + } + if (candidate.id !== input.activeProfileId) { + throw new InvalidInputError('activeProfileId must match profile.id'); + } + if (!candidate.title || !candidate.model) { + throw new InvalidInputError('profile title and model are required'); + } + + return new EmbeddingProfile({ + id: candidate.id, + providerKind: candidate.providerKind, + title: candidate.title, + enabled: true, + isDefault: true, + model: candidate.model, + dimensions: candidate.dimensions, + config: candidate.config, + createdAt: this.getCreatedAt(candidate.id, now), + updatedAt: now + }); + } + + private getCreatedAt(id: string, fallback: number): number { + return ( + this.db + .prepare<[string], { created_at: number }>('SELECT created_at FROM embedding_profiles WHERE id = ?') + .get(id)?.created_at ?? fallback + ); + } + + private persistProfile(profile: EmbeddingProfile): void { + this.db + .prepare( + `INSERT INTO embedding_profiles + (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + provider_kind = excluded.provider_kind, + title = excluded.title, + enabled = excluded.enabled, + is_default = excluded.is_default, + model = excluded.model, + dimensions = excluded.dimensions, + config = excluded.config, + updated_at = excluded.updated_at` + ) + .run( + profile.id, + profile.providerKind, + profile.title, + profile.enabled ? 1 : 0, + profile.isDefault ? 1 : 0, + profile.model, + profile.dimensions, + JSON.stringify(profile.config), + profile.createdAt, + profile.updatedAt + ); + } +} \ No newline at end of file diff --git a/src/lib/server/services/repository.service.test.ts b/src/lib/server/services/repository.service.test.ts index bbf14f3..e5e4aa8 100644 --- a/src/lib/server/services/repository.service.test.ts +++ b/src/lib/server/services/repository.service.test.ts @@ -27,16 +27,20 @@ function createTestDb(): Database.Database { client.pragma('foreign_keys = ON'); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); - const migrationSql = readFileSync(join(migrationsFolder, '0000_large_master_chief.sql'), 'utf-8'); - // Drizzle migration files use `--> statement-breakpoint` as separator. - const statements = migrationSql - .split('--> statement-breakpoint') - .map((s) => s.trim()) - .filter(Boolean); + for (const migration of [ + '0000_large_master_chief.sql', + '0001_quick_nighthawk.sql', + '0002_silky_stellaris.sql' + ]) { + const statements = readFileSync(join(migrationsFolder, migration), 'utf-8') + .split('--> statement-breakpoint') + .map((statement) => statement.trim()) + .filter(Boolean); - for (const stmt of statements) { - client.exec(stmt); + for (const statement of statements) { + client.exec(statement); + } } return client; @@ -408,6 +412,83 @@ describe('RepositoryService.getVersions()', () => { }); }); +// --------------------------------------------------------------------------- +// getIndexSummary() +// --------------------------------------------------------------------------- + +describe('RepositoryService.getIndexSummary()', () => { + let client: Database.Database; + let service: RepositoryService; + + beforeEach(() => { + client = createTestDb(); + service = makeService(client); + service.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react', branch: 'main' }); + }); + + it('returns embedding counts and indexed version labels', () => { + const now = Math.floor(Date.now() / 1000); + const docId = crypto.randomUUID(); + const versionDocId = crypto.randomUUID(); + const snippetId = crypto.randomUUID(); + const versionSnippetId = crypto.randomUUID(); + + client + .prepare( + `INSERT INTO repository_versions (id, repository_id, tag, state, created_at) + VALUES (?, '/facebook/react', ?, 'indexed', ?)` + ) + .run('/facebook/react/v18.3.0', 'v18.3.0', now); + + client + .prepare( + `INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at) + VALUES (?, '/facebook/react', NULL, 'README.md', 'base', ?)` + ) + .run(docId, now); + + client + .prepare( + `INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at) + VALUES (?, '/facebook/react', ?, 'README.md', 'version', ?)` + ) + .run(versionDocId, '/facebook/react/v18.3.0', now); + + client + .prepare( + `INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at) + VALUES (?, ?, '/facebook/react', NULL, 'info', 'base snippet', ?)` + ) + .run(snippetId, docId, now); + + client + .prepare( + `INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at) + VALUES (?, ?, '/facebook/react', ?, 'info', 'version snippet', ?)` + ) + .run(versionSnippetId, versionDocId, '/facebook/react/v18.3.0', now); + + client + .prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', 2, ?, ?)` + ) + .run(snippetId, Buffer.from(Float32Array.from([1, 0]).buffer), now); + + client + .prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', 2, ?, ?)` + ) + .run(versionSnippetId, Buffer.from(Float32Array.from([0, 1]).buffer), now); + + expect(service.getIndexSummary('/facebook/react')).toEqual({ + embeddingCount: 2, + indexedVersions: ['main', 'v18.3.0'] + }); + }); +}); + // --------------------------------------------------------------------------- // createIndexingJob() // --------------------------------------------------------------------------- diff --git a/src/lib/server/services/repository.service.ts b/src/lib/server/services/repository.service.ts index 4b20fc2..b8e5e16 100644 --- a/src/lib/server/services/repository.service.ts +++ b/src/lib/server/services/repository.service.ts @@ -39,6 +39,11 @@ export interface RepositoryStats { lastIndexedAt: Date | null; } +export interface RepositoryIndexSummary { + embeddingCount: number; + indexedVersions: string[]; +} + export class RepositoryService { constructor(private readonly db: Database.Database) {} @@ -266,6 +271,49 @@ export class RepositoryService { return rows.map((r) => r.tag); } + getIndexSummary(repositoryId: string): RepositoryIndexSummary { + const repository = this.get(repositoryId); + if (!repository) throw new NotFoundError(`Repository ${repositoryId} not found`); + + const embeddingRow = this.db + .prepare( + `SELECT COUNT(*) AS count + FROM snippet_embeddings se + INNER JOIN snippets s ON s.id = se.snippet_id + WHERE s.repository_id = ?` + ) + .get(repositoryId) as { count: number }; + + const versionRows = this.db + .prepare( + `SELECT tag FROM repository_versions + WHERE repository_id = ? AND state = 'indexed' + ORDER BY created_at DESC` + ) + .all(repositoryId) as { tag: string }[]; + + const hasDefaultBranchIndex = Boolean( + this.db + .prepare( + `SELECT 1 AS found + FROM documents + WHERE repository_id = ? AND version_id IS NULL + LIMIT 1` + ) + .get(repositoryId) + ); + + const indexedVersions = [ + ...(hasDefaultBranchIndex ? [repository.branch ?? 'default branch'] : []), + ...versionRows.map((row) => row.tag) + ]; + + return { + embeddingCount: embeddingRow.count, + indexedVersions: Array.from(new Set(indexedVersions)) + }; + } + /** * Create an indexing job for a repository. * If a job is already running, returns the existing job. diff --git a/src/lib/types.ts b/src/lib/types.ts index 37da65c..236bd01 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -32,6 +32,7 @@ export type RepositoryState = 'pending' | 'indexing' | 'indexed' | 'error'; export type SnippetType = 'code' | 'info'; export type JobStatus = 'queued' | 'running' | 'done' | 'failed'; export type VersionState = 'pending' | 'indexing' | 'indexed' | 'error'; +export type EmbeddingProviderKind = 'local-transformers' | 'openai-compatible'; // --------------------------------------------------------------------------- // API / service layer types diff --git a/src/routes/api/v1/api-contract.integration.test.ts b/src/routes/api/v1/api-contract.integration.test.ts index 58c7e09..5f6bce0 100644 --- a/src/routes/api/v1/api-contract.integration.test.ts +++ b/src/routes/api/v1/api-contract.integration.test.ts @@ -34,6 +34,7 @@ vi.mock('$lib/server/embeddings/registry.js', () => ({ })); import { POST as postLibraries } from './libs/+server.js'; +import { GET as getLibraries } from './libs/+server.js'; import { GET as getLibrary } from './libs/[id]/+server.js'; import { GET as getJobs } from './jobs/+server.js'; import { GET as getJob } from './jobs/[id]/+server.js'; @@ -186,6 +187,16 @@ function seedSnippet( return snippetId; } +function seedEmbedding(client: Database.Database, snippetId: string, values: number[]): void { + client + .prepare( + `INSERT INTO snippet_embeddings + (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, 'local-default', 'Xenova/all-MiniLM-L6-v2', ?, ?, ?)` + ) + .run(snippetId, values.length, Buffer.from(Float32Array.from(values).buffer), NOW_S); +} + function seedRules(client: Database.Database, repositoryId: string, rules: string[]) { client .prepare( @@ -249,6 +260,36 @@ describe('API contract integration', () => { expect(body).not.toHaveProperty('total_snippets'); }); + it('GET /api/v1/libs includes embedding counts and indexed versions per repository', async () => { + const repositoryId = seedRepo(db); + const versionId = seedVersion(db, repositoryId, 'v18.3.0'); + const baseDocId = seedDocument(db, repositoryId); + const versionDocId = seedDocument(db, repositoryId, versionId); + const baseSnippetId = seedSnippet(db, { + documentId: baseDocId, + repositoryId, + content: 'Base branch snippet' + }); + const versionSnippetId = seedSnippet(db, { + documentId: versionDocId, + repositoryId, + versionId, + content: 'Versioned snippet' + }); + seedEmbedding(db, baseSnippetId, [1, 0]); + seedEmbedding(db, versionSnippetId, [0, 1]); + + const response = await getLibraries({ + url: new URL('http://test/api/v1/libs') + } as never); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.libraries).toHaveLength(1); + expect(body.libraries[0].embeddingCount).toBe(2); + expect(body.libraries[0].indexedVersions).toEqual(['main', 'v18.3.0']); + }); + it('GET /api/v1/jobs and /api/v1/jobs/:id return job DTOs in camelCase', async () => { const repoService = new RepositoryService(db); repoService.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' }); diff --git a/src/routes/api/v1/context/+server.ts b/src/routes/api/v1/context/+server.ts index 784182c..1f26375 100644 --- a/src/routes/api/v1/context/+server.ts +++ b/src/routes/api/v1/context/+server.ts @@ -17,7 +17,11 @@ import { dtoJsonResponse } from '$lib/server/api/dto-response'; import { SearchService } from '$lib/server/search/search.service'; import { HybridSearchService } from '$lib/server/search/hybrid.search.service'; import { createProviderFromProfile } from '$lib/server/embeddings/registry'; -import type { EmbeddingProfile } from '$lib/server/db/schema'; +import { + EmbeddingProfileEntity, + type EmbeddingProfileEntityProps +} from '$lib/server/models/embedding-profile'; +import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper'; import { parseLibraryId } from '$lib/server/api/library-id'; import { selectSnippetsWithinBudget, DEFAULT_TOKEN_BUDGET } from '$lib/server/api/token-budget'; import { formatContextJson, formatContextTxt, CORS_HEADERS } from '$lib/server/api/formatters'; @@ -32,16 +36,18 @@ function getServices(db: ReturnType) { // Load the active embedding profile from the database const profileRow = db - .prepare< - [], - EmbeddingProfile - >('SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1') + .prepare<[], EmbeddingProfileEntityProps>( + 'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1' + ) .get(); - const provider = profileRow ? createProviderFromProfile(profileRow) : null; + const profile = profileRow + ? EmbeddingProfileMapper.fromEntity(new EmbeddingProfileEntity(profileRow)) + : null; + const provider = profile ? createProviderFromProfile(profile) : null; const hybridService = new HybridSearchService(db, searchService, provider); - return { db, searchService, hybridService, profileId: profileRow?.id }; + return { db, searchService, hybridService, profileId: profile?.id }; } interface RawRepoConfig { diff --git a/src/routes/api/v1/libs/+server.ts b/src/routes/api/v1/libs/+server.ts index 8e31eee..c3db516 100644 --- a/src/routes/api/v1/libs/+server.ts +++ b/src/routes/api/v1/libs/+server.ts @@ -32,7 +32,8 @@ export const GET: RequestHandler = ({ url }) => { const enriched = libraries.map((repo) => ({ ...RepositoryMapper.toDto(repo), - versions: service.getVersions(repo.id) + versions: service.getVersions(repo.id), + ...service.getIndexSummary(repo.id) })); return json({ libraries: enriched, total, limit, offset }); diff --git a/src/routes/api/v1/libs/[id]/+server.ts b/src/routes/api/v1/libs/[id]/+server.ts index d40f0db..e13c68c 100644 --- a/src/routes/api/v1/libs/[id]/+server.ts +++ b/src/routes/api/v1/libs/[id]/+server.ts @@ -23,7 +23,7 @@ export const GET: RequestHandler = ({ params }) => { return json({ error: 'Repository not found', code: 'NOT_FOUND' }, { status: 404 }); } const versions = service.getVersions(id); - return json({ ...RepositoryMapper.toDto(repo), versions }); + return json({ ...RepositoryMapper.toDto(repo), versions, ...service.getIndexSummary(id) }); } catch (err) { return handleServiceError(err); } diff --git a/src/routes/api/v1/settings/embedding/+server.ts b/src/routes/api/v1/settings/embedding/+server.ts index fa0b5af..1374ca9 100644 --- a/src/routes/api/v1/settings/embedding/+server.ts +++ b/src/routes/api/v1/settings/embedding/+server.ts @@ -1,30 +1,25 @@ /** - * GET /api/v1/settings/embedding — retrieve all embedding profiles - * POST /api/v1/settings/embedding — create or update an embedding profile - * PUT /api/v1/settings/embedding — alias for POST (backward compat) + * GET /api/v1/settings/embedding — retrieve embedding settings + * POST /api/v1/settings/embedding — update active embedding settings + * PUT /api/v1/settings/embedding — alias for POST */ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; +import type { EmbeddingSettingsUpdateDto } from '$lib/dtos/embedding-settings.js'; import { getClient } from '$lib/server/db/client'; -import { createProviderFromProfile } from '$lib/server/embeddings/registry'; -import type { EmbeddingProfile, NewEmbeddingProfile } from '$lib/server/db/schema'; +import { EmbeddingSettingsDtoMapper } from '$lib/server/mappers/embedding-settings.dto.mapper.js'; +import { EmbeddingSettingsService } from '$lib/server/services/embedding-settings.service.js'; import { handleServiceError, InvalidInputError } from '$lib/server/utils/validation'; // --------------------------------------------------------------------------- -// GET — Return all profiles +// GET — Return embedding settings // --------------------------------------------------------------------------- export const GET: RequestHandler = () => { try { - const db = getClient(); - const profiles = db - .prepare('SELECT * FROM embedding_profiles ORDER BY is_default DESC, created_at ASC') - .all() as EmbeddingProfile[]; - - // Sanitize: remove sensitive config fields like apiKey - const safeProfiles = profiles.map(sanitizeProfile); - return json({ profiles: safeProfiles }); + const service = new EmbeddingSettingsService(getClient()); + return json(EmbeddingSettingsDtoMapper.toDto(service.getSettings())); } catch (err) { return handleServiceError(err); } @@ -34,116 +29,23 @@ export const GET: RequestHandler = () => { // POST/PUT — Create or update a profile // --------------------------------------------------------------------------- -async function upsertProfile(body: unknown) { +async function upsertSettings(body: unknown) { if (typeof body !== 'object' || body === null) { throw new InvalidInputError('Request body must be a JSON object'); } - const obj = body as Record; - - // Required fields - if (typeof obj.id !== 'string' || !obj.id) { - throw new InvalidInputError('id is required'); - } - if (typeof obj.providerKind !== 'string' || !obj.providerKind) { - throw new InvalidInputError('providerKind is required'); - } - if (typeof obj.title !== 'string' || !obj.title) { - throw new InvalidInputError('title is required'); - } - if (typeof obj.model !== 'string' || !obj.model) { - throw new InvalidInputError('model is required'); - } - if (typeof obj.dimensions !== 'number') { - throw new InvalidInputError('dimensions must be a number'); - } - - const profile: NewEmbeddingProfile = { - id: obj.id, - providerKind: obj.providerKind, - title: obj.title, - enabled: typeof obj.enabled === 'boolean' ? obj.enabled : true, - isDefault: typeof obj.isDefault === 'boolean' ? obj.isDefault : false, - model: obj.model, - dimensions: obj.dimensions, - config: (obj.config as Record) ?? {}, - createdAt: Date.now(), - updatedAt: Date.now() - }; - - // Validate provider availability before persisting - const provider = createProviderFromProfile(profile as EmbeddingProfile); - const available = await provider.isAvailable(); - if (!available) { - throw new InvalidInputError( - `Could not connect to the "${profile.providerKind}" provider. Check your configuration.` - ); - } - - const db = getClient(); - - // If setting as default, clear other defaults first - if (profile.isDefault) { - db.prepare('UPDATE embedding_profiles SET is_default = 0').run(); - } - - // Upsert the profile - db.prepare( - `INSERT INTO embedding_profiles - (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - provider_kind = excluded.provider_kind, - title = excluded.title, - enabled = excluded.enabled, - is_default = excluded.is_default, - model = excluded.model, - dimensions = excluded.dimensions, - config = excluded.config, - updated_at = excluded.updated_at` - ).run( - profile.id, - profile.providerKind, - profile.title, - profile.enabled ? 1 : 0, - profile.isDefault ? 1 : 0, - profile.model, - profile.dimensions, - JSON.stringify(profile.config), - profile.createdAt, - profile.updatedAt - ); - - const inserted = db - .prepare('SELECT * FROM embedding_profiles WHERE id = ?') - .get(profile.id) as EmbeddingProfile; - - return sanitizeProfile(inserted); + const service = new EmbeddingSettingsService(getClient()); + const settings = await service.updateSettings(body as EmbeddingSettingsUpdateDto); + return EmbeddingSettingsDtoMapper.toDto(settings); } export const POST: RequestHandler = async ({ request }) => { try { const body = await request.json(); - const profile = await upsertProfile(body); - return json(profile); + return json(await upsertSettings(body)); } catch (err) { return handleServiceError(err); } }; -// Backward compat alias export const PUT: RequestHandler = POST; - -// --------------------------------------------------------------------------- -// Sanitize — remove sensitive config fields before returning to clients -// --------------------------------------------------------------------------- - -function sanitizeProfile(profile: EmbeddingProfile): EmbeddingProfile { - const config = profile.config as Record; - if (config && config.apiKey) { - const rest = { ...config }; - delete rest.apiKey; - return { ...profile, config: rest }; - } - return profile; -} diff --git a/src/routes/api/v1/settings/embedding/server.test.ts b/src/routes/api/v1/settings/embedding/server.test.ts new file mode 100644 index 0000000..100f336 --- /dev/null +++ b/src/routes/api/v1/settings/embedding/server.test.ts @@ -0,0 +1,183 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +let db: Database.Database; + +vi.mock('$lib/server/db/client', () => ({ + getClient: () => db +})); + +vi.mock('$lib/server/db/client.js', () => ({ + getClient: () => db +})); + +vi.mock('$lib/server/embeddings/registry', () => ({ + createProviderFromProfile: () => ({ + isAvailable: async () => true + }) +})); + +vi.mock('$lib/server/embeddings/registry.js', () => ({ + createProviderFromProfile: () => ({ + isAvailable: async () => true + }) +})); + +vi.mock('$lib/server/embeddings/local.provider', () => ({ + LocalEmbeddingProvider: class { + readonly model = 'Xenova/all-MiniLM-L6-v2'; + readonly dimensions = 384; + + async isAvailable() { + return true; + } + } +})); + +vi.mock('$lib/server/embeddings/local.provider.js', () => ({ + LocalEmbeddingProvider: class { + readonly model = 'Xenova/all-MiniLM-L6-v2'; + readonly dimensions = 384; + + async isAvailable() { + return true; + } + } +})); + +import { GET as getEmbeddingSettings, PUT as putEmbeddingSettings } from './+server.js'; +import { GET as getEmbeddingTest } from './test/+server.js'; + +function createTestDb(): Database.Database { + const client = new Database(':memory:'); + client.pragma('foreign_keys = ON'); + + const migrationsFolder = join(import.meta.dirname, '../../../../../lib/server/db/migrations'); + const ftsFile = join(import.meta.dirname, '../../../../../lib/server/db/fts.sql'); + + for (const migration of [ + '0000_large_master_chief.sql', + '0001_quick_nighthawk.sql', + '0002_silky_stellaris.sql' + ]) { + const statements = readFileSync(join(migrationsFolder, migration), 'utf-8') + .split('--> statement-breakpoint') + .map((statement) => statement.trim()) + .filter(Boolean); + + for (const statement of statements) { + client.exec(statement); + } + } + + client.exec(readFileSync(ftsFile, 'utf-8')); + + return client; +} + +describe('embedding settings routes', () => { + beforeEach(() => { + db = createTestDb(); + }); + + it('GET /api/v1/settings/embedding returns profile-based settings for the seeded default profile', async () => { + const response = await getEmbeddingSettings({} as never); + expect(response.status).toBe(200); + + const body = await response.json(); + expect(body.activeProfileId).toBe('local-default'); + expect(body.activeProfile).toMatchObject({ + id: 'local-default', + providerKind: 'local-transformers', + title: 'Local (Xenova/all-MiniLM-L6-v2)' + }); + expect(body.profiles).toHaveLength(1); + expect(body.profiles[0].providerKind).toBe('local-transformers'); + expect(body.profiles[0].isDefault).toBe(true); + }); + + it('PUT /api/v1/settings/embedding persists a clean profile-based OpenAI payload', async () => { + const response = await putEmbeddingSettings({ + request: new Request('http://test/api/v1/settings/embedding', { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + activeProfileId: 'openai-default', + profile: { + id: 'openai-default', + providerKind: 'openai-compatible', + title: 'OpenAI-compatible', + model: 'text-embedding-3-small', + dimensions: 1536, + config: { + baseUrl: 'https://api.openai.com/v1', + apiKey: 'sk-test', + model: 'text-embedding-3-small' + } + } + }) + }) + } as never); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.activeProfileId).toBe('openai-default'); + expect(body.activeProfile).toMatchObject({ + id: 'openai-default', + providerKind: 'openai-compatible' + }); + expect(body.activeProfile.config).toEqual({ + baseUrl: 'https://api.openai.com/v1', + model: 'text-embedding-3-small' + }); + expect(body.activeProfile.configEntries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'apiKey', value: '[redacted]', redacted: true }) + ]) + ); + expect(body.profiles).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'openai-default', + providerKind: 'openai-compatible', + model: 'text-embedding-3-small', + dimensions: 1536, + isDefault: true + }) + ]) + ); + + const activeProfile = db + .prepare( + 'SELECT id, provider_kind, is_default, enabled, model, dimensions FROM embedding_profiles WHERE is_default = 1 LIMIT 1' + ) + .get() as Record; + + expect(activeProfile).toMatchObject({ + id: 'openai-default', + provider_kind: 'openai-compatible', + is_default: 1, + enabled: 1, + model: 'text-embedding-3-small', + dimensions: 1536 + }); + }); + + it('GET /api/v1/settings/embedding/test checks local-provider availability directly', async () => { + const response = await getEmbeddingTest({} as never); + expect(response.status).toBe(200); + + const body = await response.json(); + expect(body).toEqual({ + available: true, + profile: { + id: 'local-default', + providerKind: 'local-transformers', + model: 'Xenova/all-MiniLM-L6-v2', + dimensions: 384 + } + }); + }); +}); \ No newline at end of file diff --git a/src/routes/api/v1/settings/embedding/test/+server.ts b/src/routes/api/v1/settings/embedding/test/+server.ts index 9f25e5d..a306883 100644 --- a/src/routes/api/v1/settings/embedding/test/+server.ts +++ b/src/routes/api/v1/settings/embedding/test/+server.ts @@ -7,35 +7,24 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import { getClient } from '$lib/server/db/client'; +import { LocalEmbeddingProvider } from '$lib/server/embeddings/local.provider'; import { createProviderFromProfile } from '$lib/server/embeddings/registry'; -import type { EmbeddingProfile } from '$lib/server/db/schema'; +import { EmbeddingProfileEntity } from '$lib/server/models/embedding-profile'; +import { EmbeddingProfileMapper } from '$lib/server/mappers/embedding-profile.mapper'; import { handleServiceError } from '$lib/server/utils/validation'; export const GET: RequestHandler = async () => { try { - const db = getClient(); - const profile = db - .prepare< - [], - EmbeddingProfile - >('SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1') - .get(); - - if (!profile) { - return json({ available: false, error: 'No active embedding profile configured' }); - } - - const provider = createProviderFromProfile(profile); + const provider = new LocalEmbeddingProvider(); const available = await provider.isAvailable(); return json({ available, profile: { - id: profile.id, - providerKind: profile.providerKind, - model: profile.model, - dimensions: profile.dimensions + id: 'local-default', + providerKind: 'local-transformers', + model: provider.model, + dimensions: provider.dimensions } }); } catch (err) { @@ -46,19 +35,43 @@ export const GET: RequestHandler = async () => { export const POST: RequestHandler = async ({ request }) => { try { const body = await request.json(); - const config = validateConfig(body); - - if (config.provider === 'none') { - throw new InvalidInputError('Cannot test the "none" provider — no backend is configured.'); + if (typeof body !== 'object' || body === null) { + throw new Error('Request body must be a JSON object'); } - const provider = createProviderFromConfig(config); + const candidate = body as Record; + if (candidate.providerKind !== 'openai-compatible') { + throw new Error('Only openai-compatible providers can be tested via this endpoint'); + } + if (typeof candidate.model !== 'string' || typeof candidate.dimensions !== 'number') { + throw new Error('model and dimensions are required'); + } + + const provider = createProviderFromProfile( + EmbeddingProfileMapper.fromEntity( + new EmbeddingProfileEntity({ + id: typeof candidate.id === 'string' ? candidate.id : 'test-openai-profile', + provider_kind: 'openai-compatible', + title: typeof candidate.title === 'string' ? candidate.title : 'Test Provider', + enabled: true, + is_default: false, + model: candidate.model, + dimensions: candidate.dimensions, + config: + typeof candidate.config === 'object' && candidate.config !== null + ? (candidate.config as Record) + : {}, + created_at: Date.now(), + updated_at: Date.now() + }) + ) + ); const available = await provider.isAvailable(); if (!available) { return new Response( JSON.stringify({ - error: `Provider "${config.provider}" is not available. Check your configuration.` + error: 'Provider is not available. Check your configuration.' }), { status: 400, headers: { 'Content-Type': 'application/json' } } ); diff --git a/src/routes/repos/[id]/+page.svelte b/src/routes/repos/[id]/+page.svelte index 39c4193..056c3ca 100644 --- a/src/routes/repos/[id]/+page.svelte +++ b/src/routes/repos/[id]/+page.svelte @@ -2,22 +2,24 @@ import { goto } from '$app/navigation'; import { resolve as resolveRoute } from '$app/paths'; import type { PageData } from './$types'; - import type { Repository, RepositoryVersion, IndexingJob } from '$lib/types'; + import type { Repository, IndexingJob } from '$lib/types'; import ConfirmDialog from '$lib/components/ConfirmDialog.svelte'; import IndexingProgress from '$lib/components/IndexingProgress.svelte'; import StatBadge from '$lib/components/StatBadge.svelte'; let { data }: { data: PageData } = $props(); - // Initialized empty; $effect syncs from data prop on every navigation/reload. - let repo = $state( - {} as Repository & { versions?: RepositoryVersion[] } + let repoOverride = $state< + (Repository & { indexedVersions?: string[]; embeddingCount?: number }) | null + >(null); + const repo = $derived( + repoOverride ?? + ((data.repo ?? {}) as Repository & { + indexedVersions?: string[]; + embeddingCount?: number; + }) ); - let recentJobs = $state([]); - $effect(() => { - if (data.repo) repo = data.repo; - recentJobs = data.recentJobs ?? []; - }); + const recentJobs = $derived((data.recentJobs ?? []) as IndexingJob[]); let showDeleteConfirm = $state(false); let activeJobId = $state(null); let errorMessage = $state(null); @@ -41,7 +43,7 @@ try { const res = await fetch(`/api/v1/libs/${encodeURIComponent(repo.id)}`); if (res.ok) { - repo = await res.json(); + repoOverride = await res.json(); } } catch { // ignore @@ -92,7 +94,8 @@ return new Date(ts as string).toLocaleString(); } - const versions = $derived(repo.versions ?? []); + const indexedVersions = $derived(repo.indexedVersions ?? []); + const embeddingCount = $derived(repo.embeddingCount ?? 0); const totalSnippets = $derived(repo.totalSnippets ?? 0); const totalTokens = $derived(repo.totalTokens ?? 0); const trustScore = $derived(repo.trustScore ?? 0); @@ -180,6 +183,7 @@
+ {#if repo.stars != null} @@ -210,31 +214,17 @@
- -{#if versions.length > 0} + +{#if indexedVersions.length > 0}

Indexed Versions

-
- {#each versions as version (version.id)} -
-
- {version.tag} - {#if version.title} - {version.title} - {/if} -
-
- - {stateLabels[version.state] ?? version.state} - - {#if version.indexedAt} - {formatDate(version.indexedAt)} - {/if} -
-
+
+ {#each indexedVersions as versionTag (versionTag)} + + {versionTag} + {/each}
diff --git a/src/routes/repos/[id]/page.server.test.ts b/src/routes/repos/[id]/page.server.test.ts index db1db3e..c25a1c9 100644 --- a/src/routes/repos/[id]/page.server.test.ts +++ b/src/routes/repos/[id]/page.server.test.ts @@ -8,7 +8,11 @@ describe('/repos/[id] page server load', () => { .mockResolvedValueOnce({ ok: true, status: 200, - json: async () => ({ id: '/facebook/react', title: 'React' }) + json: async () => ({ + id: '/facebook/react', + title: 'React', + indexedVersions: ['main', 'v18.3.0'] + }) }) .mockResolvedValueOnce({ ok: true, @@ -27,7 +31,11 @@ describe('/repos/[id] page server load', () => { '/api/v1/jobs?repositoryId=%2Ffacebook%2Freact&limit=5' ); expect(result).toEqual({ - repo: { id: '/facebook/react', title: 'React' }, + repo: { + id: '/facebook/react', + title: 'React', + indexedVersions: ['main', 'v18.3.0'] + }, recentJobs: [{ id: 'job-1', repositoryId: '/facebook/react' }] }); }); diff --git a/src/routes/settings/+page.server.ts b/src/routes/settings/+page.server.ts new file mode 100644 index 0000000..9b1f617 --- /dev/null +++ b/src/routes/settings/+page.server.ts @@ -0,0 +1,22 @@ +import type { PageServerLoad } from './$types'; +import { getClient } from '$lib/server/db/client.js'; +import { LocalEmbeddingProvider } from '$lib/server/embeddings/local.provider.js'; +import { EmbeddingSettingsDtoMapper } from '$lib/server/mappers/embedding-settings.dto.mapper.js'; +import { EmbeddingSettingsService } from '$lib/server/services/embedding-settings.service.js'; + +export const load: PageServerLoad = async () => { + const service = new EmbeddingSettingsService(getClient()); + const settings = EmbeddingSettingsDtoMapper.toDto(service.getSettings()); + + let localProviderAvailable = false; + try { + localProviderAvailable = await new LocalEmbeddingProvider().isAvailable(); + } catch { + localProviderAvailable = false; + } + + return { + settings, + localProviderAvailable + }; +}; \ No newline at end of file diff --git a/src/routes/settings/+page.svelte b/src/routes/settings/+page.svelte index 59a6836..9790bfb 100644 --- a/src/routes/settings/+page.svelte +++ b/src/routes/settings/+page.svelte @@ -1,5 +1,12 @@ @@ -175,17 +237,109 @@

Configure TrueRef embedding and indexing options

- +
+
+

Current Active Profile

+

+ This is the profile used for semantic indexing and retrieval right now. +

+ + {#if activeProfile} +
+
+
+

{activeProfile.title}

+

Profile ID: {activeProfile.id}

+
+ +
+
+
Provider
+
{activeProfile.providerKind}
+
Model
+
{activeProfile.model}
+
Dimensions
+
{activeProfile.dimensions}
+
+ +
+
Enabled
+
{activeProfile.enabled ? 'Yes' : 'No'}
+
Default
+
{activeProfile.isDefault ? 'Yes' : 'No'}
+
Updated
+
{formatTimestamp(activeProfile.updatedAt)}
+
+
+
+ +
+

Provider configuration

+

+ These are the provider-specific settings currently saved for the active profile. +

+ + {#if activeConfigEntries.length > 0} +
    + {#each activeConfigEntries as entry (entry.key)} +
  • + {entry.key} + {entry.value} +
    • + {/each} +
+ {:else} +

+ No provider-specific configuration is stored for this profile. +

+

+ For OpenAI-compatible profiles, edit the + settings in the Embedding Provider form + below. The built-in Local Model profile + does not currently expose extra configurable fields. +

+ {/if} +
+
+ {:else} +
+ Embeddings are currently disabled. Keyword search remains available, but no embedding profile is active. +
+ {/if} +
+ +
+

Profile Inventory

+

Profiles stored in the database and available for activation.

+
+ + +
+
+ {#each currentSettings.profiles as profile (profile.id)} +
+
+
+

{profile.title}

+

{profile.id}

+
+ {#if profile.id === currentSettings.activeProfileId} + Active + {/if} +
+
+ {/each} +
+
+
+

Embedding Provider

Embeddings enable semantic search. Without them, only keyword search (FTS5) is used.

- {#if loading} -

Loading current configuration…

- {:else} -
+
{#each ['none', 'openai', 'local'] as p (p)} @@ -314,9 +468,7 @@

Local ONNX model via @xenova/transformers

Model: Xenova/all-MiniLM-L6-v2 · 384 dimensions

- {#if localAvailable === null} -

Checking availability…

- {:else if localAvailable} + {#if getInitialLocalProviderAvailability()}

@xenova/transformers is installed and ready.

{:else}

@@ -381,8 +533,7 @@ {saving ? 'Saving…' : 'Save Settings'}

- - {/if} +
diff --git a/src/routes/settings/page.server.test.ts b/src/routes/settings/page.server.test.ts new file mode 100644 index 0000000..3795eb4 --- /dev/null +++ b/src/routes/settings/page.server.test.ts @@ -0,0 +1,103 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +let db: Database.Database; + +vi.mock('$lib/server/db/client.js', () => ({ + getClient: () => db +})); + +vi.mock('$lib/server/embeddings/local.provider.js', () => ({ + LocalEmbeddingProvider: class { + async isAvailable() { + return true; + } + } +})); + +import { load } from './+page.server.js'; + +function createTestDb(): Database.Database { + const client = new Database(':memory:'); + client.pragma('foreign_keys = ON'); + + const migrationsFolder = join(import.meta.dirname, '../../lib/server/db/migrations'); + const ftsFile = join(import.meta.dirname, '../../lib/server/db/fts.sql'); + + for (const migration of [ + '0000_large_master_chief.sql', + '0001_quick_nighthawk.sql', + '0002_silky_stellaris.sql' + ]) { + const statements = readFileSync(join(migrationsFolder, migration), 'utf-8') + .split('--> statement-breakpoint') + .map((statement) => statement.trim()) + .filter(Boolean); + + for (const statement of statements) { + client.exec(statement); + } + } + + client.exec(readFileSync(ftsFile, 'utf-8')); + return client; +} + +describe('/settings page server load', () => { + beforeEach(() => { + db = createTestDb(); + }); + + it('returns the active profile and local provider availability', async () => { + db.prepare( + `INSERT INTO embedding_profiles + (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + 'openai-default', + 'openai-compatible', + 'OpenAI-compatible', + 1, + 1, + 'text-embedding-3-small', + 1536, + JSON.stringify({ + baseUrl: 'https://api.openai.com/v1', + apiKey: 'sk-test', + model: 'text-embedding-3-small' + }), + 1710000000, + 1710000000 + ); + db.prepare('UPDATE embedding_profiles SET is_default = 0 WHERE id = ?').run('local-default'); + + const result = (await load({} as never)) as { + localProviderAvailable: boolean; + settings: { + activeProfileId: string | null; + activeProfile: { + config: Record; + configEntries: Array<{ key: string; value: string; redacted: boolean }>; + } | null; + }; + }; + + expect(result.localProviderAvailable).toBe(true); + expect(result.settings.activeProfileId).toBe('openai-default'); + expect(result.settings.activeProfile).toMatchObject({ + id: 'openai-default', + providerKind: 'openai-compatible' + }); + expect(result.settings.activeProfile?.config).toEqual({ + baseUrl: 'https://api.openai.com/v1', + model: 'text-embedding-3-small' + }); + expect(result.settings.activeProfile?.configEntries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ key: 'apiKey', value: '[redacted]', redacted: true }) + ]) + ); + }); +}); \ No newline at end of file From 1c5b634ea4c5170b612f223ba53d7daa6f4c4a6d Mon Sep 17 00:00:00 2001 From: Giancarmine Salucci Date: Sat, 28 Mar 2026 09:32:27 +0100 Subject: [PATCH 3/7] =?UTF-8?q?fix(MULTIVERSION-0001):=20fix=20multi-versi?= =?UTF-8?q?on=20indexing=20=E2=80=94=20jobs=20never=20created=20or=20trigg?= =?UTF-8?q?ered=20for=20secondary=20versions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs prevented secondary versions from ever being indexed: 1. JobQueue.enqueue() and RepositoryService.createIndexingJob() deduplication only checked repository_id, so a queued default-branch job blocked all version-specific jobs for the same repo. Fix: include version_id in the WHERE clause so only exact (repository_id, version_id) pairs are deduped. 2. POST /api/v1/libs/:id/versions used repoService.createIndexingJob() which inserts a job record but never triggers queue processing. Fix: use queue.enqueue() (same fallback pattern as the libs endpoint) so setImmediate fires processNext() after the job is inserted. Co-Authored-By: Claude Sonnet 4.6 --- src/lib/server/pipeline/job-queue.ts | 11 ++++--- .../services/repository.service.test.ts | 20 ++++++++++++ src/lib/server/services/repository.service.ts | 9 ++++-- .../api/v1/api-contract.integration.test.ts | 31 +++++++++++++++++++ .../api/v1/libs/[id]/versions/+server.ts | 6 +++- 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/lib/server/pipeline/job-queue.ts b/src/lib/server/pipeline/job-queue.ts index 6f8febe..587849b 100644 --- a/src/lib/server/pipeline/job-queue.ts +++ b/src/lib/server/pipeline/job-queue.ts @@ -36,14 +36,17 @@ export class JobQueue { * existing job instead of creating a duplicate. */ enqueue(repositoryId: string, versionId?: string): IndexingJob { - // Return early if there's already an active job for this repo. + // Return early if there's already an active job for this exact (repo, version) pair. + const resolvedVersionId = versionId ?? null; const activeRaw = this.db - .prepare<[string], IndexingJobEntity>( + .prepare<[string, string | null, string | null], IndexingJobEntity>( `${JOB_SELECT} - WHERE repository_id = ? AND status IN ('queued', 'running') + WHERE repository_id = ? + AND (version_id = ? OR (version_id IS NULL AND ? IS NULL)) + AND status IN ('queued', 'running') ORDER BY created_at DESC LIMIT 1` ) - .get(repositoryId); + .get(repositoryId, resolvedVersionId, resolvedVersionId); if (activeRaw) { // Ensure the queue is draining even if enqueue was called concurrently. diff --git a/src/lib/server/services/repository.service.test.ts b/src/lib/server/services/repository.service.test.ts index e5e4aa8..4b420bc 100644 --- a/src/lib/server/services/repository.service.test.ts +++ b/src/lib/server/services/repository.service.test.ts @@ -529,4 +529,24 @@ describe('RepositoryService.createIndexingJob()', () => { const job = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0'); expect(job.versionId).toBe('/facebook/react/v18.3.0'); }); + + it('allows separate jobs for the same repo but different versions', () => { + const defaultJob = service.createIndexingJob('/facebook/react'); + const versionJob = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0'); + expect(versionJob.id).not.toBe(defaultJob.id); + expect(defaultJob.versionId).toBeNull(); + expect(versionJob.versionId).toBe('/facebook/react/v18.3.0'); + }); + + it('returns the existing job when the same (repo, version) pair is already queued', () => { + const job1 = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0'); + const job2 = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0'); + expect(job2.id).toBe(job1.id); + }); + + it('returns the existing default-branch job when called again without a versionId', () => { + const job1 = service.createIndexingJob('/facebook/react'); + const job2 = service.createIndexingJob('/facebook/react'); + expect(job2.id).toBe(job1.id); + }); }); diff --git a/src/lib/server/services/repository.service.ts b/src/lib/server/services/repository.service.ts index b8e5e16..b9cac2b 100644 --- a/src/lib/server/services/repository.service.ts +++ b/src/lib/server/services/repository.service.ts @@ -319,14 +319,17 @@ export class RepositoryService { * If a job is already running, returns the existing job. */ createIndexingJob(repositoryId: string, versionId?: string): IndexingJob { - // Check for running job + // Check for an existing queued/running job for this exact (repo, version) pair. + const resolvedVersionId = versionId ?? null; const runningJob = this.db .prepare( `SELECT * FROM indexing_jobs - WHERE repository_id = ? AND status IN ('queued', 'running') + WHERE repository_id = ? + AND (version_id = ? OR (version_id IS NULL AND ? IS NULL)) + AND status IN ('queued', 'running') ORDER BY created_at DESC LIMIT 1` ) - .get(repositoryId) as IndexingJobEntity | undefined; + .get(repositoryId, resolvedVersionId, resolvedVersionId) as IndexingJobEntity | undefined; if (runningJob) return IndexingJobMapper.fromEntity(new IndexingJobEntity(runningJob)); diff --git a/src/routes/api/v1/api-contract.integration.test.ts b/src/routes/api/v1/api-contract.integration.test.ts index 5f6bce0..e509f6c 100644 --- a/src/routes/api/v1/api-contract.integration.test.ts +++ b/src/routes/api/v1/api-contract.integration.test.ts @@ -348,6 +348,37 @@ describe('API contract integration', () => { expect(getBody.versions[0]).not.toHaveProperty('total_snippets'); }); + it('POST /api/v1/libs/:id/versions creates distinct jobs for different versions of the same repo', async () => { + const repoService = new RepositoryService(db); + repoService.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' }); + + const postV1 = await postVersions({ + params: { id: encodeURIComponent('/facebook/react') }, + request: new Request('http://test/api/v1/libs/%2Ffacebook%2Freact/versions', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ tag: 'v18.3.0', autoIndex: true }) + }) + } as never); + const bodyV1 = await postV1.json(); + + const postV2 = await postVersions({ + params: { id: encodeURIComponent('/facebook/react') }, + request: new Request('http://test/api/v1/libs/%2Ffacebook%2Freact/versions', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ tag: 'v17.0.2', autoIndex: true }) + }) + } as never); + const bodyV2 = await postV2.json(); + + expect(postV1.status).toBe(201); + expect(postV2.status).toBe(201); + expect(bodyV1.job.id).not.toBe(bodyV2.job.id); + expect(bodyV1.job.versionId).toBe('/facebook/react/v18.3.0'); + expect(bodyV2.job.versionId).toBe('/facebook/react/v17.0.2'); + }); + it('GET /api/v1/context returns informative txt output for empty results', async () => { const repositoryId = seedRepo(db); diff --git a/src/routes/api/v1/libs/[id]/versions/+server.ts b/src/routes/api/v1/libs/[id]/versions/+server.ts index 0046d06..60a6a6f 100644 --- a/src/routes/api/v1/libs/[id]/versions/+server.ts +++ b/src/routes/api/v1/libs/[id]/versions/+server.ts @@ -10,6 +10,7 @@ import { RepositoryVersionMapper } from '$lib/server/mappers/repository-version. import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js'; import { RepositoryService } from '$lib/server/services/repository.service'; import { VersionService } from '$lib/server/services/version.service'; +import { getQueue } from '$lib/server/pipeline/startup'; import { handleServiceError, NotFoundError, InvalidInputError } from '$lib/server/utils/validation'; function getServices() { @@ -78,7 +79,10 @@ export const POST: RequestHandler = async ({ params, request }) => { let job: ReturnType | undefined; if (autoIndex) { - const indexingJob = repoService.createIndexingJob(repositoryId, version.id); + const queue = getQueue(); + const indexingJob = queue + ? queue.enqueue(repositoryId, version.id) + : repoService.createIndexingJob(repositoryId, version.id); job = IndexingJobMapper.toDto(indexingJob); } From 1c6823c05242a1284a01be63e416d08ce6244b12 Mon Sep 17 00:00:00 2001 From: Giancarmine Salucci Date: Sat, 28 Mar 2026 09:43:06 +0100 Subject: [PATCH 4/7] feat(MULTIVERSION-0001): add version management UI and auto-enqueue versions on re-index - Add POST /api/v1/libs/:id/versions/discover endpoint that calls versionService.discoverTags() for local repos and returns empty tags gracefully for GitHub repos or git failures - Enhance POST /api/v1/libs/:id/index to also enqueue jobs for all registered versions on default-branch re-index, returning versionJobs in the response - Replace read-only Indexed Versions section with interactive Versions panel in the repo detail page: per-version state badges, Index/Remove buttons, inline Add version form, and Discover tags flow for local repos - Add unit tests for both new/changed backend endpoints (8 new test cases) Co-Authored-By: Claude Sonnet 4.6 --- src/routes/api/v1/libs/[id]/index/+server.ts | 23 +- .../api/v1/libs/[id]/index/server.test.ts | 182 ++++++++++ .../v1/libs/[id]/versions/discover/+server.ts | 63 ++++ .../[id]/versions/discover/server.test.ts | 160 +++++++++ src/routes/repos/[id]/+page.svelte | 334 +++++++++++++++++- 5 files changed, 741 insertions(+), 21 deletions(-) create mode 100644 src/routes/api/v1/libs/[id]/index/server.test.ts create mode 100644 src/routes/api/v1/libs/[id]/versions/discover/+server.ts create mode 100644 src/routes/api/v1/libs/[id]/versions/discover/server.test.ts diff --git a/src/routes/api/v1/libs/[id]/index/+server.ts b/src/routes/api/v1/libs/[id]/index/+server.ts index 0ecebb4..3101795 100644 --- a/src/routes/api/v1/libs/[id]/index/+server.ts +++ b/src/routes/api/v1/libs/[id]/index/+server.ts @@ -1,17 +1,23 @@ /** * POST /api/v1/libs/:id/index — trigger an indexing job for a repository. + * + * Also enqueues jobs for all registered versions so that re-indexing a repo + * automatically covers its secondary versions. */ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import { getClient } from '$lib/server/db/client'; import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js'; import { RepositoryService } from '$lib/server/services/repository.service'; +import { VersionService } from '$lib/server/services/version.service'; import { getQueue } from '$lib/server/pipeline/startup'; import { handleServiceError, NotFoundError } from '$lib/server/utils/validation'; export const POST: RequestHandler = async ({ params, request }) => { try { - const service = new RepositoryService(getClient()); + const db = getClient(); + const service = new RepositoryService(db); + const versionService = new VersionService(db); const id = decodeURIComponent(params.id); const repo = service.get(id); @@ -30,7 +36,20 @@ export const POST: RequestHandler = async ({ params, request }) => { const queue = getQueue(); const job = queue ? queue.enqueue(id, versionId) : service.createIndexingJob(id, versionId); - return json({ job: IndexingJobMapper.toDto(job) }, { status: 202 }); + // Also enqueue jobs for all registered versions (dedup in queue makes this safe). + // Only when this is a default-branch re-index (no explicit versionId requested). + let versionJobs: ReturnType[] = []; + if (!versionId) { + const versions = versionService.list(id); + versionJobs = versions.map((version) => { + const vJob = queue + ? queue.enqueue(id, version.id) + : service.createIndexingJob(id, version.id); + return IndexingJobMapper.toDto(vJob); + }); + } + + return json({ job: IndexingJobMapper.toDto(job), versionJobs }, { status: 202 }); } catch (err) { return handleServiceError(err); } diff --git a/src/routes/api/v1/libs/[id]/index/server.test.ts b/src/routes/api/v1/libs/[id]/index/server.test.ts new file mode 100644 index 0000000..6dfe321 --- /dev/null +++ b/src/routes/api/v1/libs/[id]/index/server.test.ts @@ -0,0 +1,182 @@ +/** + * Unit tests for POST /api/v1/libs/:id/index + * + * Verifies: + * - Default-branch re-index also enqueues jobs for all registered versions + * - versionJobs array is returned in the response + * - Explicit versionId request does NOT trigger extra version jobs + * - Returns 404 when repo does not exist + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { RepositoryService } from '$lib/server/services/repository.service'; +import { VersionService } from '$lib/server/services/version.service'; + +let db: Database.Database; +let mockQueue: { enqueue: ReturnType } | null = null; + +vi.mock('$lib/server/db/client', () => ({ + getClient: () => db +})); + +vi.mock('$lib/server/db/client.js', () => ({ + getClient: () => db +})); + +vi.mock('$lib/server/pipeline/startup', () => ({ + getQueue: () => mockQueue +})); + +vi.mock('$lib/server/pipeline/startup.js', () => ({ + getQueue: () => mockQueue +})); + +vi.mock('$lib/server/embeddings/registry', () => ({ + createProviderFromProfile: () => null +})); + +vi.mock('$lib/server/embeddings/registry.js', () => ({ + createProviderFromProfile: () => null +})); + +import { POST as postIndex } from './+server.js'; + +const NOW_S = Math.floor(Date.now() / 1000); + +function createTestDb(): Database.Database { + const client = new Database(':memory:'); + client.pragma('foreign_keys = ON'); + + const migrationsFolder = join(import.meta.dirname, '../../../../../../lib/server/db/migrations'); + const ftsFile = join(import.meta.dirname, '../../../../../../lib/server/db/fts.sql'); + + const migration0 = readFileSync(join(migrationsFolder, '0000_large_master_chief.sql'), 'utf-8'); + const migration1 = readFileSync(join(migrationsFolder, '0001_quick_nighthawk.sql'), 'utf-8'); + const migration2 = readFileSync(join(migrationsFolder, '0002_silky_stellaris.sql'), 'utf-8'); + + for (const migration of [migration0, migration1, migration2]) { + for (const stmt of migration + .split('--> statement-breakpoint') + .map((s) => s.trim()) + .filter(Boolean)) { + client.exec(stmt); + } + } + + client.exec(readFileSync(ftsFile, 'utf-8')); + return client; +} + +function makeEnqueueJob(repositoryId: string, versionId?: string) { + return { + id: `job-${Math.random().toString(36).slice(2)}`, + repositoryId, + versionId: versionId ?? null, + status: 'queued' as const, + processedFiles: 0, + totalFiles: 0, + error: null, + startedAt: null, + completedAt: null, + createdAt: new Date(NOW_S * 1000) + }; +} + +describe('POST /api/v1/libs/:id/index', () => { + beforeEach(() => { + db = createTestDb(); + mockQueue = null; + }); + + it('returns 404 when repo does not exist', async () => { + const response = await postIndex({ + params: { id: encodeURIComponent('/nonexistent/repo') }, + request: new Request('http://test', { method: 'POST' }) + } as never); + + expect(response.status).toBe(404); + }); + + it('returns job and empty versionJobs when no versions are registered', async () => { + const repoService = new RepositoryService(db); + repoService.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' }); + + const response = await postIndex({ + params: { id: encodeURIComponent('/facebook/react') }, + request: new Request('http://test', { method: 'POST' }) + } as never); + + expect(response.status).toBe(202); + const body = await response.json(); + expect(body.job).toBeDefined(); + expect(body.job.repositoryId).toBe('/facebook/react'); + expect(body.versionJobs).toEqual([]); + }); + + it('enqueues jobs for all registered versions on default-branch re-index', async () => { + const repoService = new RepositoryService(db); + const versionService = new VersionService(db); + + repoService.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' }); + versionService.add('/facebook/react', 'v18.3.0', 'React v18.3.0'); + versionService.add('/facebook/react', 'v17.0.0', 'React v17.0.0'); + + const enqueue = vi.fn().mockImplementation( + (repositoryId: string, versionId?: string) => makeEnqueueJob(repositoryId, versionId) + ); + mockQueue = { enqueue }; + + const response = await postIndex({ + params: { id: encodeURIComponent('/facebook/react') }, + request: new Request('http://test', { method: 'POST' }) + } as never); + + expect(response.status).toBe(202); + const body = await response.json(); + + // Main job enqueued (no versionId) + expect(body.job).toBeDefined(); + expect(body.job.repositoryId).toBe('/facebook/react'); + + // Two version jobs enqueued + expect(body.versionJobs).toHaveLength(2); + expect(enqueue).toHaveBeenCalledTimes(3); // 1 main + 2 versions + + // Version IDs should be the registered version IDs + const enqueuedVersionIds = enqueue.mock.calls.slice(1).map((call) => call[1]); + expect(enqueuedVersionIds).toContain('/facebook/react/v18.3.0'); + expect(enqueuedVersionIds).toContain('/facebook/react/v17.0.0'); + }); + + it('does NOT enqueue version jobs when an explicit versionId is provided', async () => { + const repoService = new RepositoryService(db); + const versionService = new VersionService(db); + + repoService.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' }); + versionService.add('/facebook/react', 'v18.3.0', 'React v18.3.0'); + + const enqueue = vi.fn().mockImplementation( + (repositoryId: string, versionId?: string) => makeEnqueueJob(repositoryId, versionId) + ); + mockQueue = { enqueue }; + + const response = await postIndex({ + params: { id: encodeURIComponent('/facebook/react') }, + request: new Request('http://test', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ version: '/facebook/react/v18.3.0' }) + }) + } as never); + + expect(response.status).toBe(202); + const body = await response.json(); + + // Only one call — the explicit version, no extra version enumeration + expect(enqueue).toHaveBeenCalledTimes(1); + expect(body.versionJobs).toEqual([]); + }); +}); diff --git a/src/routes/api/v1/libs/[id]/versions/discover/+server.ts b/src/routes/api/v1/libs/[id]/versions/discover/+server.ts new file mode 100644 index 0000000..4952e65 --- /dev/null +++ b/src/routes/api/v1/libs/[id]/versions/discover/+server.ts @@ -0,0 +1,63 @@ +/** + * POST /api/v1/libs/:id/versions/discover — discover git tags for a local repository. + * + * Returns { tags: Array<{ tag: string; commitHash: string }> }. + * For GitHub repositories or when tag discovery fails, returns { tags: [] } (not an error). + * Returns 404 if the repository does not exist. + */ + +import { json } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { getClient } from '$lib/server/db/client'; +import { RepositoryService } from '$lib/server/services/repository.service'; +import { VersionService } from '$lib/server/services/version.service'; +import { handleServiceError, NotFoundError } from '$lib/server/utils/validation'; + +function getServices() { + const db = getClient(); + return { + repoService: new RepositoryService(db), + versionService: new VersionService(db) + }; +} + +// --------------------------------------------------------------------------- +// POST /api/v1/libs/:id/versions/discover +// --------------------------------------------------------------------------- + +export const POST: RequestHandler = ({ params }) => { + try { + const { repoService, versionService } = getServices(); + const repositoryId = decodeURIComponent(params.id); + + const repo = repoService.get(repositoryId); + if (!repo) { + throw new NotFoundError(`Repository ${repositoryId} not found`); + } + + try { + const tags = versionService.discoverTags(repositoryId); + return json({ tags }); + } catch { + // GitHub repos or git errors — return empty tags gracefully + return json({ tags: [] }); + } + } catch (err) { + return handleServiceError(err); + } +}; + +// --------------------------------------------------------------------------- +// OPTIONS preflight +// --------------------------------------------------------------------------- + +export const OPTIONS: RequestHandler = () => { + return new Response(null, { + status: 204, + headers: { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST, OPTIONS', + 'Access-Control-Allow-Headers': 'Content-Type, Authorization' + } + }); +}; diff --git a/src/routes/api/v1/libs/[id]/versions/discover/server.test.ts b/src/routes/api/v1/libs/[id]/versions/discover/server.test.ts new file mode 100644 index 0000000..160edcc --- /dev/null +++ b/src/routes/api/v1/libs/[id]/versions/discover/server.test.ts @@ -0,0 +1,160 @@ +/** + * Unit tests for POST /api/v1/libs/:id/versions/discover + * + * Verifies: + * - Local repo returns discovered tags + * - GitHub repo returns empty tags gracefully (no error) + * - Non-existent repo returns 404 + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import Database from 'better-sqlite3'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +let db: Database.Database; + +vi.mock('$lib/server/db/client', () => ({ + getClient: () => db +})); + +vi.mock('$lib/server/db/client.js', () => ({ + getClient: () => db +})); + +vi.mock('$lib/server/pipeline/startup', () => ({ + getQueue: () => null +})); + +vi.mock('$lib/server/pipeline/startup.js', () => ({ + getQueue: () => null +})); + +// Mock git utilities so tests don't require a real git repo +vi.mock('$lib/server/utils/git', () => ({ + discoverVersionTags: vi.fn(), + resolveTagToCommit: vi.fn() +})); + +vi.mock('$lib/server/utils/git.js', () => ({ + discoverVersionTags: vi.fn(), + resolveTagToCommit: vi.fn() +})); + +import { POST as postDiscover } from './+server.js'; + +const NOW_S = Math.floor(Date.now() / 1000); + +function createTestDb(): Database.Database { + const client = new Database(':memory:'); + client.pragma('foreign_keys = ON'); + + const migrationsFolder = join(import.meta.dirname, '../../../../../../../lib/server/db/migrations'); + const ftsFile = join(import.meta.dirname, '../../../../../../../lib/server/db/fts.sql'); + + const migration0 = readFileSync(join(migrationsFolder, '0000_large_master_chief.sql'), 'utf-8'); + const migration1 = readFileSync(join(migrationsFolder, '0001_quick_nighthawk.sql'), 'utf-8'); + const migration2 = readFileSync(join(migrationsFolder, '0002_silky_stellaris.sql'), 'utf-8'); + + for (const migration of [migration0, migration1, migration2]) { + for (const stmt of migration + .split('--> statement-breakpoint') + .map((s) => s.trim()) + .filter(Boolean)) { + client.exec(stmt); + } + } + + client.exec(readFileSync(ftsFile, 'utf-8')); + return client; +} + +function seedRepo( + client: Database.Database, + overrides: { id?: string; source?: 'github' | 'local'; sourceUrl?: string } = {} +): string { + const id = overrides.id ?? '/facebook/react'; + client + .prepare( + `INSERT INTO repositories + (id, title, source, source_url, state, created_at, updated_at) + VALUES (?, ?, ?, ?, 'indexed', ?, ?)` + ) + .run( + id, + 'React', + overrides.source ?? 'github', + overrides.sourceUrl ?? 'https://github.com/facebook/react', + NOW_S, + NOW_S + ); + return id; +} + +describe('POST /api/v1/libs/:id/versions/discover', () => { + beforeEach(async () => { + db = createTestDb(); + const git = await import('$lib/server/utils/git'); + vi.mocked(git.discoverVersionTags).mockReset(); + vi.mocked(git.resolveTagToCommit).mockReset(); + }); + + it('returns 404 when repo does not exist', async () => { + const response = await postDiscover({ + params: { id: encodeURIComponent('/nonexistent/repo') } + } as never); + + expect(response.status).toBe(404); + const body = await response.json(); + expect(body.error).toBeDefined(); + }); + + it('returns discovered tags for a local repository', async () => { + const { discoverVersionTags, resolveTagToCommit } = await import('$lib/server/utils/git'); + vi.mocked(discoverVersionTags).mockReturnValue(['v2.0.0', 'v1.0.0']); + vi.mocked(resolveTagToCommit).mockImplementation(({ tag }) => + tag === 'v2.0.0' ? 'abc12345' : 'def67890' + ); + + seedRepo(db, { source: 'local', sourceUrl: '/home/user/myrepo' }); + + const response = await postDiscover({ + params: { id: encodeURIComponent('/facebook/react') } + } as never); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.tags).toHaveLength(2); + expect(body.tags[0]).toEqual({ tag: 'v2.0.0', commitHash: 'abc12345' }); + expect(body.tags[1]).toEqual({ tag: 'v1.0.0', commitHash: 'def67890' }); + }); + + it('returns empty tags for a GitHub repository (no error)', async () => { + seedRepo(db, { source: 'github', sourceUrl: 'https://github.com/facebook/react' }); + + const response = await postDiscover({ + params: { id: encodeURIComponent('/facebook/react') } + } as never); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.tags).toEqual([]); + }); + + it('returns empty tags when git discovery throws', async () => { + const { discoverVersionTags } = await import('$lib/server/utils/git'); + vi.mocked(discoverVersionTags).mockImplementation(() => { + throw new Error('git command failed'); + }); + + seedRepo(db, { source: 'local', sourceUrl: '/home/user/myrepo' }); + + const response = await postDiscover({ + params: { id: encodeURIComponent('/facebook/react') } + } as never); + + expect(response.status).toBe(200); + const body = await response.json(); + expect(body.tags).toEqual([]); + }); +}); diff --git a/src/routes/repos/[id]/+page.svelte b/src/routes/repos/[id]/+page.svelte index 056c3ca..cdb1b5a 100644 --- a/src/routes/repos/[id]/+page.svelte +++ b/src/routes/repos/[id]/+page.svelte @@ -1,6 +1,7 @@