From 9525c58e9af280e452b560c9719e82fee4d8a923 Mon Sep 17 00:00:00 2001 From: Giancarmine Salucci Date: Wed, 1 Apr 2026 14:09:19 +0200 Subject: [PATCH 1/2] feat(TRUEREF-0023): add sqlite-vec search pipeline --- docs/ARCHITECTURE.md | 189 ++-- docs/FINDINGS.md | 171 +++- package-lock.json | 754 ++++++++++++-- package.json | 1 + scripts/build-workers.mjs | 3 +- src/lib/components/IndexingProgress.svelte | 52 +- src/lib/components/admin/JobSkeleton.svelte | 19 + .../components/admin/JobStatusBadge.svelte | 8 +- src/lib/components/admin/Toast.svelte | 77 ++ .../components/admin/WorkerStatusPanel.svelte | 81 ++ src/lib/server/db/client.ts | 7 + src/lib/server/db/index.ts | 7 + .../migrations/0006_yielding_centennial.sql | 6 + .../db/migrations/meta/0006_snapshot.json | 948 ++++++++++++++++++ .../server/db/migrations/meta/_journal.json | 7 + src/lib/server/db/schema.test.ts | 29 +- src/lib/server/db/schema.ts | 17 +- src/lib/server/db/sqlite-vec.ts | 49 + src/lib/server/db/vectors.sql | 2 + .../embeddings/embedding.service.test.ts | 43 + .../server/embeddings/embedding.service.ts | 15 +- src/lib/server/pipeline/embed-worker-entry.ts | 5 + .../server/pipeline/indexing.pipeline.test.ts | 108 +- src/lib/server/pipeline/indexing.pipeline.ts | 15 +- src/lib/server/pipeline/job-queue.ts | 78 +- .../pipeline/progress-broadcaster.test.ts | 21 + .../server/pipeline/progress-broadcaster.ts | 19 + src/lib/server/pipeline/startup.ts | 31 +- src/lib/server/pipeline/worker-entry.ts | 5 + src/lib/server/pipeline/worker-pool.ts | 148 ++- src/lib/server/pipeline/worker-types.ts | 54 + src/lib/server/pipeline/write-worker-entry.ts | 93 ++ .../search/hybrid.search.service.test.ts | 46 +- .../server/search/hybrid.search.service.ts | 36 +- src/lib/server/search/sqlite-vec.store.ts | 394 ++++++++ src/lib/server/search/vector.search.ts | 60 +- .../services/repository.service.test.ts | 42 +- src/lib/server/services/repository.service.ts | 7 +- .../server/services/version.service.test.ts | 78 +- src/lib/server/services/version.service.ts | 11 +- src/routes/admin/jobs/+page.svelte | 666 +++++++----- src/routes/api/v1/jobs/+server.ts | 32 +- src/routes/api/v1/jobs/[id]/stream/+server.ts | 26 +- .../v1/sse-and-settings.integration.test.ts | 147 ++- src/routes/api/v1/workers/+server.ts | 16 + 45 files changed, 4009 insertions(+), 614 deletions(-) create mode 100644 src/lib/components/admin/JobSkeleton.svelte create mode 100644 src/lib/components/admin/Toast.svelte create mode 100644 src/lib/components/admin/WorkerStatusPanel.svelte create mode 100644 src/lib/server/db/migrations/0006_yielding_centennial.sql create mode 100644 src/lib/server/db/migrations/meta/0006_snapshot.json create mode 100644 src/lib/server/db/sqlite-vec.ts create mode 100644 src/lib/server/db/vectors.sql create mode 100644 src/lib/server/pipeline/write-worker-entry.ts create mode 100644 src/lib/server/search/sqlite-vec.store.ts create mode 100644 src/routes/api/v1/workers/+server.ts diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 323f3ee..8379e7b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,171 +1,168 @@ # Architecture -Last Updated: 2026-03-30T00:00:00.000Z +Last Updated: 2026-04-01T12:05:23.000Z ## Overview -TrueRef is a TypeScript-first, self-hosted documentation retrieval platform built on SvelteKit. The repository contains a Node-targeted web application, a REST API, a Model Context Protocol server, and a multi-threaded server-side indexing pipeline backed by SQLite via better-sqlite3 and Drizzle ORM. +TrueRef is a TypeScript-first, self-hosted documentation retrieval platform built on SvelteKit. The repository contains a Node-targeted web application, a REST API, a Model Context Protocol server, and a worker-threaded indexing pipeline backed by SQLite via better-sqlite3, Drizzle ORM, FTS5, and sqlite-vec. -- Primary language: TypeScript (141 files) with a small amount of JavaScript configuration (2 files) -- Application type: Full-stack SvelteKit application with worker-threaded indexing and retrieval services +- Primary language: TypeScript (147 `.ts` files) with a small amount of JavaScript configuration and build code (2 `.js` files), excluding generated output and dependencies +- Application type: Full-stack SvelteKit application with server-side indexing, retrieval, and MCP integration - Runtime framework: SvelteKit with adapter-node -- Storage: SQLite (WAL mode) with Drizzle-managed schema plus hand-written FTS5 setup -- Concurrency: Node.js worker_threads for parse and embedding work -- Testing: Vitest with separate client and server projects +- Storage: SQLite in WAL mode with Drizzle-managed relational schema, FTS5 full-text indexes, and sqlite-vec virtual tables for vector lookup +- Concurrency: Node.js `worker_threads` for parse, embed, and auxiliary write-worker infrastructure +- Testing: Vitest for unit and integration coverage ## Project Structure -- src/routes: SvelteKit pages and HTTP endpoints, including the public UI and /api/v1 surface -- src/lib/server: Backend implementation grouped by concern: api, config, crawler, db, embeddings, mappers, models, parser, pipeline, search, services, utils -- src/mcp: Standalone MCP server entry point and tool handlers -- static: Static assets such as robots.txt -- docs/features: Feature-level implementation notes and product documentation -- build: Generated SvelteKit output +- `src/routes`: SvelteKit pages and HTTP endpoints, including the public UI and `/api/v1` surface +- `src/lib/server`: Backend implementation grouped by concern: `api`, `config`, `crawler`, `db`, `embeddings`, `mappers`, `models`, `parser`, `pipeline`, `search`, `services`, `utils` +- `src/mcp`: Standalone MCP server entry point, client, tests, and tool handlers +- `scripts`: Build helpers, including worker bundling +- `static`: Static assets such as `robots.txt` +- `docs/features`: Feature-level implementation notes and product documentation +- `build`: Generated SvelteKit output and bundled worker entrypoints ## Key Directories -### src/routes +### `src/routes` -Contains the UI entry points and API routes. The API tree under src/routes/api/v1 is the public HTTP contract for repository management, indexing jobs, search/context retrieval, settings, filesystem browsing, JSON schema discovery, real-time SSE progress streaming, and job control (pause/resume/cancel). +Contains the UI entry points and API routes. The API tree under `src/routes/api/v1` is the public HTTP contract for repository management, version discovery, indexing jobs, search/context retrieval, embedding settings, indexing settings, filesystem browsing, worker-status inspection, and SSE progress streaming. -### src/lib/server/db +### `src/lib/server/db` -Owns SQLite schema definitions, migration bootstrapping, and FTS initialization. Database startup runs through initializeDatabase(), which executes Drizzle migrations and then applies FTS5 SQL that cannot be expressed directly in the ORM. +Owns SQLite schema definitions, relational migrations, connection bootstrapping, and sqlite-vec loading. Database startup goes through `initializeDatabase()` and `getClient()`, both of which configure WAL-mode pragmas and ensure sqlite-vec is loaded on each connection before vector-backed queries run. -### src/lib/server/pipeline +### `src/lib/server/search` -Coordinates crawl, parse, chunk, store, and optional embedding generation work using a worker thread pool. The pipeline module consists of: +Implements keyword, vector, and hybrid retrieval. Keyword search uses SQLite FTS5 and BM25-style ranking. Vector search uses `SqliteVecStore` to maintain per-profile sqlite-vec `vec0` tables plus rowid mapping tables, and hybrid search blends FTS and vector candidates through reciprocal rank fusion. -- **WorkerPool** (`worker-pool.ts`): Manages a configurable number of Node.js `worker_threads` for parse jobs and an optional dedicated embed worker. Dispatches jobs round-robin to idle workers, enforces per-repository serialisation (one active job per repo), auto-respawns crashed workers, and supports runtime concurrency adjustment via `setMaxConcurrency()`. Falls back to main-thread execution when worker scripts are not found. -- **Parse worker** (`worker-entry.ts`): Runs in a worker thread. Opens its own `better-sqlite3` connection (WAL mode, `busy_timeout = 5000`), constructs a local `IndexingPipeline` instance, and processes jobs by posting `progress`, `done`, or `failed` messages back to the parent. -- **Embed worker** (`embed-worker-entry.ts`): Dedicated worker for embedding generation. Loads the embedding profile from the database, creates an `EmbeddingService`, and processes embed requests after the parse worker finishes a job. -- **ProgressBroadcaster** (`progress-broadcaster.ts`): Server-side pub/sub for real-time SSE streaming. Supports per-job, per-repository, and global subscriptions. Caches the last event per job for reconnect support. -- **Worker types** (`worker-types.ts`): Shared TypeScript discriminated union types for `ParseWorkerRequest`/`ParseWorkerResponse` and `EmbedWorkerRequest`/`EmbedWorkerResponse` message protocols. -- **Startup** (`startup.ts`): Recovers stale jobs, constructs singleton `JobQueue`, `IndexingPipeline`, `WorkerPool`, and `ProgressBroadcaster` instances, reads concurrency settings from the database, and drains queued work after restart. -- **JobQueue** (`job-queue.ts`): SQLite-backed queue that delegates to the `WorkerPool` when available, with pause/resume/cancel support. +### `src/lib/server/pipeline` -### src/lib/server/search +Coordinates crawl, diff, parse, store, embed, and job-state broadcasting. The pipeline module consists of: -Implements keyword, vector, and hybrid retrieval. The keyword path uses SQLite FTS5 and BM25; the hybrid path blends FTS and vector search with reciprocal rank fusion. +- `IndexingPipeline`: orchestrates crawl, diff, parse, transactional replacement, optional embedding generation, and repository statistics updates +- `WorkerPool`: manages parse workers, an optional embed worker, an optional write worker, per-repository-and-version serialization, worker respawn, and runtime concurrency changes +- `worker-entry.ts`: parse worker that opens its own `better-sqlite3` connection, runs the indexing pipeline, and reports progress back to the parent +- `embed-worker-entry.ts`: embedding worker that loads the active profile, creates an `EmbeddingService`, and generates vectors after parse completion +- `write-worker-entry.ts`: batch-write worker with a `write`/`write_ack`/`write_error` message protocol for document and snippet persistence +- `progress-broadcaster.ts`: server-side pub/sub for per-job, per-repository, global, and worker-status SSE streams +- `startup.ts`: recovers stale jobs, constructs singleton queue/pipeline/pool/broadcaster instances, loads concurrency settings, and drains queued work after restart +- `worker-types.ts`: shared TypeScript discriminated unions for parse, embed, and write worker protocols -### src/lib/server/crawler and src/lib/server/parser +### `src/lib/server/crawler` and `src/lib/server/parser` -Convert GitHub repositories and local folders into normalized snippet records. Crawlers fetch repository contents, parsers split Markdown, code, config, HTML-like, and plain-text files into chunks, and downstream services persist searchable content. +Convert GitHub repositories and local folders into normalized snippet records. Crawlers fetch repository contents and configuration, parsers split Markdown, code, config, HTML-like, and plain-text files into searchable snippet records, and downstream services persist searchable content and embeddings. -### src/mcp +### `src/mcp` -Provides a thin compatibility layer over the HTTP API. The MCP server exposes resolve-library-id and query-docs over stdio or HTTP and forwards work to local tool handlers. +Provides a thin compatibility layer over the HTTP API. The MCP server exposes `resolve-library-id` and `query-docs` over stdio or HTTP and forwards work to local handlers that reuse the application retrieval stack. ## Design Patterns -- The WorkerPool implements an **observer/callback pattern**: the pool owner provides `onProgress`, `onJobDone`, `onJobFailed`, `onEmbedDone`, and `onEmbedFailed` callbacks at construction time, and the pool invokes them when workers post messages. -- ProgressBroadcaster implements a **pub/sub pattern** with three subscription tiers (per-job, per-repository, global) and last-event caching for SSE reconnect. -- The implementation consistently uses **service classes** such as RepositoryService, SearchService, and HybridSearchService for business logic. -- Mapping and entity layers separate raw database rows from domain objects through **mapper/entity pairs** such as RepositoryMapper and RepositoryEntity. -- Pipeline startup uses **module-level singletons** for JobQueue, IndexingPipeline, WorkerPool, and ProgressBroadcaster lifecycle management, with accessor functions (getQueue, getPool, getBroadcaster) for route handlers. -- Worker message protocols use **TypeScript discriminated unions** (`type` field) for type-safe worker ↔ parent communication. +- **Service layer**: business logic lives in classes such as `RepositoryService`, `VersionService`, `SearchService`, `HybridSearchService`, and `EmbeddingService` +- **Factory pattern**: embedding providers are created from persisted profile records through registry/factory helpers +- **Mapper/entity separation**: mappers translate between raw database rows and domain entities such as `RepositoryEntity`, `RepositoryVersionEntity`, and `EmbeddingProfileEntity` +- **Module-level singletons**: pipeline startup owns lifecycle for `JobQueue`, `IndexingPipeline`, `WorkerPool`, and `ProgressBroadcaster`, with accessor functions for route handlers +- **Pub/sub**: `ProgressBroadcaster` maintains job, repository, global, and worker-status subscriptions for SSE delivery +- **Discriminated unions**: worker message protocols use a `type` field for type-safe parent/worker communication ## Key Components ### SvelteKit server bootstrap -src/hooks.server.ts initializes the database, loads persisted embedding configuration, creates the optional EmbeddingService, reads indexing concurrency settings from the database, starts the indexing pipeline with WorkerPool and ProgressBroadcaster via `initializePipeline(db, embeddingService, { concurrency, dbPath })`, and applies CORS headers to all /api routes. +`src/hooks.server.ts` initializes the relational database, opens the shared raw SQLite client, loads the default embedding profile, creates the optional `EmbeddingService`, reads indexing concurrency from the `settings` table, and initializes the queue/pipeline/worker infrastructure. ### Database layer -src/lib/server/db/schema.ts defines repositories, repository_versions, documents, snippets, embedding_profiles, snippet_embeddings, indexing_jobs, repository_configs, and settings. This schema models the indexed library catalog, retrieval corpus, embedding state, and job tracking. +`src/lib/server/db/schema.ts` defines repositories, repository versions, documents, snippets, embedding profiles, relational embedding metadata, indexing jobs, repository configs, and generic settings. Relational embedding rows keep canonical model metadata and raw float buffers, while sqlite-vec virtual tables are managed separately per profile through `SqliteVecStore`. + +### sqlite-vec integration + +`src/lib/server/db/sqlite-vec.ts` centralizes sqlite-vec loading and deterministic per-profile table naming. `SqliteVecStore` creates `vec0` tables plus rowid mapping tables, backfills missing rows from `snippet_embeddings`, removes stale vector references, and executes nearest-neighbor queries constrained by repository, optional version, and profile. ### Retrieval API -src/routes/api/v1/context/+server.ts validates input, resolves repository and optional version IDs, chooses keyword, semantic, or hybrid retrieval, applies token budgeting that skips oversized snippets instead of stopping early, prepends repository rules, and formats JSON or text responses with repository and version metadata. +`src/routes/api/v1/context/+server.ts` validates input, resolves repository and optional version scope, chooses keyword, semantic, or hybrid retrieval, applies token budgeting, and formats JSON or text responses. `/api/v1/libs/search` handles repository-level lookup, while MCP tool handlers expose the same retrieval behavior over stdio or HTTP transports. ### Search engine -src/lib/server/search/search.service.ts preprocesses raw user input into FTS5-safe MATCH expressions before keyword search and repository lookup. src/lib/server/search/hybrid.search.service.ts supports explicit keyword, semantic, and hybrid modes, falls back to vector retrieval when FTS yields no candidates and an embedding provider is configured, and uses reciprocal rank fusion for blended ranking. +`SearchService` preprocesses raw user input into FTS5-safe expressions before keyword search. `HybridSearchService` supports explicit keyword, semantic, and hybrid modes, falls back to vector retrieval when keyword search yields no candidates and an embedding provider is configured, and uses reciprocal rank fusion to merge ranked lists. `VectorSearch` delegates KNN execution to `SqliteVecStore` instead of doing brute-force in-memory cosine scoring. -### Repository management +### Repository and version management -src/lib/server/services/repository.service.ts provides CRUD and statistics for indexed repositories, including canonical ID generation for GitHub and local sources. +`RepositoryService` and `VersionService` provide CRUD, indexing-status, cleanup, and statistics logic for indexed repositories and tagged versions, including sqlite-vec cleanup when repository-scoped or version-scoped content is removed. -### MCP surface +### Worker-threaded indexing -src/mcp/index.ts creates the MCP server, registers the two supported tools, and exposes them over stdio or streamable HTTP. +The active indexing path is parse-worker-first: queued jobs are dispatched to parse workers, progress is written to SQLite and broadcast over SSE, and successful parse completion can enqueue embedding work on the dedicated embed worker. The worker pool also exposes status snapshots through `/api/v1/workers`. Write-worker infrastructure exists in the current architecture and is bundled at build time, but parse/embed flow remains the primary live path described by `IndexingPipeline` and `WorkerPool`. -### Worker thread pool +### SSE streaming and job control -src/lib/server/pipeline/worker-pool.ts manages a pool of Node.js worker threads. Parse workers run the full crawl → parse → store pipeline inside isolated threads with their own better-sqlite3 connections (WAL mode enables concurrent readers). An optional embed worker handles embedding generation in a separate thread. The pool enforces per-repository serialisation, auto-respawns crashed workers, and supports runtime concurrency changes persisted through the settings table. - -### SSE streaming - -src/lib/server/pipeline/progress-broadcaster.ts provides real-time Server-Sent Event streaming of indexing progress. Route handlers in src/routes/api/v1/jobs/stream and src/routes/api/v1/jobs/[id]/stream expose SSE endpoints. The broadcaster supports per-job, per-repository, and global subscriptions, with last-event caching for reconnect via the `Last-Event-ID` header. - -### Job control - -src/routes/api/v1/jobs/[id]/pause, resume, and cancel endpoints allow runtime control of indexing jobs. The JobQueue supports pause/resume/cancel state transitions persisted to SQLite. +`progress-broadcaster.ts` provides real-time Server-Sent Event streaming of indexing progress. Route handlers under `/api/v1/jobs/stream` and `/api/v1/jobs/[id]/stream` expose SSE endpoints, and `/api/v1/workers` exposes worker-pool status. Job control endpoints support pause, resume, and cancel transitions backed by SQLite job state. ### Indexing settings -src/routes/api/v1/settings/indexing exposes GET and PUT for indexing concurrency. PUT validates and clamps the value to `max(cpus - 1, 1)`, persists it to the settings table, and live-updates the WorkerPool via `setMaxConcurrency()`. +`/api/v1/settings/indexing` exposes GET and PUT for indexing concurrency. The value is persisted in the `settings` table and applied live to the `WorkerPool` through `setMaxConcurrency()`. ## Dependencies ### Production -- @modelcontextprotocol/sdk: MCP server transport and protocol types -- @xenova/transformers: local embedding support -- better-sqlite3: synchronous SQLite driver -- zod: runtime input validation for MCP tools and server helpers +- `@modelcontextprotocol/sdk`: MCP server transport and protocol types +- `@xenova/transformers`: local embedding support +- `better-sqlite3`: synchronous SQLite driver used by the main app and workers +- `sqlite-vec`: SQLite vector extension used for `vec0` storage and nearest-neighbor queries +- `zod`: runtime validation for MCP tools and server helpers ### Development -- @sveltejs/kit and @sveltejs/adapter-node: application framework and Node deployment target -- drizzle-kit and drizzle-orm: schema management and typed database access -- esbuild: worker thread entry point bundling (build/workers/) -- vite and @tailwindcss/vite: bundling and Tailwind integration -- vitest and @vitest/browser-playwright: server and browser test execution -- eslint, typescript-eslint, eslint-plugin-svelte, prettier, prettier-plugin-svelte, prettier-plugin-tailwindcss: linting and formatting -- typescript and @types/node: type-checking and Node typings +- `@sveltejs/kit` and `@sveltejs/adapter-node`: application framework and Node deployment target +- `drizzle-kit` and `drizzle-orm`: schema management and typed database access +- `esbuild`: worker entrypoint bundling into `build/workers` +- `vite` and `@tailwindcss/vite`: application bundling and Tailwind integration +- `vitest` and `@vitest/browser-playwright`: server and browser test execution +- `eslint`, `typescript-eslint`, `eslint-plugin-svelte`, `prettier`, `prettier-plugin-svelte`, `prettier-plugin-tailwindcss`: linting and formatting +- `typescript` and `@types/node`: type-checking and Node typings ## Module Organization -The backend is organized by responsibility rather than by route. HTTP handlers in src/routes/api/v1 are intentionally thin and delegate to library modules in src/lib/server. Within src/lib/server, concerns are separated into: +The backend is organized by responsibility rather than by route. HTTP handlers under `src/routes/api/v1` are intentionally thin and delegate to modules in `src/lib/server`. Within `src/lib/server`, concerns are separated into: -- models and mappers for entity translation -- services for repository/version operations -- search for retrieval strategies -- crawler and parser for indexing input transformation -- pipeline for orchestration and job execution -- embeddings for provider abstraction and embedding generation -- api and utils for response formatting, validation, and shared helpers +- `models` and `mappers` for entity translation +- `services` for repository/version operations +- `search` for keyword, vector, and hybrid retrieval strategies +- `crawler` and `parser` for indexing input transformation +- `pipeline` for orchestration, workers, and job execution +- `embeddings` for provider abstraction and vector generation +- `db`, `api`, `config`, and `utils` for persistence, response formatting, validation, and shared helpers -The frontend and backend share the same SvelteKit repository, but most non-UI behavior is implemented on the server side. +The frontend and backend live in the same SvelteKit repository, but most non-UI behavior is implemented on the server side. ## Data Flow ### Indexing flow -1. Server startup runs initializeDatabase() and initializePipeline() from src/hooks.server.ts, which creates the WorkerPool, ProgressBroadcaster, and JobQueue singletons. -2. The pipeline recovers stale jobs (marks running → failed, indexing → error), reads concurrency settings, and resumes queued work. -3. When a job is enqueued, the JobQueue delegates to the WorkerPool, which dispatches work to an idle parse worker thread. -4. Each parse worker opens its own better-sqlite3 connection (WAL mode) and runs the full crawl → parse → store pipeline, posting progress messages back to the parent thread. -5. The parent thread updates job progress in the database and broadcasts SSE events through the ProgressBroadcaster. -6. On parse completion, if an embedding provider is configured, the WorkerPool enqueues an embed request to the dedicated embed worker, which generates vectors in its own thread. -7. Job control endpoints allow pausing, resuming, or cancelling jobs at runtime. +1. Server startup runs database initialization, opens the shared client, loads sqlite-vec, and initializes the pipeline singletons. +2. Startup recovery marks interrupted jobs as failed, resets repositories stuck in `indexing`, reads persisted concurrency settings, and drains queued jobs. +3. `JobQueue` dispatches eligible work to the `WorkerPool`, which serializes by `(repositoryId, versionId)` and posts jobs to idle parse workers. +4. Each parse worker opens its own SQLite connection, crawls the source, computes differential work, parses files into snippets, and persists replacement data through the indexing pipeline. +5. The parent thread updates job progress in SQLite and broadcasts SSE progress and worker-status events. +6. If an embedding provider is configured, the completed parse job triggers embed work that stores canonical embedding blobs and synchronizes sqlite-vec profile tables for nearest-neighbor lookup. +7. Repository/version statistics and job status are finalized in SQLite, and control endpoints can pause, resume, or cancel subsequent queued work. ### Retrieval flow -1. Clients call /api/v1/libs/search, /api/v1/context, or the MCP tools. -2. Route handlers validate input and load the SQLite client. -3. Keyword search uses FTS5 via SearchService; hybrid search optionally adds vector results via HybridSearchService. -4. Query preprocessing normalizes punctuation-heavy or code-like input before FTS search, while semantic mode bypasses FTS and auto or hybrid mode can fall back to vector retrieval when keyword search produces no candidates. -5. Token budgeting walks ranked snippets in order and skips individual over-budget snippets so later matches can still be returned. -6. Formatters emit repository and version metadata in JSON responses and origin-aware or explicit no-result text output for plain-text responses. -7. MCP handlers expose the same retrieval behavior over stdio or HTTP transports. +1. Clients call `/api/v1/libs/search`, `/api/v1/context`, or the MCP tools. +2. Route handlers validate input and use the shared SQLite client. +3. Keyword search uses FTS5 through `SearchService`; semantic search uses sqlite-vec KNN through `VectorSearch`; hybrid search merges both paths with reciprocal rank fusion. +4. Retrieval is scoped by repository and optional version, and semantic/hybrid paths can fall back when keyword search yields no usable candidates. +5. Token budgeting selects ranked snippets for the response formatter, which emits repository-aware JSON or text payloads. ## Build System -- Build command: npm run build (runs `vite build` then `node scripts/build-workers.mjs`) -- Worker bundling: scripts/build-workers.mjs uses esbuild to compile worker-entry.ts and embed-worker-entry.ts into build/workers/ as ESM bundles (.mjs), with $lib path aliases resolved and better-sqlite3/@xenova/transformers marked external -- Test command: npm run test -- Primary local run command from package.json: npm run dev -- MCP entry points: npm run mcp:start and npm run mcp:http +- Build command: `npm run build` (runs `vite build` then `node scripts/build-workers.mjs`) +- Worker bundling: `scripts/build-workers.mjs` uses esbuild to compile `worker-entry.ts`, `embed-worker-entry.ts`, and `write-worker-entry.ts` into `build/workers/` as ESM bundles +- Test command: `npm test` +- Primary local run command: `npm run dev` +- MCP entry points: `npm run mcp:start` and `npm run mcp:http` diff --git a/docs/FINDINGS.md b/docs/FINDINGS.md index 788a99d..6fa234e 100644 --- a/docs/FINDINGS.md +++ b/docs/FINDINGS.md @@ -1,29 +1,28 @@ # Findings -Last Updated: 2026-03-30T00:00:00.000Z +Last Updated: 2026-04-01T12:05:23.000Z ## Initializer Summary -- JIRA: TRUEREF-0022 +- JIRA: TRUEREF-0023 - Refresh mode: REFRESH_IF_REQUIRED -- Result: Refreshed ARCHITECTURE.md and FINDINGS.md. CODE_STYLE.md remained trusted — new worker thread code follows established conventions. +- Result: Refreshed ARCHITECTURE.md and FINDINGS.md. CODE_STYLE.md remained trusted — sqlite-vec, worker-status, and write-worker additions follow the established conventions already documented. ## Research Performed -- Discovered 141 TypeScript/JavaScript source files (up from 110), with new pipeline worker, broadcaster, and SSE endpoint files. -- Read worker-pool.ts, worker-entry.ts, embed-worker-entry.ts, worker-types.ts, progress-broadcaster.ts, startup.ts, job-queue.ts to understand the new worker thread architecture. -- Read SSE endpoints (jobs/stream, jobs/[id]/stream) and job control endpoints (pause, resume, cancel). -- Read indexing settings endpoint and hooks.server.ts to verify startup wiring changes. -- Read build-workers.mjs and package.json to verify build system and dependency changes. -- Compared trusted cache state with current codebase to identify ARCHITECTURE.md as stale. -- Confirmed CODE_STYLE.md conventions still match the codebase — new code uses PascalCase classes, camelCase functions, tab indentation, ESM imports, and TypeScript discriminated unions consistent with existing style. +- Counted 149 TypeScript/JavaScript source files in the repository-wide scan and verified the live, non-generated source mix as 147 `.ts` files and 2 `.js` files. +- Read `package.json`, `.prettierrc`, and `eslint.config.js` to verify dependencies, formatting rules, and linting conventions. +- Read `sqlite-vec.ts`, `sqlite-vec.store.ts`, `vector.search.ts`, `hybrid.search.service.ts`, `schema.ts`, `client.ts`, and startup wiring to verify the accepted sqlite-vec implementation and current retrieval architecture. +- Read `worker-pool.ts`, `worker-types.ts`, `write-worker-entry.ts`, and `/api/v1/workers/+server.ts` to verify the current worker topology and status surface. +- Compared `docs/docs_cache_state.yaml` against the live docs and codebase to identify stale cache evidence and architecture drift. +- Confirmed `CODE_STYLE.md` still matches the codebase: tabs, single quotes, `trailingComma: none`, ESM imports with `node:` built-ins, flat ESLint config, and descriptive PascalCase/camelCase naming remain consistent. ## Open Questions For Planner -- Verify whether the retrieval response contract should document the new repository and version metadata fields formally in a public API reference beyond the architecture summary. -- Verify whether parser chunking should evolve further from file-level and declaration-level boundaries to member-level semantic chunks for class-heavy codebases. -- Verify whether the SSE streaming contract (event names, data shapes) should be documented in a dedicated API reference for external consumers. -- Assess whether the WorkerPool fallback mode (main-thread execution when worker scripts are missing) needs explicit test coverage or should be removed in favour of a hard build requirement. +- Verify whether the write-worker protocol should become part of the active indexing flow or remain documented as optional infrastructure only. +- Verify whether worker-status and SSE event payloads should be documented in a dedicated API reference for external consumers. +- Verify whether sqlite-vec operational details such as per-profile vec-table lifecycle and backfill behavior should move into a separate persistence document if the subsystem grows further. +- Assess whether the WorkerPool fallback mode (main-thread execution when worker scripts are missing) still belongs in the runtime contract or should be removed in favour of a hard build requirement. ## Planner Notes Template @@ -37,6 +36,41 @@ Add subsequent research below this section. - Findings: - Risks / follow-ups: +### 2026-04-01 — TRUEREF-0023 initializer refresh audit + +- Task: Refresh only stale or invalid documentation after the accepted sqlite-vec implementation. +- Files inspected: + - `docs/docs_cache_state.yaml` + - `docs/ARCHITECTURE.md` + - `docs/CODE_STYLE.md` + - `docs/FINDINGS.md` + - `package.json` + - `.prettierrc` + - `eslint.config.js` + - `src/hooks.server.ts` + - `src/lib/server/db/client.ts` + - `src/lib/server/db/schema.ts` + - `src/lib/server/db/sqlite-vec.ts` + - `src/lib/server/search/sqlite-vec.store.ts` + - `src/lib/server/search/vector.search.ts` + - `src/lib/server/search/hybrid.search.service.ts` + - `src/lib/server/pipeline/startup.ts` + - `src/lib/server/pipeline/worker-pool.ts` + - `src/lib/server/pipeline/worker-types.ts` + - `src/lib/server/pipeline/write-worker-entry.ts` + - `src/routes/api/v1/workers/+server.ts` + - `scripts/build-workers.mjs` +- Findings: + - The trusted cache metadata was no longer reliable as evidence for planning: `docs/docs_cache_state.yaml` still referenced 2026-03-27 hashes while `ARCHITECTURE.md` and `FINDINGS.md` had been edited later. + - `ARCHITECTURE.md` was stale. It still described only parse and embed worker concurrency, omitted the `sqlite-vec` production dependency, and did not document the current per-profile vec-table storage layer, worker-status endpoint, or write-worker infrastructure. + - The current retrieval stack uses sqlite-vec concretely: `loadSqliteVec()` bootstraps connections, `SqliteVecStore` manages vec0 tables plus rowid mapping tables, and `VectorSearch` delegates nearest-neighbor lookup to that store instead of brute-force scoring. + - The worker architecture now includes parse, embed, and write worker protocols in `worker-types.ts`, build-time bundling for all three entries, and a `/api/v1/workers` route that returns `WorkerPool` status snapshots. + - `CODE_STYLE.md` remained valid and did not require refresh. The observed source and config files still use tabs, single quotes, `trailingComma: none`, flat ESLint config, ESM imports, PascalCase class names, and camelCase helpers exactly as already documented. + - `FINDINGS.md` itself was stale because the initializer summary still referred to `TRUEREF-0022` instead of the requested `TRUEREF-0023` refresh. +- Risks / follow-ups: + - The write-worker protocol exists and is bundled, but the active indexing path is still centered on parse plus optional embed flow. Future documentation should keep distinguishing implemented infrastructure from the currently exercised path. + - Cache validity should continue to be driven by deterministic hash evidence rather than document timestamps or trust text alone. + ### 2026-03-27 — FEEDBACK-0001 initializer refresh audit - Task: Refresh only stale documentation after changes to retrieval, formatters, token budgeting, and parser behavior. @@ -192,3 +226,112 @@ Add subsequent research below this section. - Risks / follow-ups: - The fix should preserve the existing `/repos/[id]` route shape instead of redesigning it to a rest route unless a broader navigation contract change is explicitly requested. - Any normalization helper introduced for the repo detail page should be reused consistently across server load and client event handlers to avoid mixed encoded and decoded repository IDs during navigation and fetches. + +### 2026-04-01 — TRUEREF-0023 sqlite-vec replanning research + +- Task: Replan the rejected libSQL-native vector iteration around sqlite-vec using the current worktree and verified runtime constraints. +- Files inspected: + - `package.json` + - `docs/docs_cache_state.yaml` + - `prompts/TRUEREF-0023/prompt.yaml` + - `prompts/TRUEREF-0023/progress.yaml` + - `prompts/TRUEREF-0023/iteration_0/review_report.yaml` + - `prompts/TRUEREF-0023/iteration_0/plan.md` + - `prompts/TRUEREF-0023/iteration_0/tasks.yaml` + - `src/lib/server/db/client.ts` + - `src/lib/server/db/index.ts` + - `src/lib/server/db/schema.ts` + - `src/lib/server/db/fts.sql` + - `src/lib/server/db/vectors.sql` + - `src/lib/server/db/schema.test.ts` + - `src/lib/server/search/vector.search.ts` + - `src/lib/server/search/hybrid.search.service.test.ts` + - `src/lib/server/embeddings/embedding.service.ts` + - `src/lib/server/embeddings/embedding.service.test.ts` + - `src/lib/server/pipeline/job-queue.ts` + - `src/lib/server/pipeline/progress-broadcaster.ts` + - `src/lib/server/pipeline/progress-broadcaster.test.ts` + - `src/lib/server/pipeline/worker-pool.ts` + - `src/lib/server/pipeline/worker-entry.ts` + - `src/lib/server/pipeline/embed-worker-entry.ts` + - `src/lib/server/pipeline/worker-types.ts` + - `src/lib/server/pipeline/startup.ts` + - `src/lib/server/pipeline/indexing.pipeline.ts` + - `src/lib/server/pipeline/indexing.pipeline.test.ts` + - `src/routes/api/v1/jobs/+server.ts` + - `src/routes/api/v1/jobs/stream/+server.ts` + - `src/routes/api/v1/jobs/[id]/stream/+server.ts` + - `src/routes/api/v1/sse-and-settings.integration.test.ts` + - `src/routes/admin/jobs/+page.svelte` + - `src/lib/components/IndexingProgress.svelte` + - `src/lib/components/admin/JobStatusBadge.svelte` + - `src/lib/components/admin/JobSkeleton.svelte` + - `src/lib/components/admin/Toast.svelte` + - `src/lib/components/admin/WorkerStatusPanel.svelte` + - `scripts/build-workers.mjs` + - `node_modules/libsql/types/index.d.ts` + - `node_modules/libsql/index.js` +- Findings: + - Iteration 0 already changed the workspace materially: direct DB imports were switched from `better-sqlite3` to `libsql`, the extra WAL-related pragmas were added in the main DB clients and embed worker, composite indexes plus `vec_embedding` were added to the Drizzle schema and migration metadata, `IndexingProgress.svelte` now uses SSE, the admin jobs page was overhauled, and `WorkerPool` now serializes on `(repositoryId, versionId)` instead of repository only. + - The rejected vector implementation is still invalid in the current tree. `src/lib/server/db/vectors.sql` contains the rejected libSQL-native assumptions, including a dangling `USING libsql_vector_idx(...)` clause with no valid `CREATE INDEX` statement, and `src/lib/server/search/vector.search.ts` still performs full-table JS cosine scoring over `snippet_embeddings` instead of true in-database KNN. + - `sqlite-vec` is not currently present in `package.json` or the lockfile, and there is no existing `sqliteVec.load(...)`, `db.loadExtension(...)`, `vec0`, or extension bootstrap code anywhere under `src/`. + - Context7 sqlite-vec docs confirm the supported Node integration path is `import * as sqliteVec from 'sqlite-vec'; sqliteVec.load(db);`, storing vectors in a `vec0` virtual table and querying with `WHERE embedding MATCH ? ORDER BY distance LIMIT ?`. The docs also show vec0 metadata columns can be filtered directly, which fits the repositoryId, versionId, and profileId requirements. + - Context7 `better-sqlite3` v12.6.2 docs confirm `db.loadExtension(path)` exists. The installed `libsql` package in this workspace also exposes `loadExtension(path): this` in `node_modules/libsql/types/index.d.ts` and `loadExtension(...args)` in `node_modules/libsql/index.js`, so extension loading is not obviously blocked by the driver API surface alone. + - The review report remains the only verified runtime evidence for the current libsql path: `vector_from_float32(...)` is unavailable and `libsql_vector_idx` DDL is rejected in this environment. That invalidates the original native-vector approach but does not by itself prove sqlite-vec extension loading succeeds through the current `libsql` package alias, so the replan must include explicit connection-bootstrap and test coverage for real extension loading on the main DB client and worker-owned connections. + - Two iteration-0 deliverables referenced in the rejected plan do not exist in the current worktree: `src/lib/server/pipeline/write-worker-entry.ts` and `src/routes/api/v1/workers/+server.ts`. `scripts/build-workers.mjs` and the admin `WorkerStatusPanel.svelte` already reference those missing paths, so iteration 1 must either create them or revert those dangling references as part of a consistent plan. + - The existing admin/SSE work is largely salvageable. `src/routes/api/v1/jobs/stream/+server.ts`, `src/routes/api/v1/jobs/[id]/stream/+server.ts`, `src/lib/server/pipeline/progress-broadcaster.ts`, `src/lib/components/IndexingProgress.svelte`, `src/lib/components/admin/JobSkeleton.svelte`, `src/lib/components/admin/Toast.svelte`, and `src/lib/components/admin/WorkerStatusPanel.svelte` provide a usable foundation, but `src/routes/admin/jobs/+page.svelte` still contains `confirm(...)` and the queue API still only supports exact `repository_id = ?` and single-status filtering. + - The existing tests still encode the rejected pre-sqlite-vec model: `embedding.service.test.ts`, `schema.test.ts`, `hybrid.search.service.test.ts`, and `indexing.pipeline.test.ts` seed and assert against `snippet_embeddings.embedding` blobs only. The sqlite-vec replan therefore needs new DB bootstrap helpers, vec-table lifecycle assertions, and vector-search tests that validate actual vec0 writes and filtered KNN queries. +- Risks / follow-ups: + - The current worktree is dirty with iteration-0 partial changes and generated migration metadata, so iteration-1 tasks must explicitly distinguish keep/revise/revert work to avoid sibling tasks fighting over the same files. + - Because the current `libsql` package appears to expose `loadExtension`, the replan should avoid assuming an immediate full revert to upstream `better-sqlite3`; instead it should sequence a driver/bootstrap compatibility decision around actual sqlite-vec extension loading behavior with testable acceptance criteria. + +### 2026-04-01 — TRUEREF-0023 iteration-2 current-worktree verification + +- Task: Replan iteration 2 against the post-iteration-1 workspace state so the first validation unit no longer leaves a known vec_embedding mismatch behind. +- Files inspected: + - `package.json` + - `package-lock.json` + - `scripts/build-workers.mjs` + - `src/lib/server/db/client.ts` + - `src/lib/server/db/index.ts` + - `src/lib/server/db/schema.ts` + - `src/lib/server/db/vectors.sql` + - `src/lib/server/db/migrations/0006_yielding_centennial.sql` + - `src/lib/server/db/schema.test.ts` + - `src/lib/server/embeddings/embedding.service.ts` + - `src/lib/server/embeddings/embedding.service.test.ts` + - `src/lib/server/search/vector.search.ts` + - `src/lib/server/search/hybrid.search.service.ts` + - `src/lib/server/search/hybrid.search.service.test.ts` + - `src/lib/server/pipeline/job-queue.ts` + - `src/lib/server/pipeline/worker-pool.ts` + - `src/lib/server/pipeline/worker-entry.ts` + - `src/lib/server/pipeline/embed-worker-entry.ts` + - `src/lib/server/pipeline/worker-types.ts` + - `src/lib/server/pipeline/indexing.pipeline.ts` + - `src/lib/server/pipeline/indexing.pipeline.test.ts` + - `src/lib/server/pipeline/startup.ts` + - `src/lib/server/pipeline/progress-broadcaster.ts` + - `src/routes/api/v1/jobs/+server.ts` + - `src/routes/api/v1/jobs/stream/+server.ts` + - `src/routes/api/v1/sse-and-settings.integration.test.ts` + - `src/routes/admin/jobs/+page.svelte` + - `src/lib/components/IndexingProgress.svelte` + - `src/lib/components/admin/JobStatusBadge.svelte` + - `src/lib/components/admin/JobSkeleton.svelte` + - `src/lib/components/admin/Toast.svelte` + - `src/lib/components/admin/WorkerStatusPanel.svelte` + - `prompts/TRUEREF-0023/iteration_1/plan.md` + - `prompts/TRUEREF-0023/iteration_1/tasks.yaml` +- Findings: + - Iteration 1 already completed the direct-driver reset in the working tree: `package.json` and `package-lock.json` now contain real `better-sqlite3` plus `sqlite-vec`, and the current production/test files read in this pass import `better-sqlite3`, not `libsql`. + - The remaining failing intermediate state is exactly the schema/write mismatch called out in the review report: `src/lib/server/db/schema.ts` and `src/lib/server/db/migrations/0006_yielding_centennial.sql` still declare `vec_embedding`, `src/lib/server/db/index.ts` still executes `vectors.sql`, and `src/lib/server/embeddings/embedding.service.ts` still inserts `(embedding, vec_embedding)` into `snippet_embeddings`. + - `src/lib/server/db/vectors.sql` is still invalid startup SQL. It contains a dangling `USING libsql_vector_idx(...)` clause with no enclosing `CREATE INDEX`, so leaving it in the initialization path keeps the rejected libSQL-native design alive. + - The first iteration-1 task boundary was therefore wrong for the current baseline: the package/import reset is already present, but it only becomes a valid foundation once the relational `vec_embedding` artifacts and `EmbeddingService` insert path are cleaned up in the same validation unit. + - The current search path is still the pre-sqlite-vec implementation. `src/lib/server/search/vector.search.ts` reads every candidate embedding blob and scores in JavaScript; no `vec0`, `sqliteVec.load(db)`, or sqlite-vec KNN query exists anywhere under `src/` yet. + - The write worker and worker-status backend are still missing in the live tree even though they are already referenced elsewhere: `scripts/build-workers.mjs` includes `src/lib/server/pipeline/write-worker-entry.ts`, `src/lib/components/admin/WorkerStatusPanel.svelte` fetches `/api/v1/workers`, and `src/routes/api/v1/jobs/stream/+server.ts` currently has no worker-status event source. + - The admin jobs page remains incomplete but salvageable: `src/routes/admin/jobs/+page.svelte` still uses `confirm(...)` and `alert(...)`, while `JobSkeleton.svelte`, `Toast.svelte`, `WorkerStatusPanel.svelte`, `JobStatusBadge.svelte`, and `IndexingProgress.svelte` already provide the intended UI foundation. + - `src/lib/server/pipeline/job-queue.ts` still only supports exact `repository_id = ?` and single `status = ?` filtering, so API-side filter work remains a separate backend task and does not need to block the vector-storage implementation. +- Risks / follow-ups: + - Iteration 2 task decomposition must treat the current dirty code files from iterations 0 and 1 as the validation baseline, otherwise the executor will keep rediscovering pre-existing worktree drift instead of new task deltas. + - The sqlite-vec bootstrap helper and the relational cleanup should be planned as one acceptance unit before any downstream vec0, worker-status, or admin-page tasks, because that is the smallest unit that removes the known broken intermediate state. diff --git a/package-lock.json b/package-lock.json index 68410fa..7c2de1f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "@modelcontextprotocol/sdk": "^1.27.1", "@xenova/transformers": "^2.17.2", "better-sqlite3": "^12.6.2", + "sqlite-vec": "^0.1.9", "zod": "^4.3.6" }, "devDependencies": { @@ -25,6 +26,7 @@ "@vitest/browser-playwright": "^4.1.0", "drizzle-kit": "^0.31.8", "drizzle-orm": "^0.45.1", + "esbuild": "^0.24.0", "eslint": "^9.39.2", "eslint-config-prettier": "^10.1.8", "eslint-plugin-svelte": "^3.14.0", @@ -494,9 +496,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", - "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.2.tgz", + "integrity": "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==", "cpu": [ "ppc64" ], @@ -511,9 +513,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", - "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.2.tgz", + "integrity": "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==", "cpu": [ "arm" ], @@ -528,9 +530,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", - "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.2.tgz", + "integrity": "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==", "cpu": [ "arm64" ], @@ -545,9 +547,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", - "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.2.tgz", + "integrity": "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==", "cpu": [ "x64" ], @@ -562,9 +564,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", - "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.2.tgz", + "integrity": "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==", "cpu": [ "arm64" ], @@ -579,9 +581,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", - "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.2.tgz", + "integrity": "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==", "cpu": [ "x64" ], @@ -596,9 +598,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", - "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.2.tgz", + "integrity": "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==", "cpu": [ "arm64" ], @@ -613,9 +615,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", - "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.2.tgz", + "integrity": "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==", "cpu": [ "x64" ], @@ -630,9 +632,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", - "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.2.tgz", + "integrity": "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==", "cpu": [ "arm" ], @@ -647,9 +649,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", - "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.2.tgz", + "integrity": "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==", "cpu": [ "arm64" ], @@ -664,9 +666,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", - "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.2.tgz", + "integrity": "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==", "cpu": [ "ia32" ], @@ -681,9 +683,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", - "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.2.tgz", + "integrity": "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==", "cpu": [ "loong64" ], @@ -698,9 +700,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", - "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.2.tgz", + "integrity": "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==", "cpu": [ "mips64el" ], @@ -715,9 +717,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", - "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.2.tgz", + "integrity": "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==", "cpu": [ "ppc64" ], @@ -732,9 +734,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", - "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.2.tgz", + "integrity": "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==", "cpu": [ "riscv64" ], @@ -749,9 +751,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", - "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.2.tgz", + "integrity": "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==", "cpu": [ "s390x" ], @@ -766,9 +768,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", - "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.2.tgz", + "integrity": "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==", "cpu": [ "x64" ], @@ -783,9 +785,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", - "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.24.2.tgz", + "integrity": "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==", "cpu": [ "arm64" ], @@ -800,9 +802,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", - "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.2.tgz", + "integrity": "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==", "cpu": [ "x64" ], @@ -817,9 +819,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", - "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.2.tgz", + "integrity": "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==", "cpu": [ "arm64" ], @@ -834,9 +836,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", - "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.2.tgz", + "integrity": "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==", "cpu": [ "x64" ], @@ -868,9 +870,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", - "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.2.tgz", + "integrity": "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==", "cpu": [ "x64" ], @@ -885,9 +887,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", - "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.2.tgz", + "integrity": "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==", "cpu": [ "arm64" ], @@ -902,9 +904,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", - "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.2.tgz", + "integrity": "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==", "cpu": [ "ia32" ], @@ -919,9 +921,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", - "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.2.tgz", + "integrity": "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==", "cpu": [ "x64" ], @@ -3537,6 +3539,473 @@ "drizzle-kit": "bin.cjs" } }, + "node_modules/drizzle-kit/node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/drizzle-kit/node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, "node_modules/drizzle-orm": { "version": "0.45.1", "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-0.45.1.tgz", @@ -3753,9 +4222,9 @@ } }, "node_modules/esbuild": { - "version": "0.25.12", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", - "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.2.tgz", + "integrity": "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3766,32 +4235,31 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.12", - "@esbuild/android-arm": "0.25.12", - "@esbuild/android-arm64": "0.25.12", - "@esbuild/android-x64": "0.25.12", - "@esbuild/darwin-arm64": "0.25.12", - "@esbuild/darwin-x64": "0.25.12", - "@esbuild/freebsd-arm64": "0.25.12", - "@esbuild/freebsd-x64": "0.25.12", - "@esbuild/linux-arm": "0.25.12", - "@esbuild/linux-arm64": "0.25.12", - "@esbuild/linux-ia32": "0.25.12", - "@esbuild/linux-loong64": "0.25.12", - "@esbuild/linux-mips64el": "0.25.12", - "@esbuild/linux-ppc64": "0.25.12", - "@esbuild/linux-riscv64": "0.25.12", - "@esbuild/linux-s390x": "0.25.12", - "@esbuild/linux-x64": "0.25.12", - "@esbuild/netbsd-arm64": "0.25.12", - "@esbuild/netbsd-x64": "0.25.12", - "@esbuild/openbsd-arm64": "0.25.12", - "@esbuild/openbsd-x64": "0.25.12", - "@esbuild/openharmony-arm64": "0.25.12", - "@esbuild/sunos-x64": "0.25.12", - "@esbuild/win32-arm64": "0.25.12", - "@esbuild/win32-ia32": "0.25.12", - "@esbuild/win32-x64": "0.25.12" + "@esbuild/aix-ppc64": "0.24.2", + "@esbuild/android-arm": "0.24.2", + "@esbuild/android-arm64": "0.24.2", + "@esbuild/android-x64": "0.24.2", + "@esbuild/darwin-arm64": "0.24.2", + "@esbuild/darwin-x64": "0.24.2", + "@esbuild/freebsd-arm64": "0.24.2", + "@esbuild/freebsd-x64": "0.24.2", + "@esbuild/linux-arm": "0.24.2", + "@esbuild/linux-arm64": "0.24.2", + "@esbuild/linux-ia32": "0.24.2", + "@esbuild/linux-loong64": "0.24.2", + "@esbuild/linux-mips64el": "0.24.2", + "@esbuild/linux-ppc64": "0.24.2", + "@esbuild/linux-riscv64": "0.24.2", + "@esbuild/linux-s390x": "0.24.2", + "@esbuild/linux-x64": "0.24.2", + "@esbuild/netbsd-arm64": "0.24.2", + "@esbuild/netbsd-x64": "0.24.2", + "@esbuild/openbsd-arm64": "0.24.2", + "@esbuild/openbsd-x64": "0.24.2", + "@esbuild/sunos-x64": "0.24.2", + "@esbuild/win32-arm64": "0.24.2", + "@esbuild/win32-ia32": "0.24.2", + "@esbuild/win32-x64": "0.24.2" } }, "node_modules/escape-html": { @@ -6527,6 +6995,84 @@ "source-map": "^0.6.0" } }, + "node_modules/sqlite-vec": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec/-/sqlite-vec-0.1.9.tgz", + "integrity": "sha512-L7XJWRIBNvR9O5+vh1FQ+IGkh/3D2AzVksW5gdtk28m78Hy8skFD0pqReKH1Yp0/BUKRGcffgKvyO/EON5JXpA==", + "license": "MIT OR Apache", + "optionalDependencies": { + "sqlite-vec-darwin-arm64": "0.1.9", + "sqlite-vec-darwin-x64": "0.1.9", + "sqlite-vec-linux-arm64": "0.1.9", + "sqlite-vec-linux-x64": "0.1.9", + "sqlite-vec-windows-x64": "0.1.9" + } + }, + "node_modules/sqlite-vec-darwin-arm64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-darwin-arm64/-/sqlite-vec-darwin-arm64-0.1.9.tgz", + "integrity": "sha512-jSsZpE42OfBkGL/ItyJTVCUwl6o6Ka3U5rc4j+UBDIQzC1ulSSKMEhQLthsOnF/MdAf1MuAkYhkdKmmcjaIZQg==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sqlite-vec-darwin-x64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-darwin-x64/-/sqlite-vec-darwin-x64-0.1.9.tgz", + "integrity": "sha512-KDlVyqQT7pnOhU1ymB9gs7dMbSoVmKHitT+k1/xkjarcX8bBqPxWrGlK/R+C5WmWkfvWwyq5FfXfiBYCBs6PlA==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sqlite-vec-linux-arm64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-linux-arm64/-/sqlite-vec-linux-arm64-0.1.9.tgz", + "integrity": "sha512-5wXVJ9c9kR4CHm/wVqXb/R+XUHTdpZ4nWbPHlS+gc9qQFVHs92Km4bPnCKX4rtcPMzvNis+SIzMJR1SCEwpuUw==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/sqlite-vec-linux-x64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-linux-x64/-/sqlite-vec-linux-x64-0.1.9.tgz", + "integrity": "sha512-w3tCH8xK2finW8fQJ/m8uqKodXUZ9KAuAar2UIhz4BHILfpE0WM/MTGCRfa7RjYbrYim5Luk3guvMOGI7T7JQA==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/sqlite-vec-windows-x64": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/sqlite-vec-windows-x64/-/sqlite-vec-windows-x64-0.1.9.tgz", + "integrity": "sha512-y3gEIyy/17bq2QFPQOWLE68TYWcRZkBQVA2XLrTPHNTOp55xJi/BBBmOm40tVMDMjtP+Elpk6UBUXdaq+46b0Q==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", diff --git a/package.json b/package.json index 3b7a3a1..8d914f4 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@modelcontextprotocol/sdk": "^1.27.1", "@xenova/transformers": "^2.17.2", "better-sqlite3": "^12.6.2", + "sqlite-vec": "^0.1.9", "zod": "^4.3.6" } } diff --git a/scripts/build-workers.mjs b/scripts/build-workers.mjs index 331e2ad..1082a4e 100644 --- a/scripts/build-workers.mjs +++ b/scripts/build-workers.mjs @@ -3,7 +3,8 @@ import { existsSync } from 'node:fs'; const entries = [ 'src/lib/server/pipeline/worker-entry.ts', - 'src/lib/server/pipeline/embed-worker-entry.ts' + 'src/lib/server/pipeline/embed-worker-entry.ts', + 'src/lib/server/pipeline/write-worker-entry.ts' ]; try { diff --git a/src/lib/components/IndexingProgress.svelte b/src/lib/components/IndexingProgress.svelte index 40080b5..8a08bae 100644 --- a/src/lib/components/IndexingProgress.svelte +++ b/src/lib/components/IndexingProgress.svelte @@ -7,39 +7,33 @@ $effect(() => { job = null; - let stopped = false; - let completeFired = false; + const es = new EventSource(`/api/v1/jobs/${jobId}/stream`); - async function poll() { - if (stopped) return; - try { - const res = await fetch(`/api/v1/jobs/${jobId}`); - if (res.ok) { - const data = await res.json(); - job = data.job; - if (!completeFired && (job?.status === 'done' || job?.status === 'failed')) { - completeFired = true; - oncomplete?.(); - } - } - } catch { - // ignore transient errors - } - } + es.addEventListener('job-progress', (event) => { + const data = JSON.parse(event.data); + job = { ...job, ...data } as IndexingJob; + }); - void poll(); - const interval = setInterval(() => { - if (job?.status === 'done' || job?.status === 'failed') { - clearInterval(interval); - return; - } - void poll(); - }, 2000); + es.addEventListener('job-done', () => { + void fetch(`/api/v1/jobs/${jobId}`) + .then(r => r.json()) + .then(d => { job = d.job; oncomplete?.(); }); + es.close(); + }); - return () => { - stopped = true; - clearInterval(interval); + es.addEventListener('job-failed', (event) => { + const data = JSON.parse(event.data); + if (job) job = { ...job, status: 'failed', error: data.error ?? 'Unknown error' } as IndexingJob; + oncomplete?.(); + es.close(); + }); + + es.onerror = () => { + es.close(); + void fetch(`/api/v1/jobs/${jobId}`).then(r => r.json()).then(d => { job = d.job; }); }; + + return () => es.close(); }); const progress = $derived(job?.progress ?? 0); diff --git a/src/lib/components/admin/JobSkeleton.svelte b/src/lib/components/admin/JobSkeleton.svelte new file mode 100644 index 0000000..67edb21 --- /dev/null +++ b/src/lib/components/admin/JobSkeleton.svelte @@ -0,0 +1,19 @@ + + +{#each Array(rows) as _, i (i)} + + +
+
+ +
+
+
+
+
+ +{/each} diff --git a/src/lib/components/admin/JobStatusBadge.svelte b/src/lib/components/admin/JobStatusBadge.svelte index c667dbf..ae118d1 100644 --- a/src/lib/components/admin/JobStatusBadge.svelte +++ b/src/lib/components/admin/JobStatusBadge.svelte @@ -1,9 +1,10 @@ + +
+ {#each toasts as toast (toast.id)} +
+ {toast.message} + +
+ {/each} +
diff --git a/src/lib/components/admin/WorkerStatusPanel.svelte b/src/lib/components/admin/WorkerStatusPanel.svelte new file mode 100644 index 0000000..0cdbda9 --- /dev/null +++ b/src/lib/components/admin/WorkerStatusPanel.svelte @@ -0,0 +1,81 @@ + + +{#if status.concurrency > 0} +
+
+

Workers

+ {status.active} / {status.concurrency} active +
+
+ {#each status.workers as worker (worker.index)} +
+ + Worker {worker.index} + {#if worker.state === 'running' && worker.repositoryId} + {worker.repositoryId}{worker.versionId ? ' / ' + worker.versionId : ''} + {:else} + idle + {/if} +
+ {/each} +
+
+{/if} diff --git a/src/lib/server/db/client.ts b/src/lib/server/db/client.ts index 4b5c6e4..a9ab565 100644 --- a/src/lib/server/db/client.ts +++ b/src/lib/server/db/client.ts @@ -4,6 +4,7 @@ */ import Database from 'better-sqlite3'; import { env } from '$env/dynamic/private'; +import { loadSqliteVec } from './sqlite-vec'; let _client: Database.Database | null = null; @@ -14,6 +15,12 @@ export function getClient(): Database.Database { _client.pragma('journal_mode = WAL'); _client.pragma('foreign_keys = ON'); _client.pragma('busy_timeout = 5000'); + _client.pragma('synchronous = NORMAL'); + _client.pragma('cache_size = -65536'); + _client.pragma('temp_store = MEMORY'); + _client.pragma('mmap_size = 268435456'); + _client.pragma('wal_autocheckpoint = 1000'); + loadSqliteVec(_client); } return _client; } diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 4ff9a87..7f828af 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -5,6 +5,7 @@ import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { join, dirname } from 'node:path'; import * as schema from './schema'; +import { loadSqliteVec } from './sqlite-vec'; import { env } from '$env/dynamic/private'; if (!env.DATABASE_URL) throw new Error('DATABASE_URL is not set'); @@ -19,6 +20,12 @@ client.pragma('foreign_keys = ON'); // Prevents SQLITE_BUSY errors when the indexing pipeline holds the write lock // and an HTTP request arrives simultaneously. client.pragma('busy_timeout = 5000'); +client.pragma('synchronous = NORMAL'); +client.pragma('cache_size = -65536'); +client.pragma('temp_store = MEMORY'); +client.pragma('mmap_size = 268435456'); +client.pragma('wal_autocheckpoint = 1000'); +loadSqliteVec(client); export const db = drizzle(client, { schema }); diff --git a/src/lib/server/db/migrations/0006_yielding_centennial.sql b/src/lib/server/db/migrations/0006_yielding_centennial.sql new file mode 100644 index 0000000..dd71f9e --- /dev/null +++ b/src/lib/server/db/migrations/0006_yielding_centennial.sql @@ -0,0 +1,6 @@ +CREATE INDEX `idx_embeddings_profile` ON `snippet_embeddings` (`profile_id`,`snippet_id`);--> statement-breakpoint +CREATE INDEX `idx_documents_repo_version` ON `documents` (`repository_id`,`version_id`);--> statement-breakpoint +CREATE INDEX `idx_jobs_repo_status` ON `indexing_jobs` (`repository_id`,`status`);--> statement-breakpoint +CREATE INDEX `idx_repositories_state` ON `repositories` (`state`);--> statement-breakpoint +CREATE INDEX `idx_snippets_repo_version` ON `snippets` (`repository_id`,`version_id`);--> statement-breakpoint +CREATE INDEX `idx_snippets_repo_type` ON `snippets` (`repository_id`,`type`); \ No newline at end of file diff --git a/src/lib/server/db/migrations/meta/0006_snapshot.json b/src/lib/server/db/migrations/meta/0006_snapshot.json new file mode 100644 index 0000000..295b2fb --- /dev/null +++ b/src/lib/server/db/migrations/meta/0006_snapshot.json @@ -0,0 +1,948 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b8998bda-f89b-41bc-b923-3f676d153c79", + "prevId": "c326dcbe-1771-4a90-a566-0ebd1eca47ec", + "tables": { + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "indexed_at": { + "name": "indexed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_documents_repo_version": { + "name": "idx_documents_repo_version", + "columns": [ + "repository_id", + "version_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "documents_repository_id_repositories_id_fk": { + "name": "documents_repository_id_repositories_id_fk", + "tableFrom": "documents", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "documents_version_id_repository_versions_id_fk": { + "name": "documents_version_id_repository_versions_id_fk", + "tableFrom": "documents", + "tableTo": "repository_versions", + "columnsFrom": [ + "version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "embedding_profiles": { + "name": "embedding_profiles", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_kind": { + "name": "provider_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "indexing_jobs": { + "name": "indexing_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "total_files": { + "name": "total_files", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "processed_files": { + "name": "processed_files", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "stage_detail": { + "name": "stage_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_jobs_repo_status": { + "name": "idx_jobs_repo_status", + "columns": [ + "repository_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "indexing_jobs_repository_id_repositories_id_fk": { + "name": "indexing_jobs_repository_id_repositories_id_fk", + "tableFrom": "indexing_jobs", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'main'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "total_snippets": { + "name": "total_snippets", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "trust_score": { + "name": "trust_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "benchmark_score": { + "name": "benchmark_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "stars": { + "name": "stars", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_token": { + "name": "github_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_indexed_at": { + "name": "last_indexed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_repositories_state": { + "name": "idx_repositories_state", + "columns": [ + "state" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repository_configs": { + "name": "repository_configs", + "columns": { + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "project_title": { + "name": "project_title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "folders": { + "name": "folders", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exclude_folders": { + "name": "exclude_folders", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exclude_files": { + "name": "exclude_files", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rules": { + "name": "rules", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "previous_versions": { + "name": "previous_versions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "uniq_repo_config_base": { + "name": "uniq_repo_config_base", + "columns": [ + "repository_id" + ], + "isUnique": true, + "where": "\"repository_configs\".\"version_id\" IS NULL" + }, + "uniq_repo_config_version": { + "name": "uniq_repo_config_version", + "columns": [ + "repository_id", + "version_id" + ], + "isUnique": true, + "where": "\"repository_configs\".\"version_id\" IS NOT NULL" + } + }, + "foreignKeys": { + "repository_configs_repository_id_repositories_id_fk": { + "name": "repository_configs_repository_id_repositories_id_fk", + "tableFrom": "repository_configs", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repository_versions": { + "name": "repository_versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_hash": { + "name": "commit_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "total_snippets": { + "name": "total_snippets", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "indexed_at": { + "name": "indexed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "repository_versions_repository_id_repositories_id_fk": { + "name": "repository_versions_repository_id_repositories_id_fk", + "tableFrom": "repository_versions", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "snippet_embeddings": { + "name": "snippet_embeddings", + "columns": { + "snippet_id": { + "name": "snippet_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "embedding": { + "name": "embedding", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_embeddings_profile": { + "name": "idx_embeddings_profile", + "columns": [ + "profile_id", + "snippet_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "snippet_embeddings_snippet_id_snippets_id_fk": { + "name": "snippet_embeddings_snippet_id_snippets_id_fk", + "tableFrom": "snippet_embeddings", + "tableTo": "snippets", + "columnsFrom": [ + "snippet_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snippet_embeddings_profile_id_embedding_profiles_id_fk": { + "name": "snippet_embeddings_profile_id_embedding_profiles_id_fk", + "tableFrom": "snippet_embeddings", + "tableTo": "embedding_profiles", + "columnsFrom": [ + "profile_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "snippet_embeddings_snippet_id_profile_id_pk": { + "columns": [ + "snippet_id", + "profile_id" + ], + "name": "snippet_embeddings_snippet_id_profile_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "snippets": { + "name": "snippets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "breadcrumb": { + "name": "breadcrumb", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_snippets_repo_version": { + "name": "idx_snippets_repo_version", + "columns": [ + "repository_id", + "version_id" + ], + "isUnique": false + }, + "idx_snippets_repo_type": { + "name": "idx_snippets_repo_type", + "columns": [ + "repository_id", + "type" + ], + "isUnique": false + } + }, + "foreignKeys": { + "snippets_document_id_documents_id_fk": { + "name": "snippets_document_id_documents_id_fk", + "tableFrom": "snippets", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snippets_repository_id_repositories_id_fk": { + "name": "snippets_repository_id_repositories_id_fk", + "tableFrom": "snippets", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snippets_version_id_repository_versions_id_fk": { + "name": "snippets_version_id_repository_versions_id_fk", + "tableFrom": "snippets", + "tableTo": "repository_versions", + "columnsFrom": [ + "version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/lib/server/db/migrations/meta/_journal.json b/src/lib/server/db/migrations/meta/_journal.json index b46bd60..fec5877 100644 --- a/src/lib/server/db/migrations/meta/_journal.json +++ b/src/lib/server/db/migrations/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1774890536284, "tag": "0005_fix_stage_defaults", "breakpoints": true + }, + { + "idx": 6, + "version": "6", + "when": 1775038799913, + "tag": "0006_yielding_centennial", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/lib/server/db/schema.test.ts b/src/lib/server/db/schema.test.ts index e71bf94..292b87a 100644 --- a/src/lib/server/db/schema.test.ts +++ b/src/lib/server/db/schema.test.ts @@ -6,6 +6,7 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { eq } from 'drizzle-orm'; import * as schema from './schema'; +import { loadSqliteVec, sqliteVecRowidTableName, sqliteVecTableName } from './sqlite-vec'; import { repositories, repositoryVersions, @@ -24,6 +25,7 @@ import { function createTestDb() { const client = new Database(':memory:'); client.pragma('foreign_keys = ON'); + loadSqliteVec(client); const db = drizzle(client, { schema }); @@ -266,10 +268,11 @@ describe('snippets table', () => { describe('snippet_embeddings table', () => { let db: ReturnType['db']; + let client: Database.Database; let snippetId: string; beforeEach(() => { - ({ db } = createTestDb()); + ({ db, client } = createTestDb()); db.insert(repositories).values(makeRepo()).run(); const docId = crypto.randomUUID(); db.insert(documents) @@ -344,6 +347,30 @@ describe('snippet_embeddings table', () => { const result = db.select().from(snippetEmbeddings).all(); expect(result).toHaveLength(0); }); + + it('keeps the relational schema free of vec_embedding and retains the profile index', () => { + const columns = client + .prepare("PRAGMA table_info('snippet_embeddings')") + .all() as Array<{ name: string }>; + expect(columns.map((column) => column.name)).not.toContain('vec_embedding'); + + const indexes = client + .prepare("PRAGMA index_list('snippet_embeddings')") + .all() as Array<{ name: string }>; + expect(indexes.map((index) => index.name)).toContain('idx_embeddings_profile'); + }); + + it('loads sqlite-vec idempotently and derives deterministic per-profile table names', () => { + expect(() => loadSqliteVec(client)).not.toThrow(); + const tableName = sqliteVecTableName('local-default'); + const rowidTableName = sqliteVecRowidTableName('local-default'); + + expect(tableName).toMatch(/^snippet_embeddings_vec_local_default_[0-9a-f]{8}$/); + expect(rowidTableName).toMatch(/^snippet_embeddings_vec_rowids_local_default_[0-9a-f]{8}$/); + expect(sqliteVecTableName('local-default')).toBe(tableName); + expect(sqliteVecRowidTableName('local-default')).toBe(rowidTableName); + expect(sqliteVecTableName('local-default')).not.toBe(sqliteVecTableName('openai/custom')); + }); }); describe('indexing_jobs table', () => { diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index 9a549c8..1bb9254 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -1,6 +1,7 @@ import { sql } from 'drizzle-orm'; import { blob, + index, integer, primaryKey, real, @@ -34,7 +35,7 @@ export const repositories = sqliteTable('repositories', { lastIndexedAt: integer('last_indexed_at', { mode: 'timestamp' }), createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull() -}); +}, (t) => [index('idx_repositories_state').on(t.state)]); // --------------------------------------------------------------------------- // repository_versions @@ -72,7 +73,7 @@ export const documents = sqliteTable('documents', { tokenCount: integer('token_count').default(0), checksum: text('checksum').notNull(), // SHA-256 of file content indexedAt: integer('indexed_at', { mode: 'timestamp' }).notNull() -}); +}, (t) => [index('idx_documents_repo_version').on(t.repositoryId, t.versionId)]); // --------------------------------------------------------------------------- // snippets @@ -93,7 +94,10 @@ export const snippets = sqliteTable('snippets', { breadcrumb: text('breadcrumb'), // e.g. "Installation > Getting Started" tokenCount: integer('token_count').default(0), createdAt: integer('created_at', { mode: 'timestamp' }).notNull() -}); +}, (t) => [ + index('idx_snippets_repo_version').on(t.repositoryId, t.versionId), + index('idx_snippets_repo_type').on(t.repositoryId, t.type), +]); // --------------------------------------------------------------------------- // embedding_profiles @@ -128,7 +132,10 @@ export const snippetEmbeddings = sqliteTable( embedding: blob('embedding').notNull(), // Float32Array as binary blob createdAt: integer('created_at').notNull() }, - (table) => [primaryKey({ columns: [table.snippetId, table.profileId] })] + (table) => [ + primaryKey({ columns: [table.snippetId, table.profileId] }), + index('idx_embeddings_profile').on(table.profileId, table.snippetId), + ] ); // --------------------------------------------------------------------------- @@ -154,7 +161,7 @@ export const indexingJobs = sqliteTable('indexing_jobs', { startedAt: integer('started_at', { mode: 'timestamp' }), completedAt: integer('completed_at', { mode: 'timestamp' }), createdAt: integer('created_at', { mode: 'timestamp' }).notNull() -}); +}, (t) => [index('idx_jobs_repo_status').on(t.repositoryId, t.status)]); // --------------------------------------------------------------------------- // repository_configs diff --git a/src/lib/server/db/sqlite-vec.ts b/src/lib/server/db/sqlite-vec.ts new file mode 100644 index 0000000..65d57c8 --- /dev/null +++ b/src/lib/server/db/sqlite-vec.ts @@ -0,0 +1,49 @@ +import type Database from 'better-sqlite3'; +import * as sqliteVec from 'sqlite-vec'; + +const loadedConnections = new WeakSet(); + +function stableHash(value: string): string { + let hash = 2166136261; + + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + + return (hash >>> 0).toString(16).padStart(8, '0'); +} + +function sanitizeIdentifierPart(value: string): string { + const sanitized = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_+|_+$/g, ''); + + return sanitized.length > 0 ? sanitized.slice(0, 32) : 'profile'; +} + +export function sqliteVecTableSuffix(profileId: string): string { + return `${sanitizeIdentifierPart(profileId)}_${stableHash(profileId)}`; +} + +export function sqliteVecTableName(profileId: string): string { + return `snippet_embeddings_vec_${sqliteVecTableSuffix(profileId)}`; +} + +export function sqliteVecRowidTableName(profileId: string): string { + return `snippet_embeddings_vec_rowids_${sqliteVecTableSuffix(profileId)}`; +} + +export function quoteSqliteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; +} + +export function loadSqliteVec(db: Database.Database): void { + if (loadedConnections.has(db)) { + return; + } + + sqliteVec.load(db); + loadedConnections.add(db); +} \ No newline at end of file diff --git a/src/lib/server/db/vectors.sql b/src/lib/server/db/vectors.sql new file mode 100644 index 0000000..3e8b4a4 --- /dev/null +++ b/src/lib/server/db/vectors.sql @@ -0,0 +1,2 @@ +-- Relational vec_embedding bootstrap removed in iteration 2. +-- Downstream sqlite-vec vec0 tables are created on demand in application code. diff --git a/src/lib/server/embeddings/embedding.service.test.ts b/src/lib/server/embeddings/embedding.service.test.ts index 4d49b7c..534fdc1 100644 --- a/src/lib/server/embeddings/embedding.service.test.ts +++ b/src/lib/server/embeddings/embedding.service.test.ts @@ -12,6 +12,12 @@ import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import * as schema from '../db/schema.js'; +import { + loadSqliteVec, + sqliteVecRowidTableName, + sqliteVecTableName +} from '../db/sqlite-vec.js'; +import { SqliteVecStore } from '../search/sqlite-vec.store.js'; import { NoopEmbeddingProvider, EmbeddingError, type EmbeddingVector } from './provider.js'; import { OpenAIEmbeddingProvider } from './openai.provider.js'; @@ -31,6 +37,7 @@ import { createProviderFromProfile } from './registry.js'; function createTestDb() { const client = new Database(':memory:'); client.pragma('foreign_keys = ON'); + loadSqliteVec(client); const db = drizzle(client, { schema }); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); @@ -387,10 +394,19 @@ describe('EmbeddingService', () => { embedding: Buffer; profile_id: string; }; + expect((row as Record).vec_embedding).toBeUndefined(); expect(row.model).toBe('test-model'); expect(row.dimensions).toBe(4); expect(row.profile_id).toBe('local-default'); expect(row.embedding).toBeInstanceOf(Buffer); + + const queryEmbedding = service.getEmbedding(snippetId, 'local-default'); + const matches = new SqliteVecStore(client).queryNearestNeighbors(queryEmbedding!, { + repositoryId: '/test/embed-repo', + profileId: 'local-default', + limit: 5 + }); + expect(matches[0]?.snippetId).toBe(snippetId); }); it('stores embeddings as retrievable Float32Array blobs', async () => { @@ -436,6 +452,22 @@ describe('EmbeddingService', () => { .prepare('SELECT profile_id FROM snippet_embeddings WHERE snippet_id = ?') .get(snippetId) as { profile_id: string }; expect(row.profile_id).toBe('openai-custom'); + + const queryEmbedding = service.getEmbedding(snippetId, 'openai-custom'); + const store = new SqliteVecStore(client); + const customMatches = store.queryNearestNeighbors(queryEmbedding!, { + repositoryId: '/test/embed-repo', + profileId: 'openai-custom', + limit: 5 + }); + const defaultMatches = store.queryNearestNeighbors(new Float32Array([1, 0, 0, 0]), { + repositoryId: '/test/embed-repo', + profileId: 'local-default', + limit: 5 + }); + + expect(customMatches[0]?.snippetId).toBe(snippetId); + expect(defaultMatches).toHaveLength(0); }); it('is idempotent — re-embedding replaces the existing row', async () => { @@ -450,6 +482,17 @@ describe('EmbeddingService', () => { .prepare('SELECT COUNT(*) as cnt FROM snippet_embeddings WHERE snippet_id = ?') .get(snippetId) as { cnt: number }; expect(rows.cnt).toBe(1); + + const vecTable = sqliteVecTableName('local-default'); + const rowidTable = sqliteVecRowidTableName('local-default'); + const vecRows = client.prepare(`SELECT COUNT(*) as cnt FROM "${vecTable}"`).get() as { + cnt: number; + }; + const rowidRows = client.prepare(`SELECT COUNT(*) as cnt FROM "${rowidTable}"`).get() as { + cnt: number; + }; + expect(vecRows.cnt).toBe(1); + expect(rowidRows.cnt).toBe(1); }); it('calls onProgress after each batch', async () => { diff --git a/src/lib/server/embeddings/embedding.service.ts b/src/lib/server/embeddings/embedding.service.ts index 1dc18d5..5fe55ee 100644 --- a/src/lib/server/embeddings/embedding.service.ts +++ b/src/lib/server/embeddings/embedding.service.ts @@ -5,6 +5,7 @@ import type Database from 'better-sqlite3'; import type { EmbeddingProvider } from './provider.js'; +import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js'; interface SnippetRow { id: string; @@ -17,11 +18,15 @@ const BATCH_SIZE = 50; const TEXT_MAX_CHARS = 2048; export class EmbeddingService { + private readonly sqliteVecStore: SqliteVecStore; + constructor( private readonly db: Database.Database, private readonly provider: EmbeddingProvider, private readonly profileId: string = 'local-default' - ) {} + ) { + this.sqliteVecStore = new SqliteVecStore(db); + } findSnippetIdsMissingEmbeddings(repositoryId: string, versionId: string | null): string[] { if (versionId) { @@ -104,13 +109,19 @@ export class EmbeddingService { for (let j = 0; j < batchSnippets.length; j++) { const snippet = batchSnippets[j]; const embedding = embeddings[j]; + insert.run( snippet.id, this.profileId, embedding.model, embedding.dimensions, - Buffer.from(embedding.values.buffer) + Buffer.from( + embedding.values.buffer, + embedding.values.byteOffset, + embedding.values.byteLength + ) ); + this.sqliteVecStore.upsertEmbedding(this.profileId, snippet.id, embedding.values); } }); insertMany(); diff --git a/src/lib/server/pipeline/embed-worker-entry.ts b/src/lib/server/pipeline/embed-worker-entry.ts index 6514d65..9af2656 100644 --- a/src/lib/server/pipeline/embed-worker-entry.ts +++ b/src/lib/server/pipeline/embed-worker-entry.ts @@ -21,6 +21,11 @@ const db = new Database(dbPath); db.pragma('journal_mode = WAL'); db.pragma('foreign_keys = ON'); db.pragma('busy_timeout = 5000'); +db.pragma('synchronous = NORMAL'); +db.pragma('cache_size = -65536'); +db.pragma('temp_store = MEMORY'); +db.pragma('mmap_size = 268435456'); +db.pragma('wal_autocheckpoint = 1000'); // Load the embedding profile from DB const rawProfile = db.prepare('SELECT * FROM embedding_profiles WHERE id = ?').get(embeddingProfileId); diff --git a/src/lib/server/pipeline/indexing.pipeline.test.ts b/src/lib/server/pipeline/indexing.pipeline.test.ts index 766ea46..d22cd8a 100644 --- a/src/lib/server/pipeline/indexing.pipeline.test.ts +++ b/src/lib/server/pipeline/indexing.pipeline.test.ts @@ -13,6 +13,9 @@ import { JobQueue } from './job-queue.js'; import { IndexingPipeline } from './indexing.pipeline.js'; import { recoverStaleJobs } from './startup.js'; import { EmbeddingService } from '$lib/server/embeddings/embedding.service.js'; +import { loadSqliteVec } from '$lib/server/db/sqlite-vec.js'; +import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js'; +import { sqliteVecRowidTableName, sqliteVecTableName } from '$lib/server/db/sqlite-vec.js'; import * as diffStrategy from './differential-strategy.js'; // --------------------------------------------------------------------------- @@ -22,6 +25,7 @@ import * as diffStrategy from './differential-strategy.js'; function createTestDb(): Database.Database { const client = new Database(':memory:'); client.pragma('foreign_keys = ON'); + loadSqliteVec(client); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); for (const migrationFile of [ @@ -29,7 +33,9 @@ function createTestDb(): Database.Database { '0001_quick_nighthawk.sql', '0002_silky_stellaris.sql', '0003_multiversion_config.sql', - '0004_complete_sentry.sql' + '0004_complete_sentry.sql', + '0005_fix_stage_defaults.sql', + '0006_yielding_centennial.sql' ]) { const migrationSql = readFileSync(join(migrationsFolder, migrationFile), 'utf-8'); @@ -539,6 +545,52 @@ describe('IndexingPipeline', () => { expect(finalChecksum).toBe('sha-v2'); }); + it('removes derived vec rows when changed documents are replaced', async () => { + const docId = crypto.randomUUID(); + const snippetId = crypto.randomUUID(); + const embedding = Float32Array.from([1, 0, 0]); + const vecStore = new SqliteVecStore(db); + + db.prepare( + `INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at) + VALUES (?, '/test/repo', NULL, 'README.md', 'stale-doc', ?)` + ).run(docId, now); + db.prepare( + `INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at) + VALUES (?, ?, '/test/repo', NULL, 'info', 'stale snippet', ?)` + ).run(snippetId, docId, now); + db.prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, 'local-default', 'test-model', 3, ?, ?)` + ).run(snippetId, Buffer.from(embedding.buffer), now); + vecStore.upsertEmbedding('local-default', snippetId, embedding); + + const pipeline = makePipeline({ + files: [ + { + path: 'README.md', + content: '# Updated\n\nFresh content.', + sha: 'sha-fresh', + language: 'markdown' + } + ], + totalFiles: 1 + }); + const job = makeJob(); + + await pipeline.run(job as never); + + const vecTable = sqliteVecTableName('local-default'); + const rowidTable = sqliteVecRowidTableName('local-default'); + const vecCount = db.prepare(`SELECT COUNT(*) as n FROM "${vecTable}"`).get() as { n: number }; + const rowidCount = db.prepare(`SELECT COUNT(*) as n FROM "${rowidTable}"`).get() as { + n: number; + }; + + expect(vecCount.n).toBe(0); + expect(rowidCount.n).toBe(0); + }); + it('updates job progress as files are processed', async () => { const files = Array.from({ length: 5 }, (_, i) => ({ path: `file${i}.md`, @@ -700,6 +752,60 @@ describe('IndexingPipeline', () => { expect(version.indexed_at).not.toBeNull(); }); + it('clones ancestor embeddings into the derived vec store for differential indexing', async () => { + const ancestorVersionId = insertVersion(db, { tag: 'v1.0.0', state: 'indexed' }); + const targetVersionId = insertVersion(db, { tag: 'v1.1.0', state: 'pending' }); + const vecStore = new SqliteVecStore(db); + const docId = crypto.randomUUID(); + const snippetId = crypto.randomUUID(); + const embedding = Float32Array.from([0.2, 0.4, 0.6]); + + db.prepare( + `INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at) + VALUES (?, '/test/repo', ?, 'README.md', 'ancestor-doc', ?)` + ).run(docId, ancestorVersionId, now); + db.prepare( + `INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at) + VALUES (?, ?, '/test/repo', ?, 'info', 'ancestor snippet', ?)` + ).run(snippetId, docId, ancestorVersionId, now); + db.prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, 'local-default', 'test-model', 3, ?, ?)` + ).run(snippetId, Buffer.from(embedding.buffer), now); + vecStore.upsertEmbedding('local-default', snippetId, embedding); + + vi.spyOn(diffStrategy, 'buildDifferentialPlan').mockResolvedValue({ + ancestorTag: 'v1.0.0', + ancestorVersionId, + changedPaths: new Set(), + unchangedPaths: new Set(['README.md']) + }); + + const pipeline = makePipeline({ files: [], totalFiles: 0 }); + const job = makeJob('/test/repo', targetVersionId); + + await pipeline.run(job as never); + + const targetRows = db + .prepare( + `SELECT se.snippet_id, se.embedding + FROM snippet_embeddings se + INNER JOIN snippets s ON s.id = se.snippet_id + WHERE s.version_id = ?` + ) + .all(targetVersionId) as Array<{ snippet_id: string; embedding: Buffer }>; + + expect(targetRows).toHaveLength(1); + const matches = vecStore.queryNearestNeighbors(embedding, { + repositoryId: '/test/repo', + versionId: targetVersionId, + profileId: 'local-default', + limit: 5 + }); + + expect(matches[0]?.snippetId).toBe(targetRows[0].snippet_id); + }); + it('updates repository_versions state to error when pipeline throws and job has versionId', async () => { const versionId = insertVersion(db, { tag: 'v1.0.0', state: 'pending' }); const errorCrawl = vi.fn().mockRejectedValue(new Error('crawl failed')); diff --git a/src/lib/server/pipeline/indexing.pipeline.ts b/src/lib/server/pipeline/indexing.pipeline.ts index 99d0363..9d7ee8b 100644 --- a/src/lib/server/pipeline/indexing.pipeline.ts +++ b/src/lib/server/pipeline/indexing.pipeline.ts @@ -22,6 +22,7 @@ import type { EmbeddingService } from '$lib/server/embeddings/embedding.service. import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js'; import { IndexingJob } from '$lib/server/models/indexing-job.js'; import { Repository, RepositoryEntity } from '$lib/server/models/repository.js'; +import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js'; import { resolveConfig, type ParsedConfig } from '$lib/server/config/config-parser.js'; import { parseFile } from '$lib/server/parser/index.js'; import { computeTrustScore } from '$lib/server/search/trust-score.js'; @@ -63,12 +64,16 @@ function sha256(content: string): string { // --------------------------------------------------------------------------- export class IndexingPipeline { + private readonly sqliteVecStore: SqliteVecStore; + constructor( private readonly db: Database.Database, private readonly githubCrawl: typeof GithubCrawlFn, private readonly localCrawler: LocalCrawler, private readonly embeddingService: EmbeddingService | null - ) {} + ) { + this.sqliteVecStore = new SqliteVecStore(db); + } // ------------------------------------------------------------------------- // Public — run a job end to end @@ -593,6 +598,12 @@ export class IndexingPipeline { emb.embedding, emb.created_at ); + this.sqliteVecStore.upsertEmbeddingBuffer( + emb.profile_id, + newSnippetId, + emb.embedding, + emb.dimensions + ); } } })(); @@ -623,6 +634,8 @@ export class IndexingPipeline { ); this.db.transaction(() => { + this.sqliteVecStore.deleteEmbeddingsForDocumentIds(changedDocIds); + // Delete stale documents (cascade deletes their snippets via FK). if (changedDocIds.length > 0) { const placeholders = changedDocIds.map(() => '?').join(','); diff --git a/src/lib/server/pipeline/job-queue.ts b/src/lib/server/pipeline/job-queue.ts index 0bb84a0..e755855 100644 --- a/src/lib/server/pipeline/job-queue.ts +++ b/src/lib/server/pipeline/job-queue.ts @@ -17,6 +17,54 @@ import type { WorkerPool } from './worker-pool.js'; const JOB_SELECT = `SELECT * FROM indexing_jobs`; +type JobStatusFilter = IndexingJob['status'] | Array; + +function escapeLikePattern(value: string): string { + return value.replaceAll('\\', '\\\\').replaceAll('%', '\\%').replaceAll('_', '\\_'); +} + +function isSpecificRepositoryId(repositoryId: string): boolean { + return repositoryId.split('/').filter(Boolean).length >= 2; +} + +function normalizeStatuses(status?: JobStatusFilter): Array { + if (!status) { + return []; + } + + const statuses = Array.isArray(status) ? status : [status]; + return [...new Set(statuses)]; +} + +function buildJobFilterQuery(options?: { + repositoryId?: string; + status?: JobStatusFilter; +}): { where: string; params: unknown[] } { + const conditions: string[] = []; + const params: unknown[] = []; + + if (options?.repositoryId) { + if (isSpecificRepositoryId(options.repositoryId)) { + conditions.push('repository_id = ?'); + params.push(options.repositoryId); + } else { + conditions.push(`(repository_id = ? OR repository_id LIKE ? ESCAPE '\\')`); + params.push(options.repositoryId, `${escapeLikePattern(options.repositoryId)}/%`); + } + } + + const statuses = normalizeStatuses(options?.status); + if (statuses.length > 0) { + conditions.push(`status IN (${statuses.map(() => '?').join(', ')})`); + params.push(...statuses); + } + + return { + where: conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '', + params + }; +} + export class JobQueue { private workerPool: WorkerPool | null = null; @@ -144,23 +192,11 @@ export class JobQueue { */ listJobs(options?: { repositoryId?: string; - status?: IndexingJob['status']; + status?: JobStatusFilter; limit?: number; }): IndexingJob[] { const limit = Math.min(options?.limit ?? 20, 200); - const conditions: string[] = []; - const params: unknown[] = []; - - if (options?.repositoryId) { - conditions.push('repository_id = ?'); - params.push(options.repositoryId); - } - if (options?.status) { - conditions.push('status = ?'); - params.push(options.status); - } - - const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; + const { where, params } = buildJobFilterQuery(options); const sql = `${JOB_SELECT} ${where} ORDER BY created_at DESC LIMIT ?`; params.push(limit); @@ -194,19 +230,7 @@ export class JobQueue { * Count all jobs matching optional filters. */ countJobs(options?: { repositoryId?: string; status?: IndexingJob['status'] }): number { - const conditions: string[] = []; - const params: unknown[] = []; - - if (options?.repositoryId) { - conditions.push('repository_id = ?'); - params.push(options.repositoryId); - } - if (options?.status) { - conditions.push('status = ?'); - params.push(options.status); - } - - const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND')}` : ''; + const { where, params } = buildJobFilterQuery(options); const sql = `SELECT COUNT(*) as n FROM indexing_jobs ${where}`; const row = this.db.prepare(sql).get(...params); return row?.n ?? 0; diff --git a/src/lib/server/pipeline/progress-broadcaster.test.ts b/src/lib/server/pipeline/progress-broadcaster.test.ts index e1572b6..0760b95 100644 --- a/src/lib/server/pipeline/progress-broadcaster.test.ts +++ b/src/lib/server/pipeline/progress-broadcaster.test.ts @@ -171,4 +171,25 @@ describe('ProgressBroadcaster', () => { reader1.cancel(); reader2.cancel(); }); + + it('broadcastWorkerStatus sends worker-status events to global subscribers', async () => { + const broadcaster = new ProgressBroadcaster(); + const stream = broadcaster.subscribeAll(); + const reader = stream.getReader(); + + broadcaster.broadcastWorkerStatus({ + concurrency: 2, + active: 1, + idle: 1, + workers: [{ index: 0, state: 'running', jobId: 'job-1', repositoryId: '/repo/1', versionId: null }] + }); + + const { value } = await reader.read(); + const text = value as string; + + expect(text).toContain('event: worker-status'); + expect(text).toContain('"active":1'); + + reader.cancel(); + }); }); diff --git a/src/lib/server/pipeline/progress-broadcaster.ts b/src/lib/server/pipeline/progress-broadcaster.ts index f7fa60b..6425c2c 100644 --- a/src/lib/server/pipeline/progress-broadcaster.ts +++ b/src/lib/server/pipeline/progress-broadcaster.ts @@ -10,6 +10,7 @@ export class ProgressBroadcaster { private allSubscribers = new Set>(); private lastEventCache = new Map(); private eventCounters = new Map(); + private globalEventCounter = 0; subscribe(jobId: string): ReadableStream { return new ReadableStream({ @@ -135,6 +136,24 @@ export class ProgressBroadcaster { } } + broadcastWorkerStatus(data: object): void { + this.globalEventCounter += 1; + const event: SSEEvent = { + id: this.globalEventCounter, + event: 'worker-status', + data: JSON.stringify(data) + }; + const sse = this.formatSSE(event); + + for (const controller of this.allSubscribers) { + try { + controller.enqueue(sse); + } catch { + // Controller might be closed or errored + } + } + } + getLastEvent(jobId: string): SSEEvent | null { return this.lastEventCache.get(jobId) ?? null; } diff --git a/src/lib/server/pipeline/startup.ts b/src/lib/server/pipeline/startup.ts index 70b6f02..1b539ca 100644 --- a/src/lib/server/pipeline/startup.ts +++ b/src/lib/server/pipeline/startup.ts @@ -16,7 +16,6 @@ import { LocalCrawler } from '$lib/server/crawler/local.crawler.js'; import { IndexingPipeline } from './indexing.pipeline.js'; import { JobQueue } from './job-queue.js'; import { WorkerPool } from './worker-pool.js'; -import type { ParseWorkerResponse } from './worker-types.js'; import { initBroadcaster } from './progress-broadcaster.js'; import type { ProgressBroadcaster } from './progress-broadcaster.js'; import path from 'node:path'; @@ -90,17 +89,28 @@ export function initializePipeline( if (options?.dbPath) { _broadcaster = initBroadcaster(); + const getRepositoryIdForJob = (jobId: string): string => { + const row = db + .prepare<[string], { repository_id: string }>( + `SELECT repository_id FROM indexing_jobs WHERE id = ?` + ) + .get(jobId); + return row?.repository_id ?? ''; + }; + // Resolve worker script paths relative to this file (build/workers/ directory) const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const workerScript = path.join(__dirname, '../../../build/workers/worker-entry.mjs'); const embedWorkerScript = path.join(__dirname, '../../../build/workers/embed-worker-entry.mjs'); + const writeWorkerScript = path.join(__dirname, '../../../build/workers/write-worker-entry.mjs'); try { _pool = new WorkerPool({ concurrency: options.concurrency ?? 2, workerScript, embedWorkerScript, + writeWorkerScript, dbPath: options.dbPath, onProgress: (jobId, msg) => { // Update DB with progress @@ -112,7 +122,10 @@ export function initializePipeline( // Broadcast progress event if (_broadcaster) { - _broadcaster.broadcast(jobId, '', 'progress', msg); + _broadcaster.broadcast(jobId, getRepositoryIdForJob(jobId), 'job-progress', { + ...msg, + status: 'running' + }); } }, onJobDone: (jobId: string) => { @@ -123,7 +136,10 @@ export function initializePipeline( // Broadcast done event if (_broadcaster) { - _broadcaster.broadcast(jobId, '', 'job-done', { jobId }); + _broadcaster.broadcast(jobId, getRepositoryIdForJob(jobId), 'job-done', { + jobId, + status: 'done' + }); } }, onJobFailed: (jobId: string, error: string) => { @@ -134,7 +150,11 @@ export function initializePipeline( // Broadcast failed event if (_broadcaster) { - _broadcaster.broadcast(jobId, '', 'job-failed', { jobId, error }); + _broadcaster.broadcast(jobId, getRepositoryIdForJob(jobId), 'job-failed', { + jobId, + status: 'failed', + error + }); } }, onEmbedDone: (jobId: string) => { @@ -142,6 +162,9 @@ export function initializePipeline( }, onEmbedFailed: (jobId: string, error: string) => { console.error('[WorkerPool] Embedding failed for job:', jobId, error); + }, + onWorkerStatus: (status) => { + _broadcaster?.broadcastWorkerStatus(status); } }); diff --git a/src/lib/server/pipeline/worker-entry.ts b/src/lib/server/pipeline/worker-entry.ts index 71eec35..56fe473 100644 --- a/src/lib/server/pipeline/worker-entry.ts +++ b/src/lib/server/pipeline/worker-entry.ts @@ -13,6 +13,11 @@ const db = new Database(dbPath); db.pragma('journal_mode = WAL'); db.pragma('foreign_keys = ON'); db.pragma('busy_timeout = 5000'); +db.pragma('synchronous = NORMAL'); +db.pragma('cache_size = -65536'); +db.pragma('temp_store = MEMORY'); +db.pragma('mmap_size = 268435456'); +db.pragma('wal_autocheckpoint = 1000'); const pipeline = new IndexingPipeline(db, githubCrawl, new LocalCrawler(), null); let currentJobId: string | null = null; diff --git a/src/lib/server/pipeline/worker-pool.ts b/src/lib/server/pipeline/worker-pool.ts index e5fb7d2..47e590f 100644 --- a/src/lib/server/pipeline/worker-pool.ts +++ b/src/lib/server/pipeline/worker-pool.ts @@ -1,11 +1,19 @@ import { Worker } from 'node:worker_threads'; import { existsSync } from 'node:fs'; -import type { ParseWorkerRequest, ParseWorkerResponse, EmbedWorkerRequest, EmbedWorkerResponse, WorkerInitData } from './worker-types.js'; +import type { + ParseWorkerRequest, + ParseWorkerResponse, + EmbedWorkerRequest, + EmbedWorkerResponse, + WorkerInitData, + WriteWorkerResponse +} from './worker-types.js'; export interface WorkerPoolOptions { concurrency: number; workerScript: string; embedWorkerScript: string; + writeWorkerScript?: string; dbPath: string; embeddingProfileId?: string; onProgress: (jobId: string, msg: Extract) => void; @@ -13,6 +21,22 @@ export interface WorkerPoolOptions { onJobFailed: (jobId: string, error: string) => void; onEmbedDone: (jobId: string) => void; onEmbedFailed: (jobId: string, error: string) => void; + onWorkerStatus?: (status: WorkerPoolStatus) => void; +} + +export interface WorkerStatusEntry { + index: number; + state: 'idle' | 'running'; + jobId: string | null; + repositoryId: string | null; + versionId: string | null; +} + +export interface WorkerPoolStatus { + concurrency: number; + active: number; + idle: number; + workers: WorkerStatusEntry[]; } interface QueuedJob { @@ -24,6 +48,7 @@ interface QueuedJob { interface RunningJob { jobId: string; repositoryId: string; + versionId?: string | null; } interface EmbedQueuedJob { @@ -36,10 +61,12 @@ export class WorkerPool { private workers: Worker[] = []; private idleWorkers: Worker[] = []; private embedWorker: Worker | null = null; + private writeWorker: Worker | null = null; private embedReady = false; + private writeReady = false; private jobQueue: QueuedJob[] = []; private runningJobs = new Map(); - private runningRepoIds = new Set(); + private runningJobKeys = new Set(); private embedQueue: EmbedQueuedJob[] = []; private options: WorkerPoolOptions; private fallbackMode = false; @@ -66,6 +93,12 @@ export class WorkerPool { if (options.embeddingProfileId && existsSync(options.embedWorkerScript)) { this.embedWorker = this.spawnEmbedWorker(); } + + if (options.writeWorkerScript && existsSync(options.writeWorkerScript)) { + this.writeWorker = this.spawnWriteWorker(options.writeWorkerScript); + } + + this.emitStatusChanged(); } private spawnParseWorker(): Worker { @@ -94,6 +127,22 @@ export class WorkerPool { return worker; } + private spawnWriteWorker(writeWorkerScript: string): Worker { + const worker = new Worker(writeWorkerScript, { + workerData: { + dbPath: this.options.dbPath + } satisfies WorkerInitData + }); + + worker.on('message', (msg: WriteWorkerResponse) => this.onWriteWorkerMessage(msg)); + worker.on('exit', () => { + this.writeReady = false; + this.writeWorker = null; + }); + + return worker; + } + public enqueue(jobId: string, repositoryId: string, versionId?: string | null): void { if (this.shuttingDown) { console.warn('WorkerPool is shutting down, ignoring enqueue request'); @@ -109,10 +158,18 @@ export class WorkerPool { this.dispatch(); } + private static jobKey(repositoryId: string, versionId?: string | null): string { + return `${repositoryId}:${versionId ?? ''}`; + } + private dispatch(): void { + let statusChanged = false; + while (this.idleWorkers.length > 0 && this.jobQueue.length > 0) { - // Find first job whose repositoryId is not currently running - const jobIdx = this.jobQueue.findIndex((j) => !this.runningRepoIds.has(j.repositoryId)); + // Find first job whose (repositoryId, versionId) compound key is not currently running + const jobIdx = this.jobQueue.findIndex( + (j) => !this.runningJobKeys.has(WorkerPool.jobKey(j.repositoryId, j.versionId)) + ); if (jobIdx === -1) { // No eligible job found (all repos have running jobs) @@ -122,12 +179,17 @@ export class WorkerPool { const job = this.jobQueue.splice(jobIdx, 1)[0]; const worker = this.idleWorkers.pop()!; - this.runningJobs.set(worker, { jobId: job.jobId, repositoryId: job.repositoryId }); - this.runningRepoIds.add(job.repositoryId); + this.runningJobs.set(worker, { jobId: job.jobId, repositoryId: job.repositoryId, versionId: job.versionId }); + this.runningJobKeys.add(WorkerPool.jobKey(job.repositoryId, job.versionId)); + statusChanged = true; const msg: ParseWorkerRequest = { type: 'run', jobId: job.jobId }; worker.postMessage(msg); } + + if (statusChanged) { + this.emitStatusChanged(); + } } private onWorkerMessage(worker: Worker, msg: ParseWorkerResponse): void { @@ -137,15 +199,20 @@ export class WorkerPool { const runningJob = this.runningJobs.get(worker); if (runningJob) { this.runningJobs.delete(worker); - this.runningRepoIds.delete(runningJob.repositoryId); + this.runningJobKeys.delete(WorkerPool.jobKey(runningJob.repositoryId, runningJob.versionId)); } this.idleWorkers.push(worker); this.options.onJobDone(msg.jobId); + this.emitStatusChanged(); // If embedding configured, enqueue embed request if (this.embedWorker && this.options.embeddingProfileId) { - const runningJobData = runningJob || { jobId: msg.jobId, repositoryId: '' }; - this.enqueueEmbed(msg.jobId, runningJobData.repositoryId, null); + const runningJobData = runningJob || { jobId: msg.jobId, repositoryId: '', versionId: null }; + this.enqueueEmbed( + msg.jobId, + runningJobData.repositoryId, + runningJobData.versionId ?? null + ); } this.dispatch(); @@ -153,10 +220,11 @@ export class WorkerPool { const runningJob = this.runningJobs.get(worker); if (runningJob) { this.runningJobs.delete(worker); - this.runningRepoIds.delete(runningJob.repositoryId); + this.runningJobKeys.delete(WorkerPool.jobKey(runningJob.repositoryId, runningJob.versionId)); } this.idleWorkers.push(worker); this.options.onJobFailed(msg.jobId, msg.error); + this.emitStatusChanged(); this.dispatch(); } } @@ -176,13 +244,15 @@ export class WorkerPool { const runningJob = this.runningJobs.get(worker); if (runningJob && code !== 0) { this.runningJobs.delete(worker); - this.runningRepoIds.delete(runningJob.repositoryId); + this.runningJobKeys.delete(WorkerPool.jobKey(runningJob.repositoryId, runningJob.versionId)); this.options.onJobFailed(runningJob.jobId, `Worker crashed with code ${code}`); } else if (runningJob) { this.runningJobs.delete(worker); - this.runningRepoIds.delete(runningJob.repositoryId); + this.runningJobKeys.delete(WorkerPool.jobKey(runningJob.repositoryId, runningJob.versionId)); } + this.emitStatusChanged(); + // Remove from workers array const workerIdx = this.workers.indexOf(worker); if (workerIdx !== -1) { @@ -212,6 +282,17 @@ export class WorkerPool { } } + private onWriteWorkerMessage(msg: WriteWorkerResponse): void { + if (msg.type === 'ready') { + this.writeReady = true; + return; + } + + if (msg.type === 'write_error') { + console.error('[WorkerPool] Write worker failed for job:', msg.jobId, msg.error); + } + } + private processEmbedQueue(): void { if (!this.embedWorker || !this.embedReady) { return; @@ -250,6 +331,7 @@ export class WorkerPool { } public setMaxConcurrency(n: number): void { + this.options.concurrency = n; const current = this.workers.length; if (n > current) { @@ -274,6 +356,8 @@ export class WorkerPool { } } } + + this.emitStatusChanged(); } public async shutdown(): Promise { @@ -300,6 +384,14 @@ export class WorkerPool { } } + if (this.writeWorker) { + try { + this.writeWorker.postMessage({ type: 'shutdown' }); + } catch { + // Worker might already be exited + } + } + // Wait for workers to exit with timeout const timeout = 5000; const startTime = Date.now(); @@ -329,9 +421,41 @@ export class WorkerPool { } } + if (this.writeWorker) { + try { + this.writeWorker.terminate(); + } catch { + // Already terminated + } + } + this.workers = []; this.idleWorkers = []; this.embedWorker = null; + this.writeWorker = null; + this.emitStatusChanged(); + } + + public getStatus(): WorkerPoolStatus { + return { + concurrency: this.options.concurrency, + active: this.runningJobs.size, + idle: this.idleWorkers.length, + workers: this.workers.map((worker, index) => { + const runningJob = this.runningJobs.get(worker); + return { + index, + state: runningJob ? 'running' : 'idle', + jobId: runningJob?.jobId ?? null, + repositoryId: runningJob?.repositoryId ?? null, + versionId: runningJob?.versionId ?? null + }; + }) + }; + } + + private emitStatusChanged(): void { + this.options.onWorkerStatus?.(this.getStatus()); } public get isFallbackMode(): boolean { diff --git a/src/lib/server/pipeline/worker-types.ts b/src/lib/server/pipeline/worker-types.ts index 3696027..af74b7a 100644 --- a/src/lib/server/pipeline/worker-types.ts +++ b/src/lib/server/pipeline/worker-types.ts @@ -19,7 +19,61 @@ export type EmbedWorkerResponse = | { type: 'embed-done'; jobId: string } | { type: 'embed-failed'; jobId: string; error: string }; +export type WriteWorkerRequest = WriteRequest | { type: 'shutdown' }; + +export type WriteWorkerResponse = + | { type: 'ready' } + | WriteAck + | WriteError; + export interface WorkerInitData { dbPath: string; embeddingProfileId?: string; } + +// Write worker message types (Phase 6) +export interface SerializedDocument { + id: string; + repositoryId: string; + versionId: string | null; + filePath: string; + title: string | null; + language: string | null; + tokenCount: number; + checksum: string; + indexedAt: number; +} + +export interface SerializedSnippet { + id: string; + documentId: string; + repositoryId: string; + versionId: string | null; + type: 'code' | 'info'; + title: string | null; + content: string; + language: string | null; + breadcrumb: string | null; + tokenCount: number; + createdAt: number; +} + +export type WriteRequest = { + type: 'write'; + jobId: string; + documents: SerializedDocument[]; + snippets: SerializedSnippet[]; +}; + +export type WriteAck = { + type: 'write_ack'; + jobId: string; + documentCount: number; + snippetCount: number; +}; + +export type WriteError = { + type: 'write_error'; + jobId: string; + error: string; +}; diff --git a/src/lib/server/pipeline/write-worker-entry.ts b/src/lib/server/pipeline/write-worker-entry.ts new file mode 100644 index 0000000..4d623f1 --- /dev/null +++ b/src/lib/server/pipeline/write-worker-entry.ts @@ -0,0 +1,93 @@ +import { workerData, parentPort } from 'node:worker_threads'; +import Database from 'better-sqlite3'; +import type { + SerializedDocument, + SerializedSnippet, + WorkerInitData, + WriteWorkerRequest, + WriteWorkerResponse +} from './worker-types.js'; + +const { dbPath } = workerData as WorkerInitData; +const db = new Database(dbPath); +db.pragma('journal_mode = WAL'); +db.pragma('foreign_keys = ON'); +db.pragma('busy_timeout = 5000'); +db.pragma('synchronous = NORMAL'); +db.pragma('cache_size = -65536'); +db.pragma('temp_store = MEMORY'); +db.pragma('mmap_size = 268435456'); +db.pragma('wal_autocheckpoint = 1000'); + +const insertDocument = db.prepare( + `INSERT OR REPLACE INTO documents + (id, repository_id, version_id, file_path, title, language, token_count, checksum, indexed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` +); + +const insertSnippet = db.prepare( + `INSERT OR REPLACE INTO snippets + (id, document_id, repository_id, version_id, type, title, content, language, breadcrumb, token_count, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` +); + +const writeBatch = db.transaction((documents: SerializedDocument[], snippets: SerializedSnippet[]) => { + for (const document of documents) { + insertDocument.run( + document.id, + document.repositoryId, + document.versionId, + document.filePath, + document.title, + document.language, + document.tokenCount, + document.checksum, + document.indexedAt + ); + } + + for (const snippet of snippets) { + insertSnippet.run( + snippet.id, + snippet.documentId, + snippet.repositoryId, + snippet.versionId, + snippet.type, + snippet.title, + snippet.content, + snippet.language, + snippet.breadcrumb, + snippet.tokenCount, + snippet.createdAt + ); + } +}); + +parentPort?.postMessage({ type: 'ready' } satisfies WriteWorkerResponse); + +parentPort?.on('message', (msg: WriteWorkerRequest) => { + if (msg.type === 'shutdown') { + db.close(); + process.exit(0); + } + + if (msg.type !== 'write') { + return; + } + + try { + writeBatch(msg.documents, msg.snippets); + parentPort?.postMessage({ + type: 'write_ack', + jobId: msg.jobId, + documentCount: msg.documents.length, + snippetCount: msg.snippets.length + } satisfies WriteWorkerResponse); + } catch (error) { + parentPort?.postMessage({ + type: 'write_error', + jobId: msg.jobId, + error: error instanceof Error ? error.message : String(error) + } satisfies WriteWorkerResponse); + } +}); \ No newline at end of file diff --git a/src/lib/server/search/hybrid.search.service.test.ts b/src/lib/server/search/hybrid.search.service.test.ts index db194fe..c8f42b3 100644 --- a/src/lib/server/search/hybrid.search.service.test.ts +++ b/src/lib/server/search/hybrid.search.service.test.ts @@ -15,6 +15,8 @@ import { HybridSearchService } from './hybrid.search.service.js'; import { VectorSearch, cosineSimilarity } from './vector.search.js'; import { reciprocalRankFusion } from './rrf.js'; import type { EmbeddingProvider, EmbeddingVector } from '../embeddings/provider.js'; +import { loadSqliteVec } from '../db/sqlite-vec.js'; +import { SqliteVecStore } from './sqlite-vec.store.js'; // --------------------------------------------------------------------------- // In-memory DB factory @@ -23,6 +25,7 @@ import type { EmbeddingProvider, EmbeddingVector } from '../embeddings/provider. function createTestDb(): Database.Database { const client = new Database(':memory:'); client.pragma('foreign_keys = ON'); + loadSqliteVec(client); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); @@ -30,7 +33,11 @@ function createTestDb(): Database.Database { const migrations = [ '0000_large_master_chief.sql', '0001_quick_nighthawk.sql', - '0002_silky_stellaris.sql' + '0002_silky_stellaris.sql', + '0003_multiversion_config.sql', + '0004_complete_sentry.sql', + '0005_fix_stage_defaults.sql', + '0006_yielding_centennial.sql' ]; for (const migrationFile of migrations) { const migrationSql = readFileSync(join(migrationsFolder, migrationFile), 'utf-8'); @@ -121,6 +128,7 @@ function seedEmbedding( VALUES (?, ?, ?, ?, ?, ?)` ) .run(snippetId, profileId, model, values.length, Buffer.from(f32.buffer), NOW_S); + new SqliteVecStore(client).upsertEmbedding(profileId, snippetId, f32); } // --------------------------------------------------------------------------- @@ -368,6 +376,42 @@ describe('VectorSearch', () => { const results = vs.vectorSearch(new Float32Array([-0.5, 0.5]), { repositoryId: repoId }); expect(results[0].score).toBeCloseTo(1.0, 4); }); + + it('filters by profileId using per-profile vec tables', () => { + client + .prepare( + `INSERT INTO embedding_profiles (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run('secondary-profile', 'local-transformers', 'Secondary', 1, 0, 'test-model', 2, '{}', NOW_S, NOW_S); + + const defaultSnippet = seedSnippet(client, { + repositoryId: repoId, + documentId: docId, + content: 'default profile snippet' + }); + const secondarySnippet = seedSnippet(client, { + repositoryId: repoId, + documentId: docId, + content: 'secondary profile snippet' + }); + + seedEmbedding(client, defaultSnippet, [1, 0], 'local-default'); + seedEmbedding(client, secondarySnippet, [1, 0], 'secondary-profile'); + + const vs = new VectorSearch(client); + const defaultResults = vs.vectorSearch(new Float32Array([1, 0]), { + repositoryId: repoId, + profileId: 'local-default' + }); + const secondaryResults = vs.vectorSearch(new Float32Array([1, 0]), { + repositoryId: repoId, + profileId: 'secondary-profile' + }); + + expect(defaultResults.map((result) => result.snippetId)).toEqual([defaultSnippet]); + expect(secondaryResults.map((result) => result.snippetId)).toEqual([secondarySnippet]); + }); }); // =========================================================================== diff --git a/src/lib/server/search/hybrid.search.service.ts b/src/lib/server/search/hybrid.search.service.ts index 79c7c05..f4dd9c6 100644 --- a/src/lib/server/search/hybrid.search.service.ts +++ b/src/lib/server/search/hybrid.search.service.ts @@ -148,7 +148,12 @@ export class HybridSearchService { const topIds = vectorResults.slice(0, limit).map((r) => r.snippetId); return { - results: this.fetchSnippetsByIds(topIds, options.repositoryId, options.type), + results: this.fetchSnippetsByIds( + topIds, + options.repositoryId, + options.versionId, + options.type + ), searchModeUsed: 'semantic' }; } @@ -194,7 +199,12 @@ export class HybridSearchService { const topIds = vectorResults.slice(0, limit).map((r) => r.snippetId); return { - results: this.fetchSnippetsByIds(topIds, options.repositoryId, options.type), + results: this.fetchSnippetsByIds( + topIds, + options.repositoryId, + options.versionId, + options.type + ), searchModeUsed: 'keyword_fallback' }; } @@ -220,7 +230,12 @@ export class HybridSearchService { if (alpha === 1) { const topIds = vectorResults.slice(0, limit).map((r) => r.snippetId); return { - results: this.fetchSnippetsByIds(topIds, options.repositoryId, options.type), + results: this.fetchSnippetsByIds( + topIds, + options.repositoryId, + options.versionId, + options.type + ), searchModeUsed: 'semantic' }; } @@ -234,7 +249,12 @@ export class HybridSearchService { const topIds = fused.slice(0, limit).map((r) => r.id); return { - results: this.fetchSnippetsByIds(topIds, options.repositoryId, options.type), + results: this.fetchSnippetsByIds( + topIds, + options.repositoryId, + options.versionId, + options.type + ), searchModeUsed: 'hybrid' }; } @@ -253,13 +273,19 @@ export class HybridSearchService { private fetchSnippetsByIds( ids: string[], repositoryId: string, + versionId?: string, type?: 'code' | 'info' ): SnippetSearchResult[] { if (ids.length === 0) return []; const placeholders = ids.map(() => '?').join(', '); const params: unknown[] = [...ids, repositoryId]; + let versionClause = ''; let typeClause = ''; + if (versionId !== undefined) { + versionClause = ' AND s.version_id = ?'; + params.push(versionId); + } if (type !== undefined) { typeClause = ' AND s.type = ?'; params.push(type); @@ -276,7 +302,7 @@ export class HybridSearchService { FROM snippets s JOIN repositories r ON r.id = s.repository_id WHERE s.id IN (${placeholders}) - AND s.repository_id = ?${typeClause}` + AND s.repository_id = ?${versionClause}${typeClause}` ) .all(...params) as RawSnippetById[]; diff --git a/src/lib/server/search/sqlite-vec.store.ts b/src/lib/server/search/sqlite-vec.store.ts new file mode 100644 index 0000000..cf97111 --- /dev/null +++ b/src/lib/server/search/sqlite-vec.store.ts @@ -0,0 +1,394 @@ +import type Database from 'better-sqlite3'; +import { + loadSqliteVec, + quoteSqliteIdentifier, + sqliteVecRowidTableName, + sqliteVecTableName +} from '$lib/server/db/sqlite-vec.js'; + +export interface SqliteVecQueryOptions { + repositoryId: string; + versionId?: string; + profileId?: string; + limit?: number; +} + +export interface SqliteVecQueryResult { + snippetId: string; + score: number; + distance: number; +} + +interface ProfileDimensionsRow { + dimensions: number; +} + +interface StoredDimensionsRow { + count: number; + min_dimensions: number | null; + max_dimensions: number | null; +} + +interface SnippetRowidRow { + rowid: number; +} + +interface RawKnnRow { + snippet_id: string; + distance: number; +} + +interface CanonicalEmbeddingRow { + snippet_id: string; + embedding: Buffer; +} + +interface StoredEmbeddingRef { + profile_id: string; + snippet_id: string; +} + +interface ProfileStoreTables { + vectorTableName: string; + rowidTableName: string; + quotedVectorTableName: string; + quotedRowidTableName: string; + dimensions: number; +} + +function toEmbeddingBuffer(values: Float32Array): Buffer { + return Buffer.from(values.buffer, values.byteOffset, values.byteLength); +} + +function distanceToScore(distance: number): number { + return 1 / (1 + distance); +} + +export class SqliteVecStore { + constructor(private readonly db: Database.Database) {} + + ensureProfileStore(profileId: string, preferredDimensions?: number): number { + const tables = this.getProfileStoreTables(profileId, preferredDimensions); + + this.db.exec(` + CREATE TABLE IF NOT EXISTS ${tables.quotedRowidTableName} ( + rowid INTEGER PRIMARY KEY, + snippet_id TEXT NOT NULL UNIQUE REFERENCES snippets(id) ON DELETE CASCADE + ); + `); + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS ${tables.quotedVectorTableName} + USING vec0(embedding float[${tables.dimensions}]); + `); + + return tables.dimensions; + } + + upsertEmbedding(profileId: string, snippetId: string, embedding: Float32Array): void { + const tables = this.getProfileStoreTables(profileId, embedding.length); + + this.ensureProfileStore(profileId, tables.dimensions); + + const existingRow = this.db + .prepare<[string], SnippetRowidRow>( + `SELECT rowid FROM ${tables.quotedRowidTableName} WHERE snippet_id = ?` + ) + .get(snippetId); + + const embeddingBuffer = toEmbeddingBuffer(embedding); + if (existingRow) { + this.db + .prepare<[Buffer, number]>( + `UPDATE ${tables.quotedVectorTableName} SET embedding = ? WHERE rowid = ?` + ) + .run(embeddingBuffer, existingRow.rowid); + return; + } + + const insertResult = this.db + .prepare<[Buffer]>(`INSERT INTO ${tables.quotedVectorTableName} (embedding) VALUES (?)`) + .run(embeddingBuffer); + this.db + .prepare<[number, string]>( + `INSERT INTO ${tables.quotedRowidTableName} (rowid, snippet_id) VALUES (?, ?)` + ) + .run(Number(insertResult.lastInsertRowid), snippetId); + } + + upsertEmbeddingBuffer( + profileId: string, + snippetId: string, + embedding: Buffer, + dimensions?: number + ): void { + const vector = new Float32Array( + embedding.buffer, + embedding.byteOffset, + dimensions ?? Math.floor(embedding.byteLength / Float32Array.BYTES_PER_ELEMENT) + ); + this.upsertEmbedding(profileId, snippetId, vector); + } + + deleteEmbedding(profileId: string, snippetId: string): void { + const tables = this.getProfileStoreTables(profileId); + this.ensureProfileStore(profileId); + + const existingRow = this.db + .prepare<[string], SnippetRowidRow>( + `SELECT rowid FROM ${tables.quotedRowidTableName} WHERE snippet_id = ?` + ) + .get(snippetId); + + if (!existingRow) { + return; + } + + this.db + .prepare<[number]>(`DELETE FROM ${tables.quotedVectorTableName} WHERE rowid = ?`) + .run(existingRow.rowid); + this.db + .prepare<[string]>(`DELETE FROM ${tables.quotedRowidTableName} WHERE snippet_id = ?`) + .run(snippetId); + } + + deleteEmbeddingsForDocumentIds(documentIds: string[]): void { + if (documentIds.length === 0) { + return; + } + + const placeholders = documentIds.map(() => '?').join(', '); + const rows = this.db + .prepare( + `SELECT DISTINCT se.profile_id, se.snippet_id + FROM snippet_embeddings se + INNER JOIN snippets s ON s.id = se.snippet_id + WHERE s.document_id IN (${placeholders})` + ) + .all(...documentIds); + + this.deleteEmbeddingRefs(rows); + } + + deleteEmbeddingsForRepository(repositoryId: string): void { + const rows = this.db + .prepare<[string], StoredEmbeddingRef>( + `SELECT DISTINCT se.profile_id, se.snippet_id + FROM snippet_embeddings se + INNER JOIN snippets s ON s.id = se.snippet_id + WHERE s.repository_id = ?` + ) + .all(repositoryId); + + this.deleteEmbeddingRefs(rows); + } + + deleteEmbeddingsForVersion(repositoryId: string, versionId: string): void { + const rows = this.db + .prepare<[string, string], StoredEmbeddingRef>( + `SELECT DISTINCT se.profile_id, se.snippet_id + FROM snippet_embeddings se + INNER JOIN snippets s ON s.id = se.snippet_id + WHERE s.repository_id = ? AND s.version_id = ?` + ) + .all(repositoryId, versionId); + + this.deleteEmbeddingRefs(rows); + } + + queryNearestNeighbors( + queryEmbedding: Float32Array, + options: SqliteVecQueryOptions + ): SqliteVecQueryResult[] { + const { repositoryId, versionId, profileId = 'local-default', limit = 50 } = options; + if (limit <= 0) { + return []; + } + + const tables = this.getProfileStoreTables(profileId, queryEmbedding.length); + + this.ensureProfileStore(profileId, tables.dimensions); + const totalRows = this.synchronizeProfileStore(profileId, tables); + if (totalRows === 0) { + return []; + } + + let sql = ` + SELECT rowids.snippet_id, vec.distance + FROM ${tables.quotedVectorTableName} vec + JOIN ${tables.quotedRowidTableName} rowids ON rowids.rowid = vec.rowid + JOIN snippets s ON s.id = rowids.snippet_id + WHERE vec.embedding MATCH ? + AND vec.k = ? + AND s.repository_id = ? + `; + const params: unknown[] = [toEmbeddingBuffer(queryEmbedding), totalRows, repositoryId]; + + if (versionId !== undefined) { + sql += ' AND s.version_id = ?'; + params.push(versionId); + } + + sql += ' ORDER BY vec.distance ASC LIMIT ?'; + params.push(limit); + + const rows = this.db.prepare(sql).all(...params); + return rows.map((row) => ({ + snippetId: row.snippet_id, + score: distanceToScore(row.distance), + distance: row.distance + })); + } + + private synchronizeProfileStore(profileId: string, tables: ProfileStoreTables): number { + this.db + .prepare<[string, number]>( + `DELETE FROM ${tables.quotedRowidTableName} + WHERE rowid IN ( + SELECT rowids.rowid + FROM ${tables.quotedRowidTableName} rowids + LEFT JOIN snippet_embeddings se + ON se.snippet_id = rowids.snippet_id + AND se.profile_id = ? + AND se.dimensions = ? + LEFT JOIN ${tables.quotedVectorTableName} vec ON vec.rowid = rowids.rowid + WHERE se.snippet_id IS NULL OR vec.rowid IS NULL + )` + ) + .run(profileId, tables.dimensions); + + this.db + .prepare( + `DELETE FROM ${tables.quotedVectorTableName} + WHERE rowid NOT IN (SELECT rowid FROM ${tables.quotedRowidTableName})` + ) + .run(); + + const missingRows = this.db + .prepare<[string, number], CanonicalEmbeddingRow>( + `SELECT se.snippet_id, se.embedding + FROM snippet_embeddings se + LEFT JOIN ${tables.quotedRowidTableName} rowids ON rowids.snippet_id = se.snippet_id + WHERE se.profile_id = ? + AND se.dimensions = ? + AND rowids.snippet_id IS NULL` + ) + .all(profileId, tables.dimensions); + + if (missingRows.length > 0) { + const backfill = this.db.transaction((rows: CanonicalEmbeddingRow[]) => { + for (const row of rows) { + this.upsertEmbedding( + profileId, + row.snippet_id, + new Float32Array( + row.embedding.buffer, + row.embedding.byteOffset, + tables.dimensions + ) + ); + } + }); + backfill(missingRows); + } + + return ( + this.db + .prepare<[], { count: number }>( + `SELECT COUNT(*) AS count + FROM ${tables.quotedVectorTableName} vec + JOIN ${tables.quotedRowidTableName} rowids ON rowids.rowid = vec.rowid` + ) + .get()?.count ?? 0 + ); + } + + private deleteEmbeddingRefs(rows: StoredEmbeddingRef[]): void { + if (rows.length === 0) { + return; + } + + const removeRows = this.db.transaction((refs: StoredEmbeddingRef[]) => { + for (const ref of refs) { + this.deleteEmbedding(ref.profile_id, ref.snippet_id); + } + }); + + removeRows(rows); + } + + private getProfileStoreTables( + profileId: string, + preferredDimensions?: number + ): ProfileStoreTables { + loadSqliteVec(this.db); + + const dimensionsRow = this.db + .prepare<[string], ProfileDimensionsRow>( + 'SELECT dimensions FROM embedding_profiles WHERE id = ?' + ) + .get(profileId); + if (!dimensionsRow) { + throw new Error(`Embedding profile not found: ${profileId}`); + } + + const storedDimensions = this.db + .prepare<[string], StoredDimensionsRow>( + `SELECT + COUNT(*) AS count, + MIN(dimensions) AS min_dimensions, + MAX(dimensions) AS max_dimensions + FROM snippet_embeddings + WHERE profile_id = ?` + ) + .get(profileId); + + const effectiveDimensions = this.resolveDimensions( + profileId, + dimensionsRow.dimensions, + storedDimensions, + preferredDimensions + ); + + const vectorTableName = sqliteVecTableName(profileId); + const rowidTableName = sqliteVecRowidTableName(profileId); + + return { + vectorTableName, + rowidTableName, + quotedVectorTableName: quoteSqliteIdentifier(vectorTableName), + quotedRowidTableName: quoteSqliteIdentifier(rowidTableName), + dimensions: effectiveDimensions + }; + } + + private resolveDimensions( + profileId: string, + profileDimensions: number, + storedDimensions: StoredDimensionsRow | undefined, + preferredDimensions?: number + ): number { + if (storedDimensions && storedDimensions.count > 0) { + if (storedDimensions.min_dimensions !== storedDimensions.max_dimensions) { + throw new Error(`Stored embedding dimensions are inconsistent for profile ${profileId}`); + } + + const canonicalDimensions = storedDimensions.min_dimensions; + if (canonicalDimensions === null) { + throw new Error(`Stored embedding dimensions are missing for profile ${profileId}`); + } + + if ( + preferredDimensions !== undefined && + preferredDimensions !== canonicalDimensions + ) { + throw new Error( + `Embedding dimension mismatch for profile ${profileId}: expected ${canonicalDimensions}, received ${preferredDimensions}` + ); + } + + return canonicalDimensions; + } + + return preferredDimensions ?? profileDimensions; + } +} \ No newline at end of file diff --git a/src/lib/server/search/vector.search.ts b/src/lib/server/search/vector.search.ts index b57c48f..4d85576 100644 --- a/src/lib/server/search/vector.search.ts +++ b/src/lib/server/search/vector.search.ts @@ -1,16 +1,12 @@ /** * Vector similarity search over stored snippet embeddings. * - * SQLite does not natively support vector operations, so cosine similarity is - * computed in JavaScript after loading candidate embeddings from the - * snippet_embeddings table. - * - * Performance note: For repositories with > 50k snippets, pre-filtering by - * FTS5 candidates before computing cosine similarity is recommended. For v1, - * in-memory computation is acceptable. + * Uses sqlite-vec vector_top_k() for ANN search instead of in-memory cosine + * similarity computation over all embeddings. */ import type Database from 'better-sqlite3'; +import { SqliteVecStore } from './sqlite-vec.store.js'; // --------------------------------------------------------------------------- // Types @@ -28,12 +24,6 @@ export interface VectorSearchOptions { limit?: number; } -/** Raw DB row from snippet_embeddings joined with snippets. */ -interface RawEmbeddingRow { - snippet_id: string; - embedding: Buffer; -} - // --------------------------------------------------------------------------- // Math helpers // --------------------------------------------------------------------------- @@ -69,46 +59,26 @@ export function cosineSimilarity(a: Float32Array, b: Float32Array): number { // --------------------------------------------------------------------------- export class VectorSearch { - constructor(private readonly db: Database.Database) {} + private readonly sqliteVecStore: SqliteVecStore; + + constructor(private readonly db: Database.Database) { + this.sqliteVecStore = new SqliteVecStore(db); + } /** * Search stored embeddings by cosine similarity to the query embedding. * + * Uses in-memory cosine similarity computation. The vec_embedding column + * stores raw Float32 bytes for forward compatibility with vector-capable + * libSQL builds; scoring is performed in JS using the same bytes. + * * @param queryEmbedding - The embedded representation of the search query. * @param options - Search options including repositoryId, optional versionId, profileId, and limit. * @returns Results sorted by descending cosine similarity score. */ vectorSearch(queryEmbedding: Float32Array, options: VectorSearchOptions): VectorSearchResult[] { - const { repositoryId, versionId, profileId = 'local-default', limit = 50 } = options; - - let sql = ` - SELECT se.snippet_id, se.embedding - FROM snippet_embeddings se - JOIN snippets s ON s.id = se.snippet_id - WHERE s.repository_id = ? - AND se.profile_id = ? - `; - const params: unknown[] = [repositoryId, profileId]; - - if (versionId) { - sql += ' AND s.version_id = ?'; - params.push(versionId); - } - - const rows = this.db.prepare(sql).all(...params); - - const scored: VectorSearchResult[] = rows.map((row) => { - const embedding = new Float32Array( - row.embedding.buffer, - row.embedding.byteOffset, - row.embedding.byteLength / 4 - ); - return { - snippetId: row.snippet_id, - score: cosineSimilarity(queryEmbedding, embedding) - }; - }); - - return scored.sort((a, b) => b.score - a.score).slice(0, limit); + return this.sqliteVecStore + .queryNearestNeighbors(queryEmbedding, options) + .map((result) => ({ snippetId: result.snippetId, score: result.score })); } } diff --git a/src/lib/server/services/repository.service.test.ts b/src/lib/server/services/repository.service.test.ts index c98cb7c..ebe04e9 100644 --- a/src/lib/server/services/repository.service.test.ts +++ b/src/lib/server/services/repository.service.test.ts @@ -11,6 +11,8 @@ import Database from 'better-sqlite3'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { RepositoryService } from './repository.service'; +import { loadSqliteVec, sqliteVecRowidTableName, sqliteVecTableName } from '$lib/server/db/sqlite-vec.js'; +import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js'; import { AlreadyExistsError, InvalidInputError, @@ -25,6 +27,7 @@ import { function createTestDb(): Database.Database { const client = new Database(':memory:'); client.pragma('foreign_keys = ON'); + loadSqliteVec(client); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); @@ -33,7 +36,9 @@ function createTestDb(): Database.Database { '0001_quick_nighthawk.sql', '0002_silky_stellaris.sql', '0003_multiversion_config.sql', - '0004_complete_sentry.sql' + '0004_complete_sentry.sql', + '0005_fix_stage_defaults.sql', + '0006_yielding_centennial.sql' ]) { const statements = readFileSync(join(migrationsFolder, migration), 'utf-8') .split('--> statement-breakpoint') @@ -331,6 +336,41 @@ describe('RepositoryService.remove()', () => { it('throws NotFoundError when the repository does not exist', () => { expect(() => service.remove('/not/found')).toThrow(NotFoundError); }); + + it('removes derived vec rows before the repository cascade deletes snippets', () => { + const docId = crypto.randomUUID(); + const snippetId = crypto.randomUUID(); + const embedding = Float32Array.from([1, 0, 0]); + const vecStore = new SqliteVecStore((service as unknown as { db: Database.Database }).db); + const db = (service as unknown as { db: Database.Database }).db; + const now = Math.floor(Date.now() / 1000); + + db.prepare( + `INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at) + VALUES (?, '/facebook/react', NULL, 'README.md', 'repo-doc', ?)` + ).run(docId, now); + db.prepare( + `INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at) + VALUES (?, ?, '/facebook/react', NULL, 'info', 'repo snippet', ?)` + ).run(snippetId, docId, now); + db.prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, 'local-default', 'test-model', 3, ?, ?)` + ).run(snippetId, Buffer.from(embedding.buffer), now); + vecStore.upsertEmbedding('local-default', snippetId, embedding); + + service.remove('/facebook/react'); + + const vecTable = sqliteVecTableName('local-default'); + const rowidTable = sqliteVecRowidTableName('local-default'); + const vecCount = db.prepare(`SELECT COUNT(*) as n FROM "${vecTable}"`).get() as { n: number }; + const rowidCount = db.prepare(`SELECT COUNT(*) as n FROM "${rowidTable}"`).get() as { + n: number; + }; + + expect(vecCount.n).toBe(0); + expect(rowidCount.n).toBe(0); + }); }); // --------------------------------------------------------------------------- diff --git a/src/lib/server/services/repository.service.ts b/src/lib/server/services/repository.service.ts index 6ce0c53..f2fa174 100644 --- a/src/lib/server/services/repository.service.ts +++ b/src/lib/server/services/repository.service.ts @@ -8,6 +8,7 @@ import { RepositoryMapper } from '$lib/server/mappers/repository.mapper.js'; import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js'; import { Repository, RepositoryEntity } from '$lib/server/models/repository.js'; import { IndexingJob, IndexingJobEntity } from '$lib/server/models/indexing-job.js'; +import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js'; import { resolveGitHubId, resolveLocalId } from '$lib/server/utils/id-resolver'; import { AlreadyExistsError, @@ -230,7 +231,11 @@ export class RepositoryService { const existing = this.get(id); if (!existing) throw new NotFoundError(`Repository ${id} not found`); - this.db.prepare(`DELETE FROM repositories WHERE id = ?`).run(id); + const sqliteVecStore = new SqliteVecStore(this.db); + this.db.transaction(() => { + sqliteVecStore.deleteEmbeddingsForRepository(id); + this.db.prepare(`DELETE FROM repositories WHERE id = ?`).run(id); + })(); } /** diff --git a/src/lib/server/services/version.service.test.ts b/src/lib/server/services/version.service.test.ts index 6def4fe..33bd881 100644 --- a/src/lib/server/services/version.service.test.ts +++ b/src/lib/server/services/version.service.test.ts @@ -10,6 +10,8 @@ import { describe, it, expect } from 'vitest'; import Database from 'better-sqlite3'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; +import { loadSqliteVec, sqliteVecRowidTableName, sqliteVecTableName } from '$lib/server/db/sqlite-vec.js'; +import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js'; import { VersionService } from './version.service'; import { RepositoryService } from './repository.service'; import { AlreadyExistsError, NotFoundError } from '$lib/server/utils/validation'; @@ -21,31 +23,27 @@ import { AlreadyExistsError, NotFoundError } from '$lib/server/utils/validation' function createTestDb(): Database.Database { const client = new Database(':memory:'); client.pragma('foreign_keys = ON'); + loadSqliteVec(client); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); - // Apply all migration files in order - const migration0 = readFileSync(join(migrationsFolder, '0000_large_master_chief.sql'), 'utf-8'); - const migration1 = readFileSync(join(migrationsFolder, '0001_quick_nighthawk.sql'), 'utf-8'); + for (const migration of [ + '0000_large_master_chief.sql', + '0001_quick_nighthawk.sql', + '0002_silky_stellaris.sql', + '0003_multiversion_config.sql', + '0004_complete_sentry.sql', + '0005_fix_stage_defaults.sql', + '0006_yielding_centennial.sql' + ]) { + const statements = readFileSync(join(migrationsFolder, migration), 'utf-8') + .split('--> statement-breakpoint') + .map((statement) => statement.trim()) + .filter(Boolean); - // Apply first migration - const statements0 = migration0 - .split('--> statement-breakpoint') - .map((s) => s.trim()) - .filter(Boolean); - - for (const stmt of statements0) { - client.exec(stmt); - } - - // Apply second migration - const statements1 = migration1 - .split('--> statement-breakpoint') - .map((s) => s.trim()) - .filter(Boolean); - - for (const stmt of statements1) { - client.exec(stmt); + for (const statement of statements) { + client.exec(statement); + } } return client; @@ -198,6 +196,44 @@ describe('VersionService.remove()', () => { const doc = client.prepare(`SELECT id FROM documents WHERE id = ?`).get(docId); expect(doc).toBeUndefined(); }); + + it('removes derived vec rows before deleting the version', () => { + const { client, versionService } = setup(); + const version = versionService.add('/facebook/react', 'v18.3.0'); + const docId = crypto.randomUUID(); + const snippetId = crypto.randomUUID(); + const embedding = Float32Array.from([0.5, 0.25, 0.125]); + const now = Math.floor(Date.now() / 1000); + const vecStore = new SqliteVecStore(client); + + client.prepare( + `INSERT INTO documents (id, repository_id, version_id, file_path, checksum, indexed_at) + VALUES (?, '/facebook/react', ?, 'README.md', 'version-doc', ?)` + ).run(docId, version.id, now); + client.prepare( + `INSERT INTO snippets (id, document_id, repository_id, version_id, type, content, created_at) + VALUES (?, ?, '/facebook/react', ?, 'info', 'version snippet', ?)` + ).run(snippetId, docId, version.id, now); + client.prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, 'local-default', 'test-model', 3, ?, ?)` + ).run(snippetId, Buffer.from(embedding.buffer), now); + vecStore.upsertEmbedding('local-default', snippetId, embedding); + + versionService.remove('/facebook/react', 'v18.3.0'); + + const vecTable = sqliteVecTableName('local-default'); + const rowidTable = sqliteVecRowidTableName('local-default'); + const vecCount = client.prepare(`SELECT COUNT(*) as n FROM "${vecTable}"`).get() as { + n: number; + }; + const rowidCount = client.prepare(`SELECT COUNT(*) as n FROM "${rowidTable}"`).get() as { + n: number; + }; + + expect(vecCount.n).toBe(0); + expect(rowidCount.n).toBe(0); + }); }); // --------------------------------------------------------------------------- diff --git a/src/lib/server/services/version.service.ts b/src/lib/server/services/version.service.ts index 006f514..54a60d3 100644 --- a/src/lib/server/services/version.service.ts +++ b/src/lib/server/services/version.service.ts @@ -11,6 +11,7 @@ import { RepositoryVersion, RepositoryVersionEntity } from '$lib/server/models/repository-version.js'; +import { SqliteVecStore } from '$lib/server/search/sqlite-vec.store.js'; import { AlreadyExistsError, NotFoundError } from '$lib/server/utils/validation'; import { resolveTagToCommit, discoverVersionTags } from '$lib/server/utils/git.js'; @@ -99,9 +100,13 @@ export class VersionService { throw new NotFoundError(`Version ${tag} not found for repository ${repositoryId}`); } - this.db - .prepare(`DELETE FROM repository_versions WHERE repository_id = ? AND tag = ?`) - .run(repositoryId, tag); + const sqliteVecStore = new SqliteVecStore(this.db); + this.db.transaction(() => { + sqliteVecStore.deleteEmbeddingsForVersion(repositoryId, version.id); + this.db + .prepare(`DELETE FROM repository_versions WHERE repository_id = ? AND tag = ?`) + .run(repositoryId, tag); + })(); } /** diff --git a/src/routes/admin/jobs/+page.svelte b/src/routes/admin/jobs/+page.svelte index a8dc88a..81c800c 100644 --- a/src/routes/admin/jobs/+page.svelte +++ b/src/routes/admin/jobs/+page.svelte @@ -1,5 +1,10 @@ @@ -202,23 +314,92 @@

Monitor and control indexing jobs

- {#if loading && jobs.length === 0} -
-
-
-

Loading jobs...

+ + +
+
+
+ + +

+ Use an owner prefix like /facebook or a full repository ID like /facebook/react. +

+
+ +
+ Statuses +
+ {#each filterStatuses as status (status)} + + {/each} +
+
+ +
+ +
- {:else if error && jobs.length === 0} -
-

Error: {error}

+ + +
+

+ Showing {jobs.length} of + {total} jobs +

+ {#if hasAppliedFilters()} +

+ Active filters: + {appliedRepositoryFilter || 'all repositories'} + {#if appliedStatuses.length > 0} + · {appliedStatuses.join(', ')} + {:else} + · all statuses + {/if} +

+ {/if} +
+ + {#if error} +
+ {error}
- {:else if jobs.length === 0} + {/if} + + {#if !loading && jobs.length === 0}

- No jobs found. Jobs will appear here when repositories are indexed. + {hasAppliedFilters() + ? 'No jobs match the current filters.' + : 'No jobs found. Jobs will appear here when repositories are indexed.'}

{:else} @@ -259,86 +440,117 @@ - {#each jobs as job (job.id)} + {#if loading && jobs.length === 0} + + {:else} + {#each jobs as job (job.id)} {job.repositoryId} {#if job.versionId} @{job.versionId} {/if} +
{job.id}
- + - -
- {getStageLabel(job.stage)} - {#if job.stageDetail} - {job.stageDetail} - {/if} -
- - -
- {job.progress}% -
-
+ +
+ {getStageLabel(job.stage)} + {#if job.stageDetail} + {job.stageDetail} + {/if}
- {#if job.totalFiles > 0} - - {job.processedFiles}/{job.totalFiles} files - - {/if} -
- + + +
+
+ {job.progress}% +
+
+
+
+ {#if job.totalFiles > 0} +
+ {job.processedFiles}/{job.totalFiles} files processed +
+ {/if} +
+ {formatDate(job.createdAt)}
- {#if canPause(job.status)} + {#if pendingCancelJobId === job.id} - {/if} - {#if canResume(job.status)} - {/if} - {#if canCancel(job.status)} - - {/if} - {#if !canPause(job.status) && !canResume(job.status) && !canCancel(job.status)} - — + {:else} + {#if canPause(job.status)} + + {/if} + {#if canResume(job.status)} + + {/if} + {#if canCancel(job.status)} + + {/if} + {#if !canPause(job.status) && !canResume(job.status) && !canCancel(job.status)} + — + {/if} {/if}
- {/each} + {/each} + {/if}
- {#if loading} + {#if refreshing}
Refreshing...
{/if} {/if}
+ + diff --git a/src/routes/api/v1/jobs/+server.ts b/src/routes/api/v1/jobs/+server.ts index e906015..cf8fa5f 100644 --- a/src/routes/api/v1/jobs/+server.ts +++ b/src/routes/api/v1/jobs/+server.ts @@ -15,15 +15,39 @@ import { JobQueue } from '$lib/server/pipeline/job-queue.js'; import { handleServiceError } from '$lib/server/utils/validation.js'; import type { IndexingJob } from '$lib/types'; +const VALID_JOB_STATUSES: ReadonlySet = new Set([ + 'queued', + 'running', + 'done', + 'failed' +]); + +function parseStatusFilter(searchValue: string | null): IndexingJob['status'] | Array | undefined { + if (!searchValue) { + return undefined; + } + + const statuses = [...new Set( + searchValue + .split(',') + .map((value) => value.trim()) + .filter((value): value is IndexingJob['status'] => VALID_JOB_STATUSES.has(value as IndexingJob['status'])) + )]; + + if (statuses.length === 0) { + return undefined; + } + + return statuses.length === 1 ? statuses[0] : statuses; +} + export const GET: RequestHandler = ({ url }) => { try { const db = getClient(); const queue = new JobQueue(db); - const repositoryId = url.searchParams.get('repositoryId') ?? undefined; - const status = (url.searchParams.get('status') ?? undefined) as - | IndexingJob['status'] - | undefined; + const repositoryId = url.searchParams.get('repositoryId')?.trim() || undefined; + const status = parseStatusFilter(url.searchParams.get('status')); const limit = Math.min(parseInt(url.searchParams.get('limit') ?? '20', 10) || 20, 1000); const jobs = queue.listJobs({ repositoryId, status, limit }); diff --git a/src/routes/api/v1/jobs/[id]/stream/+server.ts b/src/routes/api/v1/jobs/[id]/stream/+server.ts index d1a2861..c521d4d 100644 --- a/src/routes/api/v1/jobs/[id]/stream/+server.ts +++ b/src/routes/api/v1/jobs/[id]/stream/+server.ts @@ -44,7 +44,7 @@ export const GET: RequestHandler = ({ params, request }) => { status: job.status, error: job.error }; - controller.enqueue(`data: ${JSON.stringify(initialData)}\n\n`); + controller.enqueue(`event: job-progress\ndata: ${JSON.stringify(initialData)}\n\n`); // Check for Last-Event-ID header for reconnect const lastEventId = request.headers.get('Last-Event-ID'); @@ -57,6 +57,13 @@ export const GET: RequestHandler = ({ params, request }) => { // Check if job is already done or failed - close immediately after first event if (job.status === 'done' || job.status === 'failed') { + if (job.status === 'done') { + controller.enqueue(`event: job-done\ndata: ${JSON.stringify({ jobId })}\n\n`); + } else { + controller.enqueue( + `event: job-failed\ndata: ${JSON.stringify({ jobId, error: job.error })}\n\n` + ); + } controller.close(); return; } @@ -73,18 +80,29 @@ export const GET: RequestHandler = ({ params, request }) => { controller.enqueue(value); // Check if the incoming event indicates job completion - if (value.includes('event: done') || value.includes('event: failed')) { + if ( + value.includes('event: job-done') || + value.includes('event: job-failed') + ) { controller.close(); break; } } } finally { reader.releaseLock(); - controller.close(); + try { + controller.close(); + } catch { + // Stream may already be closed after a terminal event. + } } } catch (err) { console.error('SSE stream error:', err); - controller.close(); + try { + controller.close(); + } catch { + // Stream may already be closed. + } } } }); diff --git a/src/routes/api/v1/sse-and-settings.integration.test.ts b/src/routes/api/v1/sse-and-settings.integration.test.ts index ee6544a..305ab5c 100644 --- a/src/routes/api/v1/sse-and-settings.integration.test.ts +++ b/src/routes/api/v1/sse-and-settings.integration.test.ts @@ -18,6 +18,7 @@ import type { ProgressBroadcaster as BroadcasterType } from '$lib/server/pipelin let db: Database.Database; // Closed over by the vi.mock factory below. let mockBroadcaster: BroadcasterType | null = null; +let mockPool: { getStatus: () => object; setMaxConcurrency?: (value: number) => void } | null = null; vi.mock('$lib/server/db/client', () => ({ getClient: () => db @@ -29,12 +30,12 @@ vi.mock('$lib/server/db/client.js', () => ({ vi.mock('$lib/server/pipeline/startup', () => ({ getQueue: () => null, - getPool: () => null + getPool: () => mockPool })); vi.mock('$lib/server/pipeline/startup.js', () => ({ getQueue: () => null, - getPool: () => null + getPool: () => mockPool })); vi.mock('$lib/server/pipeline/progress-broadcaster', async (importOriginal) => { @@ -58,9 +59,11 @@ vi.mock('$lib/server/pipeline/progress-broadcaster.js', async (importOriginal) = // --------------------------------------------------------------------------- import { ProgressBroadcaster } from '$lib/server/pipeline/progress-broadcaster.js'; +import { GET as getJobsList } from './jobs/+server.js'; import { GET as getJobStream } from './jobs/[id]/stream/+server.js'; import { GET as getJobsStream } from './jobs/stream/+server.js'; import { GET as getIndexingSettings, PUT as putIndexingSettings } from './settings/indexing/+server.js'; +import { GET as getWorkers } from './workers/+server.js'; // --------------------------------------------------------------------------- // DB factory @@ -306,6 +309,25 @@ describe('GET /api/v1/jobs/:id/stream', () => { // The replay event should include the cached event data expect(fullText).toContain('progress'); }); + + it('closes after receiving the broadcaster job-done event', async () => { + seedRepo(db); + const jobId = seedJob(db, { status: 'running', stage: 'parsing', progress: 10 }); + + const response = await getJobStream(makeEvent({ params: { id: jobId } })); + const reader = response.body!.getReader(); + + const initialChunk = await reader.read(); + expect(String(initialChunk.value ?? '')).toContain('event: job-progress'); + + mockBroadcaster!.broadcast(jobId, '/test/repo', 'job-done', { jobId, status: 'done' }); + + const completionChunk = await reader.read(); + expect(String(completionChunk.value ?? '')).toContain('event: job-done'); + + const closed = await reader.read(); + expect(closed.done).toBe(true); + }); }); // --------------------------------------------------------------------------- @@ -377,12 +399,125 @@ describe('GET /api/v1/jobs/stream', () => { }); // --------------------------------------------------------------------------- -// Test group 3: GET /api/v1/settings/indexing +// Test group 3: GET /api/v1/jobs +// --------------------------------------------------------------------------- + +describe('GET /api/v1/jobs', () => { + beforeEach(() => { + db = createTestDb(); + }); + + it('supports repository prefix and comma-separated status filters', async () => { + seedRepo(db, '/facebook/react'); + seedRepo(db, '/facebook/react-native'); + seedRepo(db, '/vitejs/vite'); + + seedJob(db, { repository_id: '/facebook/react', status: 'queued' }); + seedJob(db, { repository_id: '/facebook/react-native', status: 'running' }); + seedJob(db, { repository_id: '/facebook/react', status: 'done' }); + seedJob(db, { repository_id: '/vitejs/vite', status: 'queued' }); + + const response = await getJobsList( + makeEvent[0]>({ + url: 'http://localhost/api/v1/jobs?repositoryId=%2Ffacebook&status=queued,%20running' + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.total).toBe(2); + expect(body.jobs).toHaveLength(2); + expect(body.jobs.map((job: { repositoryId: string }) => job.repositoryId).sort()).toEqual([ + '/facebook/react', + '/facebook/react-native' + ]); + expect(body.jobs.map((job: { status: string }) => job.status).sort()).toEqual([ + 'queued', + 'running' + ]); + }); + + it('keeps exact-match behavior for specific repository IDs', async () => { + seedRepo(db, '/facebook/react'); + seedRepo(db, '/facebook/react-native'); + + seedJob(db, { repository_id: '/facebook/react', status: 'queued' }); + seedJob(db, { repository_id: '/facebook/react-native', status: 'queued' }); + + const response = await getJobsList( + makeEvent[0]>({ + url: 'http://localhost/api/v1/jobs?repositoryId=%2Ffacebook%2Freact&status=queued' + }) + ); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.total).toBe(1); + expect(body.jobs).toHaveLength(1); + expect(body.jobs[0].repositoryId).toBe('/facebook/react'); + }); +}); + +// --------------------------------------------------------------------------- +// Test group 4: GET /api/v1/workers +// --------------------------------------------------------------------------- + +describe('GET /api/v1/workers', () => { + beforeEach(() => { + mockPool = null; + }); + + it('returns 503 when the worker pool is not initialized', async () => { + const response = await getWorkers(makeEvent[0]>({})); + + expect(response.status).toBe(503); + }); + + it('returns the current worker status snapshot', async () => { + mockPool = { + getStatus: () => ({ + concurrency: 2, + active: 1, + idle: 1, + workers: [ + { + index: 0, + state: 'running', + jobId: 'job-1', + repositoryId: '/test/repo', + versionId: null + }, + { + index: 1, + state: 'idle', + jobId: null, + repositoryId: null, + versionId: null + } + ] + }) + }; + + const response = await getWorkers(makeEvent[0]>({})); + const body = await response.json(); + + expect(response.status).toBe(200); + expect(body.active).toBe(1); + expect(body.workers[0].jobId).toBe('job-1'); + }); +}); + +// --------------------------------------------------------------------------- +// Test group 5: GET /api/v1/settings/indexing // --------------------------------------------------------------------------- describe('GET /api/v1/settings/indexing', () => { beforeEach(() => { db = createTestDb(); + mockPool = { + getStatus: () => ({ concurrency: 2, active: 0, idle: 2, workers: [] }), + setMaxConcurrency: vi.fn() + }; }); it('returns { concurrency: 2 } when no setting exists in DB', async () => { @@ -417,12 +552,16 @@ describe('GET /api/v1/settings/indexing', () => { }); // --------------------------------------------------------------------------- -// Test group 4: PUT /api/v1/settings/indexing +// Test group 6: PUT /api/v1/settings/indexing // --------------------------------------------------------------------------- describe('PUT /api/v1/settings/indexing', () => { beforeEach(() => { db = createTestDb(); + mockPool = { + getStatus: () => ({ concurrency: 2, active: 0, idle: 2, workers: [] }), + setMaxConcurrency: vi.fn() + }; }); function makePutEvent(body: unknown) { diff --git a/src/routes/api/v1/workers/+server.ts b/src/routes/api/v1/workers/+server.ts new file mode 100644 index 0000000..0a5b563 --- /dev/null +++ b/src/routes/api/v1/workers/+server.ts @@ -0,0 +1,16 @@ +import type { RequestHandler } from './$types'; +import { getPool } from '$lib/server/pipeline/startup.js'; +import { handleServiceError } from '$lib/server/utils/validation.js'; + +export const GET: RequestHandler = () => { + try { + const pool = getPool(); + if (!pool) { + return new Response('Service unavailable', { status: 503 }); + } + + return Response.json(pool.getStatus()); + } catch (error) { + return handleServiceError(error); + } +}; \ No newline at end of file -- 2.49.1 From f86be4106b6843acf727cc71a18bb86ce785f813 Mon Sep 17 00:00:00 2001 From: Giancarmine Salucci Date: Thu, 2 Apr 2026 09:49:38 +0200 Subject: [PATCH 2/2] TRUEREF-0023 rewrite indexing pipeline - parallel reads - serialized writes --- README.md | 20 +- docs/FINDINGS.md | 44 + docs/features/TRUEREF-0021.md | 28 +- docs/features/TRUEREF-0022.md | 86 +- docs/features/TRUEREF-0023.md | 955 +++++++++ scripts/build-workers.mjs | 4 +- scripts/build.mjs | 2 +- src/hooks.server.ts | 14 +- src/lib/components/IndexingProgress.svelte | 16 +- .../components/RepositoryCard.svelte.test.ts | 2 +- src/lib/components/admin/JobSkeleton.svelte | 3 +- src/lib/components/admin/Toast.svelte | 6 +- src/lib/dtos/embedding-settings.ts | 2 +- src/lib/server/crawler/github-compare.test.ts | 26 +- src/lib/server/db/client.ts | 10 +- src/lib/server/db/connection.ts | 14 + src/lib/server/db/index.ts | 15 +- .../db/migrations/meta/0004_snapshot.json | 1742 ++++++++-------- .../db/migrations/meta/0006_snapshot.json | 1829 ++++++++--------- .../server/db/migrations/meta/_journal.json | 108 +- src/lib/server/db/schema.test.ts | 12 +- src/lib/server/db/schema.ts | 186 +- src/lib/server/db/sqlite-vec.ts | 2 +- .../embeddings/embedding.service.test.ts | 36 +- .../server/embeddings/embedding.service.ts | 53 +- .../mappers/embedding-profile.mapper.ts | 7 +- .../mappers/embedding-settings.dto.mapper.ts | 2 +- src/lib/server/models/embedding-profile.ts | 2 +- src/lib/server/models/embedding-settings.ts | 2 +- .../pipeline/differential-strategy.test.ts | 26 +- .../server/pipeline/differential-strategy.ts | 4 +- src/lib/server/pipeline/embed-worker-entry.ts | 91 +- .../server/pipeline/indexing.pipeline.test.ts | 66 +- src/lib/server/pipeline/indexing.pipeline.ts | 372 ++-- src/lib/server/pipeline/job-queue.ts | 16 +- .../pipeline/progress-broadcaster.test.ts | 4 +- src/lib/server/pipeline/startup.ts | 51 +- src/lib/server/pipeline/worker-entry.ts | 183 +- src/lib/server/pipeline/worker-pool.test.ts | 97 +- src/lib/server/pipeline/worker-pool.ts | 113 +- src/lib/server/pipeline/worker-types.ts | 119 +- src/lib/server/pipeline/write-operations.ts | 343 ++++ src/lib/server/pipeline/write-worker-entry.ts | 226 +- .../search/hybrid.search.service.test.ts | 13 +- src/lib/server/search/sqlite-vec.store.ts | 46 +- .../services/embedding-settings.service.ts | 12 +- .../services/repository.service.test.ts | 12 +- .../server/services/version.service.test.ts | 30 +- src/lib/server/utils/tag-order.test.ts | 17 +- src/lib/types.ts | 11 +- src/routes/+layout.svelte | 3 + src/routes/admin/jobs/+page.svelte | 196 +- src/routes/api/v1/context/+server.ts | 26 +- src/routes/api/v1/jobs/+server.ts | 20 +- src/routes/api/v1/jobs/[id]/stream/+server.ts | 11 +- src/routes/api/v1/jobs/stream/+server.ts | 2 +- .../api/v1/libs/[id]/index/server.test.ts | 16 +- .../[id]/versions/discover/server.test.ts | 5 +- .../api/v1/settings/embedding/server.test.ts | 2 +- .../api/v1/settings/indexing/+server.ts | 17 +- .../v1/sse-and-settings.integration.test.ts | 54 +- src/routes/api/v1/workers/+server.ts | 2 +- src/routes/repos/[id]/+page.svelte | 235 ++- src/routes/repos/[id]/page.server.test.ts | 2 +- src/routes/route-file-conventions.test.ts | 2 +- src/routes/settings/+page.server.ts | 10 +- src/routes/settings/+page.svelte | 488 ++--- src/routes/settings/page.server.test.ts | 2 +- 68 files changed, 5042 insertions(+), 3131 deletions(-) create mode 100644 docs/features/TRUEREF-0023.md create mode 100644 src/lib/server/db/connection.ts create mode 100644 src/lib/server/pipeline/write-operations.ts diff --git a/README.md b/README.md index d2a0e17..ec9e026 100644 --- a/README.md +++ b/README.md @@ -214,16 +214,16 @@ For GitHub repositories, TrueRef fetches the file from the default branch root. ### Fields -| Field | Type | Required | Description | -|---|---|---|---| -| `$schema` | string | No | URL to the live JSON Schema for editor validation | -| `projectTitle` | string | No | Display name override (max 100 chars) | -| `description` | string | No | Library description used for search ranking (10–500 chars) | -| `folders` | string[] | No | Path prefixes or regex strings to **include** (max 50 items). If absent, all folders are included | -| `excludeFolders` | string[] | No | Path prefixes or regex strings to **exclude** after the `folders` allowlist (max 50 items) | -| `excludeFiles` | string[] | No | Exact filenames to skip — no path, no glob (max 100 items) | -| `rules` | string[] | No | Best-practice rules prepended to every `query-docs` response (max 20 rules, 5–500 chars each) | -| `previousVersions` | object[] | No | Version tags to register when the repository is indexed (max 50 entries) | +| Field | Type | Required | Description | +| ------------------ | -------- | -------- | ------------------------------------------------------------------------------------------------- | +| `$schema` | string | No | URL to the live JSON Schema for editor validation | +| `projectTitle` | string | No | Display name override (max 100 chars) | +| `description` | string | No | Library description used for search ranking (10–500 chars) | +| `folders` | string[] | No | Path prefixes or regex strings to **include** (max 50 items). If absent, all folders are included | +| `excludeFolders` | string[] | No | Path prefixes or regex strings to **exclude** after the `folders` allowlist (max 50 items) | +| `excludeFiles` | string[] | No | Exact filenames to skip — no path, no glob (max 100 items) | +| `rules` | string[] | No | Best-practice rules prepended to every `query-docs` response (max 20 rules, 5–500 chars each) | +| `previousVersions` | object[] | No | Version tags to register when the repository is indexed (max 50 entries) | `previousVersions` entries each require a `tag` (e.g. `"v1.2.3"`) and a `title` (e.g. `"Version 1.2.3"`). diff --git a/docs/FINDINGS.md b/docs/FINDINGS.md index 6fa234e..c852def 100644 --- a/docs/FINDINGS.md +++ b/docs/FINDINGS.md @@ -335,3 +335,47 @@ Add subsequent research below this section. - Risks / follow-ups: - Iteration 2 task decomposition must treat the current dirty code files from iterations 0 and 1 as the validation baseline, otherwise the executor will keep rediscovering pre-existing worktree drift instead of new task deltas. - The sqlite-vec bootstrap helper and the relational cleanup should be planned as one acceptance unit before any downstream vec0, worker-status, or admin-page tasks, because that is the smallest unit that removes the known broken intermediate state. + +### 2026-04-01T00:00:00.000Z — TRUEREF-0023 iteration 3 navbar follow-up planning research + +- Task: Plan the accepted follow-up request to add an admin route to the main navbar. +- Files inspected: + - `prompts/TRUEREF-0023/progress.yaml` + - `prompts/TRUEREF-0023/iteration_2/review_report.yaml` + - `prompts/TRUEREF-0023/prompt.yaml` + - `package.json` + - `src/routes/+layout.svelte` + - `src/routes/admin/jobs/+page.svelte` +- Findings: + - The accepted iteration-2 workspace is green: `review_report.yaml` records passing build, passing tests, and no workspace diagnostics, so this request is a narrow additive follow-up rather than a rework of the sqlite-vec/admin jobs implementation. + - The main navbar is defined entirely in `src/routes/+layout.svelte` and already uses base-aware SvelteKit navigation via `resolve as resolveRoute` from `$app/paths` for the existing `Repositories`, `Search`, and `Settings` links. + - The existing admin surface already lives at `src/routes/admin/jobs/+page.svelte`, which sets the page title to `Job Queue - TrueRef Admin`; adding a navbar entry can therefore target `/admin/jobs` directly without introducing new routes, loaders, or components. + - Repository findings from the earlier lint planning work already confirm the codebase expectation to avoid root-relative internal navigation in SvelteKit pages and components, so the new navbar link should follow the existing `resolveRoute('/...')` anchor pattern. + - No dedicated test file currently covers the shared navbar. The appropriate validation for this follow-up remains repository-level `npm run build` and `npm test` after the single layout edit. +- Risks / follow-ups: + - The follow-up navigation request should stay isolated to the shared layout so it does not reopen the accepted sqlite-vec implementation surface. + - Build and test validation remain the appropriate regression checks because no dedicated navbar test currently exists. + +### 2026-04-01T12:05:23.000Z — TRUEREF-0023 iteration 5 tabs filter and bulk reprocess planning research + +- Task: Plan the follow-up repo-detail UI change to filter version rows in the tabs/tags view and add a bulk action that reprocesses all errored tags without adding a new backend endpoint. +- Files inspected: + - `prompts/TRUEREF-0023/progress.yaml` + - `prompts/TRUEREF-0023/prompt.yaml` + - `prompts/TRUEREF-0023/iteration_2/plan.md` + - `prompts/TRUEREF-0023/iteration_2/tasks.yaml` + - `src/routes/repos/[id]/+page.svelte` + - `src/routes/api/v1/libs/[id]/versions/[tag]/index/+server.ts` + - `src/routes/api/v1/api-contract.integration.test.ts` + - `package.json` +- Findings: + - The relevant UI surface is entirely in `src/routes/repos/[id]/+page.svelte`; the page already loads `versions`, renders per-version state badges, and exposes per-tag `Index` and `Remove` buttons. + - Version states are concretely `pending`, `indexing`, `indexed`, and `error`, and the page already centralizes their labels and color classes in `stateLabels` and `stateColors`. + - Existing per-tag reprocessing is implemented by `handleIndexVersion(tag)`, which POSTs to `/api/v1/libs/:id/versions/:tag/index`; the corresponding backend route exists and returns a queued job DTO with status `202`. + - No bulk reprocess endpoint exists, so the lowest-risk implementation is a UI-only bulk action that iterates the existing per-tag route. + - The page already contains a bounded batching pattern in `handleRegisterSelected()` with `BATCH_SIZE = 5`, which provides a concrete local precedent for bulk tag operations without inventing a new concurrency model. + - There is no existing page-component or browser test targeting `src/routes/repos/[id]/+page.svelte`; nearby automated coverage is API-contract focused, so this iteration should rely on `npm run build` and `npm test` regression validation unless a developer discovers an existing Svelte page harness during implementation. + - Context7 lookup for Svelte and SvelteKit could not be completed in this environment because the configured API key is invalid; planning therefore relied on installed versions from `package.json` (`svelte` `^5.51.0`, `@sveltejs/kit` `^2.50.2`) and the live page patterns already present in the repository. +- Risks / follow-ups: + - Bulk reprocessing must avoid queuing duplicate jobs for tags already shown as `indexing` or already tracked in `activeVersionJobs`. + - Filter state should be implemented as local UI state only and must not disturb the existing `onMount(loadVersions)` fetch path or the SSE job-progress flow. diff --git a/docs/features/TRUEREF-0021.md b/docs/features/TRUEREF-0021.md index dc33d77..3036faa 100644 --- a/docs/features/TRUEREF-0021.md +++ b/docs/features/TRUEREF-0021.md @@ -47,8 +47,8 @@ Executed in `IndexingPipeline.run()` before the crawl, when the job has a `versi containing shell metacharacters). 3. **Path partitioning**: The changed-file list is split into `changedPaths` (added + modified - + renamed-destination) and `deletedPaths`. `unchangedPaths` is derived as - `ancestorFilePaths − changedPaths − deletedPaths`. + - renamed-destination) and `deletedPaths`. `unchangedPaths` is derived as + `ancestorFilePaths − changedPaths − deletedPaths`. 4. **Guard**: Returns `null` when no indexed ancestor exists, when the ancestor has no indexed documents, or when all files changed (nothing to clone). @@ -74,18 +74,18 @@ matching files are returned. This minimises GitHub API requests and local I/O. ## API Surface Changes -| Symbol | Location | Change | -|---|---|---| -| `buildDifferentialPlan` | `pipeline/differential-strategy.ts` | **New** — async function | -| `DifferentialPlan` | `pipeline/differential-strategy.ts` | **New** — interface | -| `findBestAncestorVersion` | `utils/tag-order.ts` | **New** — pure function | -| `fetchGitHubChangedFiles` | `crawler/github-compare.ts` | **New** — async function | -| `getChangedFilesBetweenRefs` | `utils/git.ts` | **New** — sync function (uses `execFileSync`) | -| `ChangedFile` | `crawler/types.ts` | **New** — interface | -| `CrawlOptions.allowedPaths` | `crawler/types.ts` | **New** — optional field | -| `IndexingPipeline.crawl()` | `pipeline/indexing.pipeline.ts` | **Modified** — added `allowedPaths` param | -| `IndexingPipeline.cloneFromAncestor()` | `pipeline/indexing.pipeline.ts` | **New** — private method | -| `IndexingPipeline.run()` | `pipeline/indexing.pipeline.ts` | **Modified** — Stage 0 added | +| Symbol | Location | Change | +| -------------------------------------- | ----------------------------------- | --------------------------------------------- | +| `buildDifferentialPlan` | `pipeline/differential-strategy.ts` | **New** — async function | +| `DifferentialPlan` | `pipeline/differential-strategy.ts` | **New** — interface | +| `findBestAncestorVersion` | `utils/tag-order.ts` | **New** — pure function | +| `fetchGitHubChangedFiles` | `crawler/github-compare.ts` | **New** — async function | +| `getChangedFilesBetweenRefs` | `utils/git.ts` | **New** — sync function (uses `execFileSync`) | +| `ChangedFile` | `crawler/types.ts` | **New** — interface | +| `CrawlOptions.allowedPaths` | `crawler/types.ts` | **New** — optional field | +| `IndexingPipeline.crawl()` | `pipeline/indexing.pipeline.ts` | **Modified** — added `allowedPaths` param | +| `IndexingPipeline.cloneFromAncestor()` | `pipeline/indexing.pipeline.ts` | **New** — private method | +| `IndexingPipeline.run()` | `pipeline/indexing.pipeline.ts` | **Modified** — Stage 0 added | --- diff --git a/docs/features/TRUEREF-0022.md b/docs/features/TRUEREF-0022.md index 00dca28..5149007 100644 --- a/docs/features/TRUEREF-0022.md +++ b/docs/features/TRUEREF-0022.md @@ -88,6 +88,7 @@ The UI currently polls `GET /api/v1/jobs?repositoryId=...` every 2 seconds. This #### Worker Thread lifecycle Each worker is a long-lived `node:worker_threads` `Worker` instance that: + 1. Opens its own `better-sqlite3` connection to the same database file. 2. Listens for `{ type: 'run', jobId }` messages from the main thread. 3. Runs `IndexingPipeline.run(job)`, emitting `postMessage` progress events at each stage boundary and every N files. @@ -100,18 +101,18 @@ Manages a pool of `concurrency` workers. ```typescript interface WorkerPoolOptions { - concurrency: number; // default: Math.max(1, os.cpus().length - 1), capped at 4 - workerScript: string; // absolute path to the compiled worker entry + concurrency: number; // default: Math.max(1, os.cpus().length - 1), capped at 4 + workerScript: string; // absolute path to the compiled worker entry } class WorkerPool { - private workers: Worker[]; - private idle: Worker[]; + private workers: Worker[]; + private idle: Worker[]; - enqueue(jobId: string): void; - private dispatch(worker: Worker, jobId: string): void; - private onWorkerMessage(msg: WorkerMessage): void; - private onWorkerExit(worker: Worker, code: number): void; + enqueue(jobId: string): void; + private dispatch(worker: Worker, jobId: string): void; + private onWorkerMessage(msg: WorkerMessage): void; + private onWorkerExit(worker: Worker, code: number): void; } ``` @@ -120,12 +121,14 @@ Workers are kept alive across jobs. If a worker crashes (non-zero exit), the poo #### Parallelism and write contention With WAL mode enabled (already the case), SQLite supports: + - **One concurrent writer** (the transaction lock) - **Many concurrent readers** The `replaceSnippets` transaction for different repositories never contends — they write different rows. The `cloneFromAncestor` operation writes to the same tables but different `version_id` values, so WAL checkpoint logic keeps them non-overlapping at the page level. Two jobs on the **same repository** (e.g. `/my-lib/v1.0.0` and `/my-lib/v2.0.0`) can run in parallel because: + - Differential indexing (TRUEREF-0021) ensures `v2.0.0` reads from `v1.0.0`'s already-committed rows. - The write transactions for each version touch disjoint `version_id` partitions. @@ -134,6 +137,7 @@ If write contention still occurs under parallel load, `busy_timeout = 5000` (alr #### Concurrency limit per repository To prevent a user from queuing 500 tags and overwhelming the worker pool, the pool enforces: + - **Max 1 running job per repository** for the default branch (re-index). - **Max `concurrency` total running jobs** across all repositories. - Version jobs for the same repository are serialised within the pool (the queue picks the oldest queued version job for a given repo only when no other version job for that repo is running). @@ -148,15 +152,15 @@ Replace the opaque integer progress with a structured stage model: ```typescript type IndexingStage = - | 'queued' - | 'differential' // computing ancestor diff - | 'crawling' // fetching files from GitHub or local FS - | 'cloning' // cloning unchanged files from ancestor (differential only) - | 'parsing' // parsing files into snippets - | 'storing' // writing documents + snippets to DB - | 'embedding' // generating vector embeddings - | 'done' - | 'failed'; + | 'queued' + | 'differential' // computing ancestor diff + | 'crawling' // fetching files from GitHub or local FS + | 'cloning' // cloning unchanged files from ancestor (differential only) + | 'parsing' // parsing files into snippets + | 'storing' // writing documents + snippets to DB + | 'embedding' // generating vector embeddings + | 'done' + | 'failed'; ``` ### Extended Job Schema @@ -172,22 +176,24 @@ The `progress` column (0–100) is retained for backward compatibility and overa ```typescript interface ProgressMessage { - type: 'progress'; - jobId: string; - stage: IndexingStage; - stageDetail?: string; // human-readable detail for the current stage - progress: number; // 0–100 overall - processedFiles: number; - totalFiles: number; + type: 'progress'; + jobId: string; + stage: IndexingStage; + stageDetail?: string; // human-readable detail for the current stage + progress: number; // 0–100 overall + processedFiles: number; + totalFiles: number; } ``` Workers emit this message: + - On every stage transition (crawl start, parse start, store start, embed start). - Every `PROGRESS_EMIT_EVERY = 10` files during the parse loop. - On job completion or failure. The main thread receives these messages and does two things: + 1. Writes the update to `indexing_jobs` in SQLite (batched — one write per message, not per file). 2. Pushes the payload to any open SSE channels for that jobId. @@ -198,6 +204,7 @@ The main thread receives these messages and does two things: ### `GET /api/v1/jobs/:id/stream` Opens an SSE connection for a specific job. The server: + 1. Sends the current job state as the first event immediately (no initial lag). 2. Pushes `ProgressMessage` events as the worker emits them. 3. Sends a final `event: done` or `event: failed` event, then closes the connection. @@ -216,7 +223,7 @@ id: 1 event: progress data: {"stage":"crawling","progress":0,"processedFiles":0,"totalFiles":0} -id: 2 +id: 2 event: progress data: {"stage":"parsing","progress":12,"processedFiles":240,"totalFiles":2000} @@ -281,7 +288,7 @@ Expose via the settings table (key `indexing.concurrency`): ```typescript interface IndexingSettings { - concurrency: number; // 1–max(cpus-1, 1); default 2 + concurrency: number; // 1–max(cpus-1, 1); default 2 } ``` @@ -362,13 +369,13 @@ The embedding stage must **not** run inside the same Worker Thread as the crawl/ ### Why a dedicated embedding worker -| Concern | Per-parse-worker model | Dedicated embedding worker | -|---|---|---| -| Memory | N × ~100 MB (model weights + WASM heap) per worker | 1 × ~100 MB regardless of concurrency | -| Model warm-up | Paid once per worker spawn; cold starts slow | Paid once at server startup | -| Batch size | Each worker batches only its own job's snippets | All in-flight jobs queue to one worker → larger batches → higher WASM throughput | -| Provider migration | Must update every worker | Update one file | -| API rate limiting | N parallel streams to the same API → N×rate-limit hits | One serial stream, naturally throttled | +| Concern | Per-parse-worker model | Dedicated embedding worker | +| ------------------ | ------------------------------------------------------ | -------------------------------------------------------------------------------- | +| Memory | N × ~100 MB (model weights + WASM heap) per worker | 1 × ~100 MB regardless of concurrency | +| Model warm-up | Paid once per worker spawn; cold starts slow | Paid once at server startup | +| Batch size | Each worker batches only its own job's snippets | All in-flight jobs queue to one worker → larger batches → higher WASM throughput | +| Provider migration | Must update every worker | Update one file | +| API rate limiting | N parallel streams to the same API → N×rate-limit hits | One serial stream, naturally throttled | With `Xenova/all-MiniLM-L6-v2`, the WASM model and weight files occupy ~90–120 MB of heap. Running three parse workers with embedded model loading costs ~300–360 MB of resident memory that can never be freed while the server is alive. A dedicated worker keeps that cost fixed at one instance. @@ -415,6 +422,7 @@ Instead, the existing `findSnippetIdsMissingEmbeddings` query is the handshake: 5. Main thread routes this to the SSE broadcaster → UI updates the embedding progress slice. This means: + - The embedding worker reads snippet text from the DB itself (no IPC serialisation of content). - The model is loaded once, stays warm, and processes batches from all repositories in FIFO order. - Parse workers are never blocked waiting for embeddings — they complete their job stages and exit immediately. @@ -424,15 +432,15 @@ This means: ```typescript // Main → Embedding worker type EmbedRequest = - | { type: 'embed'; jobId: string; repositoryId: string; versionId: string | null } - | { type: 'shutdown' }; + | { type: 'embed'; jobId: string; repositoryId: string; versionId: string | null } + | { type: 'shutdown' }; // Embedding worker → Main type EmbedResponse = - | { type: 'embed-progress'; jobId: string; done: number; total: number } - | { type: 'embed-done'; jobId: string } - | { type: 'embed-failed'; jobId: string; error: string } - | { type: 'ready' }; // emitted once after model warm-up completes + | { type: 'embed-progress'; jobId: string; done: number; total: number } + | { type: 'embed-done'; jobId: string } + | { type: 'embed-failed'; jobId: string; error: string } + | { type: 'ready' }; // emitted once after model warm-up completes ``` The `ready` message allows the server startup sequence to defer routing any embed requests until the model is loaded, preventing a race on first-run. diff --git a/docs/features/TRUEREF-0023.md b/docs/features/TRUEREF-0023.md new file mode 100644 index 0000000..e37b12c --- /dev/null +++ b/docs/features/TRUEREF-0023.md @@ -0,0 +1,955 @@ +# TRUEREF-0023 — libSQL Migration, Native Vector Search, Parallel Tag Indexing, and Performance Hardening + +**Priority:** P1 +**Status:** Draft +**Depends On:** TRUEREF-0001, TRUEREF-0022 +**Blocks:** — + +--- + +## Overview + +TrueRef currently uses `better-sqlite3` for all database access. This creates three compounding performance problems: + +1. **Vector search does not scale.** `VectorSearch.vectorSearch()` loads the entire `snippet_embeddings` table for a repository into Node.js memory and computes cosine similarity in a JavaScript loop. A repository with 100k snippets at 1536 OpenAI dimensions allocates ~600 MB per query and ties up the worker thread for seconds before returning results. +2. **Missing composite indexes cause table scans on every query.** The schema defines FK columns used in every search and embedding filter, but declares zero composite or covering indexes on them. Every call to `searchSnippets`, `findSnippetIdsMissingEmbeddings`, and `cloneFromAncestor` performs full or near-full table scans. +3. **SQLite connection is under-configured.** Critical pragmas (`synchronous`, `cache_size`, `mmap_size`, `temp_store`) are absent, leaving significant I/O throughput on the table. + +The solution is to replace `better-sqlite3` with `@libsql/better-sqlite3` — an embeddable, drop-in synchronous replacement that is a superset of the better-sqlite3 API and exposes libSQL's native vector index (`libsql_vector_idx`). Because the API is identical, no service layer or ORM code changes are needed beyond import statements and the vector search implementation. + +Two additional structural improvements are delivered in the same feature: + +4. **Per-repo job serialization is too coarse.** `WorkerPool` prevents any two jobs sharing the same `repositoryId` from running in parallel. This means indexing 200 tags of a single library is fully sequential — one tag at a time — even though different tags write to entirely disjoint row sets. The constraint should track `(repositoryId, versionId)` pairs instead. +5. **Write lock contention under parallel indexing.** When multiple parse workers flush parsed snippets simultaneously they all compete for the SQLite write lock, spending most of their time in `busy_timeout` back-off. A single dedicated write worker eliminates this: parse workers become pure CPU workers (crawl → parse → send batches over `postMessage`) and the write worker is the sole DB writer. +6. **Admin UI is unusable under load.** The job queue page has no status or repository filters, no worker status panel, no skeleton loading, uses blocking `alert()` / `confirm()` dialogs, and `IndexingProgress` still polls every 2 seconds instead of consuming the existing SSE stream. + +--- + +## Goals + +1. Replace `better-sqlite3` with `@libsql/better-sqlite3` with minimal code churn — import paths only. +2. Add a libSQL vector index on `snippet_embeddings` so that KNN queries execute inside SQLite instead of in a JavaScript loop. +3. Add the six composite and covering indexes required by the hot query paths. +4. Tune the SQLite pragma configuration for I/O performance. +5. Eliminate the leading cause of OOM risk during semantic search. +6. Keep a single embedded database file — no external server, no network. +7. Allow multiple tags of the same repository to index in parallel (unrelated version rows, no write conflict). +8. Eliminate write-lock contention between parallel parse workers by introducing a single dedicated write worker. +9. Rebuild the admin jobs page with full filtering (status, repository, free-text), a live worker status panel, skeleton loading on initial fetch, per-action inline spinners, non-blocking toast notifications, and SSE-driven real-time updates throughout. + +--- + +## Non-Goals + +- Migrating to the async `@libsql/client` package (HTTP/embedded-replica mode). +- Changing the Drizzle ORM adapter (`drizzle-orm/better-sqlite3` stays unchanged). +- Changing `drizzle.config.ts` dialect (`sqlite` is still correct for embedded libSQL). +- Adding hybrid/approximate indexing beyond the default HNSW strategy provided by `libsql_vector_idx`. +- Parallelizing embedding batches across providers (separate feature). +- Horizontally scaling across processes. +- Allowing more than one job for the exact same `(repositoryId, versionId)` pair to run concurrently (still serialized — duplicate detection in `JobQueue` is unchanged). +- A full admin authentication system (out of scope). +- Mobile-responsive redesign of the entire admin section (out of scope). + +--- + +## Problem Detail + +### 1. Vector Search — Full Table Scan in JavaScript + +**File:** `src/lib/server/search/vector.search.ts` + +```typescript +// Current: no LIMIT, loads ALL embeddings for repo into memory +const rows = this.db.prepare(sql).all(...params); + +const scored: VectorSearchResult[] = rows.map((row) => { + const embedding = new Float32Array( + row.embedding.buffer, + row.embedding.byteOffset, + row.embedding.byteLength / 4 + ); + return { snippetId: row.snippet_id, score: cosineSimilarity(queryEmbedding, embedding) }; +}); + +return scored.sort((a, b) => b.score - a.score).slice(0, limit); +``` + +For a repo with N snippets and D dimensions, this allocates `N × D × 4` bytes per query. At N=100k and D=1536, that is ~600 MB allocated synchronously. The result is sorted entirely in JS before the top-k is returned. With a native vector index, SQLite returns only the top-k rows. + +### 2. Missing Composite Indexes + +The `snippets`, `documents`, and `snippet_embeddings` tables are queried with multi-column WHERE predicates in every hot path, but no composite indexes exist: + +| Table | Filter columns | Used in | +| -------------------- | ----------------------------- | ---------------------------------------------- | +| `snippets` | `(repository_id, version_id)` | All search, diff, clone | +| `snippets` | `(repository_id, type)` | Type-filtered queries | +| `documents` | `(repository_id, version_id)` | Diff strategy, clone | +| `snippet_embeddings` | `(profile_id, snippet_id)` | `findSnippetIdsMissingEmbeddings` LEFT JOIN | +| `repositories` | `(state)` | `searchRepositories` WHERE `state = 'indexed'` | +| `indexing_jobs` | `(repository_id, status)` | Job status lookups | + +Without these indexes, SQLite performs a B-tree scan of the primary key and filters rows in memory. On a 500k-row `snippets` table this is the dominant cost of every search. + +### 4. Admin UI — Current Problems + +**File:** `src/routes/admin/jobs/+page.svelte`, `src/lib/components/IndexingProgress.svelte` + +| Problem | Location | Impact | +| -------------------------------------------------------------- | ----------------------------------------- | ------------------------------------------------------------ | +| `IndexingProgress` polls every 2 s via `setInterval` + `fetch` | `IndexingProgress.svelte` | Constant HTTP traffic; progress lags by up to 2 s | +| No status or repository filter controls | `admin/jobs/+page.svelte` | With 200 tag jobs, finding a specific one requires scrolling | +| No worker status panel | — (no endpoint exists) | Operator cannot see which workers are busy or idle | +| `alert()` for errors, `confirm()` for cancel | `admin/jobs/+page.svelte` — `showToast()` | Blocks the entire browser tab; unusable under parallel jobs | +| `actionInProgress` is a single string, not per-job | `admin/jobs/+page.svelte` | Pausing job A disables buttons on all other jobs | +| No skeleton loading — blank + spinner on first load | `admin/jobs/+page.svelte` | Layout shift; no structural preview while data loads | +| Hard-coded `limit=50` query, no pagination | `admin/jobs/+page.svelte:fetchJobs()` | Page truncates silently for large queues | + +--- + +### 3. Under-configured SQLite Connection + +**File:** `src/lib/server/db/client.ts` and `src/lib/server/db/index.ts` + +Current pragmas: + +```typescript +client.pragma('journal_mode = WAL'); +client.pragma('foreign_keys = ON'); +client.pragma('busy_timeout = 5000'); +``` + +Missing: + +- `synchronous = NORMAL` — halves fsync overhead vs the default FULL; safe with WAL +- `cache_size = -65536` — 64 MB page cache; default is 2 MB +- `temp_store = MEMORY` — temp tables and sort spills stay in RAM +- `mmap_size = 268435456` — 256 MB memory-mapped read path; bypasses system call overhead for reads +- `wal_autocheckpoint = 1000` — more frequent checkpoints prevent WAL growth + +--- + +## Architecture + +### Drop-In Replacement: `@libsql/better-sqlite3` + +`@libsql/better-sqlite3` is published by Turso and implemented as a Node.js native addon wrapping the libSQL embedded engine. The exported class is API-compatible with `better-sqlite3`: + +```typescript +// before +import Database from 'better-sqlite3'; +const db = new Database('/path/to/file.db'); +db.pragma('journal_mode = WAL'); +const rows = db.prepare('SELECT ...').all(...params); + +// after — identical code +import Database from '@libsql/better-sqlite3'; +const db = new Database('/path/to/file.db'); +db.pragma('journal_mode = WAL'); +const rows = db.prepare('SELECT ...').all(...params); +``` + +All of the following continue to work unchanged: + +- `drizzle-orm/better-sqlite3` adapter and `migrate` helper +- `drizzle-kit` with `dialect: 'sqlite'` +- Prepared statements, transactions, WAL pragmas, foreign keys +- Worker thread per-thread connections (`worker-entry.ts`, `embed-worker-entry.ts`) +- All `type Database from 'better-sqlite3'` type imports (replaced in lock-step) + +### Vector Index Design + +libSQL provides `libsql_vector_idx()` — a virtual index type stored in a shadow table alongside the main table. Once indexed, KNN queries use a SQL `vector_top_k()` function: + +```sql +-- KNN: return top-k snippet IDs closest to the query vector +SELECT snippet_id +FROM vector_top_k('idx_snippet_embeddings_vec', vector_from_float32(?), ?) +``` + +`vector_from_float32(blob)` accepts the same raw little-endian Float32 bytes currently stored in the `embedding` blob column. **No data migration is needed** — the existing blob column can be re-indexed with `libsql_vector_idx` pointing at the bytes-stored column. + +The index strategy: + +1. Add a generated `vec_embedding` column of type `F32_BLOB(dimensions)` to `snippet_embeddings`, populated from the existing `embedding` blob via a migration trigger. +2. Create the vector index: `CREATE INDEX idx_snippet_embeddings_vec ON snippet_embeddings(vec_embedding) USING libsql_vector_idx(vec_embedding)`. +3. Rewrite `VectorSearch.vectorSearch()` to use `vector_top_k()` with a two-step join instead of the in-memory loop. +4. Update `EmbeddingService.embedSnippets()` to write `vec_embedding` on insert. + +Dimensions are profile-specific. Because the index is per-column, a separate index is needed per embedding dimensionality. For v1, a single index covering the default profile's dimensions is sufficient; multi-profile KNN can be handled with a `WHERE profile_id = ?` pre-filter on the vector_top_k results. + +### Updated Vector Search Query + +```typescript +vectorSearch(queryEmbedding: Float32Array, options: VectorSearchOptions): VectorSearchResult[] { + const { repositoryId, versionId, profileId = 'local-default', limit = 50 } = options; + + // Encode query vector as raw bytes (same format as stored blobs) + const queryBytes = Buffer.from(queryEmbedding.buffer); + + // Use libSQL vector_top_k for ANN — returns ordered (rowid, distance) pairs + let sql = ` + SELECT se.snippet_id, + vector_distance_cos(se.vec_embedding, vector_from_float32(?)) AS score + FROM vector_top_k('idx_snippet_embeddings_vec', vector_from_float32(?), ?) AS knn + JOIN snippet_embeddings se ON se.rowid = knn.id + JOIN snippets s ON s.id = se.snippet_id + WHERE s.repository_id = ? + AND se.profile_id = ? + `; + const params: unknown[] = [queryBytes, queryBytes, limit * 4, repositoryId, profileId]; + + if (versionId) { + sql += ' AND s.version_id = ?'; + params.push(versionId); + } + + sql += ' ORDER BY score ASC LIMIT ?'; + params.push(limit); + + return this.db + .prepare(sql) + .all(...params) + .map((row) => ({ snippetId: row.snippet_id, score: 1 - row.score })); +} +``` + +`vector_distance_cos` returns distance (0 = identical), so `1 - distance` gives a similarity score in [0, 1] matching the existing `VectorSearchResult.score` contract. + +--- + +## Implementation Plan + +### Phase 1 — Package Swap (no logic changes) + +**Files touched:** `package.json`, all `.ts` files that import `better-sqlite3` + +1. In `package.json`: + - Remove `"better-sqlite3": "^12.6.2"` from `dependencies` + - Add `"@libsql/better-sqlite3": "^0.4.0"` to `dependencies` + - Remove `"@types/better-sqlite3": "^7.6.13"` from `devDependencies` + - `@libsql/better-sqlite3` ships its own TypeScript declarations + +2. Replace all import statements (35 occurrences across 19 files): + + | Old import | New import | + | --------------------------------------------------------------- | ---------------------------------------------------- | + | `import Database from 'better-sqlite3'` | `import Database from '@libsql/better-sqlite3'` | + | `import type Database from 'better-sqlite3'` | `import type Database from '@libsql/better-sqlite3'` | + | `import { drizzle } from 'drizzle-orm/better-sqlite3'` | unchanged | + | `import { migrate } from 'drizzle-orm/better-sqlite3/migrator'` | unchanged | + + Affected production files: + - `src/lib/server/db/index.ts` + - `src/lib/server/db/client.ts` + - `src/lib/server/embeddings/embedding.service.ts` + - `src/lib/server/pipeline/indexing.pipeline.ts` + - `src/lib/server/pipeline/job-queue.ts` + - `src/lib/server/pipeline/startup.ts` + - `src/lib/server/pipeline/worker-entry.ts` + - `src/lib/server/pipeline/embed-worker-entry.ts` + - `src/lib/server/pipeline/differential-strategy.ts` + - `src/lib/server/search/vector.search.ts` + - `src/lib/server/search/hybrid.search.service.ts` + - `src/lib/server/search/search.service.ts` + - `src/lib/server/services/repository.service.ts` + - `src/lib/server/services/version.service.ts` + - `src/lib/server/services/embedding-settings.service.ts` + + Affected test files (same mechanical replacement): + - `src/routes/api/v1/api-contract.integration.test.ts` + - `src/routes/api/v1/sse-and-settings.integration.test.ts` + - `src/routes/settings/page.server.test.ts` + - `src/lib/server/db/schema.test.ts` + - `src/lib/server/embeddings/embedding.service.test.ts` + - `src/lib/server/pipeline/indexing.pipeline.test.ts` + - `src/lib/server/pipeline/differential-strategy.test.ts` + - `src/lib/server/search/search.service.test.ts` + - `src/lib/server/search/hybrid.search.service.test.ts` + - `src/lib/server/services/repository.service.test.ts` + - `src/lib/server/services/version.service.test.ts` + - `src/routes/api/v1/settings/embedding/server.test.ts` + - `src/routes/api/v1/libs/[id]/index/server.test.ts` + - `src/routes/api/v1/libs/[id]/versions/discover/server.test.ts` + +3. Run all tests — they should pass with zero logic changes: `npm test` + +### Phase 2 — Pragma Hardening + +**Files touched:** `src/lib/server/db/client.ts`, `src/lib/server/db/index.ts` + +Add the following pragmas to both connection factories (raw client and `initializeDatabase()`): + +```typescript +client.pragma('synchronous = NORMAL'); +client.pragma('cache_size = -65536'); // 64 MB +client.pragma('temp_store = MEMORY'); +client.pragma('mmap_size = 268435456'); // 256 MB +client.pragma('wal_autocheckpoint = 1000'); +``` + +Worker threads (`worker-entry.ts`, `embed-worker-entry.ts`) open their own connections — apply the same pragmas there. + +### Phase 3 — Composite Indexes (Drizzle migration) + +**Files touched:** `src/lib/server/db/schema.ts`, new migration SQL file + +Add indexes in `schema.ts` using Drizzle's `index()` helper: + +```typescript +// snippets table +export const snippets = sqliteTable( + 'snippets', + { + /* unchanged */ + }, + (t) => [ + index('idx_snippets_repo_version').on(t.repositoryId, t.versionId), + index('idx_snippets_repo_type').on(t.repositoryId, t.type) + ] +); + +// documents table +export const documents = sqliteTable( + 'documents', + { + /* unchanged */ + }, + (t) => [index('idx_documents_repo_version').on(t.repositoryId, t.versionId)] +); + +// snippet_embeddings table +export const snippetEmbeddings = sqliteTable( + 'snippet_embeddings', + { + /* unchanged */ + }, + (table) => [ + primaryKey({ columns: [table.snippetId, table.profileId] }), // unchanged + index('idx_embeddings_profile').on(table.profileId, table.snippetId) + ] +); + +// repositories table +export const repositories = sqliteTable( + 'repositories', + { + /* unchanged */ + }, + (t) => [index('idx_repositories_state').on(t.state)] +); + +// indexing_jobs table +export const indexingJobs = sqliteTable( + 'indexing_jobs', + { + /* unchanged */ + }, + (t) => [index('idx_jobs_repo_status').on(t.repositoryId, t.status)] +); +``` + +Generate and apply migration: `npm run db:generate && npm run db:migrate` + +### Phase 4 — Vector Column and Index (Drizzle migration) + +**Files touched:** `src/lib/server/db/schema.ts`, new migration SQL, `src/lib/server/search/vector.search.ts`, `src/lib/server/embeddings/embedding.service.ts` + +#### 4a. Schema: add `vec_embedding` column + +Add `vec_embedding` to `snippet_embeddings`. Drizzle does not have a `F32_BLOB` column type helper; use a raw SQL column: + +```typescript +import { sql } from 'drizzle-orm'; +import { customType } from 'drizzle-orm/sqlite-core'; + +const f32Blob = (name: string, dimensions: number) => + customType<{ data: Buffer }>({ + dataType() { + return `F32_BLOB(${dimensions})`; + } + })(name); + +export const snippetEmbeddings = sqliteTable( + 'snippet_embeddings', + { + snippetId: text('snippet_id') + .notNull() + .references(() => snippets.id, { onDelete: 'cascade' }), + profileId: text('profile_id') + .notNull() + .references(() => embeddingProfiles.id, { onDelete: 'cascade' }), + model: text('model').notNull(), + dimensions: integer('dimensions').notNull(), + embedding: blob('embedding').notNull(), // existing blob — kept for backward compat + vecEmbedding: f32Blob('vec_embedding', 1536), // libSQL vector column (nullable during migration fill) + createdAt: integer('created_at').notNull() + }, + (table) => [ + primaryKey({ columns: [table.snippetId, table.profileId] }), + index('idx_embeddings_profile').on(table.profileId, table.snippetId) + ] +); +``` + +Because dimensionality is fixed per model, `F32_BLOB(1536)` covers OpenAI `text-embedding-3-small/large`. A follow-up can parameterize this per profile. + +#### 4b. Migration SQL: populate `vec_embedding` from existing `embedding` blob and create the vector index + +The vector index cannot be expressed in SQL DDL portable across Drizzle — it must be applied in the FTS-style custom SQL file (`src/lib/server/db/fts.sql` or an equivalent `vectors.sql`): + +```sql +-- Backfill vec_embedding from existing raw blob data +UPDATE snippet_embeddings +SET vec_embedding = vector_from_float32(embedding) +WHERE vec_embedding IS NULL AND embedding IS NOT NULL; + +-- Create the HNSW vector index (libSQL extension syntax) +CREATE INDEX IF NOT EXISTS idx_snippet_embeddings_vec +ON snippet_embeddings(vec_embedding) +USING libsql_vector_idx(vec_embedding, 'metric=cosine', 'compress_neighbors=float8', 'max_neighbors=20'); +``` + +Add a call to this SQL in `initializeDatabase()` alongside the existing `fts.sql` execution: + +```typescript +const vectorSql = readFileSync(join(__dirname, 'vectors.sql'), 'utf-8'); +client.exec(vectorSql); +``` + +#### 4c. Update `EmbeddingService.embedSnippets()` + +When inserting a new embedding, write both the blob and the vec column: + +```typescript +const insert = this.db.prepare<[string, string, string, number, Buffer, Buffer]>(` + INSERT OR REPLACE INTO snippet_embeddings + (snippet_id, profile_id, model, dimensions, embedding, vec_embedding, created_at) + VALUES (?, ?, ?, ?, ?, vector_from_float32(?), unixepoch()) +`); + +// inside the transaction: +insert.run( + snippet.id, + this.profileId, + embedding.model, + embedding.dimensions, + embeddingBuffer, + embeddingBuffer // same bytes — vector_from_float32() interprets them +); +``` + +#### 4d. Rewrite `VectorSearch.vectorSearch()` + +Replace the full-scan JS loop with `vector_top_k()`: + +```typescript +vectorSearch(queryEmbedding: Float32Array, options: VectorSearchOptions): VectorSearchResult[] { + const { repositoryId, versionId, profileId = 'local-default', limit = 50 } = options; + + const queryBytes = Buffer.from(queryEmbedding.buffer); + const candidatePool = limit * 4; // over-fetch for post-filter + + let sql = ` + SELECT se.snippet_id, + vector_distance_cos(se.vec_embedding, vector_from_float32(?)) AS distance + FROM vector_top_k('idx_snippet_embeddings_vec', vector_from_float32(?), ?) AS knn + JOIN snippet_embeddings se ON se.rowid = knn.id + JOIN snippets s ON s.id = se.snippet_id + WHERE s.repository_id = ? + AND se.profile_id = ? + `; + const params: unknown[] = [queryBytes, queryBytes, candidatePool, repositoryId, profileId]; + + if (versionId) { + sql += ' AND s.version_id = ?'; + params.push(versionId); + } + + sql += ' ORDER BY distance ASC LIMIT ?'; + params.push(limit); + + return this.db + .prepare(sql) + .all(...params) + .map((row) => ({ snippetId: row.snippet_id, score: 1 - row.distance })); +} +``` + +The `score` contract is preserved (1 = identical, 0 = orthogonal). The `cosineSimilarity` helper function is no longer called at runtime but can be kept for unit tests. + +### Phase 5 — Per-Job Serialization Key Fix + +**Files touched:** `src/lib/server/pipeline/worker-pool.ts` + +The current serialization guard uses a bare `repositoryId`: + +```typescript +// current +private runningRepoIds = new Set(); +// blocks any job whose repositoryId is already in the set +const jobIdx = this.jobQueue.findIndex((j) => !this.runningRepoIds.has(j.repositoryId)); +``` + +Different tags of the same repository write to completely disjoint rows (`version_id`-partitioned documents, snippets, and embeddings). The only genuine conflict is two jobs for the same `(repositoryId, versionId)` pair, which `JobQueue.enqueue()` already prevents via the `status IN ('queued', 'running')` deduplication check. + +Change the guard to key on the compound pair: + +```typescript +// replace Set with Set keyed on compound pair +private runningJobKeys = new Set(); + +private jobKey(repositoryId: string, versionId?: string | null): string { + return `${repositoryId}|${versionId ?? ''}`; +} +``` + +Update all four sites that read/write `runningRepoIds`: + +| Location | Old | New | +| ------------------------------------ | ----------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| `dispatch()` find | `!this.runningRepoIds.has(j.repositoryId)` | `!this.runningJobKeys.has(this.jobKey(j.repositoryId, j.versionId))` | +| `dispatch()` add | `this.runningRepoIds.add(job.repositoryId)` | `this.runningJobKeys.add(this.jobKey(job.repositoryId, job.versionId))` | +| `onWorkerMessage` done/failed delete | `this.runningRepoIds.delete(runningJob.repositoryId)` | `this.runningJobKeys.delete(this.jobKey(runningJob.repositoryId, runningJob.versionId))` | +| `onWorkerExit` delete | same | same | + +The `QueuedJob` and `RunningJob` interfaces already carry `versionId` — no type changes needed. + +The only serialized case that remains is `versionId = null` (default-branch re-index) paired with itself, which maps to the stable key `"repositoryId|"` — correctly deduplicated. + +--- + +### Phase 6 — Dedicated Write Worker (Single-Writer Pattern) + +**Files touched:** `src/lib/server/pipeline/worker-types.ts`, `src/lib/server/pipeline/write-worker-entry.ts` (new), `src/lib/server/pipeline/worker-entry.ts`, `src/lib/server/pipeline/worker-pool.ts` + +#### Motivation + +With Phase 5 in place, N tags of the same library can index in parallel. Each parse worker currently opens its own DB connection and holds the write lock while storing parsed snippets. Under N concurrent writers, each worker spends the majority of its wall-clock time waiting in `busy_timeout` back-off. The fix is the single-writer pattern: one dedicated write worker owns the only writable DB connection; parse workers become stateless CPU workers that send write batches over `postMessage`. + +``` +Parse Worker 1 ──┐ WriteRequest (docs[], snippets[]) ┌── WriteAck +Parse Worker 2 ──┼─────────────────────────────────────► Write Worker (sole DB writer) +Parse Worker N ──┘ └── single better-sqlite3 connection +``` + +#### New message types (`worker-types.ts`) + +```typescript +export interface WriteRequest { + type: 'write'; + jobId: string; + documents: SerializedDocument[]; + snippets: SerializedSnippet[]; +} + +export interface WriteAck { + type: 'write_ack'; + jobId: string; + documentCount: number; + snippetCount: number; +} + +export interface WriteError { + type: 'write_error'; + jobId: string; + error: string; +} + +// SerializedDocument / SerializedSnippet mirror the DB column shapes +// (plain objects, safe to transfer via structured clone) +``` + +#### Write worker (`write-worker-entry.ts`) + +The write worker: + +- Opens its own `Database` connection (WAL mode, all pragmas from Phase 2) +- Listens for `WriteRequest` messages +- Wraps each batch in a single transaction +- Posts `WriteAck` or `WriteError` back to the parent, which forwards the ack to the originating parse worker by `jobId` + +```typescript +import Database from '@libsql/better-sqlite3'; +import { workerData, parentPort } from 'node:worker_threads'; +import type { WriteRequest, WriteAck, WriteError } from './worker-types.js'; + +const db = new Database((workerData as WorkerInitData).dbPath); +db.pragma('journal_mode = WAL'); +db.pragma('synchronous = NORMAL'); +db.pragma('cache_size = -65536'); +db.pragma('foreign_keys = ON'); + +const insertDoc = db.prepare(`INSERT OR REPLACE INTO documents (...) VALUES (...)`); +const insertSnippet = db.prepare(`INSERT OR REPLACE INTO snippets (...) VALUES (...)`); + +const writeBatch = db.transaction((req: WriteRequest) => { + for (const doc of req.documents) insertDoc.run(doc); + for (const snip of req.snippets) insertSnippet.run(snip); +}); + +parentPort!.on('message', (req: WriteRequest) => { + try { + writeBatch(req); + const ack: WriteAck = { + type: 'write_ack', + jobId: req.jobId, + documentCount: req.documents.length, + snippetCount: req.snippets.length + }; + parentPort!.postMessage(ack); + } catch (err) { + const fail: WriteError = { type: 'write_error', jobId: req.jobId, error: String(err) }; + parentPort!.postMessage(fail); + } +}); +``` + +#### Parse worker changes (`worker-entry.ts`) + +Parse workers lose their DB connection. `IndexingPipeline` receives a `sendWrite` callback instead of a `db` instance. After parsing each file batch, the worker calls `sendWrite({ type: 'write', jobId, documents, snippets })` and awaits the `WriteAck` before continuing. This keeps back-pressure: a slow write worker naturally throttles the parse workers without additional semaphores. + +#### WorkerPool changes + +- Spawn one write worker at startup (always, regardless of embedding config) +- Route incoming `write_ack` / `write_error` messages to the correct waiting parse worker via a `Map` promise registry +- The write worker is separate from the embed worker — embed writes (`snippet_embeddings`) can still go through the write worker by adding an `EmbedWriteRequest` message type, or remain in the embed worker since embedding runs after parsing completes (no lock contention with active parse jobs) + +#### Conflict analysis with Phase 5 + +Phases 5 and 6 compose cleanly: + +- Phase 5 allows multiple `(repo, versionId)` jobs to run concurrently +- Phase 6 ensures all those concurrent jobs share a single write path — contention is eliminated by design +- The write worker is stateless with respect to job identity; it just executes batches in arrival order within a FIFO message queue (Node.js `postMessage` is ordered) +- The embed worker remains a separate process (it runs after parse completes, so it never overlaps with active parse writes for the same job) + +--- + +### Phase 7 — Admin UI Overhaul + +**Files touched:** + +- `src/routes/admin/jobs/+page.svelte` — rebuilt +- `src/routes/api/v1/workers/+server.ts` — new endpoint +- `src/lib/components/admin/JobStatusBadge.svelte` — extend with spinner variant +- `src/lib/components/admin/JobSkeleton.svelte` — new +- `src/lib/components/admin/WorkerStatusPanel.svelte` — new +- `src/lib/components/admin/Toast.svelte` — new +- `src/lib/components/IndexingProgress.svelte` — switch to SSE + +#### 7a. New API endpoint: `GET /api/v1/workers` + +The `WorkerPool` singleton tracks running jobs in `runningJobs: Map` and idle workers in `idleWorkers: Worker[]`. Expose this state as a lightweight REST snapshot: + +```typescript +// GET /api/v1/workers +// Response shape: +interface WorkersResponse { + concurrency: number; // configured max workers + active: number; // workers with a running job + idle: number; // workers waiting for work + workers: WorkerStatus[]; // one entry per spawned parse worker +} + +interface WorkerStatus { + index: number; // worker slot (0-based) + state: 'idle' | 'running'; // current state + jobId: string | null; // null when idle + repositoryId: string | null; + versionId: string | null; +} +``` + +The route handler calls `getPool().getStatus()` — add a `getStatus(): WorkersResponse` method to `WorkerPool` that reads `runningJobs` and `idleWorkers` without any DB call. This is read-only and runs on the main thread. + +The SSE stream at `/api/v1/jobs/stream` should emit a new `worker-status` event type whenever a worker transitions idle ↔ running (on `dispatch()` and job completion). This allows the worker panel to update in real-time without polling the REST endpoint. + +#### 7b. `GET /api/v1/jobs` — add `repositoryId` free-text and multi-status filter + +The existing endpoint already accepts `repositoryId` (exact match) and `status` (single value). Extend: + +- `repositoryId` to also support prefix match (e.g. `?repositoryId=/facebook` returns all `/facebook/*` repos) +- `status` to accept comma-separated values: `?status=queued,running` +- `page` and `pageSize` query params (default pageSize=50, max 200) in addition to `limit` for backwards compat + +Return `{ jobs, total, page, pageSize }` with `total` always reflecting the unfiltered-by-page count. + +#### 7c. New component: `JobSkeleton.svelte` + +A set of skeleton rows matching the job table structure. Shown during the initial fetch before any data arrives. Uses Tailwind `animate-pulse`: + +```svelte + + + +{#each Array(rows) as _, i (i)} + + +
+
+ + +
+ + +
+ + +
+ + +
+ + +
+ + +{/each} +``` + +#### 7d. New component: `Toast.svelte` + +Replaces all `alert()` / `console.log()` calls in the jobs page. Renders a fixed-position stack in the bottom-right corner. Each toast auto-dismisses after 4 seconds and can be manually closed: + +```svelte + + + +
+ {#each toasts as toast (toast.id)} + + {/each} +
+``` + +The jobs page replaces `showToast()` with pushing onto the bound `toasts` array. The `confirm()` for cancel is replaced with an inline confirmation state per job (`pendingCancelId`) that shows "Confirm cancel?" / "Yes" / "No" buttons inside the row. + +#### 7e. New component: `WorkerStatusPanel.svelte` + +A compact panel displayed above the job table showing the worker pool health. Subscribes to the `worker-status` SSE events and falls back to polling `GET /api/v1/workers` every 5 s on SSE error: + +``` +┌─────────────────────────────────────────────────────────┐ +│ Workers [2 / 4 active] ████░░░░ 50% │ +│ Worker 0 ● running /facebook/react / v18.3.0 │ +│ Worker 1 ● running /facebook/react / v17.0.2 │ +│ Worker 2 ○ idle │ +│ Worker 3 ○ idle │ +└─────────────────────────────────────────────────────────┘ +``` + +Each worker row shows: slot index, status dot (animated green pulse for running), repository ID, version tag, and a link to the job row in the table below. + +#### 7f. Filter bar on the jobs page + +Add a filter strip between the page header and the table: + +``` +[ Repository: _______________ ] [ Status: ▾ all ] [ 🔍 Apply ] [ ↺ Reset ] +``` + +- **Repository field**: free-text input, matches `repositoryId` prefix (e.g. `/facebook` shows all `/facebook/*`) +- **Status dropdown**: multi-select checkboxes for `queued`, `running`, `paused`, `cancelled`, `done`, `failed`; default = all +- Filters are applied client-side against the loaded `jobs` array for instant feedback, and also re-fetched from the API on Apply to get the correct total count +- Filter state is mirrored to URL search params (`?repo=...&status=...`) so the view is bookmarkable and survives refresh + +#### 7g. Per-job action spinner and disabled state + +Replace the single `actionInProgress: string | null` with a `Map`: + +```typescript +let actionInProgress = $state(new Map()); +``` + +Each action button shows an inline spinner (small `animate-spin` circle) and is disabled only for that row. Other rows remain fully interactive during the action. On completion the entry is deleted from the map. + +#### 7h. `IndexingProgress.svelte` — switch from polling to SSE + +The component currently uses `setInterval + fetch` at 2 s. Replace with the per-job SSE stream already available at `/api/v1/jobs/{id}/stream`: + +```typescript +// replace the $effect body +$effect(() => { + job = null; + const es = new EventSource(`/api/v1/jobs/${jobId}/stream`); + + es.addEventListener('job-progress', (event) => { + const data = JSON.parse(event.data); + job = { ...job, ...data }; + }); + + es.addEventListener('job-done', () => { + void fetch(`/api/v1/jobs/${jobId}`) + .then((r) => r.json()) + .then((d) => { + job = d.job; + oncomplete?.(); + }); + es.close(); + }); + + es.addEventListener('job-failed', (event) => { + const data = JSON.parse(event.data); + job = { ...job, status: 'failed', error: data.error }; + oncomplete?.(); + es.close(); + }); + + es.onerror = () => { + // on SSE failure fall back to a single fetch to get current state + es.close(); + void fetch(`/api/v1/jobs/${jobId}`) + .then((r) => r.json()) + .then((d) => { + job = d.job; + }); + }; + + return () => es.close(); +}); +``` + +This reduces network traffic from 1 request/2 s to zero requests during active indexing — updates arrive as server-push events. + +#### 7i. Pagination on the jobs page + +Replace the hard-coded `?limit=50` fetch with paginated requests: + +```typescript +let currentPage = $state(1); +const PAGE_SIZE = 50; + +async function fetchJobs() { + const params = new URLSearchParams({ + page: String(currentPage), + pageSize: String(PAGE_SIZE), + ...(filterRepo ? { repositoryId: filterRepo } : {}), + ...(filterStatuses.length ? { status: filterStatuses.join(',') } : {}) + }); + const data = await fetch(`/api/v1/jobs?${params}`).then((r) => r.json()); + jobs = data.jobs; + total = data.total; +} +``` + +Render a simple `« Prev Page N of M Next »` control below the table, hidden when `total <= PAGE_SIZE`. + +--- + +## Acceptance Criteria + +- [ ] `npm install` with `@libsql/better-sqlite3` succeeds; `better-sqlite3` is absent from `node_modules` +- [ ] All existing unit and integration tests pass after Phase 1 import swap +- [ ] `npm run db:migrate` applies the composite index migration cleanly against an existing database +- [ ] `npm run db:migrate` applies the vector column migration cleanly; `sql> SELECT vec_embedding FROM snippet_embeddings LIMIT 1` returns a non-NULL value for any previously-embedded snippet +- [ ] `GET /api/v1/context?libraryId=...&query=...` with a semantic-mode or hybrid-mode request returns results in ≤ 200 ms on a repository with 50k+ snippets (vs previous multi-second response) +- [ ] Memory profiled during a /context request shows no allocation spike proportional to repository size +- [ ] `EXPLAIN QUERY PLAN` on the `snippets` search query shows `SCAN snippets USING INDEX idx_snippets_repo_version` instead of `SCAN snippets` +- [ ] Worker threads (`worker-entry.ts`, `embed-worker-entry.ts`) start and complete an indexing job successfully after the package swap +- [ ] `drizzle-kit studio` connects and browses the migrated database +- [ ] Re-indexing a repository after the migration correctly populates `vec_embedding` on all new snippets +- [ ] `cosineSimilarity` unit tests still pass (function is kept) +- [ ] Starting two indexing jobs for different tags of the same repository simultaneously results in both jobs reaching `running` state concurrently (not one waiting for the other) +- [ ] Starting two indexing jobs for the **same** `(repositoryId, versionId)` pair returns the existing job (deduplication unchanged) +- [ ] With 4 parse workers and 4 concurrent tag jobs, zero `SQLITE_BUSY` errors appear in logs +- [ ] Write worker is present in the process list during active indexing (`worker_threads` inspector shows `write-worker-entry`) +- [ ] A `WriteError` from the write worker marks the originating job as `failed` with the error message propagated to the SSE stream +- [ ] `GET /api/v1/workers` returns a `WorkersResponse` JSON object with correct `active`, `idle`, and `workers[]` fields while jobs are in-flight +- [ ] The `worker-status` SSE event is emitted by `/api/v1/jobs/stream` whenever a worker transitions state +- [ ] The admin jobs page shows skeleton rows (not a blank screen) during the initial `fetchJobs()` call +- [ ] No `alert()` or `confirm()` calls exist in `admin/jobs/+page.svelte` after this change; all notifications go through `Toast.svelte` +- [ ] Pausing job A while job B is also in progress does not disable job B's action buttons +- [ ] The status filter multi-select correctly restricts the visible job list; the URL updates to reflect the filter state +- [ ] The repository prefix filter `?repositoryId=/facebook` returns all jobs whose `repositoryId` starts with `/facebook` +- [ ] Paginating past page 1 fetches the next batch from the API, not from the client-side array +- [ ] `IndexingProgress.svelte` has no `setInterval` call; it uses `EventSource` for progress updates +- [ ] The `WorkerStatusPanel` shows the correct number of running workers live during a multi-tag indexing run +- [ ] Refreshing the jobs page with `?repo=/facebook/react&status=running` pre-populates the filters and fetches with those params + +--- + +## Migration Safety + +### Backward Compatibility + +The `embedding` blob column is kept. The `vec_embedding` column is nullable during the backfill window and becomes populated as: + +1. The `UPDATE` in `vectors.sql` fills all existing rows on startup +2. New embeddings populate it at insert time + +If `vec_embedding IS NULL` for a row (e.g., a row inserted before the migration runs), the vector index silently omits that row from results. The fallback in `HybridSearchService` to FTS-only mode still applies when no embeddings exist, so degraded-but-correct behavior is preserved. + +### Rollback + +Rollback before Phase 4 (vector column): remove `@libsql/better-sqlite3`, restore `better-sqlite3`, restore imports. No schema changes have been made. + +Rollback after Phase 4: schema now has `vec_embedding` column. Drop the column with a migration reversal and restore imports. The `embedding` blob is intact throughout — no data loss. + +### SQLite File Compatibility + +libSQL embedded mode reads and writes standard SQLite 3 files. The WAL file, page size, and encoding are unchanged. An existing production database opened with `@libsql/better-sqlite3` is fully readable and writable. The vector index is stored in a shadow table `idx_snippet_embeddings_vec_shadow` which better-sqlite3 would ignore if rolled back (it is a regular table with a special name). + +--- + +## Dependencies + +| Package | Action | Reason | +| ------------------------ | ----------------------------- | ----------------------------------------------- | +| `better-sqlite3` | Remove from `dependencies` | Replaced | +| `@types/better-sqlite3` | Remove from `devDependencies` | `@libsql/better-sqlite3` ships own types | +| `@libsql/better-sqlite3` | Add to `dependencies` | Drop-in libSQL node addon | +| `drizzle-orm` | No change | `better-sqlite3` adapter works unchanged | +| `drizzle-kit` | No change | `dialect: 'sqlite'` correct for embedded libSQL | + +No new runtime dependencies beyond the package replacement. + +--- + +## Testing Strategy + +### Unit Tests + +- `src/lib/server/search/vector.search.ts`: add test asserting KNN results are correct for a seeded 3-vector table; verify memory is not proportional to table size (mock `db.prepare` to assert no unbounded `.all()` is called) +- `src/lib/server/embeddings/embedding.service.ts`: existing tests cover insert round-trips; verify `vec_embedding` column is non-NULL after `embedSnippets()` + +### Integration Tests + +- `api-contract.integration.test.ts`: existing tests already use `new Database(':memory:')` — these continue to work with `@libsql/better-sqlite3` because the in-memory path is identical +- Add one test to `api-contract.integration.test.ts`: seed a repository + multiple embeddings, call `/api/v1/context` in semantic mode, assert non-empty results and response time < 500ms on in-memory DB + +### UI Tests + +- `src/routes/admin/jobs/+page.svelte`: add Vitest browser tests (Playwright) verifying: + - Skeleton rows appear before the first fetch resolves (mock `fetch` to delay 200 ms) + - Status filter restricts displayed rows; URL param updates + - Pausing job A leaves job B's buttons enabled + - Toast appears and auto-dismisses on successful pause + - Cancel confirm flow shows inline confirmation, not `window.confirm` +- `src/lib/components/IndexingProgress.svelte`: unit test that no `setInterval` is created; verify `EventSource` is opened with the correct URL + +### Performance Regression Gate + +Add a benchmark script `scripts/bench-vector-search.mjs` that: + +1. Creates an in-memory libSQL database +2. Seeds 10000 snippet embeddings (random Float32Array, 1536 dims) +3. Runs 100 `vectorSearch()` calls +4. Asserts p99 < 50 ms + +This gates the CI check on Phase 4 correctness and speed. diff --git a/scripts/build-workers.mjs b/scripts/build-workers.mjs index 1082a4e..8af7d16 100644 --- a/scripts/build-workers.mjs +++ b/scripts/build-workers.mjs @@ -8,7 +8,7 @@ const entries = [ ]; try { - const existing = entries.filter(e => existsSync(e)); + const existing = entries.filter((e) => existsSync(e)); if (existing.length === 0) { console.log('[build-workers] No worker entry files found yet, skipping.'); process.exit(0); @@ -23,7 +23,7 @@ try { outdir: 'build/workers', outExtension: { '.js': '.mjs' }, alias: { - '$lib': './src/lib', + $lib: './src/lib', '$lib/server': './src/lib/server' }, external: ['better-sqlite3', '@xenova/transformers'], diff --git a/scripts/build.mjs b/scripts/build.mjs index 5531a15..042d94f 100644 --- a/scripts/build.mjs +++ b/scripts/build.mjs @@ -76,4 +76,4 @@ try { await runViteBuild(); } finally { await restoreReservedRouteTests(); -} \ No newline at end of file +} diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 4c43ea0..3acda3f 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -33,9 +33,10 @@ try { try { const db = getClient(); const activeProfileRow = db - .prepare<[], EmbeddingProfileEntityProps>( - 'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1' - ) + .prepare< + [], + EmbeddingProfileEntityProps + >('SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1') .get(); let embeddingService: EmbeddingService | null = null; @@ -55,9 +56,10 @@ try { let concurrency = 2; // default if (dbPath) { const concurrencyRow = db - .prepare<[], { value: string }>( - "SELECT value FROM settings WHERE key = 'indexing.concurrency' LIMIT 1" - ) + .prepare< + [], + { value: string } + >("SELECT value FROM settings WHERE key = 'indexing.concurrency' LIMIT 1") .get(); if (concurrencyRow) { try { diff --git a/src/lib/components/IndexingProgress.svelte b/src/lib/components/IndexingProgress.svelte index 8a08bae..2770cb1 100644 --- a/src/lib/components/IndexingProgress.svelte +++ b/src/lib/components/IndexingProgress.svelte @@ -16,21 +16,29 @@ es.addEventListener('job-done', () => { void fetch(`/api/v1/jobs/${jobId}`) - .then(r => r.json()) - .then(d => { job = d.job; oncomplete?.(); }); + .then((r) => r.json()) + .then((d) => { + job = d.job; + oncomplete?.(); + }); es.close(); }); es.addEventListener('job-failed', (event) => { const data = JSON.parse(event.data); - if (job) job = { ...job, status: 'failed', error: data.error ?? 'Unknown error' } as IndexingJob; + if (job) + job = { ...job, status: 'failed', error: data.error ?? 'Unknown error' } as IndexingJob; oncomplete?.(); es.close(); }); es.onerror = () => { es.close(); - void fetch(`/api/v1/jobs/${jobId}`).then(r => r.json()).then(d => { job = d.job; }); + void fetch(`/api/v1/jobs/${jobId}`) + .then((r) => r.json()) + .then((d) => { + job = d.job; + }); }; return () => es.close(); diff --git a/src/lib/components/RepositoryCard.svelte.test.ts b/src/lib/components/RepositoryCard.svelte.test.ts index e3e7d24..5fcef70 100644 --- a/src/lib/components/RepositoryCard.svelte.test.ts +++ b/src/lib/components/RepositoryCard.svelte.test.ts @@ -30,4 +30,4 @@ describe('RepositoryCard.svelte', () => { expect(text).toMatch(/1[,.\u00a0\u202f]?200 embeddings/); expect(text).toContain('Indexed: main, v18.3.0'); }); -}); \ No newline at end of file +}); diff --git a/src/lib/components/admin/JobSkeleton.svelte b/src/lib/components/admin/JobSkeleton.svelte index 67edb21..af57662 100644 --- a/src/lib/components/admin/JobSkeleton.svelte +++ b/src/lib/components/admin/JobSkeleton.svelte @@ -1,8 +1,9 @@ -{#each Array(rows) as _, i (i)} +{#each rowIndexes as i (i)}
diff --git a/src/lib/components/admin/Toast.svelte b/src/lib/components/admin/Toast.svelte index 3d7c156..efae163 100644 --- a/src/lib/components/admin/Toast.svelte +++ b/src/lib/components/admin/Toast.svelte @@ -1,5 +1,6 @@