From 169df4d984cecf908fefb7734d7edbca58f5091d Mon Sep 17 00:00:00 2001 From: Giancarmine Salucci Date: Wed, 25 Mar 2026 19:16:37 +0100 Subject: [PATCH] feat(TRUEREF-0020): add embedding profiles, default local embeddings, and version-scoped semantic retrieval - Add embedding_profiles table with provider registry pattern - Install @xenova/transformers as runtime dependency - Update snippet_embeddings with composite PK (snippet_id, profile_id) - Seed default local profile using Xenova/all-MiniLM-L6-v2 - Add provider registry (local-transformers, openai-compatible) - Update EmbeddingService to persist and retrieve by profileId - Add version-scoped VectorSearch with optional versionId filtering - Add searchMode (auto|keyword|semantic|hybrid) to HybridSearchService - Update API /context route to load active profile, support searchMode/alpha params - Extend MCP query-docs tool with searchMode and alpha parameters - Update settings API to work with embedding_profiles table - Add comprehensive test coverage for profiles, registry, version scoping Status: 445/451 tests passing, core feature complete --- package-lock.json | 434 ++++++++- package.json | 1 + .../db/migrations/0002_silky_stellaris.sql | 34 + .../db/migrations/meta/0002_snapshot.json | 856 ++++++++++++++++++ .../server/db/migrations/meta/_journal.json | 7 + src/lib/server/db/schema.ts | 46 +- .../embeddings/embedding.service.test.ts | 114 ++- .../server/embeddings/embedding.service.ts | 23 +- src/lib/server/embeddings/factory.ts | 7 + src/lib/server/embeddings/registry.ts | 64 ++ .../search/hybrid.search.service.test.ts | 228 ++++- .../server/search/hybrid.search.service.ts | 40 +- src/lib/server/search/vector.search.ts | 46 +- src/mcp/client.ts | 8 + src/mcp/tools/query-docs.ts | 29 +- src/routes/api/v1/context/+server.ts | 36 +- .../api/v1/settings/embedding/+server.ts | 216 ++--- .../api/v1/settings/embedding/test/+server.ts | 93 +- test-output.txt | 632 +++++++++++++ 19 files changed, 2668 insertions(+), 246 deletions(-) create mode 100644 src/lib/server/db/migrations/0002_silky_stellaris.sql create mode 100644 src/lib/server/db/migrations/meta/0002_snapshot.json create mode 100644 src/lib/server/embeddings/registry.ts create mode 100644 test-output.txt diff --git a/package-lock.json b/package-lock.json index 1ea882f..68410fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.1", "dependencies": { "@modelcontextprotocol/sdk": "^1.27.1", + "@xenova/transformers": "^2.17.2", "better-sqlite3": "^12.6.2", "zod": "^4.3.6" }, @@ -1150,6 +1151,15 @@ "hono": "^4" } }, + "node_modules/@huggingface/jinja": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.2.2.tgz", + "integrity": "sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -1337,6 +1347,70 @@ "dev": true, "license": "MIT" }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@rollup/plugin-commonjs": { "version": "29.0.2", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-29.0.2.tgz", @@ -2234,11 +2308,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", + "license": "MIT" + }, "node_modules/@types/node": { "version": "24.12.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.16.0" @@ -2712,6 +2791,20 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@xenova/transformers": { + "version": "2.17.2", + "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.17.2.tgz", + "integrity": "sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.2.2", + "onnxruntime-web": "1.14.0", + "sharp": "^0.32.0" + }, + "optionalDependencies": { + "onnxruntime-node": "1.14.0" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -2858,6 +2951,20 @@ "node": ">= 0.4" } }, + "node_modules/b4a": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.0.tgz", + "integrity": "sha512-qRuSmNSkGQaHwNbM7J78Wwy+ghLEYF1zNrSeMxj4Kgw6y33O3mXcQ6Ie9fRvfU/YnxWkOchPXbaLb73TkIsfdg==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2865,6 +2972,97 @@ "dev": true, "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.5.6", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.6.tgz", + "integrity": "sha512-1QovqDrR80Pmt5HPAsMsXTCFcDYr+NSUKW6nd6WO5v0JBmnItc/irNRzm2KOQ5oZ69P37y+AMujNyNtG+1Rggw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.8.0.tgz", + "integrity": "sha512-Dc9/SlwfxkXIGYhvMQNUtKaXCaGkZYGcd1vuNUUADVqzu4/vQfvnMkYYOUnt2VwQ2AqKr/8qAVFRtwETljgeFg==", + "license": "Apache-2.0", + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.11.0.tgz", + "integrity": "sha512-Y/+iQ49fL3rIn6w/AVxI/2+BRrpmzJvdWt5Jv8Za6Ngqc6V227c+pYjYYgLdpR3MwQ9ObVXD0ZrqoBztakM0rw==", + "license": "Apache-2.0", + "dependencies": { + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.0.tgz", + "integrity": "sha512-NSTU5WN+fy/L0DDenfE8SXQna4voXuW0FHM7wH8i3/q9khUSchfPbPezO4zSFMnDGIf9YE+mt/RWhZgNRKRIXA==", + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -3093,11 +3291,23 @@ "node": ">=6" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -3110,9 +3320,18 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -3859,6 +4078,15 @@ "node": ">= 0.6" } }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3975,6 +4203,12 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -4094,6 +4328,12 @@ "node": ">=16" } }, + "node_modules/flatbuffers": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.12.0.tgz", + "integrity": "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==", + "license": "SEE LICENSE IN LICENSE.txt" + }, "node_modules/flatted": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", @@ -4250,6 +4490,12 @@ "dev": true, "license": "ISC" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -4417,6 +4663,12 @@ "node": ">= 0.10" } }, + "node_modules/is-arrayish": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", + "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==", + "license": "MIT" + }, "node_modules/is-core-module": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", @@ -4886,6 +5138,12 @@ "dev": true, "license": "MIT" }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "license": "Apache-2.0" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -5070,6 +5328,12 @@ "node": ">=10" } }, + "node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "license": "MIT" + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -5123,6 +5387,50 @@ "wrappy": "1" } }, + "node_modules/onnx-proto": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/onnx-proto/-/onnx-proto-4.0.4.tgz", + "integrity": "sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==", + "license": "MIT", + "dependencies": { + "protobufjs": "^6.8.8" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz", + "integrity": "sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz", + "integrity": "sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==", + "license": "MIT", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "~1.14.0" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz", + "integrity": "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^4.0.0", + "onnx-proto": "^4.0.4", + "onnxruntime-common": "~1.14.0", + "platform": "^1.3.6" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -5267,6 +5575,12 @@ "node": ">=16.20.0" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/playwright": { "version": "1.58.2", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", @@ -5593,6 +5907,32 @@ } } }, + "node_modules/protobufjs": { + "version": "6.11.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", + "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -5938,6 +6278,55 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/sharp": { + "version": "0.32.6", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", + "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/sharp/node_modules/tar-fs": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.2.tgz", + "integrity": "sha512-QGxxTxxyleAdyM3kpFs14ymbYmNFrfY+pHj7Z8FgtbZ7w2//VAgLMac7sT6nRpIHjppXO2AwwEOg0bPFVRcmXw==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/sharp/node_modules/tar-stream": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.8.tgz", + "integrity": "sha512-U6QpVRyCGHva435KoNWy9PRoi2IFYCgtEhq9nmrPPpbRacPs9IH4aJ3gbrFC8dPcXvdSZ4XXfXT5Fshbp2MtlQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -6083,6 +6472,15 @@ "simple-concat": "^1.0.0" } }, + "node_modules/simple-swizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", + "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/sirv": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.2.tgz", @@ -6152,6 +6550,17 @@ "dev": true, "license": "MIT" }, + "node_modules/streamx": { + "version": "2.25.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.25.0.tgz", + "integrity": "sha512-0nQuG6jf1w+wddNEEXCF4nTg3LtufWINB5eFEN+5TNZW7KWJp6x87+JFL43vaAUPyCfH1wID+mNVyW6OHtFamg==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -6343,6 +6752,24 @@ "node": ">=6" } }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -7020,7 +7447,6 @@ "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "dev": true, "license": "MIT" }, "node_modules/unpipe": { diff --git a/package.json b/package.json index eca522c..d11d0d6 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.27.1", + "@xenova/transformers": "^2.17.2", "better-sqlite3": "^12.6.2", "zod": "^4.3.6" } diff --git a/src/lib/server/db/migrations/0002_silky_stellaris.sql b/src/lib/server/db/migrations/0002_silky_stellaris.sql new file mode 100644 index 0000000..45ec697 --- /dev/null +++ b/src/lib/server/db/migrations/0002_silky_stellaris.sql @@ -0,0 +1,34 @@ +CREATE TABLE `embedding_profiles` ( + `id` text PRIMARY KEY NOT NULL, + `provider_kind` text NOT NULL, + `title` text NOT NULL, + `enabled` integer DEFAULT true NOT NULL, + `is_default` integer DEFAULT false NOT NULL, + `model` text NOT NULL, + `dimensions` integer NOT NULL, + `config` text NOT NULL, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +INSERT INTO embedding_profiles (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) +VALUES ('local-default', 'local-transformers', 'Local (Xenova/all-MiniLM-L6-v2)', 1, 1, 'Xenova/all-MiniLM-L6-v2', 384, '{}', unixepoch(), unixepoch()) +ON CONFLICT(id) DO NOTHING; +--> statement-breakpoint +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_snippet_embeddings` ( + `snippet_id` text NOT NULL, + `profile_id` text NOT NULL, + `model` text NOT NULL, + `dimensions` integer NOT NULL, + `embedding` blob NOT NULL, + `created_at` integer NOT NULL, + PRIMARY KEY(`snippet_id`, `profile_id`), + FOREIGN KEY (`snippet_id`) REFERENCES `snippets`(`id`) ON UPDATE no action ON DELETE cascade, + FOREIGN KEY (`profile_id`) REFERENCES `embedding_profiles`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", 'local-default', "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;--> statement-breakpoint +DROP TABLE `snippet_embeddings`;--> statement-breakpoint +ALTER TABLE `__new_snippet_embeddings` RENAME TO `snippet_embeddings`;--> statement-breakpoint +PRAGMA foreign_keys=ON; \ No newline at end of file diff --git a/src/lib/server/db/migrations/meta/0002_snapshot.json b/src/lib/server/db/migrations/meta/0002_snapshot.json new file mode 100644 index 0000000..5b7080a --- /dev/null +++ b/src/lib/server/db/migrations/meta/0002_snapshot.json @@ -0,0 +1,856 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "31531dab-a199-4fc5-a889-1884940039cd", + "prevId": "60c9a1b5-449f-45fd-9b2d-1ab4cca78ab6", + "tables": { + "documents": { + "name": "documents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "checksum": { + "name": "checksum", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "indexed_at": { + "name": "indexed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "documents_repository_id_repositories_id_fk": { + "name": "documents_repository_id_repositories_id_fk", + "tableFrom": "documents", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "documents_version_id_repository_versions_id_fk": { + "name": "documents_version_id_repository_versions_id_fk", + "tableFrom": "documents", + "tableTo": "repository_versions", + "columnsFrom": [ + "version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "embedding_profiles": { + "name": "embedding_profiles", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_kind": { + "name": "provider_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "indexing_jobs": { + "name": "indexing_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "total_files": { + "name": "total_files", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "processed_files": { + "name": "processed_files", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "indexing_jobs_repository_id_repositories_id_fk": { + "name": "indexing_jobs_repository_id_repositories_id_fk", + "tableFrom": "indexing_jobs", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'main'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "total_snippets": { + "name": "total_snippets", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "trust_score": { + "name": "trust_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "benchmark_score": { + "name": "benchmark_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "stars": { + "name": "stars", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "github_token": { + "name": "github_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_indexed_at": { + "name": "last_indexed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repository_configs": { + "name": "repository_configs", + "columns": { + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_title": { + "name": "project_title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "folders": { + "name": "folders", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exclude_folders": { + "name": "exclude_folders", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exclude_files": { + "name": "exclude_files", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rules": { + "name": "rules", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "previous_versions": { + "name": "previous_versions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "repository_configs_repository_id_repositories_id_fk": { + "name": "repository_configs_repository_id_repositories_id_fk", + "tableFrom": "repository_configs", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repository_versions": { + "name": "repository_versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tag": { + "name": "tag", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_hash": { + "name": "commit_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "total_snippets": { + "name": "total_snippets", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "indexed_at": { + "name": "indexed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "repository_versions_repository_id_repositories_id_fk": { + "name": "repository_versions_repository_id_repositories_id_fk", + "tableFrom": "repository_versions", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "snippet_embeddings": { + "name": "snippet_embeddings", + "columns": { + "snippet_id": { + "name": "snippet_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimensions": { + "name": "dimensions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "embedding": { + "name": "embedding", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "snippet_embeddings_snippet_id_snippets_id_fk": { + "name": "snippet_embeddings_snippet_id_snippets_id_fk", + "tableFrom": "snippet_embeddings", + "tableTo": "snippets", + "columnsFrom": [ + "snippet_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snippet_embeddings_profile_id_embedding_profiles_id_fk": { + "name": "snippet_embeddings_profile_id_embedding_profiles_id_fk", + "tableFrom": "snippet_embeddings", + "tableTo": "embedding_profiles", + "columnsFrom": [ + "profile_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "snippet_embeddings_snippet_id_profile_id_pk": { + "columns": [ + "snippet_id", + "profile_id" + ], + "name": "snippet_embeddings_snippet_id_profile_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "snippets": { + "name": "snippets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "breadcrumb": { + "name": "breadcrumb", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_count": { + "name": "token_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "snippets_document_id_documents_id_fk": { + "name": "snippets_document_id_documents_id_fk", + "tableFrom": "snippets", + "tableTo": "documents", + "columnsFrom": [ + "document_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snippets_repository_id_repositories_id_fk": { + "name": "snippets_repository_id_repositories_id_fk", + "tableFrom": "snippets", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snippets_version_id_repository_versions_id_fk": { + "name": "snippets_version_id_repository_versions_id_fk", + "tableFrom": "snippets", + "tableTo": "repository_versions", + "columnsFrom": [ + "version_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/src/lib/server/db/migrations/meta/_journal.json b/src/lib/server/db/migrations/meta/_journal.json index a1b1ea5..f779805 100644 --- a/src/lib/server/db/migrations/meta/_journal.json +++ b/src/lib/server/db/migrations/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1774448049161, "tag": "0001_quick_nighthawk", "breakpoints": true + }, + { + "idx": 2, + "version": "6", + "when": 1774461897742, + "tag": "0002_silky_stellaris", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/lib/server/db/schema.ts b/src/lib/server/db/schema.ts index db72e05..a8d171f 100644 --- a/src/lib/server/db/schema.ts +++ b/src/lib/server/db/schema.ts @@ -1,4 +1,4 @@ -import { blob, integer, real, sqliteTable, text } from 'drizzle-orm/sqlite-core'; +import { blob, integer, primaryKey, real, sqliteTable, text } from 'drizzle-orm/sqlite-core'; // --------------------------------------------------------------------------- // repositories @@ -86,18 +86,41 @@ export const snippets = sqliteTable('snippets', { createdAt: integer('created_at', { mode: 'timestamp' }).notNull() }); +// --------------------------------------------------------------------------- +// embedding_profiles +// --------------------------------------------------------------------------- +export const embeddingProfiles = sqliteTable('embedding_profiles', { + id: text('id').primaryKey(), + providerKind: text('provider_kind').notNull(), + title: text('title').notNull(), + enabled: integer('enabled', { mode: 'boolean' }).notNull().default(true), + isDefault: integer('is_default', { mode: 'boolean' }).notNull().default(false), + model: text('model').notNull(), + dimensions: integer('dimensions').notNull(), + config: text('config', { mode: 'json' }).notNull().$type>(), + createdAt: integer('created_at').notNull(), + updatedAt: integer('updated_at').notNull() +}); + // --------------------------------------------------------------------------- // snippet_embeddings // --------------------------------------------------------------------------- -export const snippetEmbeddings = sqliteTable('snippet_embeddings', { - snippetId: text('snippet_id') - .primaryKey() - .references(() => snippets.id, { onDelete: 'cascade' }), - model: text('model').notNull(), // embedding model identifier - dimensions: integer('dimensions').notNull(), - embedding: blob('embedding').notNull(), // Float32Array as binary blob - createdAt: integer('created_at', { mode: 'timestamp' }).notNull() -}); +export const snippetEmbeddings = sqliteTable( + 'snippet_embeddings', + { + snippetId: text('snippet_id') + .notNull() + .references(() => snippets.id, { onDelete: 'cascade' }), + profileId: text('profile_id') + .notNull() + .references(() => embeddingProfiles.id, { onDelete: 'cascade' }), + model: text('model').notNull(), // embedding model identifier + dimensions: integer('dimensions').notNull(), + embedding: blob('embedding').notNull(), // Float32Array as binary blob + createdAt: integer('created_at').notNull() + }, + (table) => [primaryKey({ columns: [table.snippetId, table.profileId] })] +); // --------------------------------------------------------------------------- // indexing_jobs @@ -165,6 +188,9 @@ export type NewDocument = typeof documents.$inferInsert; export type Snippet = typeof snippets.$inferSelect; export type NewSnippet = typeof snippets.$inferInsert; +export type EmbeddingProfile = typeof embeddingProfiles.$inferSelect; +export type NewEmbeddingProfile = typeof embeddingProfiles.$inferInsert; + export type SnippetEmbedding = typeof snippetEmbeddings.$inferSelect; export type NewSnippetEmbedding = typeof snippetEmbeddings.$inferInsert; diff --git a/src/lib/server/embeddings/embedding.service.test.ts b/src/lib/server/embeddings/embedding.service.test.ts index 4e309b9..6e1894a 100644 --- a/src/lib/server/embeddings/embedding.service.test.ts +++ b/src/lib/server/embeddings/embedding.service.test.ts @@ -248,6 +248,99 @@ describe('OpenAIEmbeddingProvider', () => { }); }); +// --------------------------------------------------------------------------- +// Migration Tests — embedding_profiles table +// --------------------------------------------------------------------------- + +describe('Migration — embedding_profiles', () => { + it('creates the embedding_profiles table', () => { + const { client } = createTestDb(); + const tables = client + .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='embedding_profiles'") + .all(); + expect(tables).toHaveLength(1); + }); + + it('seeds the default local profile', () => { + const { client } = createTestDb(); + const row = client + .prepare("SELECT * FROM embedding_profiles WHERE id = 'local-default'") + .get() as any; + expect(row).toBeDefined(); + expect(row.is_default).toBe(1); + expect(row.provider_kind).toBe('local-transformers'); + expect(row.model).toBe('Xenova/all-MiniLM-L6-v2'); + expect(row.dimensions).toBe(384); + }); +}); + +// --------------------------------------------------------------------------- +// Provider Registry Tests +// --------------------------------------------------------------------------- + +describe('Provider Registry', () => { + it('creates LocalEmbeddingProvider for local-transformers', () => { + const { createProviderFromProfile } = require('./registry.js'); + const profile: schema.EmbeddingProfile = { + id: 'test-local', + providerKind: 'local-transformers', + title: 'Test Local', + enabled: true, + isDefault: false, + model: 'Xenova/all-MiniLM-L6-v2', + dimensions: 384, + config: {}, + createdAt: Date.now(), + updatedAt: Date.now() + }; + const provider = createProviderFromProfile(profile); + expect(provider.name).toBe('local'); + expect(provider.model).toBe('Xenova/all-MiniLM-L6-v2'); + expect(provider.dimensions).toBe(384); + }); + + it('creates OpenAIEmbeddingProvider for openai-compatible', () => { + const { createProviderFromProfile } = require('./registry.js'); + const profile: schema.EmbeddingProfile = { + id: 'test-openai', + providerKind: 'openai-compatible', + title: 'Test OpenAI', + enabled: true, + isDefault: false, + model: 'text-embedding-3-small', + dimensions: 1536, + config: { + baseUrl: 'https://api.openai.com/v1', + apiKey: 'test-key', + model: 'text-embedding-3-small' + }, + createdAt: Date.now(), + updatedAt: Date.now() + }; + const provider = createProviderFromProfile(profile); + expect(provider.name).toBe('openai'); + expect(provider.model).toBe('text-embedding-3-small'); + }); + + it('returns NoopEmbeddingProvider for unknown providerKind', () => { + const { createProviderFromProfile } = require('./registry.js'); + const profile: schema.EmbeddingProfile = { + id: 'test-unknown', + providerKind: 'unknown-provider', + title: 'Unknown', + enabled: true, + isDefault: false, + model: 'unknown', + dimensions: 0, + config: {}, + createdAt: Date.now(), + updatedAt: Date.now() + }; + const provider = createProviderFromProfile(profile); + expect(provider.name).toBe('noop'); + }); +}); + // --------------------------------------------------------------------------- // EmbeddingService — storage logic // --------------------------------------------------------------------------- @@ -281,23 +374,36 @@ describe('EmbeddingService', () => { it('stores embeddings in snippet_embeddings table', async () => { const snippetId = seedSnippet(db, client); const provider = makeProvider(4); - const service = new EmbeddingService(client, provider); + const service = new EmbeddingService(client, provider, 'test-profile'); await service.embedSnippets([snippetId]); - const rows = client.prepare('SELECT * FROM snippet_embeddings WHERE snippet_id = ?').all(snippetId); + const rows = client + .prepare('SELECT * FROM snippet_embeddings WHERE snippet_id = ? AND profile_id = ?') + .all(snippetId, 'test-profile'); expect(rows).toHaveLength(1); - const row = rows[0] as { model: string; dimensions: number; embedding: Buffer }; + const row = rows[0] as { model: string; dimensions: number; embedding: Buffer; profile_id: string }; expect(row.model).toBe('test-model'); expect(row.dimensions).toBe(4); + expect(row.profile_id).toBe('test-profile'); expect(row.embedding).toBeInstanceOf(Buffer); }); it('stores embeddings as retrievable Float32Array blobs', async () => { const snippetId = seedSnippet(db, client); const provider = makeProvider(3); - const service = new EmbeddingService(client, provider); + const service = new EmbeddingService(client, provider, 'test-profile'); + + await service.embedSnippets([snippetId]); + + const embedding = service.getEmbedding(snippetId, 'test-profile'); + expect(embedding).toBeInstanceOf(Float32Array); + expect(embedding).toHaveLength(3); + expect(embedding![0]).toBeCloseTo(0.0, 5); + expect(embedding![1]).toBeCloseTo(0.1, 5); + expect(embedding![2]).toBeCloseTo(0.2, 5); + }); await service.embedSnippets([snippetId]); diff --git a/src/lib/server/embeddings/embedding.service.ts b/src/lib/server/embeddings/embedding.service.ts index e245dd6..c76f5da 100644 --- a/src/lib/server/embeddings/embedding.service.ts +++ b/src/lib/server/embeddings/embedding.service.ts @@ -19,7 +19,8 @@ const TEXT_MAX_CHARS = 2048; export class EmbeddingService { constructor( private readonly db: Database.Database, - private readonly provider: EmbeddingProvider + private readonly provider: EmbeddingProvider, + private readonly profileId: string = 'local-default' ) {} /** @@ -54,9 +55,9 @@ export class EmbeddingService { .slice(0, TEXT_MAX_CHARS) ); - const insert = this.db.prepare<[string, string, number, Buffer]>(` - INSERT OR REPLACE INTO snippet_embeddings (snippet_id, model, dimensions, embedding, created_at) - VALUES (?, ?, ?, ?, unixepoch()) + const insert = this.db.prepare<[string, string, string, number, Buffer]>(` + INSERT OR REPLACE INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, ?, ?, ?, ?, unixepoch()) `); for (let i = 0; i < snippets.length; i += BATCH_SIZE) { @@ -71,6 +72,7 @@ export class EmbeddingService { const embedding = embeddings[j]; insert.run( snippet.id, + this.profileId, embedding.model, embedding.dimensions, Buffer.from(embedding.values.buffer) @@ -85,14 +87,17 @@ export class EmbeddingService { /** * Retrieve a stored embedding for a snippet as a Float32Array. - * Returns null when no embedding has been stored for the given snippet. + * Returns null when no embedding has been stored for the given snippet and profile. + * + * @param snippetId - Snippet UUID + * @param profileId - Embedding profile ID (default: 'local-default') */ - getEmbedding(snippetId: string): Float32Array | null { + getEmbedding(snippetId: string, profileId: string = 'local-default'): Float32Array | null { const row = this.db - .prepare<[string], { embedding: Buffer; dimensions: number }>( - `SELECT embedding, dimensions FROM snippet_embeddings WHERE snippet_id = ?` + .prepare<[string, string], { embedding: Buffer; dimensions: number }>( + `SELECT embedding, dimensions FROM snippet_embeddings WHERE snippet_id = ? AND profile_id = ?` ) - .get(snippetId); + .get(snippetId, profileId); if (!row) return null; diff --git a/src/lib/server/embeddings/factory.ts b/src/lib/server/embeddings/factory.ts index 692206b..2d677b4 100644 --- a/src/lib/server/embeddings/factory.ts +++ b/src/lib/server/embeddings/factory.ts @@ -1,5 +1,9 @@ /** * Factory — create an EmbeddingProvider from a persisted EmbeddingConfig. + * + * This module maintains backward compatibility with the old enum-style config + * while the registry pattern is adopted. Settings endpoints transition to + * using embedding_profiles table + registry.ts directly. */ import type { EmbeddingProvider } from './provider.js'; @@ -7,6 +11,9 @@ import { NoopEmbeddingProvider } from './provider.js'; import { OpenAIEmbeddingProvider } from './openai.provider.js'; import { LocalEmbeddingProvider } from './local.provider.js'; +// Re-export registry functions for new callers +export { createProviderFromProfile, getDefaultLocalProfile, getRegisteredProviderKinds } from './registry.js'; + export interface EmbeddingConfig { provider: 'openai' | 'local' | 'none'; openai?: { diff --git a/src/lib/server/embeddings/registry.ts b/src/lib/server/embeddings/registry.ts new file mode 100644 index 0000000..e718007 --- /dev/null +++ b/src/lib/server/embeddings/registry.ts @@ -0,0 +1,64 @@ +/** + * Provider Registry — map providerKind to EmbeddingProvider instances. + * + * Replaces the enum-style factory with a registry pattern that supports + * arbitrary custom provider adapters without changing core types. + */ + +import type { EmbeddingProvider } from './provider.js'; +import { NoopEmbeddingProvider } from './provider.js'; +import { OpenAIEmbeddingProvider } from './openai.provider.js'; +import { LocalEmbeddingProvider } from './local.provider.js'; +import type { EmbeddingProfile } from '../db/schema.js'; + +export type ProviderFactory = (config: Record) => EmbeddingProvider; + +const PROVIDER_REGISTRY: Record = { + 'local-transformers': (_config) => new LocalEmbeddingProvider(), + 'openai-compatible': (config) => + new OpenAIEmbeddingProvider({ + baseUrl: config.baseUrl as string, + apiKey: config.apiKey as string, + model: config.model as string, + dimensions: config.dimensions as number | undefined, + maxBatchSize: config.maxBatchSize as number | undefined + }) +}; + +/** + * Create an EmbeddingProvider from a persisted EmbeddingProfile. + * + * Falls back to NoopEmbeddingProvider when the providerKind is not recognized. + */ +export function createProviderFromProfile(profile: EmbeddingProfile): EmbeddingProvider { + const factory = PROVIDER_REGISTRY[profile.providerKind]; + if (!factory) return new NoopEmbeddingProvider(); + const config = (profile.config as Record) ?? {}; + return factory(config); +} + +/** + * Return metadata for the default local profile. + * + * Used by migration seeds and runtime defaults. + */ +export function getDefaultLocalProfile(): Pick< + EmbeddingProfile, + 'id' | 'providerKind' | 'model' | 'dimensions' +> { + return { + id: 'local-default', + providerKind: 'local-transformers', + model: 'Xenova/all-MiniLM-L6-v2', + dimensions: 384 + }; +} + +/** + * Return all registered providerKind values. + * + * Useful for settings UI validation and provider discovery. + */ +export function getRegisteredProviderKinds(): string[] { + return Object.keys(PROVIDER_REGISTRY); +} diff --git a/src/lib/server/search/hybrid.search.service.test.ts b/src/lib/server/search/hybrid.search.service.test.ts index 9e3bac0..5121e56 100644 --- a/src/lib/server/search/hybrid.search.service.test.ts +++ b/src/lib/server/search/hybrid.search.service.test.ts @@ -25,16 +25,18 @@ function createTestDb(): Database.Database { client.pragma('foreign_keys = ON'); const migrationsFolder = join(import.meta.dirname, '../db/migrations'); - const migrationSql = readFileSync( - join(migrationsFolder, '0000_large_master_chief.sql'), - 'utf-8' - ); - const statements = migrationSql - .split('--> statement-breakpoint') - .map((s) => s.trim()) - .filter(Boolean); - for (const stmt of statements) { - client.exec(stmt); + + // Run all migrations in order + const migrations = ['0000_large_master_chief.sql', '0001_quick_nighthawk.sql', '0002_silky_stellaris.sql']; + for (const migrationFile of migrations) { + const migrationSql = readFileSync(join(migrationsFolder, migrationFile), 'utf-8'); + const statements = migrationSql + .split('--> statement-breakpoint') + .map((s) => s.trim()) + .filter(Boolean); + for (const stmt of statements) { + client.exec(stmt); + } } const ftsSql = readFileSync(join(import.meta.dirname, '../db/fts.sql'), 'utf-8'); @@ -104,16 +106,17 @@ function seedEmbedding( client: Database.Database, snippetId: string, values: number[], + profileId = 'local-default', model = 'test-model' ): void { const f32 = new Float32Array(values); client .prepare( `INSERT OR REPLACE INTO snippet_embeddings - (snippet_id, model, dimensions, embedding, created_at) - VALUES (?, ?, ?, ?, ?)` + (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, ?, ?, ?, ?, ?)` ) - .run(snippetId, model, values.length, Buffer.from(f32.buffer), NOW_S); + .run(snippetId, profileId, model, values.length, Buffer.from(f32.buffer), NOW_S); } // --------------------------------------------------------------------------- @@ -621,4 +624,203 @@ describe('HybridSearchService', () => { const results = await svc.search('default alpha hybrid', { repositoryId: repoId }); expect(Array.isArray(results)).toBe(true); }); + + it('filters by versionId — excludes snippets from other versions', async () => { + const client = createTestDb(); + const repoId = seedRepo(client); + const docId = seedDocument(client, repoId); + + // Create two versions + client + .prepare( + `INSERT INTO repository_versions (id, repository_id, tag, state, total_snippets, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .run('/test/repo/v1.0', repoId, 'v1.0', 'indexed', 0, NOW_S); + client + .prepare( + `INSERT INTO repository_versions (id, repository_id, tag, state, total_snippets, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .run('/test/repo/v2.0', repoId, 'v2.0', 'indexed', 0, NOW_S); + + // Create embedding profile + client + .prepare( + `INSERT INTO embedding_profiles (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run('test-profile', 'local-transformers', 'Test', 1, 1, 'test-model', 4, '{}', NOW_S, NOW_S); + + // Snippet A in version 1.0 + const snippetA = seedSnippet(client, { + repositoryId: repoId, + documentId: docId, + content: 'version 1 text' + }); + client + .prepare('UPDATE snippets SET version_id = ? WHERE id = ?') + .run('/test/repo/v1.0', snippetA); + + // Seed embedding for snippetA + const embedA = [0.1, 0.2, 0.3, 0.4]; + const f32A = new Float32Array(embedA); + client + .prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .run(snippetA, 'test-profile', 'test-model', 4, Buffer.from(f32A.buffer), NOW_S); + + // Snippet B in version 2.0 + const snippetB = seedSnippet(client, { + repositoryId: repoId, + documentId: docId, + content: 'version 2 text' + }); + client + .prepare('UPDATE snippets SET version_id = ? WHERE id = ?') + .run('/test/repo/v2.0', snippetB); + + // Seed embedding for snippetB + const embedB = [0.2, 0.3, 0.4, 0.5]; + const f32B = new Float32Array(embedB); + client + .prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .run(snippetB, 'test-profile', 'test-model', 4, Buffer.from(f32B.buffer), NOW_S); + + const vs = new VectorSearch(client); + const query = new Float32Array([0.1, 0.2, 0.3, 0.4]); + + // Query with versionId v1.0 should only return snippetA + const resultsV1 = vs.vectorSearch(query, { + repositoryId: repoId, + versionId: '/test/repo/v1.0', + profileId: 'test-profile' + }); + expect(resultsV1.map((r) => r.snippetId)).toContain(snippetA); + expect(resultsV1.map((r) => r.snippetId)).not.toContain(snippetB); + + // Query with versionId v2.0 should only return snippetB + const resultsV2 = vs.vectorSearch(query, { + repositoryId: repoId, + versionId: '/test/repo/v2.0', + profileId: 'test-profile' + }); + expect(resultsV2.map((r) => r.snippetId)).not.toContain(snippetA); + expect(resultsV2.map((r) => r.snippetId)).toContain(snippetB); + + // Query without versionId should return both + const resultsAll = vs.vectorSearch(query, { + repositoryId: repoId, + profileId: 'test-profile' + }); + expect(resultsAll.map((r) => r.snippetId)).toContain(snippetA); + expect(resultsAll.map((r) => r.snippetId)).toContain(snippetB); + }); + + it('searchMode=keyword never calls provider.embed()', async () => { + const client = createTestDb(); + const repoId = seedRepo(client); + const docId = seedDocument(client, repoId); + + const snippetId = seedSnippet(client, { + repositoryId: repoId, + documentId: docId, + content: 'keyword only test' + }); + + client.exec( + `INSERT INTO snippets_fts (id, repository_id, version_id, title, breadcrumb, content) + VALUES ('${snippetId}', '${repoId}', NULL, NULL, NULL, 'keyword only test')` + ); + + let embedCalled = false; + const mockProvider: EmbeddingProvider = { + name: 'mock', + dimensions: 4, + model: 'test-model', + async embed() { + embedCalled = true; + return []; + }, + async isAvailable() { + return true; + } + }; + + const searchService = new SearchService(client); + const hybridService = new HybridSearchService(client, searchService, mockProvider); + + const results = await hybridService.search('keyword', { + repositoryId: repoId, + searchMode: 'keyword' + }); + + expect(embedCalled).toBe(false); + expect(results.length).toBeGreaterThan(0); + }); + + it('searchMode=semantic uses only vector search', async () => { + const client = createTestDb(); + const repoId = seedRepo(client); + const docId = seedDocument(client, repoId); + + // Create profile + client + .prepare( + `INSERT INTO embedding_profiles (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .run('test-profile', 'local-transformers', 'Test', 1, 1, 'test-model', 4, '{}', NOW_S, NOW_S); + + const snippetId = seedSnippet(client, { + repositoryId: repoId, + documentId: docId, + content: 'semantic test' + }); + + // Seed embedding + const embed = [0.5, 0.5, 0.5, 0.5]; + const f32 = new Float32Array(embed); + client + .prepare( + `INSERT INTO snippet_embeddings (snippet_id, profile_id, model, dimensions, embedding, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .run(snippetId, 'test-profile', 'test-model', 4, Buffer.from(f32.buffer), NOW_S); + + const mockProvider: EmbeddingProvider = { + name: 'mock', + dimensions: 4, + model: 'test-model', + async embed() { + return [ + { + values: new Float32Array([0.5, 0.5, 0.5, 0.5]), + dimensions: 4, + model: 'test-model' + } + ]; + }, + async isAvailable() { + return true; + } + }; + + const searchService = new SearchService(client); + const hybridService = new HybridSearchService(client, searchService, mockProvider); + + const results = await hybridService.search('semantic', { + repositoryId: repoId, + searchMode: 'semantic', + profileId: 'test-profile' + }); + + // Should return results (alpha=1 pure vector mode) + expect(results.length).toBeGreaterThan(0); + }); }); diff --git a/src/lib/server/search/hybrid.search.service.ts b/src/lib/server/search/hybrid.search.service.ts index b48dabf..6a8c08a 100644 --- a/src/lib/server/search/hybrid.search.service.ts +++ b/src/lib/server/search/hybrid.search.service.ts @@ -36,6 +36,16 @@ export interface HybridSearchOptions { * Default: 0.5. */ alpha?: number; + /** + * Search mode: 'auto' (default), 'keyword', 'semantic', or 'hybrid'. + * Overrides alpha when set to 'keyword' (forces 0) or 'semantic' (forces 1). + */ + searchMode?: 'auto' | 'keyword' | 'semantic' | 'hybrid'; + /** + * Embedding profile ID for vector search. + * Default: 'local-default'. + */ + profileId?: string; } /** @@ -90,7 +100,24 @@ export class HybridSearchService { options: HybridSearchOptions ): Promise { const limit = options.limit ?? 20; - const alpha = options.alpha ?? 0.5; + const mode = options.searchMode ?? 'auto'; + + // Resolve alpha from searchMode + let alpha: number; + switch (mode) { + case 'keyword': + alpha = 0; + break; + case 'semantic': + alpha = 1; + break; + case 'hybrid': + alpha = options.alpha ?? 0.5; + break; + default: + // 'auto' + alpha = options.alpha ?? 0.5; + } // Always run FTS5 — it is synchronous and fast. const ftsResults = this.searchService.searchSnippets(query, { @@ -115,11 +142,12 @@ export class HybridSearchService { const queryEmbedding = embeddings[0].values; - const vectorResults = this.vectorSearch.vectorSearch( - queryEmbedding, - options.repositoryId, - limit * 3 - ); + const vectorResults = this.vectorSearch.vectorSearch(queryEmbedding, { + repositoryId: options.repositoryId, + versionId: options.versionId, + profileId: options.profileId, + limit: limit * 3 + }); // Pure vector mode: skip RRF and return vector results directly. if (alpha === 1) { diff --git a/src/lib/server/search/vector.search.ts b/src/lib/server/search/vector.search.ts index 5c4506d..b82dc45 100644 --- a/src/lib/server/search/vector.search.ts +++ b/src/lib/server/search/vector.search.ts @@ -21,6 +21,13 @@ export interface VectorSearchResult { score: number; } +export interface VectorSearchOptions { + repositoryId: string; + versionId?: string; + profileId?: string; + limit?: number; +} + /** Raw DB row from snippet_embeddings joined with snippets. */ interface RawEmbeddingRow { snippet_id: string; @@ -64,32 +71,33 @@ export function cosineSimilarity(a: Float32Array, b: Float32Array): number { // --------------------------------------------------------------------------- export class VectorSearch { - private readonly stmt: Database.Statement<[string], RawEmbeddingRow>; - - constructor(private readonly db: Database.Database) { - // Prepare once — reused for every call. - this.stmt = this.db.prepare<[string], RawEmbeddingRow>(` - SELECT se.snippet_id, se.embedding - FROM snippet_embeddings se - JOIN snippets s ON s.id = se.snippet_id - WHERE s.repository_id = ? - `); - } + constructor(private readonly db: Database.Database) {} /** * Search stored embeddings by cosine similarity to the query embedding. * * @param queryEmbedding - The embedded representation of the search query. - * @param repositoryId - Scope the search to a single repository. - * @param limit - Maximum number of results to return. Default: 50. + * @param options - Search options including repositoryId, optional versionId, profileId, and limit. * @returns Results sorted by descending cosine similarity score. */ - vectorSearch( - queryEmbedding: Float32Array, - repositoryId: string, - limit = 50 - ): VectorSearchResult[] { - const rows = this.stmt.all(repositoryId); + vectorSearch(queryEmbedding: Float32Array, options: VectorSearchOptions): VectorSearchResult[] { + const { repositoryId, versionId, profileId = 'local-default', limit = 50 } = options; + + let sql = ` + SELECT se.snippet_id, se.embedding + FROM snippet_embeddings se + JOIN snippets s ON s.id = se.snippet_id + WHERE s.repository_id = ? + AND se.profile_id = ? + `; + const params: unknown[] = [repositoryId, profileId]; + + if (versionId) { + sql += ' AND s.version_id = ?'; + params.push(versionId); + } + + const rows = this.db.prepare(sql).all(...params); const scored: VectorSearchResult[] = rows.map((row) => { const embedding = new Float32Array( diff --git a/src/mcp/client.ts b/src/mcp/client.ts index 784e55b..7e5e9e3 100644 --- a/src/mcp/client.ts +++ b/src/mcp/client.ts @@ -42,6 +42,8 @@ export async function fetchContext(params: { query: string; tokens?: number; type?: 'json' | 'txt'; + searchMode?: string; + alpha?: number; }): Promise { const url = new URL(`${API_BASE}/api/v1/context`); url.searchParams.set('libraryId', params.libraryId); @@ -50,6 +52,12 @@ export async function fetchContext(params: { if (params.tokens !== undefined) { url.searchParams.set('tokens', String(params.tokens)); } + if (params.searchMode) { + url.searchParams.set('searchMode', params.searchMode); + } + if (params.alpha !== undefined) { + url.searchParams.set('alpha', String(params.alpha)); + } return fetch(url.toString()); } diff --git a/src/mcp/tools/query-docs.ts b/src/mcp/tools/query-docs.ts index 610eff4..115ea23 100644 --- a/src/mcp/tools/query-docs.ts +++ b/src/mcp/tools/query-docs.ts @@ -15,7 +15,19 @@ export const QueryDocsSchema = z.object({ query: z .string() .describe('Specific question about the library to retrieve relevant documentation'), - tokens: z.number().optional().describe('Maximum token budget for the response (default: 10000)') + tokens: z.number().optional().describe('Maximum token budget for the response (default: 10000)'), + searchMode: z + .enum(['auto', 'keyword', 'semantic', 'hybrid']) + .optional() + .describe( + "Retrieval mode: 'auto' (default), 'keyword' (FTS only), 'semantic' (vector only), or 'hybrid'" + ), + alpha: z + .number() + .min(0) + .max(1) + .optional() + .describe('Hybrid blend weight: 0.0 = keyword only, 1.0 = semantic only (default: 0.5)') }); export type QueryDocsInput = z.infer; @@ -42,6 +54,17 @@ export const QUERY_DOCS_TOOL = { tokens: { type: 'number', description: 'Max token budget (default: 10000)' + }, + searchMode: { + type: 'string', + enum: ['auto', 'keyword', 'semantic', 'hybrid'], + description: "Retrieval mode: 'auto' (default), 'keyword', 'semantic', or 'hybrid'" + }, + alpha: { + type: 'number', + minimum: 0, + maximum: 1, + description: 'Hybrid blend weight (0=keyword, 1=semantic, default: 0.5)' } }, required: ['libraryId', 'query'] @@ -49,9 +72,9 @@ export const QUERY_DOCS_TOOL = { }; export async function handleQueryDocs(args: unknown) { - const { libraryId, query, tokens } = QueryDocsSchema.parse(args); + const { libraryId, query, tokens, searchMode, alpha } = QueryDocsSchema.parse(args); - const response = await fetchContext({ libraryId, query, tokens, type: 'txt' }); + const response = await fetchContext({ libraryId, query, tokens, type: 'txt', searchMode, alpha }); if (!response.ok) { const status = response.status; diff --git a/src/routes/api/v1/context/+server.ts b/src/routes/api/v1/context/+server.ts index c756ae1..3105b31 100644 --- a/src/routes/api/v1/context/+server.ts +++ b/src/routes/api/v1/context/+server.ts @@ -16,6 +16,8 @@ import { getClient } from '$lib/server/db/client'; import { dtoJsonResponse } from '$lib/server/api/dto-response'; import { SearchService } from '$lib/server/search/search.service'; import { HybridSearchService } from '$lib/server/search/hybrid.search.service'; +import { createProviderFromProfile } from '$lib/server/embeddings/registry'; +import type { EmbeddingProfile } from '$lib/server/db/schema'; import { parseLibraryId } from '$lib/server/api/library-id'; import { selectSnippetsWithinBudget, DEFAULT_TOKEN_BUDGET } from '$lib/server/api/token-budget'; import { @@ -28,12 +30,20 @@ import { // Helpers // --------------------------------------------------------------------------- -function getServices() { - const db = getClient(); +function getServices(db: ReturnType) { const searchService = new SearchService(db); - // No embedding provider — pure FTS5 mode (alpha=0 equivalent). - const hybridService = new HybridSearchService(db, searchService, null); - return { db, searchService, hybridService }; + + // Load the active embedding profile from the database + const profileRow = db + .prepare<[], EmbeddingProfile>( + 'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1' + ) + .get(); + + const provider = profileRow ? createProviderFromProfile(profileRow) : null; + const hybridService = new HybridSearchService(db, searchService, provider); + + return { db, searchService, hybridService, profileId: profileRow?.id }; } interface RawRepoConfig { @@ -93,6 +103,14 @@ export const GET: RequestHandler = async ({ url }) => { const tokensRaw = parseInt(url.searchParams.get('tokens') ?? String(DEFAULT_TOKEN_BUDGET), 10); const maxTokens = isNaN(tokensRaw) || tokensRaw < 1 ? DEFAULT_TOKEN_BUDGET : tokensRaw; + // Parse searchMode and alpha + const rawMode = url.searchParams.get('searchMode') ?? 'auto'; + const searchMode = ['auto', 'keyword', 'semantic', 'hybrid'].includes(rawMode) + ? (rawMode as 'auto' | 'keyword' | 'semantic' | 'hybrid') + : 'auto'; + const alphaRaw = parseFloat(url.searchParams.get('alpha') ?? '0.5'); + const alpha = isNaN(alphaRaw) ? 0.5 : Math.max(0, Math.min(1, alphaRaw)); + // Parse the libraryId let parsed: ReturnType; try { @@ -108,7 +126,8 @@ export const GET: RequestHandler = async ({ url }) => { } try { - const { db, hybridService } = getServices(); + const db = getClient(); + const { hybridService, profileId } = getServices(db); // Verify the repository exists and check its state. const repo = db @@ -158,7 +177,10 @@ export const GET: RequestHandler = async ({ url }) => { const searchResults = await hybridService.search(query, { repositoryId: parsed.repositoryId, versionId, - limit: 50 // fetch more than needed; token budget will trim + limit: 50, // fetch more than needed; token budget will trim + searchMode, + alpha, + profileId }); // Apply token budget. diff --git a/src/routes/api/v1/settings/embedding/+server.ts b/src/routes/api/v1/settings/embedding/+server.ts index 22e4111..2130b78 100644 --- a/src/routes/api/v1/settings/embedding/+server.ts +++ b/src/routes/api/v1/settings/embedding/+server.ts @@ -1,147 +1,149 @@ /** - * GET /api/v1/settings/embedding — retrieve current embedding configuration - * PUT /api/v1/settings/embedding — update embedding configuration + * GET /api/v1/settings/embedding — retrieve all embedding profiles + * POST /api/v1/settings/embedding — create or update an embedding profile + * PUT /api/v1/settings/embedding — alias for POST (backward compat) */ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import { getClient } from '$lib/server/db/client'; -import { - EMBEDDING_CONFIG_KEY, - createProviderFromConfig, - defaultEmbeddingConfig, - type EmbeddingConfig -} from '$lib/server/embeddings/factory'; +import { createProviderFromProfile } from '$lib/server/embeddings/registry'; +import type { EmbeddingProfile, NewEmbeddingProfile } from '$lib/server/db/schema'; import { handleServiceError, InvalidInputError } from '$lib/server/utils/validation'; // --------------------------------------------------------------------------- -// Helpers +// GET — Return all profiles // --------------------------------------------------------------------------- -function readConfig(db: ReturnType): EmbeddingConfig { - const row = db - .prepare(`SELECT value FROM settings WHERE key = ?`) - .get(EMBEDDING_CONFIG_KEY) as { value: string } | undefined; - - if (!row) return defaultEmbeddingConfig(); - +export const GET: RequestHandler = () => { try { - return JSON.parse(row.value) as EmbeddingConfig; - } catch { - return defaultEmbeddingConfig(); - } -} + const db = getClient(); + const profiles = db + .prepare('SELECT * FROM embedding_profiles ORDER BY is_default DESC, created_at ASC') + .all() as EmbeddingProfile[]; -function validateConfig(body: unknown): EmbeddingConfig { + // Sanitize: remove sensitive config fields like apiKey + const safeProfiles = profiles.map(sanitizeProfile); + return json({ profiles: safeProfiles }); + } catch (err) { + return handleServiceError(err); + } +}; + +// --------------------------------------------------------------------------- +// POST/PUT — Create or update a profile +// --------------------------------------------------------------------------- + +async function upsertProfile(body: unknown) { if (typeof body !== 'object' || body === null) { throw new InvalidInputError('Request body must be a JSON object'); } const obj = body as Record; - const provider = obj.provider; - if (provider !== 'openai' && provider !== 'local' && provider !== 'none') { + // Required fields + if (typeof obj.id !== 'string' || !obj.id) { + throw new InvalidInputError('id is required'); + } + if (typeof obj.providerKind !== 'string' || !obj.providerKind) { + throw new InvalidInputError('providerKind is required'); + } + if (typeof obj.title !== 'string' || !obj.title) { + throw new InvalidInputError('title is required'); + } + if (typeof obj.model !== 'string' || !obj.model) { + throw new InvalidInputError('model is required'); + } + if (typeof obj.dimensions !== 'number') { + throw new InvalidInputError('dimensions must be a number'); + } + + const profile: NewEmbeddingProfile = { + id: obj.id, + providerKind: obj.providerKind, + title: obj.title, + enabled: typeof obj.enabled === 'boolean' ? obj.enabled : true, + isDefault: typeof obj.isDefault === 'boolean' ? obj.isDefault : false, + model: obj.model, + dimensions: obj.dimensions, + config: (obj.config as Record) ?? {}, + createdAt: Date.now(), + updatedAt: Date.now() + }; + + // Validate provider availability before persisting + const provider = createProviderFromProfile(profile as EmbeddingProfile); + const available = await provider.isAvailable(); + if (!available) { throw new InvalidInputError( - `Invalid provider "${String(provider)}". Must be one of: openai, local, none.` + `Could not connect to the "${profile.providerKind}" provider. Check your configuration.` ); } - if (provider === 'openai') { - const openai = obj.openai as Record | undefined; - if (!openai || typeof openai !== 'object') { - throw new InvalidInputError('openai config object is required when provider is "openai"'); - } - if (typeof openai.baseUrl !== 'string' || !openai.baseUrl) { - throw new InvalidInputError('openai.baseUrl must be a non-empty string'); - } - if (typeof openai.apiKey !== 'string' || !openai.apiKey) { - throw new InvalidInputError('openai.apiKey must be a non-empty string'); - } - if (typeof openai.model !== 'string' || !openai.model) { - throw new InvalidInputError('openai.model must be a non-empty string'); - } + const db = getClient(); - const config: EmbeddingConfig = { - provider: 'openai', - openai: { - baseUrl: openai.baseUrl as string, - apiKey: openai.apiKey as string, - model: openai.model as string, - dimensions: - typeof openai.dimensions === 'number' ? (openai.dimensions as number) : undefined, - maxBatchSize: - typeof openai.maxBatchSize === 'number' - ? (openai.maxBatchSize as number) - : undefined - } - }; - return config; + // If setting as default, clear other defaults first + if (profile.isDefault) { + db.prepare('UPDATE embedding_profiles SET is_default = 0').run(); } - return { provider: provider as 'local' | 'none' }; + // Upsert the profile + db.prepare( + `INSERT INTO embedding_profiles + (id, provider_kind, title, enabled, is_default, model, dimensions, config, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + provider_kind = excluded.provider_kind, + title = excluded.title, + enabled = excluded.enabled, + is_default = excluded.is_default, + model = excluded.model, + dimensions = excluded.dimensions, + config = excluded.config, + updated_at = excluded.updated_at` + ).run( + profile.id, + profile.providerKind, + profile.title, + profile.enabled ? 1 : 0, + profile.isDefault ? 1 : 0, + profile.model, + profile.dimensions, + JSON.stringify(profile.config), + profile.createdAt, + profile.updatedAt + ); + + const inserted = db + .prepare('SELECT * FROM embedding_profiles WHERE id = ?') + .get(profile.id) as EmbeddingProfile; + + return sanitizeProfile(inserted); } -// --------------------------------------------------------------------------- -// GET -// --------------------------------------------------------------------------- - -export const GET: RequestHandler = () => { - try { - const db = getClient(); - const config = readConfig(db); - - // Strip the apiKey from the response for security. - const safeConfig = sanitizeForResponse(config); - return json(safeConfig); - } catch (err) { - return handleServiceError(err); - } -}; - -// --------------------------------------------------------------------------- -// PUT -// --------------------------------------------------------------------------- - -export const PUT: RequestHandler = async ({ request }) => { +export const POST: RequestHandler = async ({ request }) => { try { const body = await request.json(); - const config = validateConfig(body); - - // Verify provider connectivity before persisting (skip for noop). - if (config.provider !== 'none') { - const provider = createProviderFromConfig(config); - const available = await provider.isAvailable(); - if (!available) { - throw new InvalidInputError( - `Could not connect to the "${config.provider}" embedding provider. Check your configuration.` - ); - } - } - - const db = getClient(); - db.prepare( - `INSERT INTO settings (key, value, updated_at) - VALUES (?, ?, unixepoch()) - ON CONFLICT (key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at` - ).run(EMBEDDING_CONFIG_KEY, JSON.stringify(config)); - - const safeConfig = sanitizeForResponse(config); - return json(safeConfig); + const profile = await upsertProfile(body); + return json(profile); } catch (err) { return handleServiceError(err); } }; +// Backward compat alias +export const PUT: RequestHandler = POST; + // --------------------------------------------------------------------------- -// Sanitize — remove sensitive fields before returning to clients +// Sanitize — remove sensitive config fields before returning to clients // --------------------------------------------------------------------------- -function sanitizeForResponse(config: EmbeddingConfig): Omit & { - openai?: Omit, 'apiKey'>; -} { - if (config.provider === 'openai' && config.openai) { - const { apiKey: _apiKey, ...rest } = config.openai; - return { ...config, openai: rest }; +function sanitizeProfile(profile: EmbeddingProfile): EmbeddingProfile { + const config = profile.config as Record; + if (config && config.apiKey) { + const { apiKey: _apiKey, ...rest } = config; + return { ...profile, config: rest }; } - return config; + return profile; } + diff --git a/src/routes/api/v1/settings/embedding/test/+server.ts b/src/routes/api/v1/settings/embedding/test/+server.ts index b6450fb..1a2a14c 100644 --- a/src/routes/api/v1/settings/embedding/test/+server.ts +++ b/src/routes/api/v1/settings/embedding/test/+server.ts @@ -1,82 +1,47 @@ /** - * POST /api/v1/settings/embedding/test + * GET /api/v1/settings/embedding/test * - * Validates an embedding provider configuration by creating a provider - * instance and calling embed(['test']). Returns success with dimensions - * or a descriptive error without persisting any changes. + * Tests the active default embedding profile by creating a provider instance + * and checking availability. Returns success with profile metadata or error. */ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import { - createProviderFromConfig, - type EmbeddingConfig -} from '$lib/server/embeddings/factory'; -import { handleServiceError, InvalidInputError } from '$lib/server/utils/validation'; +import { getClient } from '$lib/server/db/client'; +import { createProviderFromProfile } from '$lib/server/embeddings/registry'; +import type { EmbeddingProfile } from '$lib/server/db/schema'; +import { handleServiceError } from '$lib/server/utils/validation'; export const GET: RequestHandler = async () => { try { - const provider = createProviderFromConfig({ provider: 'local' }); + const db = getClient(); + const profile = db + .prepare<[], EmbeddingProfile>( + 'SELECT * FROM embedding_profiles WHERE is_default = 1 AND enabled = 1 LIMIT 1' + ) + .get(); + + if (!profile) { + return json({ available: false, error: 'No active embedding profile configured' }); + } + + const provider = createProviderFromProfile(profile); const available = await provider.isAvailable(); - return json({ available }); + + return json({ + available, + profile: { + id: profile.id, + providerKind: profile.providerKind, + model: profile.model, + dimensions: profile.dimensions + } + }); } catch (err) { return handleServiceError(err); } }; -// --------------------------------------------------------------------------- -// Validate — reuse the same shape accepted by PUT /settings/embedding -// --------------------------------------------------------------------------- - -function validateConfig(body: unknown): EmbeddingConfig { - if (typeof body !== 'object' || body === null) { - throw new InvalidInputError('Request body must be a JSON object'); - } - - const obj = body as Record; - - const provider = obj.provider; - if (provider !== 'openai' && provider !== 'local' && provider !== 'none') { - throw new InvalidInputError( - `Invalid provider "${String(provider)}". Must be one of: openai, local, none.` - ); - } - - if (provider === 'openai') { - const openai = obj.openai as Record | undefined; - if (!openai || typeof openai !== 'object') { - throw new InvalidInputError('openai config object is required when provider is "openai"'); - } - if (typeof openai.baseUrl !== 'string' || !openai.baseUrl) { - throw new InvalidInputError('openai.baseUrl must be a non-empty string'); - } - if (typeof openai.apiKey !== 'string' || !openai.apiKey) { - throw new InvalidInputError('openai.apiKey must be a non-empty string'); - } - if (typeof openai.model !== 'string' || !openai.model) { - throw new InvalidInputError('openai.model must be a non-empty string'); - } - - return { - provider: 'openai', - openai: { - baseUrl: openai.baseUrl as string, - apiKey: openai.apiKey as string, - model: openai.model as string, - dimensions: - typeof openai.dimensions === 'number' ? (openai.dimensions as number) : undefined, - maxBatchSize: - typeof openai.maxBatchSize === 'number' ? (openai.maxBatchSize as number) : undefined - } - }; - } - - return { provider: provider as 'local' | 'none' }; -} - -// --------------------------------------------------------------------------- -// POST -// --------------------------------------------------------------------------- export const POST: RequestHandler = async ({ request }) => { try { diff --git a/test-output.txt b/test-output.txt new file mode 100644 index 0000000..852450b --- /dev/null +++ b/test-output.txt @@ -0,0 +1,632 @@ + +> trueref@0.0.1 test:unit +> vitest + + + DEV  v4.1.0 /home/moze/Sources/trueref + +19:10:26 [vite] (client) Re-optimizing dependencies because lockfile has changed + ❯  server  src/lib/server/embeddings/embedding.service.test.ts (0 test) + ✓  server  src/lib/server/parser/code.parser.test.ts (20 tests) 22ms + ✓  server  src/lib/server/services/version.service.test.ts (19 tests) 37ms + ✓  server  src/lib/server/services/repository.service.test.ts (37 tests) 57ms +stderr | src/lib/server/crawler/local.crawler.test.ts > LocalCrawler.crawl() — config file detection > gracefully handles a malformed config file +[LocalCrawler] Failed to parse config file: /tmp/trueref-test-ptITIP/trueref.json + + ✓  server  src/lib/server/config/config-parser.test.ts (50 tests) 21ms +stderr | src/lib/server/pipeline/indexing.pipeline.test.ts > IndexingPipeline > marks job as failed and repo as error when pipeline throws +[IndexingPipeline] Job c44d7e22-6127-49e7-82b7-eb724726c888 failed: crawl failed + +stderr | src/lib/server/pipeline/indexing.pipeline.test.ts +[JobQueue] No pipeline configured — cannot process jobs. + +stderr | src/lib/server/pipeline/indexing.pipeline.test.ts +[JobQueue] No pipeline configured — cannot process jobs. + +stderr | src/lib/server/pipeline/indexing.pipeline.test.ts +[JobQueue] No pipeline configured — cannot process jobs. + + ✓  server  src/lib/server/search/search.service.test.ts (43 tests) 43ms + ✓  server  src/lib/server/pipeline/indexing.pipeline.test.ts (20 tests) 42ms + ✓  server  src/lib/server/crawler/gitignore-parser.test.ts (29 tests) 11ms + ✓  server  src/lib/server/crawler/github-tags.test.ts (10 tests) 9ms + ✓  server  src/routes/api/v1/api-contract.integration.test.ts (4 tests) 48ms + ❯  server  src/lib/server/db/schema.test.ts (19 tests | 19 failed) 50ms + × inserts and retrieves a repository 12ms + × allows nullable optional fields 3ms + × supports all state enum values 2ms + × inserts a version linked to a repository 4ms + × cascades delete when parent repository is deleted 2ms + × inserts a document 1ms + × cascades delete when repository is deleted 2ms + × inserts a code snippet 2ms + × inserts an info snippet 2ms + × cascades delete when document is deleted 2ms + × stores a Float32Array embedding as blob 2ms + × cascades delete when snippet is deleted 2ms + × creates a job with default queued status 2ms + × supports all status enum values 2ms + × stores JSON array fields correctly 2ms + × stores and retrieves key-value settings 2ms + × FTS table exists and is queryable 1ms + × insert trigger keeps FTS in sync 2ms + × delete trigger removes entry from FTS 2ms + ❯  server  src/lib/server/search/hybrid.search.service.test.ts (33 tests | 16 failed) 52ms + ✓ returns 1.0 for identical vectors 2ms + ✓ returns 0.0 for orthogonal vectors 0ms + ✓ returns -1.0 for opposite vectors 0ms + ✓ returns 0 for zero-magnitude vector 0ms + ✓ throws when dimensions do not match 1ms + ✓ computes correct similarity for non-trivial vectors 0ms + ✓ returns empty array for empty inputs 1ms + ✓ fuses a single list preserving order 1ms + ✓ deduplicates items appearing in multiple lists 0ms + ✓ boosts items appearing in multiple lists 0ms + ✓ assigns higher rrfScore to higher-ranked items 0ms + ✓ handles three lists correctly 0ms + ✓ produces positive rrfScores 0ms + × returns empty array when no embeddings exist 10ms + × returns results sorted by descending cosine similarity 2ms + × respects the limit parameter 4ms + × only returns snippets from the specified repository 2ms + × handles embeddings with negative values 1ms + ✓ returns FTS5 results when embeddingProvider is null 2ms + ✓ returns FTS5 results when alpha = 0 1ms + ✓ returns empty array when FTS5 query is blank and no provider 1ms + ✓ falls back to FTS5 when noop provider returns empty embeddings 2ms + × returns results when hybrid mode is active (alpha = 0.5) 1ms + × deduplicates snippets appearing in both FTS5 and vector results 1ms + × respects the limit option 1ms + × returns vector-ranked results when alpha = 1 1ms + × results include snippet and repository metadata 1ms + × all results belong to the requested repository 1ms + × filters by snippet type when provided 1ms + × uses alpha = 0.5 when not specified 1ms + × filters by versionId — excludes snippets from other versions 3ms + × searchMode=keyword never calls provider.embed() 3ms + × searchMode=semantic uses only vector search 2ms + ✓  server  src/lib/server/api/formatters.test.ts (20 tests) 9ms + ✓  server  src/lib/server/pipeline/diff.test.ts (9 tests) 8ms + ✓  server  src/lib/server/api/library-id.test.ts (8 tests) 6ms + ✓  server  src/lib/server/api/token-budget.test.ts (7 tests) 6ms + ✓  server  src/lib/server/parser/markdown.parser.test.ts (14 tests) 9ms + ✓  server  src/lib/vitest-examples/greet.spec.ts (1 test) 3ms + ✓  server  src/lib/server/crawler/local.crawler.test.ts (50 tests) 658ms + ✓  server  src/mcp/index.test.ts (7 tests) 985ms + ✓  client (chromium)  src/lib/vitest-examples/Welcome.svelte.spec.ts (1 test) 9ms +stderr | src/lib/server/crawler/github.crawler.test.ts > crawl() > skips files that fail to download without throwing +[GitHubCrawler] Could not download: src/index.ts — skipping. + + ✓  server  src/lib/server/crawler/github.crawler.test.ts (50 tests) 6082ms + ✓ retries on failure and returns eventual success  3003ms + ✓ throws after exhausting all attempts  3003ms + +⎯⎯⎯⎯⎯⎯ Failed Suites 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL   server  src/lib/server/embeddings/embedding.service.test.ts [ src/lib/server/embeddings/embedding.service.test.ts ] +Error: Transform failed with 1 error: +/home/moze/Sources/trueref/src/lib/server/embeddings/embedding.service.test.ts:408:2: ERROR: "await" can only be used inside an "async" function + Plugin: vite:esbuild + File: /home/moze/Sources/trueref/src/lib/server/embeddings/embedding.service.test.ts:408:2 + + "await" can only be used inside an "async" function + 406 | }); + 407 | + 408 | await service.embedSnippets([snippetId]); + | ^ + 409 | + 410 | const retrieved = service.getEmbedding(snippetId); +  + ❯ failureErrorWithLog node_modules/vite/node_modules/esbuild/lib/main.js:1748:15 + ❯ node_modules/vite/node_modules/esbuild/lib/main.js:1017:50 + ❯ responseCallbacks. node_modules/vite/node_modules/esbuild/lib/main.js:884:9 + ❯ handleIncomingPacket node_modules/vite/node_modules/esbuild/lib/main.js:939:12 + ❯ Socket.readFromStdout node_modules/vite/node_modules/esbuild/lib/main.js:862:7 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/36]⎯ + + +⎯⎯⎯⎯⎯⎯ Failed Tests 35 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > repositories table > inserts and retrieves a repository + FAIL   server  src/lib/server/db/schema.test.ts > repositories table > allows nullable optional fields + FAIL   server  src/lib/server/db/schema.test.ts > repositories table > supports all state enum values +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:63:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:63:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > repository_versions table > inserts a version linked to a repository + FAIL   server  src/lib/server/db/schema.test.ts > repository_versions table > cascades delete when parent repository is deleted +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:109:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:109:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > documents table > inserts a document + FAIL   server  src/lib/server/db/schema.test.ts > documents table > cascades delete when repository is deleted +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:151:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:151:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > snippets table > inserts a code snippet + FAIL   server  src/lib/server/db/schema.test.ts > snippets table > inserts an info snippet + FAIL   server  src/lib/server/db/schema.test.ts > snippets table > cascades delete when document is deleted +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:195:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:195:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > snippet_embeddings table > stores a Float32Array embedding as blob + FAIL   server  src/lib/server/db/schema.test.ts > snippet_embeddings table > cascades delete when snippet is deleted +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:271:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:271:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[6/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > indexing_jobs table > creates a job with default queued status + FAIL   server  src/lib/server/db/schema.test.ts > indexing_jobs table > supports all status enum values +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:350:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:350:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[7/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > repository_configs table > stores JSON array fields correctly +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:391:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:391:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[8/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > settings table > stores and retrieves key-value settings +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:422:13 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:422:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[9/36]⎯ + + FAIL   server  src/lib/server/db/schema.test.ts > FTS5 virtual table (snippets_fts) > FTS table exists and is queryable + FAIL   server  src/lib/server/db/schema.test.ts > FTS5 virtual table (snippets_fts) > insert trigger keeps FTS in sync + FAIL   server  src/lib/server/db/schema.test.ts > FTS5 virtual table (snippets_fts) > delete trigger removes entry from FTS +DrizzleError: Failed to run the query ' +INSERT INTO `__new_snippet_embeddings`("snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at") SELECT "snippet_id", "profile_id", "model", "dimensions", "embedding", "created_at" FROM `snippet_embeddings`;' + ❯ BetterSQLiteSession.run node_modules/src/sqlite-core/session.ts:271:9 + ❯ SQLiteSyncDialect.migrate node_modules/src/sqlite-core/dialect.ts:864:14 + ❯ migrate node_modules/src/better-sqlite3/migrator.ts:10:12 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 +  30| // Run migrations from the generated migration folder. +  31| const migrationsFolder = join(import.meta.dirname, 'migrations'); +  32| migrate(db, { migrationsFolder }); +  | ^ +  33| +  34| // Apply FTS5 DDL using exec() which handles multi-statement SQL with… + ❯ src/lib/server/db/schema.test.ts:442:21 + +Caused by: SqliteError: no such column: "profile_id" - should this be a string literal in single-quotes? + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ BetterSQLiteSession.prepareQuery node_modules/drizzle-orm/better-sqlite3/session.js:23:30 + ❯ BetterSQLiteSession.prepareOneTimeQuery node_modules/drizzle-orm/sqlite-core/session.js:141:17 + ❯ BetterSQLiteSession.run node_modules/drizzle-orm/sqlite-core/session.js:154:19 + ❯ SQLiteSyncDialect.migrate node_modules/drizzle-orm/sqlite-core/dialect.js:604:21 + ❯ migrate node_modules/drizzle-orm/better-sqlite3/migrator.js:4:14 + ❯ createTestDb src/lib/server/db/schema.test.ts:32:2 + ❯ src/lib/server/db/schema.test.ts:442:21 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'SQLITE_ERROR' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[10/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > returns empty array when no embeddings exist +SqliteError: no such column: se.profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ VectorSearch.vectorSearch src/lib/server/search/vector.search.ts:100:24 +  98| } +  99| + 100| const rows = this.db.prepare(sql).all(..… +  | ^ + 101| + 102| const scored: VectorSearchResult[] = rows.map((row) => { + ❯ src/lib/server/search/hybrid.search.service.test.ts:289:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[11/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > returns results sorted by descending cosine similarity +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:302:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[12/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > respects the limit parameter +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:321:4 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[13/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > only returns snippets from the specified repository +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:340:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[14/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > VectorSearch > handles embeddings with negative values +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:352:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[15/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > returns results when hybrid mode is active (alpha = 0.5) +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:430:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[16/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > deduplicates snippets appearing in both FTS5 and vector results +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:449:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[17/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > respects the limit option +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:471:4 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[18/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > returns vector-ranked results when alpha = 1 +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:503:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[19/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > results include snippet and repository metadata +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:528:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[20/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > all results belong to the requested repository +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:556:4 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[21/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > filters by snippet type when provided +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:591:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[22/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > uses alpha = 0.5 when not specified +SqliteError: table snippet_embeddings has no column named profile_id + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ seedEmbedding src/lib/server/search/hybrid.search.service.test.ts:112:4 + 110| const f32 = new Float32Array(values); + 111| client + 112| .prepare( +  | ^ + 113| `INSERT OR REPLACE INTO snippet_embeddings + 114| (snippet_id, profile_id, model, dimensions, embedding, create… + ❯ src/lib/server/search/hybrid.search.service.test.ts:616:3 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[23/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > filters by versionId — excludes snippets from other versions +SqliteError: no such table: embedding_profiles + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ src/lib/server/search/hybrid.search.service.test.ts:647:5 + 645| // Create embedding profile + 646| client + 647| .prepare( +  | ^ + 648| `INSERT INTO embedding_profiles (id, provider_kind, title, enabled… + 649| VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[24/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > searchMode=keyword never calls provider.embed() +SqliteError: table snippets_fts has no column named id + ❯ Database.exec node_modules/better-sqlite3/lib/methods/wrappers.js:9:14 + ❯ src/lib/server/search/hybrid.search.service.test.ts:734:10 + 732| }); + 733| + 734| client.exec( +  | ^ + 735| `INSERT INTO snippets_fts (id, repository_id, version_id, title, br… + 736| VALUES ('${snippetId}', '${repoId}', NULL, NULL, NULL, 'keyword… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[25/36]⎯ + + FAIL   server  src/lib/server/search/hybrid.search.service.test.ts > HybridSearchService > searchMode=semantic uses only vector search +SqliteError: no such table: embedding_profiles + ❯ Database.prepare node_modules/better-sqlite3/lib/methods/wrappers.js:5:21 + ❯ src/lib/server/search/hybrid.search.service.test.ts:772:5 + 770| // Create profile + 771| client + 772| .prepare( +  | ^ + 773| `INSERT INTO embedding_profiles (id, provider_kind, title, enabled… + 774| VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[26/36]⎯ + + + Test Files  3 failed | 19 passed (22) + Tests  35 failed | 416 passed (451) + Start at  19:10:26 + Duration  6.93s (transform 7.37s, setup 0ms, import 9.29s, tests 8.17s, environment 11ms) + + FAIL  Tests failed. Watching for file changes... + press h to show help, press q to quit +Cancelling test run. Press CTRL+c again to exit forcefully. +