chore(FEEDBACK-0001): linting

This commit is contained in:
Giancarmine Salucci
2026-03-27 02:23:01 +01:00
parent 16436bfab2
commit 5a3c27224d
102 changed files with 5108 additions and 4976 deletions

View File

@@ -31,24 +31,26 @@ Represents an indexed library source (GitHub repo or local directory).
```typescript
export const repositories = sqliteTable('repositories', {
id: text('id').primaryKey(), // e.g. "/facebook/react" or "/local/my-sdk"
title: text('title').notNull(),
description: text('description'),
source: text('source', { enum: ['github', 'local'] }).notNull(),
sourceUrl: text('source_url').notNull(), // GitHub URL or absolute local path
branch: text('branch').default('main'),
state: text('state', {
enum: ['pending', 'indexing', 'indexed', 'error']
}).notNull().default('pending'),
totalSnippets: integer('total_snippets').default(0),
totalTokens: integer('total_tokens').default(0),
trustScore: real('trust_score').default(0), // 0.010.0
benchmarkScore: real('benchmark_score').default(0), // 0.0100.0
stars: integer('stars'),
githubToken: text('github_token'), // encrypted PAT for private repos
lastIndexedAt: integer('last_indexed_at', { mode: 'timestamp' }),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
id: text('id').primaryKey(), // e.g. "/facebook/react" or "/local/my-sdk"
title: text('title').notNull(),
description: text('description'),
source: text('source', { enum: ['github', 'local'] }).notNull(),
sourceUrl: text('source_url').notNull(), // GitHub URL or absolute local path
branch: text('branch').default('main'),
state: text('state', {
enum: ['pending', 'indexing', 'indexed', 'error']
})
.notNull()
.default('pending'),
totalSnippets: integer('total_snippets').default(0),
totalTokens: integer('total_tokens').default(0),
trustScore: real('trust_score').default(0), // 0.010.0
benchmarkScore: real('benchmark_score').default(0), // 0.0100.0
stars: integer('stars'),
githubToken: text('github_token'), // encrypted PAT for private repos
lastIndexedAt: integer('last_indexed_at', { mode: 'timestamp' }),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull()
});
```
@@ -58,17 +60,20 @@ Tracks indexed git tags/branches beyond the default branch.
```typescript
export const repositoryVersions = sqliteTable('repository_versions', {
id: text('id').primaryKey(), // e.g. "/facebook/react/v18.3.0"
repositoryId: text('repository_id').notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
tag: text('tag').notNull(), // git tag or branch name
title: text('title'),
state: text('state', {
enum: ['pending', 'indexing', 'indexed', 'error']
}).notNull().default('pending'),
totalSnippets: integer('total_snippets').default(0),
indexedAt: integer('indexed_at', { mode: 'timestamp' }),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
id: text('id').primaryKey(), // e.g. "/facebook/react/v18.3.0"
repositoryId: text('repository_id')
.notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
tag: text('tag').notNull(), // git tag or branch name
title: text('title'),
state: text('state', {
enum: ['pending', 'indexing', 'indexed', 'error']
})
.notNull()
.default('pending'),
totalSnippets: integer('total_snippets').default(0),
indexedAt: integer('indexed_at', { mode: 'timestamp' }),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
});
```
@@ -78,17 +83,17 @@ A parsed source file within a repository.
```typescript
export const documents = sqliteTable('documents', {
id: text('id').primaryKey(), // UUID
repositoryId: text('repository_id').notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
versionId: text('version_id')
.references(() => repositoryVersions.id, { onDelete: 'cascade' }),
filePath: text('file_path').notNull(), // relative path within repo
title: text('title'),
language: text('language'), // e.g. "typescript", "markdown"
tokenCount: integer('token_count').default(0),
checksum: text('checksum').notNull(), // SHA-256 of file content
indexedAt: integer('indexed_at', { mode: 'timestamp' }).notNull(),
id: text('id').primaryKey(), // UUID
repositoryId: text('repository_id')
.notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
versionId: text('version_id').references(() => repositoryVersions.id, { onDelete: 'cascade' }),
filePath: text('file_path').notNull(), // relative path within repo
title: text('title'),
language: text('language'), // e.g. "typescript", "markdown"
tokenCount: integer('token_count').default(0),
checksum: text('checksum').notNull(), // SHA-256 of file content
indexedAt: integer('indexed_at', { mode: 'timestamp' }).notNull()
});
```
@@ -98,20 +103,21 @@ An indexed chunk of content, the atomic unit of search.
```typescript
export const snippets = sqliteTable('snippets', {
id: text('id').primaryKey(), // UUID
documentId: text('document_id').notNull()
.references(() => documents.id, { onDelete: 'cascade' }),
repositoryId: text('repository_id').notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
versionId: text('version_id')
.references(() => repositoryVersions.id, { onDelete: 'cascade' }),
type: text('type', { enum: ['code', 'info'] }).notNull(),
title: text('title'),
content: text('content').notNull(), // searchable text / code
language: text('language'),
breadcrumb: text('breadcrumb'), // e.g. "Installation > Getting Started"
tokenCount: integer('token_count').default(0),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
id: text('id').primaryKey(), // UUID
documentId: text('document_id')
.notNull()
.references(() => documents.id, { onDelete: 'cascade' }),
repositoryId: text('repository_id')
.notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
versionId: text('version_id').references(() => repositoryVersions.id, { onDelete: 'cascade' }),
type: text('type', { enum: ['code', 'info'] }).notNull(),
title: text('title'),
content: text('content').notNull(), // searchable text / code
language: text('language'),
breadcrumb: text('breadcrumb'), // e.g. "Installation > Getting Started"
tokenCount: integer('token_count').default(0),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
});
```
@@ -121,12 +127,13 @@ Stores vector embeddings separately to keep snippets table lean.
```typescript
export const snippetEmbeddings = sqliteTable('snippet_embeddings', {
snippetId: text('snippet_id').primaryKey()
.references(() => snippets.id, { onDelete: 'cascade' }),
model: text('model').notNull(), // embedding model identifier
dimensions: integer('dimensions').notNull(),
embedding: blob('embedding').notNull(), // Float32Array as binary blob
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
snippetId: text('snippet_id')
.primaryKey()
.references(() => snippets.id, { onDelete: 'cascade' }),
model: text('model').notNull(), // embedding model identifier
dimensions: integer('dimensions').notNull(),
embedding: blob('embedding').notNull(), // Float32Array as binary blob
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
});
```
@@ -136,20 +143,23 @@ Tracks asynchronous indexing operations.
```typescript
export const indexingJobs = sqliteTable('indexing_jobs', {
id: text('id').primaryKey(), // UUID
repositoryId: text('repository_id').notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
versionId: text('version_id'),
status: text('status', {
enum: ['queued', 'running', 'done', 'failed']
}).notNull().default('queued'),
progress: integer('progress').default(0), // 0100
totalFiles: integer('total_files').default(0),
processedFiles: integer('processed_files').default(0),
error: text('error'),
startedAt: integer('started_at', { mode: 'timestamp' }),
completedAt: integer('completed_at', { mode: 'timestamp' }),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
id: text('id').primaryKey(), // UUID
repositoryId: text('repository_id')
.notNull()
.references(() => repositories.id, { onDelete: 'cascade' }),
versionId: text('version_id'),
status: text('status', {
enum: ['queued', 'running', 'done', 'failed']
})
.notNull()
.default('queued'),
progress: integer('progress').default(0), // 0100
totalFiles: integer('total_files').default(0),
processedFiles: integer('processed_files').default(0),
error: text('error'),
startedAt: integer('started_at', { mode: 'timestamp' }),
completedAt: integer('completed_at', { mode: 'timestamp' }),
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
});
```
@@ -159,17 +169,19 @@ Stores parsed `trueref.json` / `context7.json` configuration.
```typescript
export const repositoryConfigs = sqliteTable('repository_configs', {
repositoryId: text('repository_id').primaryKey()
.references(() => repositories.id, { onDelete: 'cascade' }),
projectTitle: text('project_title'),
description: text('description'),
folders: text('folders', { mode: 'json' }).$type<string[]>(),
excludeFolders: text('exclude_folders', { mode: 'json' }).$type<string[]>(),
excludeFiles: text('exclude_files', { mode: 'json' }).$type<string[]>(),
rules: text('rules', { mode: 'json' }).$type<string[]>(),
previousVersions: text('previous_versions', { mode: 'json' })
.$type<{ tag: string; title: string }[]>(),
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
repositoryId: text('repository_id')
.primaryKey()
.references(() => repositories.id, { onDelete: 'cascade' }),
projectTitle: text('project_title'),
description: text('description'),
folders: text('folders', { mode: 'json' }).$type<string[]>(),
excludeFolders: text('exclude_folders', { mode: 'json' }).$type<string[]>(),
excludeFiles: text('exclude_files', { mode: 'json' }).$type<string[]>(),
rules: text('rules', { mode: 'json' }).$type<string[]>(),
previousVersions: text('previous_versions', { mode: 'json' }).$type<
{ tag: string; title: string }[]
>(),
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull()
});
```
@@ -179,9 +191,9 @@ Key-value store for global application settings.
```typescript
export const settings = sqliteTable('settings', {
key: text('key').primaryKey(),
value: text('value', { mode: 'json' }),
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
key: text('key').primaryKey(),
value: text('value', { mode: 'json' }),
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull()
});
```

View File

@@ -31,10 +31,12 @@ Implement the core `RepositoryService` that handles CRUD operations for reposito
## Repository ID Generation
GitHub repositories:
- Input URL: `https://github.com/facebook/react` or `github.com/facebook/react`
- Generated ID: `/facebook/react`
Local repositories:
- Input path: `/home/user/projects/my-sdk`
- Generated ID: `/local/my-sdk` (basename of path, slugified)
- Collision resolution: append `-2`, `-3`, etc.
@@ -49,44 +51,44 @@ Version-specific IDs: `/facebook/react/v18.3.0`
// src/lib/server/services/repository.service.ts
export interface AddRepositoryInput {
source: 'github' | 'local';
sourceUrl: string; // GitHub URL or absolute local path
title?: string; // override auto-detected title
description?: string;
branch?: string; // GitHub: default branch; Local: n/a
githubToken?: string; // for private GitHub repos
source: 'github' | 'local';
sourceUrl: string; // GitHub URL or absolute local path
title?: string; // override auto-detected title
description?: string;
branch?: string; // GitHub: default branch; Local: n/a
githubToken?: string; // for private GitHub repos
}
export interface UpdateRepositoryInput {
title?: string;
description?: string;
branch?: string;
githubToken?: string;
title?: string;
description?: string;
branch?: string;
githubToken?: string;
}
export class RepositoryService {
constructor(private db: BetterSQLite3.Database) {}
constructor(private db: BetterSQLite3.Database) {}
async list(options?: {
state?: Repository['state'];
limit?: number;
offset?: number;
}): Promise<Repository[]>
async list(options?: {
state?: Repository['state'];
limit?: number;
offset?: number;
}): Promise<Repository[]>;
async get(id: string): Promise<Repository | null>
async get(id: string): Promise<Repository | null>;
async add(input: AddRepositoryInput): Promise<Repository>
async add(input: AddRepositoryInput): Promise<Repository>;
async update(id: string, input: UpdateRepositoryInput): Promise<Repository>
async update(id: string, input: UpdateRepositoryInput): Promise<Repository>;
async remove(id: string): Promise<void>
async remove(id: string): Promise<void>;
async getStats(id: string): Promise<{
totalSnippets: number;
totalTokens: number;
totalDocuments: number;
lastIndexedAt: Date | null;
}>
async getStats(id: string): Promise<{
totalSnippets: number;
totalTokens: number;
totalDocuments: number;
lastIndexedAt: Date | null;
}>;
}
```
@@ -97,48 +99,52 @@ export class RepositoryService {
### `GET /api/v1/libs`
Query parameters:
- `state` (optional): filter by state (`pending`, `indexed`, `error`, etc.)
- `limit` (optional, default 50): max results
- `offset` (optional, default 0): pagination offset
Response `200`:
```json
{
"libraries": [
{
"id": "/facebook/react",
"title": "React",
"description": "...",
"source": "github",
"state": "indexed",
"totalSnippets": 1234,
"totalTokens": 98000,
"trustScore": 8.5,
"stars": 228000,
"lastIndexedAt": "2026-03-22T10:00:00Z",
"versions": ["v18.3.0", "v17.0.2"]
}
],
"total": 12,
"limit": 50,
"offset": 0
"libraries": [
{
"id": "/facebook/react",
"title": "React",
"description": "...",
"source": "github",
"state": "indexed",
"totalSnippets": 1234,
"totalTokens": 98000,
"trustScore": 8.5,
"stars": 228000,
"lastIndexedAt": "2026-03-22T10:00:00Z",
"versions": ["v18.3.0", "v17.0.2"]
}
],
"total": 12,
"limit": 50,
"offset": 0
}
```
### `POST /api/v1/libs`
Request body:
```json
{
"source": "github",
"sourceUrl": "https://github.com/facebook/react",
"branch": "main",
"githubToken": "ghp_...",
"autoIndex": true
"source": "github",
"sourceUrl": "https://github.com/facebook/react",
"branch": "main",
"githubToken": "ghp_...",
"autoIndex": true
}
```
Response `201`:
```json
{
"library": { ...Repository },
@@ -149,6 +155,7 @@ Response `201`:
`autoIndex: true` (default) immediately queues an indexing job.
Response `409` if repository already exists:
```json
{ "error": "Repository /facebook/react already exists" }
```
@@ -176,20 +183,22 @@ Response `404`: not found.
Triggers a new indexing job. If a job is already running for this repo, returns the existing job.
Request body (optional):
```json
{ "version": "v18.3.0" }
```
Response `202`:
```json
{
"job": {
"id": "uuid",
"repositoryId": "/facebook/react",
"status": "queued",
"progress": 0,
"createdAt": "2026-03-22T10:00:00Z"
}
"job": {
"id": "uuid",
"repositoryId": "/facebook/react",
"status": "queued",
"progress": 0,
"createdAt": "2026-03-22T10:00:00Z"
}
}
```
@@ -198,15 +207,17 @@ Response `202`:
## Error Response Shape
All error responses follow:
```json
{
"error": "Human-readable message",
"code": "MACHINE_READABLE_CODE",
"details": {}
"error": "Human-readable message",
"code": "MACHINE_READABLE_CODE",
"details": {}
}
```
Error codes:
- `NOT_FOUND`
- `ALREADY_EXISTS`
- `INVALID_INPUT`
@@ -219,23 +230,23 @@ Error codes:
```typescript
function resolveGitHubId(url: string): string {
// Parse owner/repo from URL variants:
// https://github.com/facebook/react
// https://github.com/facebook/react.git
// github.com/facebook/react
const match = url.match(/github\.com\/([^/]+)\/([^/\s.]+)/);
if (!match) throw new Error('Invalid GitHub URL');
return `/${match[1]}/${match[2]}`;
// Parse owner/repo from URL variants:
// https://github.com/facebook/react
// https://github.com/facebook/react.git
// github.com/facebook/react
const match = url.match(/github\.com\/([^/]+)\/([^/\s.]+)/);
if (!match) throw new Error('Invalid GitHub URL');
return `/${match[1]}/${match[2]}`;
}
function resolveLocalId(path: string, existingIds: string[]): string {
const base = slugify(path.split('/').at(-1)!);
let id = `/local/${base}`;
let counter = 2;
while (existingIds.includes(id)) {
id = `/local/${base}-${counter++}`;
}
return id;
const base = slugify(path.split('/').at(-1)!);
let id = `/local/${base}`;
let counter = 2;
while (existingIds.includes(id)) {
id = `/local/${base}-${counter++}`;
}
return id;
}
```

View File

@@ -37,17 +37,46 @@ The crawler only downloads files with these extensions:
```typescript
const INDEXABLE_EXTENSIONS = new Set([
// Documentation
'.md', '.mdx', '.txt', '.rst',
// Code
'.ts', '.tsx', '.js', '.jsx',
'.py', '.rb', '.go', '.rs', '.java', '.cs', '.cpp', '.c', '.h',
'.swift', '.kt', '.php', '.scala', '.clj', '.ex', '.exs',
'.sh', '.bash', '.zsh', '.fish',
// Config / data
'.json', '.yaml', '.yml', '.toml',
// Web
'.html', '.css', '.svelte', '.vue',
// Documentation
'.md',
'.mdx',
'.txt',
'.rst',
// Code
'.ts',
'.tsx',
'.js',
'.jsx',
'.py',
'.rb',
'.go',
'.rs',
'.java',
'.cs',
'.cpp',
'.c',
'.h',
'.swift',
'.kt',
'.php',
'.scala',
'.clj',
'.ex',
'.exs',
'.sh',
'.bash',
'.zsh',
'.fish',
// Config / data
'.json',
'.yaml',
'.yml',
'.toml',
// Web
'.html',
'.css',
'.svelte',
'.vue'
]);
const MAX_FILE_SIZE_BYTES = 500_000; // 500 KB — skip large generated files
@@ -59,28 +88,28 @@ const MAX_FILE_SIZE_BYTES = 500_000; // 500 KB — skip large generated files
```typescript
export interface CrawledFile {
path: string; // relative path within repo, e.g. "src/index.ts"
content: string; // UTF-8 file content
size: number; // bytes
sha: string; // GitHub blob SHA (used as checksum)
language: string; // detected from extension
path: string; // relative path within repo, e.g. "src/index.ts"
content: string; // UTF-8 file content
size: number; // bytes
sha: string; // GitHub blob SHA (used as checksum)
language: string; // detected from extension
}
export interface CrawlResult {
files: CrawledFile[];
totalFiles: number; // files matching filters
skippedFiles: number; // filtered out or too large
branch: string; // branch/tag that was crawled
commitSha: string; // HEAD commit SHA
files: CrawledFile[];
totalFiles: number; // files matching filters
skippedFiles: number; // filtered out or too large
branch: string; // branch/tag that was crawled
commitSha: string; // HEAD commit SHA
}
export interface CrawlOptions {
owner: string;
repo: string;
ref?: string; // branch, tag, or commit SHA; defaults to repo default branch
token?: string; // GitHub PAT for private repos
config?: RepoConfig; // parsed trueref.json
onProgress?: (processed: number, total: number) => void;
owner: string;
repo: string;
ref?: string; // branch, tag, or commit SHA; defaults to repo default branch
token?: string; // GitHub PAT for private repos
config?: RepoConfig; // parsed trueref.json
onProgress?: (processed: number, total: number) => void;
}
```
@@ -89,12 +118,14 @@ export interface CrawlOptions {
## GitHub API Usage
### Step 1: Get default branch (if ref not specified)
```
GET https://api.github.com/repos/{owner}/{repo}
→ { default_branch: "main", stargazers_count: 12345 }
```
### Step 2: Fetch file tree (recursive)
```
GET https://api.github.com/repos/{owner}/{repo}/git/trees/{ref}?recursive=1
→ {
@@ -109,12 +140,14 @@ GET https://api.github.com/repos/{owner}/{repo}/git/trees/{ref}?recursive=1
If `truncated: true`, the tree has >100k items. Use `--depth` pagination or filter top-level directories first.
### Step 3: Download file contents (parallel)
```
GET https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={ref}
→ { content: "<base64>", encoding: "base64", size: 1234, sha: "abc123" }
```
Alternative for large repos: use raw content URL:
```
GET https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}
```
@@ -124,48 +157,47 @@ GET https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}
## Filtering Logic
```typescript
function shouldIndexFile(
filePath: string,
fileSize: number,
config?: RepoConfig
): boolean {
const ext = path.extname(filePath).toLowerCase();
const base = path.basename(filePath);
function shouldIndexFile(filePath: string, fileSize: number, config?: RepoConfig): boolean {
const ext = path.extname(filePath).toLowerCase();
const base = path.basename(filePath);
// 1. Must have indexable extension
if (!INDEXABLE_EXTENSIONS.has(ext)) return false;
// 1. Must have indexable extension
if (!INDEXABLE_EXTENSIONS.has(ext)) return false;
// 2. Must not exceed size limit
if (fileSize > MAX_FILE_SIZE_BYTES) return false;
// 2. Must not exceed size limit
if (fileSize > MAX_FILE_SIZE_BYTES) return false;
// 3. Exclude lockfiles and other non-source artifacts
if (IGNORED_FILE_NAMES.has(base)) return false;
// 3. Exclude lockfiles and other non-source artifacts
if (IGNORED_FILE_NAMES.has(base)) return false;
// 4. Exclude minified and bundled assets
if (base.includes('.min.') || base.endsWith('.bundle.js') || base.endsWith('.bundle.css')) {
return false;
}
// 4. Exclude minified and bundled assets
if (base.includes('.min.') || base.endsWith('.bundle.js') || base.endsWith('.bundle.css')) {
return false;
}
// 5. Apply config excludeFiles (exact filename match)
if (config?.excludeFiles?.includes(base)) return false;
// 5. Apply config excludeFiles (exact filename match)
if (config?.excludeFiles?.includes(base)) return false;
// 6. Exclude common dependency/build/cache directories at any depth
if (isInIgnoredDirectory(filePath)) return false;
// 6. Exclude common dependency/build/cache directories at any depth
if (isInIgnoredDirectory(filePath)) return false;
// 7. Apply config excludeFolders (regex or prefix match)
if (config?.excludeFolders?.some(folder =>
filePath.startsWith(folder) || new RegExp(folder).test(filePath)
)) return false;
// 7. Apply config excludeFolders (regex or prefix match)
if (
config?.excludeFolders?.some(
(folder) => filePath.startsWith(folder) || new RegExp(folder).test(filePath)
)
)
return false;
// 8. Apply config folders allowlist (if specified, only index those paths)
if (config?.folders?.length) {
const inAllowedFolder = config.folders.some(folder =>
filePath.startsWith(folder) || new RegExp(folder).test(filePath)
);
if (!inAllowedFolder) return false;
}
// 8. Apply config folders allowlist (if specified, only index those paths)
if (config?.folders?.length) {
const inAllowedFolder = config.folders.some(
(folder) => filePath.startsWith(folder) || new RegExp(folder).test(filePath)
);
if (!inAllowedFolder) return false;
}
return true;
return true;
}
```
@@ -177,20 +209,20 @@ The shared ignored-directory list is intentionally broader than the original bas
```typescript
class GitHubRateLimiter {
private remaining = 5000;
private resetAt = Date.now();
private remaining = 5000;
private resetAt = Date.now();
updateFromHeaders(headers: Headers): void {
this.remaining = parseInt(headers.get('X-RateLimit-Remaining') ?? '5000');
this.resetAt = parseInt(headers.get('X-RateLimit-Reset') ?? '0') * 1000;
}
updateFromHeaders(headers: Headers): void {
this.remaining = parseInt(headers.get('X-RateLimit-Remaining') ?? '5000');
this.resetAt = parseInt(headers.get('X-RateLimit-Reset') ?? '0') * 1000;
}
async waitIfNeeded(): Promise<void> {
if (this.remaining <= 10) {
const waitMs = Math.max(0, this.resetAt - Date.now()) + 1000;
await sleep(waitMs);
}
}
async waitIfNeeded(): Promise<void> {
if (this.remaining <= 10) {
const waitMs = Math.max(0, this.resetAt - Date.now()) + 1000;
await sleep(waitMs);
}
}
}
```
@@ -200,14 +232,14 @@ Requests are made with a concurrency limit of 10 parallel downloads using a sema
## Error Handling
| Scenario | Behavior |
|----------|---------|
| 404 Not Found | Throw `RepositoryNotFoundError` |
| 401 Unauthorized | Throw `AuthenticationError` (invalid or missing token) |
| 403 Forbidden | If `X-RateLimit-Remaining: 0`, wait and retry; else throw `PermissionError` |
| 422 Unprocessable | Tree too large; switch to directory-by-directory traversal |
| Network error | Retry up to 3 times with exponential backoff |
| File content decode error | Skip file, log warning |
| Scenario | Behavior |
| ------------------------- | --------------------------------------------------------------------------- |
| 404 Not Found | Throw `RepositoryNotFoundError` |
| 401 Unauthorized | Throw `AuthenticationError` (invalid or missing token) |
| 403 Forbidden | If `X-RateLimit-Remaining: 0`, wait and retry; else throw `PermissionError` |
| 422 Unprocessable | Tree too large; switch to directory-by-directory traversal |
| Network error | Retry up to 3 times with exponential backoff |
| File content decode error | Skip file, log warning |
---

View File

@@ -38,9 +38,9 @@ Reuses `CrawledFile` and `CrawlResult` from TRUEREF-0003 crawler types:
```typescript
export interface LocalCrawlOptions {
rootPath: string; // absolute path to repository root
config?: RepoConfig; // parsed trueref.json
onProgress?: (processed: number, total: number) => void;
rootPath: string; // absolute path to repository root
config?: RepoConfig; // parsed trueref.json
onProgress?: (processed: number, total: number) => void;
}
```
@@ -50,75 +50,73 @@ export interface LocalCrawlOptions {
```typescript
export class LocalCrawler {
async crawl(options: LocalCrawlOptions): Promise<CrawlResult> {
// 1. Load root .gitignore if present
const gitignore = await this.loadGitignore(options.rootPath);
async crawl(options: LocalCrawlOptions): Promise<CrawlResult> {
// 1. Load root .gitignore if present
const gitignore = await this.loadGitignore(options.rootPath);
// 2. Enumerate files recursively, pruning ignored directories early
const allFiles = await this.walkDirectory(options.rootPath, '', gitignore);
// 2. Enumerate files recursively, pruning ignored directories early
const allFiles = await this.walkDirectory(options.rootPath, '', gitignore);
// 3. Look for trueref.json / context7.json first
const configFile = allFiles.find(f =>
f === 'trueref.json' || f === 'context7.json'
);
let config = options.config;
if (configFile && !config) {
config = await this.parseConfigFile(
path.join(options.rootPath, configFile)
);
}
// 3. Look for trueref.json / context7.json first
const configFile = allFiles.find((f) => f === 'trueref.json' || f === 'context7.json');
let config = options.config;
if (configFile && !config) {
config = await this.parseConfigFile(path.join(options.rootPath, configFile));
}
// 4. Filter files
const filteredFiles = allFiles.filter(relPath => {
const stat = statCache.get(relPath);
return shouldIndexFile(relPath, stat.size, config);
});
// 4. Filter files
const filteredFiles = allFiles.filter((relPath) => {
const stat = statCache.get(relPath);
return shouldIndexFile(relPath, stat.size, config);
});
// 5. Read and return file contents
const crawledFiles: CrawledFile[] = [];
for (const [i, relPath] of filteredFiles.entries()) {
const absPath = path.join(options.rootPath, relPath);
const content = await fs.readFile(absPath, 'utf-8');
const sha = computeSHA256(content);
crawledFiles.push({
path: relPath,
content,
size: Buffer.byteLength(content, 'utf-8'),
sha,
language: detectLanguage(relPath),
});
options.onProgress?.(i + 1, filteredFiles.length);
}
// 5. Read and return file contents
const crawledFiles: CrawledFile[] = [];
for (const [i, relPath] of filteredFiles.entries()) {
const absPath = path.join(options.rootPath, relPath);
const content = await fs.readFile(absPath, 'utf-8');
const sha = computeSHA256(content);
crawledFiles.push({
path: relPath,
content,
size: Buffer.byteLength(content, 'utf-8'),
sha,
language: detectLanguage(relPath)
});
options.onProgress?.(i + 1, filteredFiles.length);
}
return {
files: crawledFiles,
totalFiles: filteredFiles.length,
skippedFiles: allFiles.length - filteredFiles.length,
branch: 'local',
commitSha: computeSHA256(crawledFiles.map(f => f.sha).join('')),
};
}
return {
files: crawledFiles,
totalFiles: filteredFiles.length,
skippedFiles: allFiles.length - filteredFiles.length,
branch: 'local',
commitSha: computeSHA256(crawledFiles.map((f) => f.sha).join(''))
};
}
private async walkDirectory(dir: string, rel = '', gitignore?: GitignoreFilter): Promise<string[]> {
const entries = await fs.readdir(dir, { withFileTypes: true });
const files: string[] = [];
for (const entry of entries) {
if (!entry.isFile() && !entry.isDirectory()) continue; // skip symlinks, devices
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
if (entry.isDirectory()) {
if (shouldPruneDirectory(relPath) || gitignore?.isIgnored(relPath, true)) {
continue;
}
files.push(...await this.walkDirectory(
path.join(dir, entry.name), relPath, gitignore
));
} else {
if (gitignore?.isIgnored(relPath, false)) continue;
files.push(relPath);
}
}
return files;
}
private async walkDirectory(
dir: string,
rel = '',
gitignore?: GitignoreFilter
): Promise<string[]> {
const entries = await fs.readdir(dir, { withFileTypes: true });
const files: string[] = [];
for (const entry of entries) {
if (!entry.isFile() && !entry.isDirectory()) continue; // skip symlinks, devices
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
if (entry.isDirectory()) {
if (shouldPruneDirectory(relPath) || gitignore?.isIgnored(relPath, true)) {
continue;
}
files.push(...(await this.walkDirectory(path.join(dir, entry.name), relPath, gitignore)));
} else {
if (gitignore?.isIgnored(relPath, false)) continue;
files.push(relPath);
}
}
return files;
}
}
```
@@ -142,7 +140,7 @@ Directory pruning should happen during the walk so large dependency trees are ne
import { createHash } from 'crypto';
function computeSHA256(content: string): string {
return createHash('sha256').update(content, 'utf-8').digest('hex');
return createHash('sha256').update(content, 'utf-8').digest('hex');
}
```

View File

@@ -30,19 +30,19 @@ Implement the document parsing and chunking pipeline that transforms raw file co
## Supported File Types
| Extension | Parser Strategy |
|-----------|----------------|
| `.md`, `.mdx` | Heading-based section splitting + code block extraction |
| `.txt`, `.rst` | Paragraph-based splitting |
| `.ts`, `.tsx`, `.js`, `.jsx` | AST-free: function/class boundary detection via regex |
| `.py` | `def`/`class` boundary detection |
| `.go` | `func`/`type` boundary detection |
| `.rs` | `fn`/`impl`/`struct` boundary detection |
| `.java`, `.cs`, `.kt`, `.swift` | Class/method boundary detection |
| `.rb` | `def`/`class` boundary detection |
| `.json`, `.yaml`, `.yml`, `.toml` | Structural chunking (top-level keys) |
| `.html`, `.svelte`, `.vue` | Text content extraction + script block splitting |
| Other code | Line-count-based sliding window (200 lines per chunk) |
| Extension | Parser Strategy |
| --------------------------------- | ------------------------------------------------------- |
| `.md`, `.mdx` | Heading-based section splitting + code block extraction |
| `.txt`, `.rst` | Paragraph-based splitting |
| `.ts`, `.tsx`, `.js`, `.jsx` | AST-free: function/class boundary detection via regex |
| `.py` | `def`/`class` boundary detection |
| `.go` | `func`/`type` boundary detection |
| `.rs` | `fn`/`impl`/`struct` boundary detection |
| `.java`, `.cs`, `.kt`, `.swift` | Class/method boundary detection |
| `.rb` | `def`/`class` boundary detection |
| `.json`, `.yaml`, `.yml`, `.toml` | Structural chunking (top-level keys) |
| `.html`, `.svelte`, `.vue` | Text content extraction + script block splitting |
| Other code | Line-count-based sliding window (200 lines per chunk) |
---
@@ -52,9 +52,9 @@ Use a simple character-based approximation (no tokenizer library needed for v1):
```typescript
function estimateTokens(text: string): number {
// Empirically: ~4 chars per token for English prose
// ~3 chars per token for code (more symbols)
return Math.ceil(text.length / 3.5);
// Empirically: ~4 chars per token for English prose
// ~3 chars per token for code (more symbols)
return Math.ceil(text.length / 3.5);
}
```
@@ -74,49 +74,49 @@ The Markdown parser is the most important parser as most documentation is Markdo
```typescript
interface MarkdownSection {
headings: string[]; // heading stack at this point
content: string; // text content (sans code blocks)
codeBlocks: { language: string; code: string }[];
headings: string[]; // heading stack at this point
content: string; // text content (sans code blocks)
codeBlocks: { language: string; code: string }[];
}
function parseMarkdown(content: string, filePath: string): Snippet[] {
const sections = splitIntoSections(content);
const snippets: Snippet[] = [];
const sections = splitIntoSections(content);
const snippets: Snippet[] = [];
for (const section of sections) {
const breadcrumb = section.headings.join(' > ');
const title = section.headings.at(-1) ?? path.basename(filePath);
for (const section of sections) {
const breadcrumb = section.headings.join(' > ');
const title = section.headings.at(-1) ?? path.basename(filePath);
// Emit info snippet for text content
if (section.content.trim().length >= 20) {
const chunks = chunkText(section.content, MAX_TOKENS, OVERLAP_TOKENS);
for (const chunk of chunks) {
snippets.push({
type: 'info',
title,
content: chunk,
breadcrumb,
tokenCount: estimateTokens(chunk),
});
}
}
// Emit info snippet for text content
if (section.content.trim().length >= 20) {
const chunks = chunkText(section.content, MAX_TOKENS, OVERLAP_TOKENS);
for (const chunk of chunks) {
snippets.push({
type: 'info',
title,
content: chunk,
breadcrumb,
tokenCount: estimateTokens(chunk)
});
}
}
// Emit code snippets for each code block
for (const block of section.codeBlocks) {
if (block.code.trim().length >= 20) {
snippets.push({
type: 'code',
title,
content: block.code,
language: block.language || detectLanguage('.' + block.language),
breadcrumb,
tokenCount: estimateTokens(block.code),
});
}
}
}
// Emit code snippets for each code block
for (const block of section.codeBlocks) {
if (block.code.trim().length >= 20) {
snippets.push({
type: 'code',
title,
content: block.code,
language: block.language || detectLanguage('.' + block.language),
breadcrumb,
tokenCount: estimateTokens(block.code)
});
}
}
}
return snippets;
return snippets;
}
```
@@ -135,43 +135,41 @@ For non-Markdown code files, use regex-based function/class boundary detection.
```typescript
const BOUNDARY_PATTERNS: Record<string, RegExp> = {
typescript: /^(export\s+)?(async\s+)?(function|class|interface|type|const|let|var)\s+\w+/m,
python: /^(async\s+)?(def|class)\s+\w+/m,
go: /^(func|type|var|const)\s+\w+/m,
rust: /^(pub\s+)?(fn|impl|struct|enum|trait)\s+\w+/m,
java: /^(public|private|protected|static).*?(class|interface|enum|void|\w+)\s+\w+\s*[({]/m,
typescript: /^(export\s+)?(async\s+)?(function|class|interface|type|const|let|var)\s+\w+/m,
python: /^(async\s+)?(def|class)\s+\w+/m,
go: /^(func|type|var|const)\s+\w+/m,
rust: /^(pub\s+)?(fn|impl|struct|enum|trait)\s+\w+/m,
java: /^(public|private|protected|static).*?(class|interface|enum|void|\w+)\s+\w+\s*[({]/m
};
function parseCodeFile(
content: string,
filePath: string,
language: string
): Snippet[] {
const pattern = BOUNDARY_PATTERNS[language];
const breadcrumb = filePath;
const title = path.basename(filePath);
function parseCodeFile(content: string, filePath: string, language: string): Snippet[] {
const pattern = BOUNDARY_PATTERNS[language];
const breadcrumb = filePath;
const title = path.basename(filePath);
if (!pattern) {
// Fallback: sliding window
return slidingWindowChunks(content, filePath, language);
}
if (!pattern) {
// Fallback: sliding window
return slidingWindowChunks(content, filePath, language);
}
const chunks = splitAtBoundaries(content, pattern);
return chunks
.filter(chunk => chunk.trim().length >= 20)
.flatMap(chunk => {
if (estimateTokens(chunk) <= MAX_TOKENS) {
return [{
type: 'code' as const,
title,
content: chunk,
language,
breadcrumb,
tokenCount: estimateTokens(chunk),
}];
}
return slidingWindowChunks(chunk, filePath, language);
});
const chunks = splitAtBoundaries(content, pattern);
return chunks
.filter((chunk) => chunk.trim().length >= 20)
.flatMap((chunk) => {
if (estimateTokens(chunk) <= MAX_TOKENS) {
return [
{
type: 'code' as const,
title,
content: chunk,
language,
breadcrumb,
tokenCount: estimateTokens(chunk)
}
];
}
return slidingWindowChunks(chunk, filePath, language);
});
}
```
@@ -188,27 +186,23 @@ const MIN_CONTENT_LENGTH = 20; // characters
### Sliding Window Chunker
```typescript
function chunkText(
text: string,
maxTokens: number,
overlapTokens: number
): string[] {
const words = text.split(/\s+/);
const wordsPerToken = 0.75; // ~0.75 words per token
const maxWords = Math.floor(maxTokens * wordsPerToken);
const overlapWords = Math.floor(overlapTokens * wordsPerToken);
function chunkText(text: string, maxTokens: number, overlapTokens: number): string[] {
const words = text.split(/\s+/);
const wordsPerToken = 0.75; // ~0.75 words per token
const maxWords = Math.floor(maxTokens * wordsPerToken);
const overlapWords = Math.floor(overlapTokens * wordsPerToken);
const chunks: string[] = [];
let start = 0;
const chunks: string[] = [];
let start = 0;
while (start < words.length) {
const end = Math.min(start + maxWords, words.length);
chunks.push(words.slice(start, end).join(' '));
if (end === words.length) break;
start = end - overlapWords;
}
while (start < words.length) {
const end = Math.min(start + maxWords, words.length);
chunks.push(words.slice(start, end).join(' '));
if (end === words.length) break;
start = end - overlapWords;
}
return chunks;
return chunks;
}
```
@@ -218,34 +212,42 @@ function chunkText(
```typescript
const LANGUAGE_MAP: Record<string, string> = {
'.ts': 'typescript', '.tsx': 'typescript',
'.js': 'javascript', '.jsx': 'javascript',
'.py': 'python',
'.rb': 'ruby',
'.go': 'go',
'.rs': 'rust',
'.java': 'java',
'.cs': 'csharp',
'.cpp': 'cpp', '.c': 'c', '.h': 'c',
'.swift': 'swift',
'.kt': 'kotlin',
'.php': 'php',
'.scala': 'scala',
'.sh': 'bash', '.bash': 'bash', '.zsh': 'bash',
'.md': 'markdown', '.mdx': 'markdown',
'.json': 'json',
'.yaml': 'yaml', '.yml': 'yaml',
'.toml': 'toml',
'.html': 'html',
'.css': 'css',
'.svelte': 'svelte',
'.vue': 'vue',
'.sql': 'sql',
'.ts': 'typescript',
'.tsx': 'typescript',
'.js': 'javascript',
'.jsx': 'javascript',
'.py': 'python',
'.rb': 'ruby',
'.go': 'go',
'.rs': 'rust',
'.java': 'java',
'.cs': 'csharp',
'.cpp': 'cpp',
'.c': 'c',
'.h': 'c',
'.swift': 'swift',
'.kt': 'kotlin',
'.php': 'php',
'.scala': 'scala',
'.sh': 'bash',
'.bash': 'bash',
'.zsh': 'bash',
'.md': 'markdown',
'.mdx': 'markdown',
'.json': 'json',
'.yaml': 'yaml',
'.yml': 'yaml',
'.toml': 'toml',
'.html': 'html',
'.css': 'css',
'.svelte': 'svelte',
'.vue': 'vue',
'.sql': 'sql'
};
function detectLanguage(filePath: string): string {
const ext = path.extname(filePath).toLowerCase();
return LANGUAGE_MAP[ext] ?? 'text';
const ext = path.extname(filePath).toLowerCase();
return LANGUAGE_MAP[ext] ?? 'text';
}
```
@@ -255,32 +257,32 @@ function detectLanguage(filePath: string): string {
```typescript
export interface ParseOptions {
repositoryId: string;
documentId: string;
versionId?: string;
repositoryId: string;
documentId: string;
versionId?: string;
}
export function parseFile(
file: CrawledFile,
options: ParseOptions
): NewSnippet[] {
const language = detectLanguage(file.path);
let rawSnippets: Omit<NewSnippet, 'id' | 'repositoryId' | 'documentId' | 'versionId' | 'createdAt'>[];
export function parseFile(file: CrawledFile, options: ParseOptions): NewSnippet[] {
const language = detectLanguage(file.path);
let rawSnippets: Omit<
NewSnippet,
'id' | 'repositoryId' | 'documentId' | 'versionId' | 'createdAt'
>[];
if (language === 'markdown') {
rawSnippets = parseMarkdown(file.content, file.path);
} else {
rawSnippets = parseCodeFile(file.content, file.path, language);
}
if (language === 'markdown') {
rawSnippets = parseMarkdown(file.content, file.path);
} else {
rawSnippets = parseCodeFile(file.content, file.path, language);
}
return rawSnippets.map(s => ({
...s,
id: crypto.randomUUID(),
repositoryId: options.repositoryId,
documentId: options.documentId,
versionId: options.versionId ?? null,
createdAt: new Date(),
}));
return rawSnippets.map((s) => ({
...s,
id: crypto.randomUUID(),
repositoryId: options.repositoryId,
documentId: options.documentId,
versionId: options.versionId ?? null,
createdAt: new Date()
}));
}
```

View File

@@ -33,42 +33,37 @@ Implement the full-text search engine using SQLite's built-in FTS5 extension. Th
// src/lib/server/search/search.service.ts
export interface SnippetSearchOptions {
repositoryId: string;
versionId?: string;
type?: 'code' | 'info';
limit?: number; // default: 20
offset?: number; // default: 0
repositoryId: string;
versionId?: string;
type?: 'code' | 'info';
limit?: number; // default: 20
offset?: number; // default: 0
}
export interface SnippetSearchResult {
snippet: Snippet;
score: number; // BM25 rank (negative, lower = better)
repository: Pick<Repository, 'id' | 'title'>;
snippet: Snippet;
score: number; // BM25 rank (negative, lower = better)
repository: Pick<Repository, 'id' | 'title'>;
}
export interface LibrarySearchOptions {
libraryName: string;
query?: string; // semantic relevance hint
limit?: number; // default: 10
libraryName: string;
query?: string; // semantic relevance hint
limit?: number; // default: 10
}
export interface LibrarySearchResult {
repository: Repository;
versions: RepositoryVersion[];
score: number; // composite relevance score
repository: Repository;
versions: RepositoryVersion[];
score: number; // composite relevance score
}
export class SearchService {
constructor(private db: BetterSQLite3.Database) {}
constructor(private db: BetterSQLite3.Database) {}
searchSnippets(
query: string,
options: SnippetSearchOptions
): SnippetSearchResult[]
searchSnippets(query: string, options: SnippetSearchOptions): SnippetSearchResult[];
searchRepositories(
options: LibrarySearchOptions
): LibrarySearchResult[]
searchRepositories(options: LibrarySearchOptions): LibrarySearchResult[];
}
```
@@ -101,21 +96,21 @@ The FTS5 MATCH query uses the porter stemmer and unicode61 tokenizer (configured
```typescript
function preprocessQuery(raw: string): string {
// 1. Trim and normalize whitespace
let q = raw.trim().replace(/\s+/g, ' ');
// 1. Trim and normalize whitespace
let q = raw.trim().replace(/\s+/g, ' ');
// 2. Escape FTS5 special characters that aren't intended as operators
// Keep: * (prefix), " " (phrase), AND, OR, NOT
q = q.replace(/[()]/g, ' ');
// 2. Escape FTS5 special characters that aren't intended as operators
// Keep: * (prefix), " " (phrase), AND, OR, NOT
q = q.replace(/[()]/g, ' ');
// 3. Add prefix wildcard to last token for "typing as you go" feel
const tokens = q.split(' ');
const lastToken = tokens.at(-1) ?? '';
if (lastToken.length >= 3 && !lastToken.endsWith('*')) {
tokens[tokens.length - 1] = lastToken + '*';
}
// 3. Add prefix wildcard to last token for "typing as you go" feel
const tokens = q.split(' ');
const lastToken = tokens.at(-1) ?? '';
if (lastToken.length >= 3 && !lastToken.endsWith('*')) {
tokens[tokens.length - 1] = lastToken + '*';
}
return tokens.join(' ');
return tokens.join(' ');
}
```
@@ -174,56 +169,65 @@ searchRepositories(options: LibrarySearchOptions): LibrarySearchResult[] {
The search results must be formatted for the REST API and MCP tool responses:
### Library search response (for `resolve-library-id`):
```typescript
function formatLibraryResults(results: LibrarySearchResult[]): string {
if (results.length === 0) {
return 'No libraries found matching your search.';
}
if (results.length === 0) {
return 'No libraries found matching your search.';
}
return results.map((r, i) => {
const repo = r.repository;
const versions = r.versions.map(v => v.tag).join(', ') || 'default branch';
return [
`${i + 1}. ${repo.title}`,
` Library ID: ${repo.id}`,
` Description: ${repo.description ?? 'No description'}`,
` Snippets: ${repo.totalSnippets} | Trust Score: ${repo.trustScore.toFixed(1)}/10`,
` Available Versions: ${versions}`,
].join('\n');
}).join('\n\n');
return results
.map((r, i) => {
const repo = r.repository;
const versions = r.versions.map((v) => v.tag).join(', ') || 'default branch';
return [
`${i + 1}. ${repo.title}`,
` Library ID: ${repo.id}`,
` Description: ${repo.description ?? 'No description'}`,
` Snippets: ${repo.totalSnippets} | Trust Score: ${repo.trustScore.toFixed(1)}/10`,
` Available Versions: ${versions}`
].join('\n');
})
.join('\n\n');
}
```
### Snippet search response (for `query-docs`):
```typescript
function formatSnippetResults(
results: SnippetSearchResult[],
rules?: string[]
): string {
const parts: string[] = [];
function formatSnippetResults(results: SnippetSearchResult[], rules?: string[]): string {
const parts: string[] = [];
// Prepend repository rules if present
if (rules?.length) {
parts.push('## Library Rules\n' + rules.map(r => `- ${r}`).join('\n'));
}
// Prepend repository rules if present
if (rules?.length) {
parts.push('## Library Rules\n' + rules.map((r) => `- ${r}`).join('\n'));
}
for (const { snippet } of results) {
if (snippet.type === 'code') {
parts.push([
snippet.title ? `### ${snippet.title}` : '',
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
`\`\`\`${snippet.language ?? ''}\n${snippet.content}\n\`\`\``,
].filter(Boolean).join('\n'));
} else {
parts.push([
snippet.title ? `### ${snippet.title}` : '',
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
snippet.content,
].filter(Boolean).join('\n'));
}
}
for (const { snippet } of results) {
if (snippet.type === 'code') {
parts.push(
[
snippet.title ? `### ${snippet.title}` : '',
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
`\`\`\`${snippet.language ?? ''}\n${snippet.content}\n\`\`\``
]
.filter(Boolean)
.join('\n')
);
} else {
parts.push(
[
snippet.title ? `### ${snippet.title}` : '',
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
snippet.content
]
.filter(Boolean)
.join('\n')
);
}
}
return parts.join('\n\n---\n\n');
return parts.join('\n\n---\n\n');
}
```
@@ -235,26 +239,26 @@ Compute `trustScore` (010) when a repository is first indexed:
```typescript
function computeTrustScore(repo: Repository): number {
let score = 0;
let score = 0;
// Stars (up to 4 points): log scale, 10k stars = 4 pts
if (repo.stars) {
score += Math.min(4, Math.log10(repo.stars + 1));
}
// Stars (up to 4 points): log scale, 10k stars = 4 pts
if (repo.stars) {
score += Math.min(4, Math.log10(repo.stars + 1));
}
// Documentation coverage (up to 3 points)
score += Math.min(3, repo.totalSnippets / 500);
// Documentation coverage (up to 3 points)
score += Math.min(3, repo.totalSnippets / 500);
// Source type (1 point for GitHub, 0 for local)
if (repo.source === 'github') score += 1;
// Source type (1 point for GitHub, 0 for local)
if (repo.source === 'github') score += 1;
// Successful indexing (1 point)
if (repo.state === 'indexed') score += 1;
// Successful indexing (1 point)
if (repo.state === 'indexed') score += 1;
// Has description (1 point)
if (repo.description) score += 1;
// Has description (1 point)
if (repo.description) score += 1;
return Math.min(10, parseFloat(score.toFixed(1)));
return Math.min(10, parseFloat(score.toFixed(1)));
}
```

View File

@@ -34,18 +34,18 @@ Implement a pluggable embedding generation system that produces vector represent
// src/lib/server/embeddings/provider.ts
export interface EmbeddingVector {
values: Float32Array;
dimensions: number;
model: string;
values: Float32Array;
dimensions: number;
model: string;
}
export interface EmbeddingProvider {
readonly name: string;
readonly dimensions: number;
readonly model: string;
readonly name: string;
readonly dimensions: number;
readonly model: string;
embed(texts: string[]): Promise<EmbeddingVector[]>;
isAvailable(): Promise<boolean>;
embed(texts: string[]): Promise<EmbeddingVector[]>;
isAvailable(): Promise<boolean>;
}
```
@@ -55,51 +55,51 @@ export interface EmbeddingProvider {
```typescript
export interface OpenAIProviderConfig {
baseUrl: string; // e.g. "https://api.openai.com/v1" or "http://localhost:11434/v1"
apiKey: string;
model: string; // e.g. "text-embedding-3-small", "nomic-embed-text"
dimensions?: number; // override for models that support it (e.g. text-embedding-3-small)
maxBatchSize?: number; // default: 100
baseUrl: string; // e.g. "https://api.openai.com/v1" or "http://localhost:11434/v1"
apiKey: string;
model: string; // e.g. "text-embedding-3-small", "nomic-embed-text"
dimensions?: number; // override for models that support it (e.g. text-embedding-3-small)
maxBatchSize?: number; // default: 100
}
export class OpenAIEmbeddingProvider implements EmbeddingProvider {
constructor(private config: OpenAIProviderConfig) {}
constructor(private config: OpenAIProviderConfig) {}
async embed(texts: string[]): Promise<EmbeddingVector[]> {
// Batch into groups of maxBatchSize
const batches = chunk(texts, this.config.maxBatchSize ?? 100);
const allEmbeddings: EmbeddingVector[] = [];
async embed(texts: string[]): Promise<EmbeddingVector[]> {
// Batch into groups of maxBatchSize
const batches = chunk(texts, this.config.maxBatchSize ?? 100);
const allEmbeddings: EmbeddingVector[] = [];
for (const batch of batches) {
const response = await fetch(`${this.config.baseUrl}/embeddings`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${this.config.apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: this.config.model,
input: batch,
dimensions: this.config.dimensions,
}),
});
for (const batch of batches) {
const response = await fetch(`${this.config.baseUrl}/embeddings`, {
method: 'POST',
headers: {
Authorization: `Bearer ${this.config.apiKey}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({
model: this.config.model,
input: batch,
dimensions: this.config.dimensions
})
});
if (!response.ok) {
throw new EmbeddingError(`API error: ${response.status}`);
}
if (!response.ok) {
throw new EmbeddingError(`API error: ${response.status}`);
}
const data = await response.json();
for (const item of data.data) {
allEmbeddings.push({
values: new Float32Array(item.embedding),
dimensions: item.embedding.length,
model: this.config.model,
});
}
}
const data = await response.json();
for (const item of data.data) {
allEmbeddings.push({
values: new Float32Array(item.embedding),
dimensions: item.embedding.length,
model: this.config.model
});
}
}
return allEmbeddings;
}
return allEmbeddings;
}
}
```
@@ -110,41 +110,41 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
```typescript
// Uses @xenova/transformers — only loaded if installed
export class LocalEmbeddingProvider implements EmbeddingProvider {
private pipeline: unknown = null;
private pipeline: unknown = null;
readonly name = 'local';
readonly model = 'Xenova/all-MiniLM-L6-v2'; // 384-dim, fast, small
readonly dimensions = 384;
readonly name = 'local';
readonly model = 'Xenova/all-MiniLM-L6-v2'; // 384-dim, fast, small
readonly dimensions = 384;
async embed(texts: string[]): Promise<EmbeddingVector[]> {
if (!this.pipeline) {
const { pipeline } = await import('@xenova/transformers');
this.pipeline = await pipeline('feature-extraction', this.model);
}
async embed(texts: string[]): Promise<EmbeddingVector[]> {
if (!this.pipeline) {
const { pipeline } = await import('@xenova/transformers');
this.pipeline = await pipeline('feature-extraction', this.model);
}
const results: EmbeddingVector[] = [];
for (const text of texts) {
const output = await (this.pipeline as Function)(text, {
pooling: 'mean',
normalize: true,
});
results.push({
values: new Float32Array(output.data),
dimensions: this.dimensions,
model: this.model,
});
}
return results;
}
const results: EmbeddingVector[] = [];
for (const text of texts) {
const output = await (this.pipeline as Function)(text, {
pooling: 'mean',
normalize: true
});
results.push({
values: new Float32Array(output.data),
dimensions: this.dimensions,
model: this.model
});
}
return results;
}
async isAvailable(): Promise<boolean> {
try {
await import('@xenova/transformers');
return true;
} catch {
return false;
}
}
async isAvailable(): Promise<boolean> {
try {
await import('@xenova/transformers');
return true;
} catch {
return false;
}
}
}
```
@@ -154,53 +154,55 @@ export class LocalEmbeddingProvider implements EmbeddingProvider {
```typescript
export class EmbeddingService {
constructor(
private db: BetterSQLite3.Database,
private provider: EmbeddingProvider
) {}
constructor(
private db: BetterSQLite3.Database,
private provider: EmbeddingProvider
) {}
async embedSnippets(
snippetIds: string[],
onProgress?: (done: number, total: number) => void
): Promise<void> {
const snippets = this.db.prepare(
`SELECT id, content, type FROM snippets WHERE id IN (${snippetIds.map(() => '?').join(',')})`
).all(...snippetIds) as Snippet[];
async embedSnippets(
snippetIds: string[],
onProgress?: (done: number, total: number) => void
): Promise<void> {
const snippets = this.db
.prepare(
`SELECT id, content, type FROM snippets WHERE id IN (${snippetIds.map(() => '?').join(',')})`
)
.all(...snippetIds) as Snippet[];
// Prepare text for embedding: combine title + content
const texts = snippets.map(s =>
[s.title, s.breadcrumb, s.content].filter(Boolean).join('\n').slice(0, 2048)
);
// Prepare text for embedding: combine title + content
const texts = snippets.map((s) =>
[s.title, s.breadcrumb, s.content].filter(Boolean).join('\n').slice(0, 2048)
);
const BATCH_SIZE = 50;
const insert = this.db.prepare(`
const BATCH_SIZE = 50;
const insert = this.db.prepare(`
INSERT OR REPLACE INTO snippet_embeddings (snippet_id, model, dimensions, embedding, created_at)
VALUES (?, ?, ?, ?, unixepoch())
`);
for (let i = 0; i < snippets.length; i += BATCH_SIZE) {
const batch = snippets.slice(i, i + BATCH_SIZE);
const batchTexts = texts.slice(i, i + BATCH_SIZE);
for (let i = 0; i < snippets.length; i += BATCH_SIZE) {
const batch = snippets.slice(i, i + BATCH_SIZE);
const batchTexts = texts.slice(i, i + BATCH_SIZE);
const embeddings = await this.provider.embed(batchTexts);
const embeddings = await this.provider.embed(batchTexts);
const insertMany = this.db.transaction(() => {
for (let j = 0; j < batch.length; j++) {
const snippet = batch[j];
const embedding = embeddings[j];
insert.run(
snippet.id,
embedding.model,
embedding.dimensions,
Buffer.from(embedding.values.buffer)
);
}
});
insertMany();
const insertMany = this.db.transaction(() => {
for (let j = 0; j < batch.length; j++) {
const snippet = batch[j];
const embedding = embeddings[j];
insert.run(
snippet.id,
embedding.model,
embedding.dimensions,
Buffer.from(embedding.values.buffer)
);
}
});
insertMany();
onProgress?.(Math.min(i + BATCH_SIZE, snippets.length), snippets.length);
}
}
onProgress?.(Math.min(i + BATCH_SIZE, snippets.length), snippets.length);
}
}
}
```
@@ -212,13 +214,13 @@ Stored in the `settings` table as JSON:
```typescript
export interface EmbeddingConfig {
provider: 'openai' | 'local' | 'none';
openai?: {
baseUrl: string;
apiKey: string;
model: string;
dimensions?: number;
};
provider: 'openai' | 'local' | 'none';
openai?: {
baseUrl: string;
apiKey: string;
model: string;
dimensions?: number;
};
}
// Settings key: 'embedding_config'
@@ -227,14 +229,15 @@ export interface EmbeddingConfig {
### API Endpoints
`GET /api/v1/settings/embedding`
```json
{
"provider": "openai",
"openai": {
"baseUrl": "https://api.openai.com/v1",
"model": "text-embedding-3-small",
"dimensions": 1536
}
"provider": "openai",
"openai": {
"baseUrl": "https://api.openai.com/v1",
"model": "text-embedding-3-small",
"dimensions": 1536
}
}
```
@@ -251,11 +254,7 @@ Embeddings are stored as raw `Float32Array` binary blobs:
const buffer = Buffer.from(float32Array.buffer);
// Retrieve
const float32Array = new Float32Array(
buffer.buffer,
buffer.byteOffset,
buffer.byteLength / 4
);
const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
```
---

View File

@@ -102,21 +102,21 @@ async vectorSearch(
```typescript
function reciprocalRankFusion(
...rankings: Array<Array<{ id: string; score: number }>>
...rankings: Array<Array<{ id: string; score: number }>>
): Array<{ id: string; rrfScore: number }> {
const K = 60; // RRF constant (standard value)
const scores = new Map<string, number>();
const K = 60; // RRF constant (standard value)
const scores = new Map<string, number>();
for (const ranking of rankings) {
ranking.forEach(({ id }, rank) => {
const current = scores.get(id) ?? 0;
scores.set(id, current + 1 / (K + rank + 1));
});
}
for (const ranking of rankings) {
ranking.forEach(({ id }, rank) => {
const current = scores.get(id) ?? 0;
scores.set(id, current + 1 / (K + rank + 1));
});
}
return Array.from(scores.entries())
.map(([id, rrfScore]) => ({ id, rrfScore }))
.sort((a, b) => b.rrfScore - a.rrfScore);
return Array.from(scores.entries())
.map(([id, rrfScore]) => ({ id, rrfScore }))
.sort((a, b) => b.rrfScore - a.rrfScore);
}
```
@@ -126,65 +126,62 @@ function reciprocalRankFusion(
```typescript
export interface HybridSearchOptions {
repositoryId: string;
versionId?: string;
type?: 'code' | 'info';
limit?: number;
alpha?: number; // 0.0 = FTS5 only, 1.0 = vector only, 0.5 = balanced
repositoryId: string;
versionId?: string;
type?: 'code' | 'info';
limit?: number;
alpha?: number; // 0.0 = FTS5 only, 1.0 = vector only, 0.5 = balanced
}
export class HybridSearchService {
constructor(
private db: BetterSQLite3.Database,
private searchService: SearchService,
private embeddingProvider: EmbeddingProvider | null,
) {}
constructor(
private db: BetterSQLite3.Database,
private searchService: SearchService,
private embeddingProvider: EmbeddingProvider | null
) {}
async search(
query: string,
options: HybridSearchOptions
): Promise<SnippetSearchResult[]> {
const limit = options.limit ?? 20;
const alpha = options.alpha ?? 0.5;
async search(query: string, options: HybridSearchOptions): Promise<SnippetSearchResult[]> {
const limit = options.limit ?? 20;
const alpha = options.alpha ?? 0.5;
// Always run FTS5 search
const ftsResults = this.searchService.searchSnippets(query, {
repositoryId: options.repositoryId,
versionId: options.versionId,
type: options.type,
limit: limit * 3, // get more candidates for fusion
});
// Always run FTS5 search
const ftsResults = this.searchService.searchSnippets(query, {
repositoryId: options.repositoryId,
versionId: options.versionId,
type: options.type,
limit: limit * 3 // get more candidates for fusion
});
// If no embedding provider or alpha = 0, return FTS5 results directly
if (!this.embeddingProvider || alpha === 0) {
return ftsResults.slice(0, limit);
}
// If no embedding provider or alpha = 0, return FTS5 results directly
if (!this.embeddingProvider || alpha === 0) {
return ftsResults.slice(0, limit);
}
// Embed the query and run vector search
const [queryEmbedding] = await this.embeddingProvider.embed([query]);
const vectorResults = await this.vectorSearch(
queryEmbedding.values,
options.repositoryId,
limit * 3
);
// Embed the query and run vector search
const [queryEmbedding] = await this.embeddingProvider.embed([query]);
const vectorResults = await this.vectorSearch(
queryEmbedding.values,
options.repositoryId,
limit * 3
);
// Normalize result lists for RRF
const ftsRanked = ftsResults.map((r, i) => ({
id: r.snippet.id,
score: i,
}));
const vecRanked = vectorResults.map((r, i) => ({
id: r.snippetId,
score: i,
}));
// Normalize result lists for RRF
const ftsRanked = ftsResults.map((r, i) => ({
id: r.snippet.id,
score: i
}));
const vecRanked = vectorResults.map((r, i) => ({
id: r.snippetId,
score: i
}));
// Apply RRF
const fused = reciprocalRankFusion(ftsRanked, vecRanked);
// Apply RRF
const fused = reciprocalRankFusion(ftsRanked, vecRanked);
// Fetch full snippet data for top results
const topIds = fused.slice(0, limit).map(r => r.id);
return this.fetchSnippetsByIds(topIds, options.repositoryId);
}
// Fetch full snippet data for top results
const topIds = fused.slice(0, limit).map((r) => r.id);
return this.fetchSnippetsByIds(topIds, options.repositoryId);
}
}
```
@@ -197,9 +194,9 @@ The hybrid search alpha value can be set per-request or globally via settings:
```typescript
// Default config stored in settings table under key 'search_config'
export interface SearchConfig {
alpha: number; // 0.5 default
maxResults: number; // 20 default
enableHybrid: boolean; // true if embedding provider is configured
alpha: number; // 0.5 default
maxResults: number; // 20 default
enableHybrid: boolean; // true if embedding provider is configured
}
```

View File

@@ -56,75 +56,83 @@ Implement the end-to-end indexing pipeline that orchestrates crawling, parsing,
// src/lib/server/pipeline/job-queue.ts
export class JobQueue {
private isRunning = false;
private isRunning = false;
constructor(private db: BetterSQLite3.Database) {}
constructor(private db: BetterSQLite3.Database) {}
enqueue(repositoryId: string, versionId?: string): IndexingJob {
const job: NewIndexingJob = {
id: crypto.randomUUID(),
repositoryId,
versionId: versionId ?? null,
status: 'queued',
progress: 0,
totalFiles: 0,
processedFiles: 0,
error: null,
startedAt: null,
completedAt: null,
createdAt: new Date(),
};
enqueue(repositoryId: string, versionId?: string): IndexingJob {
const job: NewIndexingJob = {
id: crypto.randomUUID(),
repositoryId,
versionId: versionId ?? null,
status: 'queued',
progress: 0,
totalFiles: 0,
processedFiles: 0,
error: null,
startedAt: null,
completedAt: null,
createdAt: new Date()
};
this.db.prepare(`
this.db
.prepare(
`
INSERT INTO indexing_jobs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`).run(...Object.values(job));
`
)
.run(...Object.values(job));
// Kick off processing if not already running
if (!this.isRunning) {
setImmediate(() => this.processNext());
}
// Kick off processing if not already running
if (!this.isRunning) {
setImmediate(() => this.processNext());
}
return job;
}
return job;
}
private async processNext(): Promise<void> {
if (this.isRunning) return;
private async processNext(): Promise<void> {
if (this.isRunning) return;
const job = this.db.prepare(`
const job = this.db
.prepare(
`
SELECT * FROM indexing_jobs
WHERE status = 'queued'
ORDER BY created_at ASC
LIMIT 1
`).get() as IndexingJob | undefined;
`
)
.get() as IndexingJob | undefined;
if (!job) return;
if (!job) return;
this.isRunning = true;
try {
await this.pipeline.run(job);
} finally {
this.isRunning = false;
// Check for next queued job
const nextJob = this.db.prepare(
`SELECT id FROM indexing_jobs WHERE status = 'queued' LIMIT 1`
).get();
if (nextJob) setImmediate(() => this.processNext());
}
}
this.isRunning = true;
try {
await this.pipeline.run(job);
} finally {
this.isRunning = false;
// Check for next queued job
const nextJob = this.db
.prepare(`SELECT id FROM indexing_jobs WHERE status = 'queued' LIMIT 1`)
.get();
if (nextJob) setImmediate(() => this.processNext());
}
}
getJob(id: string): IndexingJob | null {
return this.db.prepare(
`SELECT * FROM indexing_jobs WHERE id = ?`
).get(id) as IndexingJob | null;
}
getJob(id: string): IndexingJob | null {
return this.db
.prepare(`SELECT * FROM indexing_jobs WHERE id = ?`)
.get(id) as IndexingJob | null;
}
listJobs(repositoryId?: string, limit = 20): IndexingJob[] {
const query = repositoryId
? `SELECT * FROM indexing_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT ?`
: `SELECT * FROM indexing_jobs ORDER BY created_at DESC LIMIT ?`;
const params = repositoryId ? [repositoryId, limit] : [limit];
return this.db.prepare(query).all(...params) as IndexingJob[];
}
listJobs(repositoryId?: string, limit = 20): IndexingJob[] {
const query = repositoryId
? `SELECT * FROM indexing_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT ?`
: `SELECT * FROM indexing_jobs ORDER BY created_at DESC LIMIT ?`;
const params = repositoryId ? [repositoryId, limit] : [limit];
return this.db.prepare(query).all(...params) as IndexingJob[];
}
}
```
@@ -136,94 +144,96 @@ export class JobQueue {
// src/lib/server/pipeline/indexing.pipeline.ts
export class IndexingPipeline {
constructor(
private db: BetterSQLite3.Database,
private githubCrawler: GitHubCrawler,
private localCrawler: LocalCrawler,
private embeddingService: EmbeddingService | null,
) {}
constructor(
private db: BetterSQLite3.Database,
private githubCrawler: GitHubCrawler,
private localCrawler: LocalCrawler,
private embeddingService: EmbeddingService | null
) {}
async run(job: IndexingJob): Promise<void> {
this.updateJob(job.id, { status: 'running', startedAt: new Date() });
async run(job: IndexingJob): Promise<void> {
this.updateJob(job.id, { status: 'running', startedAt: new Date() });
try {
const repo = this.getRepository(job.repositoryId);
if (!repo) throw new Error(`Repository ${job.repositoryId} not found`);
try {
const repo = this.getRepository(job.repositoryId);
if (!repo) throw new Error(`Repository ${job.repositoryId} not found`);
// Update repo state
this.updateRepo(repo.id, { state: 'indexing' });
// Update repo state
this.updateRepo(repo.id, { state: 'indexing' });
// Step 1: Crawl
const crawlResult = await this.crawl(repo, job);
// Step 1: Crawl
const crawlResult = await this.crawl(repo, job);
// Step 2: Parse and diff
const { newSnippets, changedDocIds, newDocuments } =
await this.parseAndDiff(crawlResult, repo, job);
// Step 2: Parse and diff
const { newSnippets, changedDocIds, newDocuments } = await this.parseAndDiff(
crawlResult,
repo,
job
);
// Step 3: Atomic replacement
this.replaceSnippets(repo.id, changedDocIds, newDocuments, newSnippets);
// Step 3: Atomic replacement
this.replaceSnippets(repo.id, changedDocIds, newDocuments, newSnippets);
// Step 4: Embeddings (async, non-blocking for job completion)
if (this.embeddingService && newSnippets.length > 0) {
await this.embeddingService.embedSnippets(
newSnippets.map(s => s.id),
(done, total) => {
// Update job progress for embedding phase
}
);
}
// Step 4: Embeddings (async, non-blocking for job completion)
if (this.embeddingService && newSnippets.length > 0) {
await this.embeddingService.embedSnippets(
newSnippets.map((s) => s.id),
(done, total) => {
// Update job progress for embedding phase
}
);
}
// Step 5: Update repo stats
const stats = this.computeStats(repo.id);
this.updateRepo(repo.id, {
state: 'indexed',
totalSnippets: stats.totalSnippets,
totalTokens: stats.totalTokens,
trustScore: computeTrustScore({ ...repo, ...stats }),
lastIndexedAt: new Date(),
});
// Step 5: Update repo stats
const stats = this.computeStats(repo.id);
this.updateRepo(repo.id, {
state: 'indexed',
totalSnippets: stats.totalSnippets,
totalTokens: stats.totalTokens,
trustScore: computeTrustScore({ ...repo, ...stats }),
lastIndexedAt: new Date()
});
this.updateJob(job.id, {
status: 'done',
progress: 100,
completedAt: new Date(),
});
this.updateJob(job.id, {
status: 'done',
progress: 100,
completedAt: new Date()
});
} catch (error) {
this.updateJob(job.id, {
status: 'failed',
error: (error as Error).message,
completedAt: new Date()
});
this.updateRepo(job.repositoryId, { state: 'error' });
throw error;
}
}
} catch (error) {
this.updateJob(job.id, {
status: 'failed',
error: (error as Error).message,
completedAt: new Date(),
});
this.updateRepo(job.repositoryId, { state: 'error' });
throw error;
}
}
private replaceSnippets(
repositoryId: string,
changedDocIds: string[],
newDocuments: NewDocument[],
newSnippets: NewSnippet[]
): void {
// Single transaction: delete old → insert new
this.db.transaction(() => {
if (changedDocIds.length > 0) {
// Cascade deletes snippets via FK constraint
this.db
.prepare(`DELETE FROM documents WHERE id IN (${changedDocIds.map(() => '?').join(',')})`)
.run(...changedDocIds);
}
private replaceSnippets(
repositoryId: string,
changedDocIds: string[],
newDocuments: NewDocument[],
newSnippets: NewSnippet[]
): void {
// Single transaction: delete old → insert new
this.db.transaction(() => {
if (changedDocIds.length > 0) {
// Cascade deletes snippets via FK constraint
this.db.prepare(
`DELETE FROM documents WHERE id IN (${changedDocIds.map(() => '?').join(',')})`
).run(...changedDocIds);
}
for (const doc of newDocuments) {
this.insertDocument(doc);
}
for (const doc of newDocuments) {
this.insertDocument(doc);
}
for (const snippet of newSnippets) {
this.insertSnippet(snippet);
}
})();
}
for (const snippet of newSnippets) {
this.insertSnippet(snippet);
}
})();
}
}
```
@@ -233,26 +243,24 @@ export class IndexingPipeline {
```typescript
function calculateProgress(
processedFiles: number,
totalFiles: number,
embeddingsDone: number,
embeddingsTotal: number,
hasEmbeddings: boolean
processedFiles: number,
totalFiles: number,
embeddingsDone: number,
embeddingsTotal: number,
hasEmbeddings: boolean
): number {
if (totalFiles === 0) return 0;
if (totalFiles === 0) return 0;
if (!hasEmbeddings) {
// Crawl + parse = 100%
return Math.round((processedFiles / totalFiles) * 100);
}
if (!hasEmbeddings) {
// Crawl + parse = 100%
return Math.round((processedFiles / totalFiles) * 100);
}
// Crawl+parse = 80%, embeddings = 20%
const parseProgress = (processedFiles / totalFiles) * 80;
const embedProgress = embeddingsTotal > 0
? (embeddingsDone / embeddingsTotal) * 20
: 0;
// Crawl+parse = 80%, embeddings = 20%
const parseProgress = (processedFiles / totalFiles) * 80;
const embedProgress = embeddingsTotal > 0 ? (embeddingsDone / embeddingsTotal) * 20 : 0;
return Math.round(parseProgress + embedProgress);
return Math.round(parseProgress + embedProgress);
}
```
@@ -263,20 +271,21 @@ function calculateProgress(
### `GET /api/v1/jobs/:id`
Response `200`:
```json
{
"job": {
"id": "uuid",
"repositoryId": "/facebook/react",
"status": "running",
"progress": 47,
"totalFiles": 342,
"processedFiles": 162,
"error": null,
"startedAt": "2026-03-22T10:00:00Z",
"completedAt": null,
"createdAt": "2026-03-22T09:59:55Z"
}
"job": {
"id": "uuid",
"repositoryId": "/facebook/react",
"status": "running",
"progress": 47,
"totalFiles": 342,
"processedFiles": 162,
"error": null,
"startedAt": "2026-03-22T10:00:00Z",
"completedAt": null,
"createdAt": "2026-03-22T09:59:55Z"
}
}
```
@@ -285,6 +294,7 @@ Response `200`:
Query params: `repositoryId` (optional), `status` (optional), `limit` (default 20).
Response `200`:
```json
{
"jobs": [...],
@@ -300,20 +310,24 @@ On application start, mark any jobs in `running` state as `failed` (they were in
```typescript
function recoverStaleJobs(db: BetterSQLite3.Database): void {
db.prepare(`
db.prepare(
`
UPDATE indexing_jobs
SET status = 'failed',
error = 'Server restarted while job was running',
completed_at = unixepoch()
WHERE status = 'running'
`).run();
`
).run();
// Also reset any repositories stuck in 'indexing' state
db.prepare(`
// Also reset any repositories stuck in 'indexing' state
db.prepare(
`
UPDATE repositories
SET state = 'error'
WHERE state = 'indexing'
`).run();
`
).run();
}
```

View File

@@ -32,33 +32,33 @@ Implement the public-facing REST API endpoints that replicate context7's `/api/v
### Query Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `libraryName` | string | Yes | Library name to search for |
| `query` | string | No | User's question for relevance ranking |
| `limit` | integer | No | Max results (default: 10, max: 50) |
| Parameter | Type | Required | Description |
| ------------- | ------- | -------- | ------------------------------------- |
| `libraryName` | string | Yes | Library name to search for |
| `query` | string | No | User's question for relevance ranking |
| `limit` | integer | No | Max results (default: 10, max: 50) |
### Response `200` (`type=json`, default):
```json
{
"results": [
{
"id": "/facebook/react",
"title": "React",
"description": "A JavaScript library for building user interfaces",
"branch": "main",
"lastUpdateDate": "2026-03-22T10:00:00Z",
"state": "finalized",
"totalTokens": 142000,
"totalSnippets": 1247,
"stars": 228000,
"trustScore": 9.2,
"benchmarkScore": 87,
"versions": ["v18.3.0", "v17.0.2"],
"source": "https://github.com/facebook/react"
}
]
"results": [
{
"id": "/facebook/react",
"title": "React",
"description": "A JavaScript library for building user interfaces",
"branch": "main",
"lastUpdateDate": "2026-03-22T10:00:00Z",
"state": "finalized",
"totalTokens": 142000,
"totalSnippets": 1247,
"stars": 228000,
"trustScore": 9.2,
"benchmarkScore": 87,
"versions": ["v18.3.0", "v17.0.2"],
"source": "https://github.com/facebook/react"
}
]
}
```
@@ -67,11 +67,11 @@ Note: `state: "finalized"` maps from TrueRef's `state: "indexed"` for compatibil
### State Mapping
| TrueRef state | context7 state |
|---------------|---------------|
| `pending` | `initial` |
| `indexing` | `initial` |
| `indexed` | `finalized` |
| `error` | `error` |
| ------------- | -------------- |
| `pending` | `initial` |
| `indexing` | `initial` |
| `indexed` | `finalized` |
| `error` | `error` |
---
@@ -81,43 +81,43 @@ Note: `state: "finalized"` maps from TrueRef's `state: "indexed"` for compatibil
### Query Parameters
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `libraryId` | string | Yes | Library ID, e.g. `/facebook/react` or `/facebook/react/v18.3.0` |
| `query` | string | Yes | Specific question about the library |
| `type` | string | No | `json` (default) or `txt` (plain text for LLM injection) |
| `tokens` | integer | No | Approximate max token count for response (default: 10000) |
| Parameter | Type | Required | Description |
| ----------- | ------- | -------- | --------------------------------------------------------------- |
| `libraryId` | string | Yes | Library ID, e.g. `/facebook/react` or `/facebook/react/v18.3.0` |
| `query` | string | Yes | Specific question about the library |
| `type` | string | No | `json` (default) or `txt` (plain text for LLM injection) |
| `tokens` | integer | No | Approximate max token count for response (default: 10000) |
### Response `200` (`type=json`):
```json
{
"snippets": [
{
"type": "code",
"title": "Basic Component",
"description": "Getting Started > Components",
"language": "tsx",
"codeList": [
{
"language": "tsx",
"code": "function MyComponent() {\n return <div>Hello</div>;\n}"
}
],
"id": "uuid",
"tokenCount": 45,
"pageTitle": "Getting Started"
},
{
"type": "info",
"text": "React components let you split the UI into independent...",
"breadcrumb": "Core Concepts > Components",
"pageId": "uuid",
"tokenCount": 120
}
],
"rules": ["Always use functional components", "..."],
"totalTokens": 2840
"snippets": [
{
"type": "code",
"title": "Basic Component",
"description": "Getting Started > Components",
"language": "tsx",
"codeList": [
{
"language": "tsx",
"code": "function MyComponent() {\n return <div>Hello</div>;\n}"
}
],
"id": "uuid",
"tokenCount": 45,
"pageTitle": "Getting Started"
},
{
"type": "info",
"text": "React components let you split the UI into independent...",
"breadcrumb": "Core Concepts > Components",
"pageId": "uuid",
"tokenCount": 120
}
],
"rules": ["Always use functional components", "..."],
"totalTokens": 2840
}
```
@@ -125,7 +125,7 @@ Note: `state: "finalized"` maps from TrueRef's `state: "indexed"` for compatibil
Plain text formatted for direct LLM context injection:
```
````
## Library Rules
- Always use functional components
- Use hooks for state management
@@ -139,15 +139,17 @@ Plain text formatted for direct LLM context injection:
function MyComponent() {
return <div>Hello</div>;
}
```
````
---
### React components let you split the UI...
*Core Concepts > Components*
_Core Concepts > Components_
React components let you split the UI into independent, reusable pieces...
```
````
---
@@ -167,7 +169,7 @@ function parseLibraryId(libraryId: string): {
version: match[3],
};
}
```
````
---
@@ -176,20 +178,17 @@ function parseLibraryId(libraryId: string): {
The `tokens` parameter limits the total response size. Snippets are added greedily until the budget is exhausted:
```typescript
function selectSnippetsWithinBudget(
snippets: Snippet[],
maxTokens: number
): Snippet[] {
const selected: Snippet[] = [];
let usedTokens = 0;
function selectSnippetsWithinBudget(snippets: Snippet[], maxTokens: number): Snippet[] {
const selected: Snippet[] = [];
let usedTokens = 0;
for (const snippet of snippets) {
if (usedTokens + (snippet.tokenCount ?? 0) > maxTokens) break;
selected.push(snippet);
usedTokens += snippet.tokenCount ?? 0;
}
for (const snippet of snippets) {
if (usedTokens + (snippet.tokenCount ?? 0) > maxTokens) break;
selected.push(snippet);
usedTokens += snippet.tokenCount ?? 0;
}
return selected;
return selected;
}
```
@@ -215,6 +214,7 @@ Default token budget: 10,000 tokens (~7,500 words) — enough for ~20 medium sni
## CORS Configuration
All API routes include:
```
Access-Control-Allow-Origin: *
Access-Control-Allow-Methods: GET, POST, PATCH, DELETE, OPTIONS

View File

@@ -32,8 +32,8 @@ Implement a Model Context Protocol (MCP) server that exposes `resolve-library-id
```json
{
"@modelcontextprotocol/sdk": "^1.25.1",
"zod": "^4.3.4"
"@modelcontextprotocol/sdk": "^1.25.1",
"zod": "^4.3.4"
}
```
@@ -46,189 +46,190 @@ Implement a Model Context Protocol (MCP) server that exposes `resolve-library-id
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import {
CallToolRequestSchema,
ListToolsRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
import { z } from 'zod';
const API_BASE = process.env.TRUEREF_API_URL ?? 'http://localhost:5173';
const server = new Server(
{
name: 'io.github.trueref/trueref',
version: '1.0.0',
},
{
capabilities: { tools: {} },
}
{
name: 'io.github.trueref/trueref',
version: '1.0.0'
},
{
capabilities: { tools: {} }
}
);
// Tool schemas — identical to context7 for drop-in compatibility
const ResolveLibraryIdSchema = z.object({
libraryName: z.string().describe(
'Library name to search for and resolve to a TrueRef library ID'
),
query: z.string().describe(
"The user's question or context to help rank results"
),
libraryName: z
.string()
.describe('Library name to search for and resolve to a TrueRef library ID'),
query: z.string().describe("The user's question or context to help rank results")
});
const QueryDocsSchema = z.object({
libraryId: z.string().describe(
'The TrueRef library ID obtained from resolve-library-id, e.g. /facebook/react'
),
query: z.string().describe(
'Specific question about the library to retrieve relevant documentation'
),
tokens: z.number().optional().describe(
'Maximum token budget for the response (default: 10000)'
),
libraryId: z
.string()
.describe('The TrueRef library ID obtained from resolve-library-id, e.g. /facebook/react'),
query: z
.string()
.describe('Specific question about the library to retrieve relevant documentation'),
tokens: z.number().optional().describe('Maximum token budget for the response (default: 10000)')
});
server.setRequestHandler(ListToolsRequestSchema, async () => ({
tools: [
{
name: 'resolve-library-id',
description: [
'Searches TrueRef to find a library matching the given name.',
'Returns a list of matching libraries with their IDs.',
'ALWAYS call this tool before query-docs to get the correct library ID.',
'Call at most 3 times per user question.',
].join(' '),
inputSchema: {
type: 'object',
properties: {
libraryName: {
type: 'string',
description: 'Library name to search for',
},
query: {
type: 'string',
description: "User's question for relevance ranking",
},
},
required: ['libraryName', 'query'],
},
},
{
name: 'query-docs',
description: [
'Fetches documentation and code examples from TrueRef for a specific library.',
'Requires a library ID obtained from resolve-library-id.',
'Returns relevant snippets formatted for LLM consumption.',
'Call at most 3 times per user question.',
].join(' '),
inputSchema: {
type: 'object',
properties: {
libraryId: {
type: 'string',
description: 'TrueRef library ID, e.g. /facebook/react',
},
query: {
type: 'string',
description: 'Specific question about the library',
},
tokens: {
type: 'number',
description: 'Max token budget (default: 10000)',
},
},
required: ['libraryId', 'query'],
},
},
],
tools: [
{
name: 'resolve-library-id',
description: [
'Searches TrueRef to find a library matching the given name.',
'Returns a list of matching libraries with their IDs.',
'ALWAYS call this tool before query-docs to get the correct library ID.',
'Call at most 3 times per user question.'
].join(' '),
inputSchema: {
type: 'object',
properties: {
libraryName: {
type: 'string',
description: 'Library name to search for'
},
query: {
type: 'string',
description: "User's question for relevance ranking"
}
},
required: ['libraryName', 'query']
}
},
{
name: 'query-docs',
description: [
'Fetches documentation and code examples from TrueRef for a specific library.',
'Requires a library ID obtained from resolve-library-id.',
'Returns relevant snippets formatted for LLM consumption.',
'Call at most 3 times per user question.'
].join(' '),
inputSchema: {
type: 'object',
properties: {
libraryId: {
type: 'string',
description: 'TrueRef library ID, e.g. /facebook/react'
},
query: {
type: 'string',
description: 'Specific question about the library'
},
tokens: {
type: 'number',
description: 'Max token budget (default: 10000)'
}
},
required: ['libraryId', 'query']
}
}
]
}));
server.setRequestHandler(CallToolRequestSchema, async (request) => {
const { name, arguments: args } = request.params;
const { name, arguments: args } = request.params;
if (name === 'resolve-library-id') {
const { libraryName, query } = ResolveLibraryIdSchema.parse(args);
if (name === 'resolve-library-id') {
const { libraryName, query } = ResolveLibraryIdSchema.parse(args);
const url = new URL(`${API_BASE}/api/v1/libs/search`);
url.searchParams.set('libraryName', libraryName);
url.searchParams.set('query', query);
url.searchParams.set('type', 'txt');
const url = new URL(`${API_BASE}/api/v1/libs/search`);
url.searchParams.set('libraryName', libraryName);
url.searchParams.set('query', query);
url.searchParams.set('type', 'txt');
const response = await fetch(url.toString());
if (!response.ok) {
return {
content: [{
type: 'text',
text: `Error searching libraries: ${response.status} ${response.statusText}`,
}],
isError: true,
};
}
const response = await fetch(url.toString());
if (!response.ok) {
return {
content: [
{
type: 'text',
text: `Error searching libraries: ${response.status} ${response.statusText}`
}
],
isError: true
};
}
const text = await response.text();
return {
content: [{ type: 'text', text }],
};
}
const text = await response.text();
return {
content: [{ type: 'text', text }]
};
}
if (name === 'query-docs') {
const { libraryId, query, tokens } = QueryDocsSchema.parse(args);
if (name === 'query-docs') {
const { libraryId, query, tokens } = QueryDocsSchema.parse(args);
const url = new URL(`${API_BASE}/api/v1/context`);
url.searchParams.set('libraryId', libraryId);
url.searchParams.set('query', query);
url.searchParams.set('type', 'txt');
if (tokens) url.searchParams.set('tokens', String(tokens));
const url = new URL(`${API_BASE}/api/v1/context`);
url.searchParams.set('libraryId', libraryId);
url.searchParams.set('query', query);
url.searchParams.set('type', 'txt');
if (tokens) url.searchParams.set('tokens', String(tokens));
const response = await fetch(url.toString());
if (!response.ok) {
const status = response.status;
if (status === 404) {
return {
content: [{
type: 'text',
text: `Library "${libraryId}" not found. Please run resolve-library-id first.`,
}],
isError: true,
};
}
if (status === 503) {
return {
content: [{
type: 'text',
text: `Library "${libraryId}" is currently being indexed. Please try again in a moment.`,
}],
isError: true,
};
}
return {
content: [{
type: 'text',
text: `Error fetching documentation: ${response.status} ${response.statusText}`,
}],
isError: true,
};
}
const response = await fetch(url.toString());
if (!response.ok) {
const status = response.status;
if (status === 404) {
return {
content: [
{
type: 'text',
text: `Library "${libraryId}" not found. Please run resolve-library-id first.`
}
],
isError: true
};
}
if (status === 503) {
return {
content: [
{
type: 'text',
text: `Library "${libraryId}" is currently being indexed. Please try again in a moment.`
}
],
isError: true
};
}
return {
content: [
{
type: 'text',
text: `Error fetching documentation: ${response.status} ${response.statusText}`
}
],
isError: true
};
}
const text = await response.text();
return {
content: [{ type: 'text', text }],
};
}
const text = await response.text();
return {
content: [{ type: 'text', text }]
};
}
return {
content: [{ type: 'text', text: `Unknown tool: ${name}` }],
isError: true,
};
return {
content: [{ type: 'text', text: `Unknown tool: ${name}` }],
isError: true
};
});
async function main() {
const transport = new StdioServerTransport();
await server.connect(transport);
// Server runs until process exits
const transport = new StdioServerTransport();
await server.connect(transport);
// Server runs until process exits
}
main().catch((err) => {
process.stderr.write(`MCP server error: ${err.message}\n`);
process.exit(1);
process.stderr.write(`MCP server error: ${err.message}\n`);
process.exit(1);
});
```
@@ -238,18 +239,19 @@ main().catch((err) => {
```json
{
"scripts": {
"mcp:start": "node --experimental-vm-modules src/mcp/index.ts"
}
"scripts": {
"mcp:start": "node --experimental-vm-modules src/mcp/index.ts"
}
}
```
Or with `tsx` for TypeScript-direct execution:
```json
{
"scripts": {
"mcp:start": "tsx src/mcp/index.ts"
}
"scripts": {
"mcp:start": "tsx src/mcp/index.ts"
}
}
```
@@ -261,30 +263,31 @@ Users add to `.mcp.json`:
```json
{
"mcpServers": {
"trueref": {
"command": "node",
"args": ["/path/to/trueref/dist/mcp/index.js"],
"env": {
"TRUEREF_API_URL": "http://localhost:5173"
}
}
}
"mcpServers": {
"trueref": {
"command": "node",
"args": ["/path/to/trueref/dist/mcp/index.js"],
"env": {
"TRUEREF_API_URL": "http://localhost:5173"
}
}
}
}
```
Or with tsx for development:
```json
{
"mcpServers": {
"trueref": {
"command": "npx",
"args": ["tsx", "/path/to/trueref/src/mcp/index.ts"],
"env": {
"TRUEREF_API_URL": "http://localhost:5173"
}
}
}
"mcpServers": {
"trueref": {
"command": "npx",
"args": ["tsx", "/path/to/trueref/src/mcp/index.ts"],
"env": {
"TRUEREF_API_URL": "http://localhost:5173"
}
}
}
}
```
@@ -295,13 +298,15 @@ Or with tsx for development:
The MCP server should include a `resources` list item (optional) or the library responses themselves prepend rules. Additionally, users should add a Claude rule file:
```markdown
<!-- .claude/rules/trueref.md -->
---
## <!-- .claude/rules/trueref.md -->
description: Use TrueRef to retrieve documentation for indexed libraries
alwaysApply: true
---
When answering questions about indexed libraries, always use the TrueRef MCP tools:
1. Call `resolve-library-id` with the library name and the user's question to get the library ID
2. Call `query-docs` with the library ID and question to retrieve relevant documentation
3. Use the returned documentation to answer the question accurately

View File

@@ -50,64 +50,64 @@ import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
const { values: args } = parseArgs({
options: {
transport: { type: 'string', default: 'stdio' },
port: { type: 'string', default: process.env.PORT ?? '3001' },
},
options: {
transport: { type: 'string', default: 'stdio' },
port: { type: 'string', default: process.env.PORT ?? '3001' }
}
});
async function startHttp(server: Server, port: number): Promise<void> {
const httpServer = createServer(async (req, res) => {
const url = new URL(req.url!, `http://localhost:${port}`);
const httpServer = createServer(async (req, res) => {
const url = new URL(req.url!, `http://localhost:${port}`);
// Health check
if (url.pathname === '/ping') {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: true }));
return;
}
// Health check
if (url.pathname === '/ping') {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ ok: true }));
return;
}
// MCP endpoint
if (url.pathname === '/mcp') {
// CORS preflight
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'POST, GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Accept');
// MCP endpoint
if (url.pathname === '/mcp') {
// CORS preflight
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'POST, GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Accept');
if (req.method === 'OPTIONS') {
res.writeHead(204);
res.end();
return;
}
if (req.method === 'OPTIONS') {
res.writeHead(204);
res.end();
return;
}
const transport = new StreamableHTTPServerTransport({
sessionIdGenerator: () => crypto.randomUUID(),
});
const transport = new StreamableHTTPServerTransport({
sessionIdGenerator: () => crypto.randomUUID()
});
await server.connect(transport);
await transport.handleRequest(req, res);
return;
}
await server.connect(transport);
await transport.handleRequest(req, res);
return;
}
res.writeHead(404);
res.end('Not Found');
});
res.writeHead(404);
res.end('Not Found');
});
httpServer.listen(port, () => {
process.stderr.write(`TrueRef MCP server listening on http://localhost:${port}/mcp\n`);
});
httpServer.listen(port, () => {
process.stderr.write(`TrueRef MCP server listening on http://localhost:${port}/mcp\n`);
});
}
async function main() {
const mcpServer = createMcpServer(); // shared server creation
const mcpServer = createMcpServer(); // shared server creation
if (args.transport === 'http') {
const port = parseInt(args.port!, 10);
await startHttp(mcpServer, port);
} else {
const transport = new StdioServerTransport();
await mcpServer.connect(transport);
}
if (args.transport === 'http') {
const port = parseInt(args.port!, 10);
await startHttp(mcpServer, port);
} else {
const transport = new StdioServerTransport();
await mcpServer.connect(transport);
}
}
```
@@ -117,10 +117,10 @@ async function main() {
```json
{
"scripts": {
"mcp:start": "tsx src/mcp/index.ts",
"mcp:http": "tsx src/mcp/index.ts --transport http --port 3001"
}
"scripts": {
"mcp:start": "tsx src/mcp/index.ts",
"mcp:http": "tsx src/mcp/index.ts --transport http --port 3001"
}
}
```
@@ -132,12 +132,12 @@ For HTTP transport, users configure Claude Code with the remote URL:
```json
{
"mcpServers": {
"trueref": {
"type": "http",
"url": "http://localhost:3001/mcp"
}
}
"mcpServers": {
"trueref": {
"type": "http",
"url": "http://localhost:3001/mcp"
}
}
}
```

View File

@@ -32,53 +32,53 @@ Support `trueref.json` configuration files placed in the root of a repository. T
// src/lib/server/config/trueref-config.schema.ts
export interface TrueRefConfig {
/**
* Override the display name for this library.
* 1100 characters.
*/
projectTitle?: string;
/**
* Override the display name for this library.
* 1100 characters.
*/
projectTitle?: string;
/**
* Description of the library for search ranking.
* 10500 characters.
*/
description?: string;
/**
* Description of the library for search ranking.
* 10500 characters.
*/
description?: string;
/**
* Folders to include in indexing (allowlist).
* Each entry is a path prefix or regex string.
* If empty/absent, all folders are included.
* Examples: ["src/", "docs/", "^packages/core"]
*/
folders?: string[];
/**
* Folders to include in indexing (allowlist).
* Each entry is a path prefix or regex string.
* If empty/absent, all folders are included.
* Examples: ["src/", "docs/", "^packages/core"]
*/
folders?: string[];
/**
* Folders to exclude from indexing.
* Applied after `folders` allowlist.
* Examples: ["test/", "fixtures/", "__mocks__"]
*/
excludeFolders?: string[];
/**
* Folders to exclude from indexing.
* Applied after `folders` allowlist.
* Examples: ["test/", "fixtures/", "__mocks__"]
*/
excludeFolders?: string[];
/**
* Exact filenames to exclude (no path, no regex).
* Examples: ["README.md", "CHANGELOG.md", "jest.config.ts"]
*/
excludeFiles?: string[];
/**
* Exact filenames to exclude (no path, no regex).
* Examples: ["README.md", "CHANGELOG.md", "jest.config.ts"]
*/
excludeFiles?: string[];
/**
* Best practices / rules to inject at the top of every query-docs response.
* Each rule: 5500 characters.
* Maximum 20 rules.
*/
rules?: string[];
/**
* Best practices / rules to inject at the top of every query-docs response.
* Each rule: 5500 characters.
* Maximum 20 rules.
*/
rules?: string[];
/**
* Previously released versions to make available for versioned queries.
*/
previousVersions?: Array<{
tag: string; // git tag (e.g. "v1.2.3")
title: string; // human-readable (e.g. "Version 1.2.3")
}>;
/**
* Previously released versions to make available for versioned queries.
*/
previousVersions?: Array<{
tag: string; // git tag (e.g. "v1.2.3")
title: string; // human-readable (e.g. "Version 1.2.3")
}>;
}
```
@@ -88,14 +88,14 @@ export interface TrueRefConfig {
```typescript
const CONFIG_CONSTRAINTS = {
projectTitle: { minLength: 1, maxLength: 100 },
description: { minLength: 10, maxLength: 500 },
folders: { maxItems: 50, maxLength: 200 }, // per entry
excludeFolders: { maxItems: 50, maxLength: 200 },
excludeFiles: { maxItems: 100, maxLength: 200 },
rules: { maxItems: 20, minLength: 5, maxLength: 500 },
previousVersions: { maxItems: 50 },
versionTag: { pattern: /^v?\d+\.\d+(\.\d+)?(-.*)?$/ },
projectTitle: { minLength: 1, maxLength: 100 },
description: { minLength: 10, maxLength: 500 },
folders: { maxItems: 50, maxLength: 200 }, // per entry
excludeFolders: { maxItems: 50, maxLength: 200 },
excludeFiles: { maxItems: 100, maxLength: 200 },
rules: { maxItems: 20, minLength: 5, maxLength: 500 },
previousVersions: { maxItems: 50 },
versionTag: { pattern: /^v?\d+\.\d+(\.\d+)?(-.*)?$/ }
};
```
@@ -107,94 +107,96 @@ const CONFIG_CONSTRAINTS = {
// src/lib/server/config/config-parser.ts
export interface ParsedConfig {
config: TrueRefConfig;
source: 'trueref.json' | 'context7.json';
warnings: string[];
config: TrueRefConfig;
source: 'trueref.json' | 'context7.json';
warnings: string[];
}
export function parseConfigFile(content: string, filename: string): ParsedConfig {
let raw: unknown;
let raw: unknown;
try {
raw = JSON.parse(content);
} catch (e) {
throw new ConfigParseError(`${filename} is not valid JSON: ${(e as Error).message}`);
}
try {
raw = JSON.parse(content);
} catch (e) {
throw new ConfigParseError(`${filename} is not valid JSON: ${(e as Error).message}`);
}
if (typeof raw !== 'object' || raw === null) {
throw new ConfigParseError(`${filename} must be a JSON object`);
}
if (typeof raw !== 'object' || raw === null) {
throw new ConfigParseError(`${filename} must be a JSON object`);
}
const config = raw as Record<string, unknown>;
const validated: TrueRefConfig = {};
const warnings: string[] = [];
const config = raw as Record<string, unknown>;
const validated: TrueRefConfig = {};
const warnings: string[] = [];
// projectTitle
if (config.projectTitle !== undefined) {
if (typeof config.projectTitle !== 'string') {
warnings.push('projectTitle must be a string, ignoring');
} else if (config.projectTitle.length > 100) {
validated.projectTitle = config.projectTitle.slice(0, 100);
warnings.push('projectTitle truncated to 100 characters');
} else {
validated.projectTitle = config.projectTitle;
}
}
// projectTitle
if (config.projectTitle !== undefined) {
if (typeof config.projectTitle !== 'string') {
warnings.push('projectTitle must be a string, ignoring');
} else if (config.projectTitle.length > 100) {
validated.projectTitle = config.projectTitle.slice(0, 100);
warnings.push('projectTitle truncated to 100 characters');
} else {
validated.projectTitle = config.projectTitle;
}
}
// description
if (config.description !== undefined) {
if (typeof config.description === 'string') {
validated.description = config.description.slice(0, 500);
}
}
// description
if (config.description !== undefined) {
if (typeof config.description === 'string') {
validated.description = config.description.slice(0, 500);
}
}
// folders / excludeFolders / excludeFiles — validated as string arrays
for (const field of ['folders', 'excludeFolders', 'excludeFiles'] as const) {
if (config[field] !== undefined) {
if (!Array.isArray(config[field])) {
warnings.push(`${field} must be an array, ignoring`);
} else {
validated[field] = (config[field] as unknown[])
.filter((item): item is string => {
if (typeof item !== 'string') {
warnings.push(`${field} entry must be a string, skipping: ${item}`);
return false;
}
return true;
})
.slice(0, field === 'excludeFiles' ? 100 : 50);
}
}
}
// folders / excludeFolders / excludeFiles — validated as string arrays
for (const field of ['folders', 'excludeFolders', 'excludeFiles'] as const) {
if (config[field] !== undefined) {
if (!Array.isArray(config[field])) {
warnings.push(`${field} must be an array, ignoring`);
} else {
validated[field] = (config[field] as unknown[])
.filter((item): item is string => {
if (typeof item !== 'string') {
warnings.push(`${field} entry must be a string, skipping: ${item}`);
return false;
}
return true;
})
.slice(0, field === 'excludeFiles' ? 100 : 50);
}
}
}
// rules
if (config.rules !== undefined) {
if (Array.isArray(config.rules)) {
validated.rules = (config.rules as unknown[])
.filter((r): r is string => typeof r === 'string' && r.length >= 5)
.map(r => r.slice(0, 500))
.slice(0, 20);
}
}
// rules
if (config.rules !== undefined) {
if (Array.isArray(config.rules)) {
validated.rules = (config.rules as unknown[])
.filter((r): r is string => typeof r === 'string' && r.length >= 5)
.map((r) => r.slice(0, 500))
.slice(0, 20);
}
}
// previousVersions
if (config.previousVersions !== undefined) {
if (Array.isArray(config.previousVersions)) {
validated.previousVersions = (config.previousVersions as unknown[])
.filter((v): v is { tag: string; title: string } =>
typeof v === 'object' && v !== null &&
typeof (v as Record<string, unknown>).tag === 'string' &&
typeof (v as Record<string, unknown>).title === 'string'
)
.slice(0, 50);
}
}
// previousVersions
if (config.previousVersions !== undefined) {
if (Array.isArray(config.previousVersions)) {
validated.previousVersions = (config.previousVersions as unknown[])
.filter(
(v): v is { tag: string; title: string } =>
typeof v === 'object' &&
v !== null &&
typeof (v as Record<string, unknown>).tag === 'string' &&
typeof (v as Record<string, unknown>).title === 'string'
)
.slice(0, 50);
}
}
return {
config: validated,
source: filename.startsWith('trueref') ? 'trueref.json' : 'context7.json',
warnings,
};
return {
config: validated,
source: filename.startsWith('trueref') ? 'trueref.json' : 'context7.json',
warnings
};
}
```
@@ -219,21 +221,15 @@ When `query-docs` returns results, `rules` from `repository_configs` are prepend
```typescript
// In formatters.ts
function buildContextResponse(
snippets: Snippet[],
config: RepositoryConfig | null
): string {
const parts: string[] = [];
function buildContextResponse(snippets: Snippet[], config: RepositoryConfig | null): string {
const parts: string[] = [];
if (config?.rules?.length) {
parts.push(
'## Library Best Practices\n' +
config.rules.map(r => `- ${r}`).join('\n')
);
}
if (config?.rules?.length) {
parts.push('## Library Best Practices\n' + config.rules.map((r) => `- ${r}`).join('\n'));
}
// ... append snippet content
return parts.join('\n\n---\n\n');
// ... append snippet content
return parts.join('\n\n---\n\n');
}
```

View File

@@ -45,34 +45,37 @@ Examples:
### `GET /api/v1/libs/:id/versions`
Response `200`:
```json
{
"versions": [
{
"id": "/facebook/react/v18.3.0",
"repositoryId": "/facebook/react",
"tag": "v18.3.0",
"title": "React v18.3.0",
"state": "indexed",
"totalSnippets": 892,
"indexedAt": "2026-03-22T10:00:00Z"
}
]
"versions": [
{
"id": "/facebook/react/v18.3.0",
"repositoryId": "/facebook/react",
"tag": "v18.3.0",
"title": "React v18.3.0",
"state": "indexed",
"totalSnippets": 892,
"indexedAt": "2026-03-22T10:00:00Z"
}
]
}
```
### `POST /api/v1/libs/:id/versions`
Request body:
```json
{
"tag": "v18.3.0",
"title": "React v18.3.0",
"autoIndex": true
"tag": "v18.3.0",
"title": "React v18.3.0",
"autoIndex": true
}
```
Response `201`:
```json
{
"version": { ...RepositoryVersion },
@@ -96,23 +99,22 @@ Response `202` with job details.
```typescript
async function listGitHubTags(
owner: string,
repo: string,
token?: string
owner: string,
repo: string,
token?: string
): Promise<Array<{ name: string; commit: { sha: string } }>> {
const headers: Record<string, string> = {
'Accept': 'application/vnd.github.v3+json',
'User-Agent': 'TrueRef/1.0',
};
if (token) headers['Authorization'] = `Bearer ${token}`;
const headers: Record<string, string> = {
Accept: 'application/vnd.github.v3+json',
'User-Agent': 'TrueRef/1.0'
};
if (token) headers['Authorization'] = `Bearer ${token}`;
const response = await fetch(
`https://api.github.com/repos/${owner}/${repo}/tags?per_page=100`,
{ headers }
);
const response = await fetch(`https://api.github.com/repos/${owner}/${repo}/tags?per_page=100`, {
headers
});
if (!response.ok) throw new GitHubApiError(response.status);
return response.json();
if (!response.ok) throw new GitHubApiError(response.status);
return response.json();
}
```
@@ -124,28 +126,26 @@ In the search/context endpoints, the `libraryId` is parsed to extract the option
```typescript
function resolveSearchTarget(libraryId: string): {
repositoryId: string;
versionId?: string;
repositoryId: string;
versionId?: string;
} {
const { repositoryId, version } = parseLibraryId(libraryId);
const { repositoryId, version } = parseLibraryId(libraryId);
if (!version) {
// Query default branch: versionId = NULL
return { repositoryId };
}
if (!version) {
// Query default branch: versionId = NULL
return { repositoryId };
}
// Look up versionId from tag
const versionRecord = db.prepare(
`SELECT id FROM repository_versions WHERE repository_id = ? AND tag = ?`
).get(repositoryId, version) as { id: string } | undefined;
// Look up versionId from tag
const versionRecord = db
.prepare(`SELECT id FROM repository_versions WHERE repository_id = ? AND tag = ?`)
.get(repositoryId, version) as { id: string } | undefined;
if (!versionRecord) {
throw new NotFoundError(
`Version "${version}" not found for library "${repositoryId}"`
);
}
if (!versionRecord) {
throw new NotFoundError(`Version "${version}" not found for library "${repositoryId}"`);
}
return { repositoryId, versionId: versionRecord.id };
return { repositoryId, versionId: versionRecord.id };
}
```
@@ -157,20 +157,20 @@ Snippets with `version_id IS NULL` belong to the default branch; snippets with a
```typescript
export class VersionService {
constructor(private db: BetterSQLite3.Database) {}
constructor(private db: BetterSQLite3.Database) {}
list(repositoryId: string): RepositoryVersion[]
list(repositoryId: string): RepositoryVersion[];
add(repositoryId: string, tag: string, title?: string): RepositoryVersion
add(repositoryId: string, tag: string, title?: string): RepositoryVersion;
remove(repositoryId: string, tag: string): void
remove(repositoryId: string, tag: string): void;
getByTag(repositoryId: string, tag: string): RepositoryVersion | null
getByTag(repositoryId: string, tag: string): RepositoryVersion | null;
registerFromConfig(
repositoryId: string,
previousVersions: { tag: string; title: string }[]
): RepositoryVersion[]
registerFromConfig(
repositoryId: string,
previousVersions: { tag: string; title: string }[]
): RepositoryVersion[];
}
```

View File

@@ -49,79 +49,79 @@ Implement the main web interface for managing repositories. Built with SvelteKit
```svelte
<!-- src/lib/components/RepositoryCard.svelte -->
<script lang="ts">
import type { Repository } from '$lib/types';
import type { Repository } from '$lib/types';
let { repo, onReindex, onDelete } = $props<{
repo: Repository;
onReindex: (id: string) => void;
onDelete: (id: string) => void;
}>();
let { repo, onReindex, onDelete } = $props<{
repo: Repository;
onReindex: (id: string) => void;
onDelete: (id: string) => void;
}>();
const stateColors = {
pending: 'bg-gray-100 text-gray-600',
indexing: 'bg-blue-100 text-blue-700',
indexed: 'bg-green-100 text-green-700',
error: 'bg-red-100 text-red-700',
};
const stateColors = {
pending: 'bg-gray-100 text-gray-600',
indexing: 'bg-blue-100 text-blue-700',
indexed: 'bg-green-100 text-green-700',
error: 'bg-red-100 text-red-700'
};
const stateLabels = {
pending: 'Pending',
indexing: 'Indexing...',
indexed: 'Indexed',
error: 'Error',
};
const stateLabels = {
pending: 'Pending',
indexing: 'Indexing...',
indexed: 'Indexed',
error: 'Error'
};
</script>
<div class="rounded-xl border border-gray-200 bg-white p-5 shadow-sm">
<div class="flex items-start justify-between">
<div>
<h3 class="font-semibold text-gray-900">{repo.title}</h3>
<p class="mt-0.5 font-mono text-sm text-gray-500">{repo.id}</p>
</div>
<span class="rounded-full px-2.5 py-0.5 text-xs font-medium {stateColors[repo.state]}">
{stateLabels[repo.state]}
</span>
</div>
<div class="flex items-start justify-between">
<div>
<h3 class="font-semibold text-gray-900">{repo.title}</h3>
<p class="mt-0.5 font-mono text-sm text-gray-500">{repo.id}</p>
</div>
<span class="rounded-full px-2.5 py-0.5 text-xs font-medium {stateColors[repo.state]}">
{stateLabels[repo.state]}
</span>
</div>
{#if repo.description}
<p class="mt-2 line-clamp-2 text-sm text-gray-600">{repo.description}</p>
{/if}
{#if repo.description}
<p class="mt-2 line-clamp-2 text-sm text-gray-600">{repo.description}</p>
{/if}
<div class="mt-4 flex gap-4 text-sm text-gray-500">
<span>{repo.totalSnippets.toLocaleString()} snippets</span>
<span>·</span>
<span>Trust: {repo.trustScore.toFixed(1)}/10</span>
{#if repo.stars}
<span>·</span>
<span>{repo.stars.toLocaleString()}</span>
{/if}
</div>
<div class="mt-4 flex gap-4 text-sm text-gray-500">
<span>{repo.totalSnippets.toLocaleString()} snippets</span>
<span>·</span>
<span>Trust: {repo.trustScore.toFixed(1)}/10</span>
{#if repo.stars}
<span>·</span>
<span>{repo.stars.toLocaleString()}</span>
{/if}
</div>
{#if repo.state === 'error'}
<p class="mt-2 text-xs text-red-600">Indexing failed. Check jobs for details.</p>
{/if}
{#if repo.state === 'error'}
<p class="mt-2 text-xs text-red-600">Indexing failed. Check jobs for details.</p>
{/if}
<div class="mt-4 flex gap-2">
<button
onclick={() => onReindex(repo.id)}
class="rounded-lg bg-blue-600 px-3 py-1.5 text-sm text-white hover:bg-blue-700"
disabled={repo.state === 'indexing'}
>
{repo.state === 'indexing' ? 'Indexing...' : 'Re-index'}
</button>
<a
href="/repos/{encodeURIComponent(repo.id)}"
class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm text-gray-700 hover:bg-gray-50"
>
Details
</a>
<button
onclick={() => onDelete(repo.id)}
class="ml-auto rounded-lg px-3 py-1.5 text-sm text-red-600 hover:bg-red-50"
>
Delete
</button>
</div>
<div class="mt-4 flex gap-2">
<button
onclick={() => onReindex(repo.id)}
class="rounded-lg bg-blue-600 px-3 py-1.5 text-sm text-white hover:bg-blue-700"
disabled={repo.state === 'indexing'}
>
{repo.state === 'indexing' ? 'Indexing...' : 'Re-index'}
</button>
<a
href="/repos/{encodeURIComponent(repo.id)}"
class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm text-gray-700 hover:bg-gray-50"
>
Details
</a>
<button
onclick={() => onDelete(repo.id)}
class="ml-auto rounded-lg px-3 py-1.5 text-sm text-red-600 hover:bg-red-50"
>
Delete
</button>
</div>
</div>
```
@@ -132,98 +132,104 @@ Implement the main web interface for managing repositories. Built with SvelteKit
```svelte
<!-- src/lib/components/AddRepositoryModal.svelte -->
<script lang="ts">
let { onClose, onAdded } = $props<{
onClose: () => void;
onAdded: () => void;
}>();
let { onClose, onAdded } = $props<{
onClose: () => void;
onAdded: () => void;
}>();
let source = $state<'github' | 'local'>('github');
let sourceUrl = $state('');
let githubToken = $state('');
let loading = $state(false);
let error = $state<string | null>(null);
let source = $state<'github' | 'local'>('github');
let sourceUrl = $state('');
let githubToken = $state('');
let loading = $state(false);
let error = $state<string | null>(null);
async function handleSubmit() {
loading = true;
error = null;
try {
const res = await fetch('/api/v1/libs', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ source, sourceUrl, githubToken: githubToken || undefined }),
});
if (!res.ok) {
const data = await res.json();
throw new Error(data.error ?? 'Failed to add repository');
}
onAdded();
onClose();
} catch (e) {
error = (e as Error).message;
} finally {
loading = false;
}
}
async function handleSubmit() {
loading = true;
error = null;
try {
const res = await fetch('/api/v1/libs', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ source, sourceUrl, githubToken: githubToken || undefined })
});
if (!res.ok) {
const data = await res.json();
throw new Error(data.error ?? 'Failed to add repository');
}
onAdded();
onClose();
} catch (e) {
error = (e as Error).message;
} finally {
loading = false;
}
}
</script>
<dialog class="modal" open>
<div class="modal-box max-w-md">
<h2 class="mb-4 text-lg font-semibold">Add Repository</h2>
<div class="modal-box max-w-md">
<h2 class="mb-4 text-lg font-semibold">Add Repository</h2>
<div class="mb-4 flex gap-2">
<button
class="flex-1 rounded-lg py-2 text-sm {source === 'github' ? 'bg-blue-600 text-white' : 'border border-gray-200 text-gray-700'}"
onclick={() => source = 'github'}
>GitHub</button>
<button
class="flex-1 rounded-lg py-2 text-sm {source === 'local' ? 'bg-blue-600 text-white' : 'border border-gray-200 text-gray-700'}"
onclick={() => source = 'local'}
>Local Path</button>
</div>
<div class="mb-4 flex gap-2">
<button
class="flex-1 rounded-lg py-2 text-sm {source === 'github'
? 'bg-blue-600 text-white'
: 'border border-gray-200 text-gray-700'}"
onclick={() => (source = 'github')}>GitHub</button
>
<button
class="flex-1 rounded-lg py-2 text-sm {source === 'local'
? 'bg-blue-600 text-white'
: 'border border-gray-200 text-gray-700'}"
onclick={() => (source = 'local')}>Local Path</button
>
</div>
<label class="block">
<span class="text-sm font-medium text-gray-700">
{source === 'github' ? 'GitHub URL' : 'Absolute Path'}
</span>
<input
type="text"
bind:value={sourceUrl}
placeholder={source === 'github'
? 'https://github.com/facebook/react'
: '/home/user/projects/my-sdk'}
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
/>
</label>
<label class="block">
<span class="text-sm font-medium text-gray-700">
{source === 'github' ? 'GitHub URL' : 'Absolute Path'}
</span>
<input
type="text"
bind:value={sourceUrl}
placeholder={source === 'github'
? 'https://github.com/facebook/react'
: '/home/user/projects/my-sdk'}
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
/>
</label>
{#if source === 'github'}
<label class="mt-3 block">
<span class="text-sm font-medium text-gray-700">GitHub Token (optional, for private repos)</span>
<input
type="password"
bind:value={githubToken}
placeholder="ghp_..."
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
/>
</label>
{/if}
{#if source === 'github'}
<label class="mt-3 block">
<span class="text-sm font-medium text-gray-700"
>GitHub Token (optional, for private repos)</span
>
<input
type="password"
bind:value={githubToken}
placeholder="ghp_..."
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
/>
</label>
{/if}
{#if error}
<p class="mt-3 text-sm text-red-600">{error}</p>
{/if}
{#if error}
<p class="mt-3 text-sm text-red-600">{error}</p>
{/if}
<div class="mt-6 flex justify-end gap-3">
<button onclick={onClose} class="rounded-lg border border-gray-200 px-4 py-2 text-sm">
Cancel
</button>
<button
onclick={handleSubmit}
disabled={loading || !sourceUrl}
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
>
{loading ? 'Adding...' : 'Add & Index'}
</button>
</div>
</div>
<div class="mt-6 flex justify-end gap-3">
<button onclick={onClose} class="rounded-lg border border-gray-200 px-4 py-2 text-sm">
Cancel
</button>
<button
onclick={handleSubmit}
disabled={loading || !sourceUrl}
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
>
{loading ? 'Adding...' : 'Add & Index'}
</button>
</div>
</div>
</dialog>
```
@@ -234,48 +240,48 @@ Implement the main web interface for managing repositories. Built with SvelteKit
```svelte
<!-- src/lib/components/IndexingProgress.svelte -->
<script lang="ts">
import { onMount, onDestroy } from 'svelte';
import type { IndexingJob } from '$lib/types';
import { onMount, onDestroy } from 'svelte';
import type { IndexingJob } from '$lib/types';
let { jobId } = $props<{ jobId: string }>();
let job = $state<IndexingJob | null>(null);
let interval: ReturnType<typeof setInterval>;
let { jobId } = $props<{ jobId: string }>();
let job = $state<IndexingJob | null>(null);
let interval: ReturnType<typeof setInterval>;
async function pollJob() {
const res = await fetch(`/api/v1/jobs/${jobId}`);
if (res.ok) {
const data = await res.json();
job = data.job;
if (job?.status === 'done' || job?.status === 'failed') {
clearInterval(interval);
}
}
}
async function pollJob() {
const res = await fetch(`/api/v1/jobs/${jobId}`);
if (res.ok) {
const data = await res.json();
job = data.job;
if (job?.status === 'done' || job?.status === 'failed') {
clearInterval(interval);
}
}
}
onMount(() => {
pollJob();
interval = setInterval(pollJob, 2000);
});
onMount(() => {
pollJob();
interval = setInterval(pollJob, 2000);
});
onDestroy(() => clearInterval(interval));
onDestroy(() => clearInterval(interval));
</script>
{#if job}
<div class="mt-2">
<div class="flex justify-between text-xs text-gray-500">
<span>{job.processedFiles} / {job.totalFiles} files</span>
<span>{job.progress}%</span>
</div>
<div class="mt-1 h-1.5 w-full rounded-full bg-gray-200">
<div
class="h-1.5 rounded-full bg-blue-600 transition-all"
style="width: {job.progress}%"
></div>
</div>
{#if job.status === 'failed'}
<p class="mt-1 text-xs text-red-600">{job.error}</p>
{/if}
</div>
<div class="mt-2">
<div class="flex justify-between text-xs text-gray-500">
<span>{job.processedFiles} / {job.totalFiles} files</span>
<span>{job.progress}%</span>
</div>
<div class="mt-1 h-1.5 w-full rounded-full bg-gray-200">
<div
class="h-1.5 rounded-full bg-blue-600 transition-all"
style="width: {job.progress}%"
></div>
</div>
{#if job.status === 'failed'}
<p class="mt-1 text-xs text-red-600">{job.error}</p>
{/if}
</div>
{/if}
```
@@ -288,9 +294,9 @@ Implement the main web interface for managing repositories. Built with SvelteKit
import type { PageServerLoad } from './$types';
export const load: PageServerLoad = async ({ fetch }) => {
const res = await fetch('/api/v1/libs');
const data = await res.json();
return { repositories: data.libraries };
const res = await fetch('/api/v1/libs');
const data = await res.json();
return { repositories: data.libraries };
};
```

View File

@@ -57,25 +57,31 @@ An interactive search interface within the web UI that lets users test the docum
```svelte
<!-- src/lib/components/search/LibraryResult.svelte -->
<script lang="ts">
let { result, onSelect } = $props<{
result: { id: string; title: string; description: string; totalSnippets: number; trustScore: number };
onSelect: (id: string) => void;
}>();
let { result, onSelect } = $props<{
result: {
id: string;
title: string;
description: string;
totalSnippets: number;
trustScore: number;
};
onSelect: (id: string) => void;
}>();
</script>
<button
onclick={() => onSelect(result.id)}
class="w-full rounded-xl border border-gray-200 bg-white p-4 text-left shadow-sm hover:border-blue-300 hover:shadow-md transition-all"
onclick={() => onSelect(result.id)}
class="w-full rounded-xl border border-gray-200 bg-white p-4 text-left shadow-sm transition-all hover:border-blue-300 hover:shadow-md"
>
<div class="flex items-center justify-between">
<span class="font-semibold text-gray-900">{result.title}</span>
<span class="text-xs text-gray-400">Trust {result.trustScore.toFixed(1)}/10</span>
</div>
<p class="font-mono text-xs text-gray-400">{result.id}</p>
{#if result.description}
<p class="mt-1.5 text-sm text-gray-600 line-clamp-2">{result.description}</p>
{/if}
<p class="mt-2 text-xs text-gray-400">{result.totalSnippets.toLocaleString()} snippets</p>
<div class="flex items-center justify-between">
<span class="font-semibold text-gray-900">{result.title}</span>
<span class="text-xs text-gray-400">Trust {result.trustScore.toFixed(1)}/10</span>
</div>
<p class="font-mono text-xs text-gray-400">{result.id}</p>
{#if result.description}
<p class="mt-1.5 line-clamp-2 text-sm text-gray-600">{result.description}</p>
{/if}
<p class="mt-2 text-xs text-gray-400">{result.totalSnippets.toLocaleString()} snippets</p>
</button>
```
@@ -86,37 +92,39 @@ An interactive search interface within the web UI that lets users test the docum
```svelte
<!-- src/lib/components/search/SnippetCard.svelte -->
<script lang="ts">
import type { Snippet } from '$lib/types';
import type { Snippet } from '$lib/types';
let { snippet } = $props<{ snippet: Snippet }>();
let { snippet } = $props<{ snippet: Snippet }>();
</script>
<div class="rounded-xl border border-gray-200 bg-white overflow-hidden">
<div class="flex items-center justify-between border-b border-gray-100 px-4 py-2.5">
<div class="flex items-center gap-2">
{#if snippet.type === 'code'}
<span class="rounded bg-purple-100 px-1.5 py-0.5 text-xs text-purple-700">code</span>
{:else}
<span class="rounded bg-blue-100 px-1.5 py-0.5 text-xs text-blue-700">info</span>
{/if}
{#if snippet.title}
<span class="text-sm font-medium text-gray-800">{snippet.title}</span>
{/if}
</div>
<span class="text-xs text-gray-400">{snippet.tokenCount} tokens</span>
</div>
<div class="overflow-hidden rounded-xl border border-gray-200 bg-white">
<div class="flex items-center justify-between border-b border-gray-100 px-4 py-2.5">
<div class="flex items-center gap-2">
{#if snippet.type === 'code'}
<span class="rounded bg-purple-100 px-1.5 py-0.5 text-xs text-purple-700">code</span>
{:else}
<span class="rounded bg-blue-100 px-1.5 py-0.5 text-xs text-blue-700">info</span>
{/if}
{#if snippet.title}
<span class="text-sm font-medium text-gray-800">{snippet.title}</span>
{/if}
</div>
<span class="text-xs text-gray-400">{snippet.tokenCount} tokens</span>
</div>
{#if snippet.breadcrumb}
<p class="bg-gray-50 px-4 py-1.5 text-xs text-gray-500 italic">{snippet.breadcrumb}</p>
{/if}
{#if snippet.breadcrumb}
<p class="bg-gray-50 px-4 py-1.5 text-xs text-gray-500 italic">{snippet.breadcrumb}</p>
{/if}
<div class="p-4">
{#if snippet.type === 'code'}
<pre class="overflow-x-auto rounded bg-gray-950 p-4 text-sm text-gray-100"><code>{snippet.content}</code></pre>
{:else}
<div class="prose prose-sm max-w-none text-gray-700">{snippet.content}</div>
{/if}
</div>
<div class="p-4">
{#if snippet.type === 'code'}
<pre class="overflow-x-auto rounded bg-gray-950 p-4 text-sm text-gray-100"><code
>{snippet.content}</code
></pre>
{:else}
<div class="prose prose-sm max-w-none text-gray-700">{snippet.content}</div>
{/if}
</div>
</div>
```
@@ -127,44 +135,44 @@ An interactive search interface within the web UI that lets users test the docum
```svelte
<!-- src/routes/search/+page.svelte -->
<script lang="ts">
import { page } from '$app/stores';
import { goto } from '$app/navigation';
import { page } from '$app/stores';
import { goto } from '$app/navigation';
let libraryName = $state('');
let selectedLibraryId = $state<string | null>(null);
let query = $state('');
let libraryResults = $state<LibrarySearchResult[]>([]);
let snippets = $state<Snippet[]>([]);
let loadingLibraries = $state(false);
let loadingSnippets = $state(false);
let libraryName = $state('');
let selectedLibraryId = $state<string | null>(null);
let query = $state('');
let libraryResults = $state<LibrarySearchResult[]>([]);
let snippets = $state<Snippet[]>([]);
let loadingLibraries = $state(false);
let loadingSnippets = $state(false);
async function searchLibraries() {
loadingLibraries = true;
const res = await fetch(
`/api/v1/libs/search?libraryName=${encodeURIComponent(libraryName)}&query=${encodeURIComponent(query)}`
);
const data = await res.json();
libraryResults = data.results;
loadingLibraries = false;
}
async function searchLibraries() {
loadingLibraries = true;
const res = await fetch(
`/api/v1/libs/search?libraryName=${encodeURIComponent(libraryName)}&query=${encodeURIComponent(query)}`
);
const data = await res.json();
libraryResults = data.results;
loadingLibraries = false;
}
async function searchDocs() {
if (!selectedLibraryId) return;
loadingSnippets = true;
const url = new URL('/api/v1/context', window.location.origin);
url.searchParams.set('libraryId', selectedLibraryId);
url.searchParams.set('query', query);
const res = await fetch(url);
const data = await res.json();
snippets = data.snippets;
loadingSnippets = false;
async function searchDocs() {
if (!selectedLibraryId) return;
loadingSnippets = true;
const url = new URL('/api/v1/context', window.location.origin);
url.searchParams.set('libraryId', selectedLibraryId);
url.searchParams.set('query', query);
const res = await fetch(url);
const data = await res.json();
snippets = data.snippets;
loadingSnippets = false;
// Update URL
goto(`/search?lib=${encodeURIComponent(selectedLibraryId)}&q=${encodeURIComponent(query)}`, {
replaceState: true,
keepFocus: true,
});
}
// Update URL
goto(`/search?lib=${encodeURIComponent(selectedLibraryId)}&q=${encodeURIComponent(query)}`, {
replaceState: true,
keepFocus: true
});
}
</script>
```
@@ -177,9 +185,9 @@ Use a minimal, zero-dependency approach for v1 — wrap code blocks in `<pre><co
```typescript
// Optional: lazy-load highlight.js only when code snippets are present
async function highlightCode(code: string, language: string): Promise<string> {
const hljs = await import('highlight.js/lib/core');
// Register only needed languages
return hljs.highlight(code, { language }).value;
const hljs = await import('highlight.js/lib/core');
// Register only needed languages
return hljs.highlight(code, { language }).value;
}
```

View File

@@ -30,39 +30,39 @@ Optimize re-indexing by skipping files that haven't changed since the last index
```typescript
interface FileDiff {
added: CrawledFile[]; // new files not in DB
modified: CrawledFile[]; // files with changed checksum
deleted: string[]; // file paths in DB but not in crawl
unchanged: string[]; // file paths with matching checksum
added: CrawledFile[]; // new files not in DB
modified: CrawledFile[]; // files with changed checksum
deleted: string[]; // file paths in DB but not in crawl
unchanged: string[]; // file paths with matching checksum
}
function computeDiff(
crawledFiles: CrawledFile[],
existingDocs: Document[] // documents currently in DB for this repo
crawledFiles: CrawledFile[],
existingDocs: Document[] // documents currently in DB for this repo
): FileDiff {
const existingMap = new Map(existingDocs.map(d => [d.filePath, d]));
const crawledMap = new Map(crawledFiles.map(f => [f.path, f]));
const existingMap = new Map(existingDocs.map((d) => [d.filePath, d]));
const crawledMap = new Map(crawledFiles.map((f) => [f.path, f]));
const added: CrawledFile[] = [];
const modified: CrawledFile[] = [];
const unchanged: string[] = [];
const added: CrawledFile[] = [];
const modified: CrawledFile[] = [];
const unchanged: string[] = [];
for (const file of crawledFiles) {
const existing = existingMap.get(file.path);
if (!existing) {
added.push(file);
} else if (existing.checksum !== file.sha) {
modified.push(file);
} else {
unchanged.push(file.path);
}
}
for (const file of crawledFiles) {
const existing = existingMap.get(file.path);
if (!existing) {
added.push(file);
} else if (existing.checksum !== file.sha) {
modified.push(file);
} else {
unchanged.push(file.path);
}
}
const deleted = existingDocs
.filter(doc => !crawledMap.has(doc.filePath))
.map(doc => doc.filePath);
const deleted = existingDocs
.filter((doc) => !crawledMap.has(doc.filePath))
.map((doc) => doc.filePath);
return { added, modified, deleted, unchanged };
return { added, modified, deleted, unchanged };
}
```
@@ -78,7 +78,7 @@ const diff = computeDiff(crawledResult.files, existingDocs);
// Log diff summary
this.updateJob(job.id, {
totalFiles: crawledResult.files.length,
totalFiles: crawledResult.files.length
});
// Process only changed/new files
@@ -89,29 +89,29 @@ const docIdsToDelete: string[] = [];
// Map modified files to their existing document IDs for deletion
for (const file of diff.modified) {
const existing = existingDocs.find(d => d.filePath === file.path);
if (existing) docIdsToDelete.push(existing.id);
const existing = existingDocs.find((d) => d.filePath === file.path);
if (existing) docIdsToDelete.push(existing.id);
}
// Map deleted file paths to document IDs
for (const filePath of diff.deleted) {
const existing = existingDocs.find(d => d.filePath === filePath);
if (existing) docIdsToDelete.push(existing.id);
const existing = existingDocs.find((d) => d.filePath === filePath);
if (existing) docIdsToDelete.push(existing.id);
}
// Parse new/modified files
for (const [i, file] of filesToProcess.entries()) {
const docId = crypto.randomUUID();
newDocuments.push({ id: docId, ...buildDocument(file, repo.id, job.versionId) });
newSnippets.push(...parseFile(file, { repositoryId: repo.id, documentId: docId }));
const docId = crypto.randomUUID();
newDocuments.push({ id: docId, ...buildDocument(file, repo.id, job.versionId) });
newSnippets.push(...parseFile(file, { repositoryId: repo.id, documentId: docId }));
// Count ALL files (including skipped) in progress
const totalProcessed = diff.unchanged.length + i + 1;
const progress = Math.round((totalProcessed / crawledResult.files.length) * 80);
this.updateJob(job.id, {
processedFiles: totalProcessed,
progress,
});
// Count ALL files (including skipped) in progress
const totalProcessed = diff.unchanged.length + i + 1;
const progress = Math.round((totalProcessed / crawledResult.files.length) * 80);
this.updateJob(job.id, {
processedFiles: totalProcessed,
progress
});
}
// Atomic replacement of only changed documents
@@ -123,6 +123,7 @@ this.replaceSnippets(repo.id, docIdsToDelete, newDocuments, newSnippets);
## Performance Impact
For a typical repository with 1,000 files where 50 changed:
- **Without incremental**: 1,000 files parsed + 1,000 embed batches
- **With incremental**: 50 files parsed + 50 embed batches
- Estimated speedup: ~20x for re-indexing

View File

@@ -32,24 +32,24 @@ A settings page within the web UI that allows users to configure the embedding p
```typescript
const PROVIDER_PRESETS = [
{
name: 'OpenAI',
baseUrl: 'https://api.openai.com/v1',
model: 'text-embedding-3-small',
dimensions: 1536,
},
{
name: 'Ollama (local)',
baseUrl: 'http://localhost:11434/v1',
model: 'nomic-embed-text',
dimensions: 768,
},
{
name: 'Azure OpenAI',
baseUrl: 'https://{resource}.openai.azure.com/openai/deployments/{deployment}/v1',
model: 'text-embedding-3-small',
dimensions: 1536,
},
{
name: 'OpenAI',
baseUrl: 'https://api.openai.com/v1',
model: 'text-embedding-3-small',
dimensions: 1536
},
{
name: 'Ollama (local)',
baseUrl: 'http://localhost:11434/v1',
model: 'nomic-embed-text',
dimensions: 768
},
{
name: 'Azure OpenAI',
baseUrl: 'https://{resource}.openai.azure.com/openai/deployments/{deployment}/v1',
model: 'text-embedding-3-small',
dimensions: 1536
}
];
```
@@ -60,133 +60,157 @@ const PROVIDER_PRESETS = [
```svelte
<!-- src/routes/settings/+page.svelte -->
<script lang="ts">
let provider = $state<'none' | 'openai' | 'local'>('none');
let baseUrl = $state('https://api.openai.com/v1');
let apiKey = $state('');
let model = $state('text-embedding-3-small');
let dimensions = $state<number | undefined>(1536);
let testStatus = $state<'idle' | 'testing' | 'ok' | 'error'>('idle');
let testError = $state<string | null>(null);
let saving = $state(false);
let provider = $state<'none' | 'openai' | 'local'>('none');
let baseUrl = $state('https://api.openai.com/v1');
let apiKey = $state('');
let model = $state('text-embedding-3-small');
let dimensions = $state<number | undefined>(1536);
let testStatus = $state<'idle' | 'testing' | 'ok' | 'error'>('idle');
let testError = $state<string | null>(null);
let saving = $state(false);
async function testConnection() {
testStatus = 'testing';
testError = null;
try {
const res = await fetch('/api/v1/settings/embedding/test', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } }),
});
if (res.ok) {
testStatus = 'ok';
} else {
const data = await res.json();
testStatus = 'error';
testError = data.error;
}
} catch (e) {
testStatus = 'error';
testError = (e as Error).message;
}
}
async function testConnection() {
testStatus = 'testing';
testError = null;
try {
const res = await fetch('/api/v1/settings/embedding/test', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } })
});
if (res.ok) {
testStatus = 'ok';
} else {
const data = await res.json();
testStatus = 'error';
testError = data.error;
}
} catch (e) {
testStatus = 'error';
testError = (e as Error).message;
}
}
async function save() {
saving = true;
await fetch('/api/v1/settings/embedding', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } }),
});
saving = false;
}
async function save() {
saving = true;
await fetch('/api/v1/settings/embedding', {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } })
});
saving = false;
}
</script>
<div class="mx-auto max-w-2xl py-8">
<h1 class="mb-6 text-2xl font-bold text-gray-900">Settings</h1>
<h1 class="mb-6 text-2xl font-bold text-gray-900">Settings</h1>
<section class="rounded-xl border border-gray-200 bg-white p-6">
<h2 class="mb-1 text-lg font-semibold">Embedding Provider</h2>
<p class="mb-4 text-sm text-gray-500">
Embeddings enable semantic search. Without them, only keyword search (FTS5) is used.
</p>
<section class="rounded-xl border border-gray-200 bg-white p-6">
<h2 class="mb-1 text-lg font-semibold">Embedding Provider</h2>
<p class="mb-4 text-sm text-gray-500">
Embeddings enable semantic search. Without them, only keyword search (FTS5) is used.
</p>
<div class="mb-4 flex gap-2">
{#each ['none', 'openai', 'local'] as p}
<button
onclick={() => provider = p}
class="rounded-lg px-4 py-2 text-sm {provider === p
? 'bg-blue-600 text-white'
: 'border border-gray-200 text-gray-700 hover:bg-gray-50'}"
>
{p === 'none' ? 'None (FTS5 only)' : p === 'openai' ? 'OpenAI-compatible' : 'Local Model'}
</button>
{/each}
</div>
<div class="mb-4 flex gap-2">
{#each ['none', 'openai', 'local'] as p}
<button
onclick={() => (provider = p)}
class="rounded-lg px-4 py-2 text-sm {provider === p
? 'bg-blue-600 text-white'
: 'border border-gray-200 text-gray-700 hover:bg-gray-50'}"
>
{p === 'none' ? 'None (FTS5 only)' : p === 'openai' ? 'OpenAI-compatible' : 'Local Model'}
</button>
{/each}
</div>
{#if provider === 'none'}
<div class="rounded-lg bg-amber-50 border border-amber-200 p-3 text-sm text-amber-700">
Search will use keyword matching only. Results may be less relevant for complex questions.
</div>
{/if}
{#if provider === 'none'}
<div class="rounded-lg border border-amber-200 bg-amber-50 p-3 text-sm text-amber-700">
Search will use keyword matching only. Results may be less relevant for complex questions.
</div>
{/if}
{#if provider === 'openai'}
<div class="space-y-3">
<!-- Preset buttons -->
<div class="flex gap-2 flex-wrap">
{#each PROVIDER_PRESETS as preset}
<button
onclick={() => { baseUrl = preset.baseUrl; model = preset.model; dimensions = preset.dimensions; }}
class="rounded border border-gray-200 px-2.5 py-1 text-xs text-gray-600 hover:bg-gray-50"
>
{preset.name}
</button>
{/each}
</div>
{#if provider === 'openai'}
<div class="space-y-3">
<!-- Preset buttons -->
<div class="flex flex-wrap gap-2">
{#each PROVIDER_PRESETS as preset}
<button
onclick={() => {
baseUrl = preset.baseUrl;
model = preset.model;
dimensions = preset.dimensions;
}}
class="rounded border border-gray-200 px-2.5 py-1 text-xs text-gray-600 hover:bg-gray-50"
>
{preset.name}
</button>
{/each}
</div>
<label class="block">
<span class="text-sm font-medium">Base URL</span>
<input type="text" bind:value={baseUrl} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" />
</label>
<label class="block">
<span class="text-sm font-medium">Base URL</span>
<input
type="text"
bind:value={baseUrl}
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
/>
</label>
<label class="block">
<span class="text-sm font-medium">API Key</span>
<input type="password" bind:value={apiKey} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" placeholder="sk-..." />
</label>
<label class="block">
<span class="text-sm font-medium">API Key</span>
<input
type="password"
bind:value={apiKey}
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
placeholder="sk-..."
/>
</label>
<label class="block">
<span class="text-sm font-medium">Model</span>
<input type="text" bind:value={model} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" />
</label>
<label class="block">
<span class="text-sm font-medium">Model</span>
<input
type="text"
bind:value={model}
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
/>
</label>
<label class="block">
<span class="text-sm font-medium">Dimensions (optional override)</span>
<input type="number" bind:value={dimensions} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" />
</label>
<label class="block">
<span class="text-sm font-medium">Dimensions (optional override)</span>
<input
type="number"
bind:value={dimensions}
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
/>
</label>
<div class="flex items-center gap-3">
<button onclick={testConnection} class="rounded-lg border border-gray-300 px-3 py-1.5 text-sm">
{testStatus === 'testing' ? 'Testing...' : 'Test Connection'}
</button>
{#if testStatus === 'ok'}
<span class="text-sm text-green-600">✓ Connection successful</span>
{:else if testStatus === 'error'}
<span class="text-sm text-red-600">{testError}</span>
{/if}
</div>
</div>
{/if}
<div class="flex items-center gap-3">
<button
onclick={testConnection}
class="rounded-lg border border-gray-300 px-3 py-1.5 text-sm"
>
{testStatus === 'testing' ? 'Testing...' : 'Test Connection'}
</button>
{#if testStatus === 'ok'}
<span class="text-sm text-green-600">✓ Connection successful</span>
{:else if testStatus === 'error'}
<span class="text-sm text-red-600">{testError}</span>
{/if}
</div>
</div>
{/if}
<div class="mt-6 flex justify-end">
<button
onclick={save}
disabled={saving}
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
>
{saving ? 'Saving...' : 'Save Settings'}
</button>
</div>
</section>
<div class="mt-6 flex justify-end">
<button
onclick={save}
disabled={saving}
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
>
{saving ? 'Saving...' : 'Save Settings'}
</button>
</div>
</section>
</div>
```

View File

@@ -80,6 +80,7 @@ git -C /path/to/repo archive <commit-hash> | tar -x -C /tmp/trueref-idx/<repo>-<
```
Advantages over `git checkout` or worktrees:
- Working directory is completely untouched
- No `.git` directory in the output (cleaner for parsing)
- Temp directory deleted after indexing with no git state to clean up
@@ -102,26 +103,26 @@ Allow commit hashes to be pinned explicitly per version, overriding tag resoluti
```json
{
"previousVersions": [
{
"tag": "v2.0.0",
"title": "Version 2.0.0",
"commitHash": "a3f9c12abc..."
}
]
"previousVersions": [
{
"tag": "v2.0.0",
"title": "Version 2.0.0",
"commitHash": "a3f9c12abc..."
}
]
}
```
### Edge Cases
| Case | Handling |
|------|----------|
| Annotated tags | `rev-parse <tag>^{commit}` peels to commit automatically |
| Mutable tags (e.g. `latest`) | Re-resolve on re-index; warn in UI if hash has changed |
| Branch as version | `rev-parse origin/<branch>^{commit}` gives tip; re-resolves on re-index |
| Shallow clone | Run `git fetch --unshallow` before `git archive` if commit is unavailable |
| Submodules | `git archive --recurse-submodules` or document as a known limitation |
| Git LFS | `git lfs pull` required after archive if LFS-tracked files are needed for indexing |
| Case | Handling |
| ---------------------------- | ---------------------------------------------------------------------------------- |
| Annotated tags | `rev-parse <tag>^{commit}` peels to commit automatically |
| Mutable tags (e.g. `latest`) | Re-resolve on re-index; warn in UI if hash has changed |
| Branch as version | `rev-parse origin/<branch>^{commit}` gives tip; re-resolves on re-index |
| Shallow clone | Run `git fetch --unshallow` before `git archive` if commit is unavailable |
| Submodules | `git archive --recurse-submodules` or document as a known limitation |
| Git LFS | `git lfs pull` required after archive if LFS-tracked files are needed for indexing |
### Acceptance Criteria
@@ -169,12 +170,12 @@ fi
Username conventions by server type:
| Server | HTTPS username | Password |
|--------|---------------|----------|
| Bitbucket Server / Data Center | `x-token-auth` | HTTP access token |
| Bitbucket Cloud | account username | App password |
| GitLab (self-hosted or cloud) | `oauth2` | Personal access token |
| GitLab deploy token | `gitlab-deploy-token` | Deploy token secret |
| Server | HTTPS username | Password |
| ------------------------------ | --------------------- | --------------------- |
| Bitbucket Server / Data Center | `x-token-auth` | HTTP access token |
| Bitbucket Cloud | account username | App password |
| GitLab (self-hosted or cloud) | `oauth2` | Personal access token |
| GitLab deploy token | `gitlab-deploy-token` | Deploy token secret |
SSH authentication is also supported and preferred for long-lived deployments. The host SSH configuration (`~/.ssh/config`) handles per-host key selection and travels into the container via volume mount.
@@ -220,7 +221,7 @@ services:
web:
build: .
ports:
- "3000:3000"
- '3000:3000'
volumes:
- trueref-data:/data
- ${USERPROFILE}/.ssh:/root/.ssh:ro
@@ -228,20 +229,20 @@ services:
- ${CORP_CA_CERT}:/certs/corp-ca.crt:ro
environment:
DATABASE_URL: /data/trueref.db
GIT_TOKEN_BITBUCKET: "${BITBUCKET_TOKEN}"
GIT_TOKEN_GITLAB: "${GITLAB_TOKEN}"
BITBUCKET_HOST: "${BITBUCKET_HOST}"
GITLAB_HOST: "${GITLAB_HOST}"
GIT_TOKEN_BITBUCKET: '${BITBUCKET_TOKEN}'
GIT_TOKEN_GITLAB: '${GITLAB_TOKEN}'
BITBUCKET_HOST: '${BITBUCKET_HOST}'
GITLAB_HOST: '${GITLAB_HOST}'
restart: unless-stopped
mcp:
build: .
command: mcp
ports:
- "3001:3001"
- '3001:3001'
environment:
TRUEREF_API_URL: http://web:3000
MCP_PORT: "3001"
MCP_PORT: '3001'
depends_on:
- web
restart: unless-stopped

View File

@@ -107,16 +107,16 @@ An embedding profile is persisted configuration selecting one provider adapter p
```typescript
interface EmbeddingProfile {
id: string;
providerKind: string;
title: string;
enabled: boolean;
isDefault: boolean;
config: Record<string, unknown>;
model: string;
dimensions: number;
createdAt: number;
updatedAt: number;
id: string;
providerKind: string;
title: string;
enabled: boolean;
isDefault: boolean;
config: Record<string, unknown>;
model: string;
dimensions: number;
createdAt: number;
updatedAt: number;
}
```
@@ -317,4 +317,4 @@ Indexing must embed snippets against the default active profile, and profile cha
## Notes
This ticket intentionally leaves `libs/search` as keyword-only. The caller is expected to identify the target library and, when needed, pass a version-qualified library ID such as `/owner/repo/v1.2.3` before requesting semantic retrieval.
This ticket intentionally leaves `libs/search` as keyword-only. The caller is expected to identify the target library and, when needed, pass a version-qualified library ID such as `/owner/repo/v1.2.3` before requesting semantic retrieval.