chore(FEEDBACK-0001): linting
This commit is contained in:
@@ -31,24 +31,26 @@ Represents an indexed library source (GitHub repo or local directory).
|
||||
|
||||
```typescript
|
||||
export const repositories = sqliteTable('repositories', {
|
||||
id: text('id').primaryKey(), // e.g. "/facebook/react" or "/local/my-sdk"
|
||||
title: text('title').notNull(),
|
||||
description: text('description'),
|
||||
source: text('source', { enum: ['github', 'local'] }).notNull(),
|
||||
sourceUrl: text('source_url').notNull(), // GitHub URL or absolute local path
|
||||
branch: text('branch').default('main'),
|
||||
state: text('state', {
|
||||
enum: ['pending', 'indexing', 'indexed', 'error']
|
||||
}).notNull().default('pending'),
|
||||
totalSnippets: integer('total_snippets').default(0),
|
||||
totalTokens: integer('total_tokens').default(0),
|
||||
trustScore: real('trust_score').default(0), // 0.0–10.0
|
||||
benchmarkScore: real('benchmark_score').default(0), // 0.0–100.0
|
||||
stars: integer('stars'),
|
||||
githubToken: text('github_token'), // encrypted PAT for private repos
|
||||
lastIndexedAt: integer('last_indexed_at', { mode: 'timestamp' }),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
|
||||
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
|
||||
id: text('id').primaryKey(), // e.g. "/facebook/react" or "/local/my-sdk"
|
||||
title: text('title').notNull(),
|
||||
description: text('description'),
|
||||
source: text('source', { enum: ['github', 'local'] }).notNull(),
|
||||
sourceUrl: text('source_url').notNull(), // GitHub URL or absolute local path
|
||||
branch: text('branch').default('main'),
|
||||
state: text('state', {
|
||||
enum: ['pending', 'indexing', 'indexed', 'error']
|
||||
})
|
||||
.notNull()
|
||||
.default('pending'),
|
||||
totalSnippets: integer('total_snippets').default(0),
|
||||
totalTokens: integer('total_tokens').default(0),
|
||||
trustScore: real('trust_score').default(0), // 0.0–10.0
|
||||
benchmarkScore: real('benchmark_score').default(0), // 0.0–100.0
|
||||
stars: integer('stars'),
|
||||
githubToken: text('github_token'), // encrypted PAT for private repos
|
||||
lastIndexedAt: integer('last_indexed_at', { mode: 'timestamp' }),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
|
||||
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
@@ -58,17 +60,20 @@ Tracks indexed git tags/branches beyond the default branch.
|
||||
|
||||
```typescript
|
||||
export const repositoryVersions = sqliteTable('repository_versions', {
|
||||
id: text('id').primaryKey(), // e.g. "/facebook/react/v18.3.0"
|
||||
repositoryId: text('repository_id').notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
tag: text('tag').notNull(), // git tag or branch name
|
||||
title: text('title'),
|
||||
state: text('state', {
|
||||
enum: ['pending', 'indexing', 'indexed', 'error']
|
||||
}).notNull().default('pending'),
|
||||
totalSnippets: integer('total_snippets').default(0),
|
||||
indexedAt: integer('indexed_at', { mode: 'timestamp' }),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
|
||||
id: text('id').primaryKey(), // e.g. "/facebook/react/v18.3.0"
|
||||
repositoryId: text('repository_id')
|
||||
.notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
tag: text('tag').notNull(), // git tag or branch name
|
||||
title: text('title'),
|
||||
state: text('state', {
|
||||
enum: ['pending', 'indexing', 'indexed', 'error']
|
||||
})
|
||||
.notNull()
|
||||
.default('pending'),
|
||||
totalSnippets: integer('total_snippets').default(0),
|
||||
indexedAt: integer('indexed_at', { mode: 'timestamp' }),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
@@ -78,17 +83,17 @@ A parsed source file within a repository.
|
||||
|
||||
```typescript
|
||||
export const documents = sqliteTable('documents', {
|
||||
id: text('id').primaryKey(), // UUID
|
||||
repositoryId: text('repository_id').notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
versionId: text('version_id')
|
||||
.references(() => repositoryVersions.id, { onDelete: 'cascade' }),
|
||||
filePath: text('file_path').notNull(), // relative path within repo
|
||||
title: text('title'),
|
||||
language: text('language'), // e.g. "typescript", "markdown"
|
||||
tokenCount: integer('token_count').default(0),
|
||||
checksum: text('checksum').notNull(), // SHA-256 of file content
|
||||
indexedAt: integer('indexed_at', { mode: 'timestamp' }).notNull(),
|
||||
id: text('id').primaryKey(), // UUID
|
||||
repositoryId: text('repository_id')
|
||||
.notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
versionId: text('version_id').references(() => repositoryVersions.id, { onDelete: 'cascade' }),
|
||||
filePath: text('file_path').notNull(), // relative path within repo
|
||||
title: text('title'),
|
||||
language: text('language'), // e.g. "typescript", "markdown"
|
||||
tokenCount: integer('token_count').default(0),
|
||||
checksum: text('checksum').notNull(), // SHA-256 of file content
|
||||
indexedAt: integer('indexed_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
@@ -98,20 +103,21 @@ An indexed chunk of content, the atomic unit of search.
|
||||
|
||||
```typescript
|
||||
export const snippets = sqliteTable('snippets', {
|
||||
id: text('id').primaryKey(), // UUID
|
||||
documentId: text('document_id').notNull()
|
||||
.references(() => documents.id, { onDelete: 'cascade' }),
|
||||
repositoryId: text('repository_id').notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
versionId: text('version_id')
|
||||
.references(() => repositoryVersions.id, { onDelete: 'cascade' }),
|
||||
type: text('type', { enum: ['code', 'info'] }).notNull(),
|
||||
title: text('title'),
|
||||
content: text('content').notNull(), // searchable text / code
|
||||
language: text('language'),
|
||||
breadcrumb: text('breadcrumb'), // e.g. "Installation > Getting Started"
|
||||
tokenCount: integer('token_count').default(0),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
|
||||
id: text('id').primaryKey(), // UUID
|
||||
documentId: text('document_id')
|
||||
.notNull()
|
||||
.references(() => documents.id, { onDelete: 'cascade' }),
|
||||
repositoryId: text('repository_id')
|
||||
.notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
versionId: text('version_id').references(() => repositoryVersions.id, { onDelete: 'cascade' }),
|
||||
type: text('type', { enum: ['code', 'info'] }).notNull(),
|
||||
title: text('title'),
|
||||
content: text('content').notNull(), // searchable text / code
|
||||
language: text('language'),
|
||||
breadcrumb: text('breadcrumb'), // e.g. "Installation > Getting Started"
|
||||
tokenCount: integer('token_count').default(0),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
@@ -121,12 +127,13 @@ Stores vector embeddings separately to keep snippets table lean.
|
||||
|
||||
```typescript
|
||||
export const snippetEmbeddings = sqliteTable('snippet_embeddings', {
|
||||
snippetId: text('snippet_id').primaryKey()
|
||||
.references(() => snippets.id, { onDelete: 'cascade' }),
|
||||
model: text('model').notNull(), // embedding model identifier
|
||||
dimensions: integer('dimensions').notNull(),
|
||||
embedding: blob('embedding').notNull(), // Float32Array as binary blob
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
|
||||
snippetId: text('snippet_id')
|
||||
.primaryKey()
|
||||
.references(() => snippets.id, { onDelete: 'cascade' }),
|
||||
model: text('model').notNull(), // embedding model identifier
|
||||
dimensions: integer('dimensions').notNull(),
|
||||
embedding: blob('embedding').notNull(), // Float32Array as binary blob
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
@@ -136,20 +143,23 @@ Tracks asynchronous indexing operations.
|
||||
|
||||
```typescript
|
||||
export const indexingJobs = sqliteTable('indexing_jobs', {
|
||||
id: text('id').primaryKey(), // UUID
|
||||
repositoryId: text('repository_id').notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
versionId: text('version_id'),
|
||||
status: text('status', {
|
||||
enum: ['queued', 'running', 'done', 'failed']
|
||||
}).notNull().default('queued'),
|
||||
progress: integer('progress').default(0), // 0–100
|
||||
totalFiles: integer('total_files').default(0),
|
||||
processedFiles: integer('processed_files').default(0),
|
||||
error: text('error'),
|
||||
startedAt: integer('started_at', { mode: 'timestamp' }),
|
||||
completedAt: integer('completed_at', { mode: 'timestamp' }),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull(),
|
||||
id: text('id').primaryKey(), // UUID
|
||||
repositoryId: text('repository_id')
|
||||
.notNull()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
versionId: text('version_id'),
|
||||
status: text('status', {
|
||||
enum: ['queued', 'running', 'done', 'failed']
|
||||
})
|
||||
.notNull()
|
||||
.default('queued'),
|
||||
progress: integer('progress').default(0), // 0–100
|
||||
totalFiles: integer('total_files').default(0),
|
||||
processedFiles: integer('processed_files').default(0),
|
||||
error: text('error'),
|
||||
startedAt: integer('started_at', { mode: 'timestamp' }),
|
||||
completedAt: integer('completed_at', { mode: 'timestamp' }),
|
||||
createdAt: integer('created_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
@@ -159,17 +169,19 @@ Stores parsed `trueref.json` / `context7.json` configuration.
|
||||
|
||||
```typescript
|
||||
export const repositoryConfigs = sqliteTable('repository_configs', {
|
||||
repositoryId: text('repository_id').primaryKey()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
projectTitle: text('project_title'),
|
||||
description: text('description'),
|
||||
folders: text('folders', { mode: 'json' }).$type<string[]>(),
|
||||
excludeFolders: text('exclude_folders', { mode: 'json' }).$type<string[]>(),
|
||||
excludeFiles: text('exclude_files', { mode: 'json' }).$type<string[]>(),
|
||||
rules: text('rules', { mode: 'json' }).$type<string[]>(),
|
||||
previousVersions: text('previous_versions', { mode: 'json' })
|
||||
.$type<{ tag: string; title: string }[]>(),
|
||||
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
|
||||
repositoryId: text('repository_id')
|
||||
.primaryKey()
|
||||
.references(() => repositories.id, { onDelete: 'cascade' }),
|
||||
projectTitle: text('project_title'),
|
||||
description: text('description'),
|
||||
folders: text('folders', { mode: 'json' }).$type<string[]>(),
|
||||
excludeFolders: text('exclude_folders', { mode: 'json' }).$type<string[]>(),
|
||||
excludeFiles: text('exclude_files', { mode: 'json' }).$type<string[]>(),
|
||||
rules: text('rules', { mode: 'json' }).$type<string[]>(),
|
||||
previousVersions: text('previous_versions', { mode: 'json' }).$type<
|
||||
{ tag: string; title: string }[]
|
||||
>(),
|
||||
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
@@ -179,9 +191,9 @@ Key-value store for global application settings.
|
||||
|
||||
```typescript
|
||||
export const settings = sqliteTable('settings', {
|
||||
key: text('key').primaryKey(),
|
||||
value: text('value', { mode: 'json' }),
|
||||
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(),
|
||||
key: text('key').primaryKey(),
|
||||
value: text('value', { mode: 'json' }),
|
||||
updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull()
|
||||
});
|
||||
```
|
||||
|
||||
|
||||
@@ -31,10 +31,12 @@ Implement the core `RepositoryService` that handles CRUD operations for reposito
|
||||
## Repository ID Generation
|
||||
|
||||
GitHub repositories:
|
||||
|
||||
- Input URL: `https://github.com/facebook/react` or `github.com/facebook/react`
|
||||
- Generated ID: `/facebook/react`
|
||||
|
||||
Local repositories:
|
||||
|
||||
- Input path: `/home/user/projects/my-sdk`
|
||||
- Generated ID: `/local/my-sdk` (basename of path, slugified)
|
||||
- Collision resolution: append `-2`, `-3`, etc.
|
||||
@@ -49,44 +51,44 @@ Version-specific IDs: `/facebook/react/v18.3.0`
|
||||
// src/lib/server/services/repository.service.ts
|
||||
|
||||
export interface AddRepositoryInput {
|
||||
source: 'github' | 'local';
|
||||
sourceUrl: string; // GitHub URL or absolute local path
|
||||
title?: string; // override auto-detected title
|
||||
description?: string;
|
||||
branch?: string; // GitHub: default branch; Local: n/a
|
||||
githubToken?: string; // for private GitHub repos
|
||||
source: 'github' | 'local';
|
||||
sourceUrl: string; // GitHub URL or absolute local path
|
||||
title?: string; // override auto-detected title
|
||||
description?: string;
|
||||
branch?: string; // GitHub: default branch; Local: n/a
|
||||
githubToken?: string; // for private GitHub repos
|
||||
}
|
||||
|
||||
export interface UpdateRepositoryInput {
|
||||
title?: string;
|
||||
description?: string;
|
||||
branch?: string;
|
||||
githubToken?: string;
|
||||
title?: string;
|
||||
description?: string;
|
||||
branch?: string;
|
||||
githubToken?: string;
|
||||
}
|
||||
|
||||
export class RepositoryService {
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
|
||||
async list(options?: {
|
||||
state?: Repository['state'];
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}): Promise<Repository[]>
|
||||
async list(options?: {
|
||||
state?: Repository['state'];
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}): Promise<Repository[]>;
|
||||
|
||||
async get(id: string): Promise<Repository | null>
|
||||
async get(id: string): Promise<Repository | null>;
|
||||
|
||||
async add(input: AddRepositoryInput): Promise<Repository>
|
||||
async add(input: AddRepositoryInput): Promise<Repository>;
|
||||
|
||||
async update(id: string, input: UpdateRepositoryInput): Promise<Repository>
|
||||
async update(id: string, input: UpdateRepositoryInput): Promise<Repository>;
|
||||
|
||||
async remove(id: string): Promise<void>
|
||||
async remove(id: string): Promise<void>;
|
||||
|
||||
async getStats(id: string): Promise<{
|
||||
totalSnippets: number;
|
||||
totalTokens: number;
|
||||
totalDocuments: number;
|
||||
lastIndexedAt: Date | null;
|
||||
}>
|
||||
async getStats(id: string): Promise<{
|
||||
totalSnippets: number;
|
||||
totalTokens: number;
|
||||
totalDocuments: number;
|
||||
lastIndexedAt: Date | null;
|
||||
}>;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -97,48 +99,52 @@ export class RepositoryService {
|
||||
### `GET /api/v1/libs`
|
||||
|
||||
Query parameters:
|
||||
|
||||
- `state` (optional): filter by state (`pending`, `indexed`, `error`, etc.)
|
||||
- `limit` (optional, default 50): max results
|
||||
- `offset` (optional, default 0): pagination offset
|
||||
|
||||
Response `200`:
|
||||
|
||||
```json
|
||||
{
|
||||
"libraries": [
|
||||
{
|
||||
"id": "/facebook/react",
|
||||
"title": "React",
|
||||
"description": "...",
|
||||
"source": "github",
|
||||
"state": "indexed",
|
||||
"totalSnippets": 1234,
|
||||
"totalTokens": 98000,
|
||||
"trustScore": 8.5,
|
||||
"stars": 228000,
|
||||
"lastIndexedAt": "2026-03-22T10:00:00Z",
|
||||
"versions": ["v18.3.0", "v17.0.2"]
|
||||
}
|
||||
],
|
||||
"total": 12,
|
||||
"limit": 50,
|
||||
"offset": 0
|
||||
"libraries": [
|
||||
{
|
||||
"id": "/facebook/react",
|
||||
"title": "React",
|
||||
"description": "...",
|
||||
"source": "github",
|
||||
"state": "indexed",
|
||||
"totalSnippets": 1234,
|
||||
"totalTokens": 98000,
|
||||
"trustScore": 8.5,
|
||||
"stars": 228000,
|
||||
"lastIndexedAt": "2026-03-22T10:00:00Z",
|
||||
"versions": ["v18.3.0", "v17.0.2"]
|
||||
}
|
||||
],
|
||||
"total": 12,
|
||||
"limit": 50,
|
||||
"offset": 0
|
||||
}
|
||||
```
|
||||
|
||||
### `POST /api/v1/libs`
|
||||
|
||||
Request body:
|
||||
|
||||
```json
|
||||
{
|
||||
"source": "github",
|
||||
"sourceUrl": "https://github.com/facebook/react",
|
||||
"branch": "main",
|
||||
"githubToken": "ghp_...",
|
||||
"autoIndex": true
|
||||
"source": "github",
|
||||
"sourceUrl": "https://github.com/facebook/react",
|
||||
"branch": "main",
|
||||
"githubToken": "ghp_...",
|
||||
"autoIndex": true
|
||||
}
|
||||
```
|
||||
|
||||
Response `201`:
|
||||
|
||||
```json
|
||||
{
|
||||
"library": { ...Repository },
|
||||
@@ -149,6 +155,7 @@ Response `201`:
|
||||
`autoIndex: true` (default) immediately queues an indexing job.
|
||||
|
||||
Response `409` if repository already exists:
|
||||
|
||||
```json
|
||||
{ "error": "Repository /facebook/react already exists" }
|
||||
```
|
||||
@@ -176,20 +183,22 @@ Response `404`: not found.
|
||||
Triggers a new indexing job. If a job is already running for this repo, returns the existing job.
|
||||
|
||||
Request body (optional):
|
||||
|
||||
```json
|
||||
{ "version": "v18.3.0" }
|
||||
```
|
||||
|
||||
Response `202`:
|
||||
|
||||
```json
|
||||
{
|
||||
"job": {
|
||||
"id": "uuid",
|
||||
"repositoryId": "/facebook/react",
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"createdAt": "2026-03-22T10:00:00Z"
|
||||
}
|
||||
"job": {
|
||||
"id": "uuid",
|
||||
"repositoryId": "/facebook/react",
|
||||
"status": "queued",
|
||||
"progress": 0,
|
||||
"createdAt": "2026-03-22T10:00:00Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -198,15 +207,17 @@ Response `202`:
|
||||
## Error Response Shape
|
||||
|
||||
All error responses follow:
|
||||
|
||||
```json
|
||||
{
|
||||
"error": "Human-readable message",
|
||||
"code": "MACHINE_READABLE_CODE",
|
||||
"details": {}
|
||||
"error": "Human-readable message",
|
||||
"code": "MACHINE_READABLE_CODE",
|
||||
"details": {}
|
||||
}
|
||||
```
|
||||
|
||||
Error codes:
|
||||
|
||||
- `NOT_FOUND`
|
||||
- `ALREADY_EXISTS`
|
||||
- `INVALID_INPUT`
|
||||
@@ -219,23 +230,23 @@ Error codes:
|
||||
|
||||
```typescript
|
||||
function resolveGitHubId(url: string): string {
|
||||
// Parse owner/repo from URL variants:
|
||||
// https://github.com/facebook/react
|
||||
// https://github.com/facebook/react.git
|
||||
// github.com/facebook/react
|
||||
const match = url.match(/github\.com\/([^/]+)\/([^/\s.]+)/);
|
||||
if (!match) throw new Error('Invalid GitHub URL');
|
||||
return `/${match[1]}/${match[2]}`;
|
||||
// Parse owner/repo from URL variants:
|
||||
// https://github.com/facebook/react
|
||||
// https://github.com/facebook/react.git
|
||||
// github.com/facebook/react
|
||||
const match = url.match(/github\.com\/([^/]+)\/([^/\s.]+)/);
|
||||
if (!match) throw new Error('Invalid GitHub URL');
|
||||
return `/${match[1]}/${match[2]}`;
|
||||
}
|
||||
|
||||
function resolveLocalId(path: string, existingIds: string[]): string {
|
||||
const base = slugify(path.split('/').at(-1)!);
|
||||
let id = `/local/${base}`;
|
||||
let counter = 2;
|
||||
while (existingIds.includes(id)) {
|
||||
id = `/local/${base}-${counter++}`;
|
||||
}
|
||||
return id;
|
||||
const base = slugify(path.split('/').at(-1)!);
|
||||
let id = `/local/${base}`;
|
||||
let counter = 2;
|
||||
while (existingIds.includes(id)) {
|
||||
id = `/local/${base}-${counter++}`;
|
||||
}
|
||||
return id;
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -37,17 +37,46 @@ The crawler only downloads files with these extensions:
|
||||
|
||||
```typescript
|
||||
const INDEXABLE_EXTENSIONS = new Set([
|
||||
// Documentation
|
||||
'.md', '.mdx', '.txt', '.rst',
|
||||
// Code
|
||||
'.ts', '.tsx', '.js', '.jsx',
|
||||
'.py', '.rb', '.go', '.rs', '.java', '.cs', '.cpp', '.c', '.h',
|
||||
'.swift', '.kt', '.php', '.scala', '.clj', '.ex', '.exs',
|
||||
'.sh', '.bash', '.zsh', '.fish',
|
||||
// Config / data
|
||||
'.json', '.yaml', '.yml', '.toml',
|
||||
// Web
|
||||
'.html', '.css', '.svelte', '.vue',
|
||||
// Documentation
|
||||
'.md',
|
||||
'.mdx',
|
||||
'.txt',
|
||||
'.rst',
|
||||
// Code
|
||||
'.ts',
|
||||
'.tsx',
|
||||
'.js',
|
||||
'.jsx',
|
||||
'.py',
|
||||
'.rb',
|
||||
'.go',
|
||||
'.rs',
|
||||
'.java',
|
||||
'.cs',
|
||||
'.cpp',
|
||||
'.c',
|
||||
'.h',
|
||||
'.swift',
|
||||
'.kt',
|
||||
'.php',
|
||||
'.scala',
|
||||
'.clj',
|
||||
'.ex',
|
||||
'.exs',
|
||||
'.sh',
|
||||
'.bash',
|
||||
'.zsh',
|
||||
'.fish',
|
||||
// Config / data
|
||||
'.json',
|
||||
'.yaml',
|
||||
'.yml',
|
||||
'.toml',
|
||||
// Web
|
||||
'.html',
|
||||
'.css',
|
||||
'.svelte',
|
||||
'.vue'
|
||||
]);
|
||||
|
||||
const MAX_FILE_SIZE_BYTES = 500_000; // 500 KB — skip large generated files
|
||||
@@ -59,28 +88,28 @@ const MAX_FILE_SIZE_BYTES = 500_000; // 500 KB — skip large generated files
|
||||
|
||||
```typescript
|
||||
export interface CrawledFile {
|
||||
path: string; // relative path within repo, e.g. "src/index.ts"
|
||||
content: string; // UTF-8 file content
|
||||
size: number; // bytes
|
||||
sha: string; // GitHub blob SHA (used as checksum)
|
||||
language: string; // detected from extension
|
||||
path: string; // relative path within repo, e.g. "src/index.ts"
|
||||
content: string; // UTF-8 file content
|
||||
size: number; // bytes
|
||||
sha: string; // GitHub blob SHA (used as checksum)
|
||||
language: string; // detected from extension
|
||||
}
|
||||
|
||||
export interface CrawlResult {
|
||||
files: CrawledFile[];
|
||||
totalFiles: number; // files matching filters
|
||||
skippedFiles: number; // filtered out or too large
|
||||
branch: string; // branch/tag that was crawled
|
||||
commitSha: string; // HEAD commit SHA
|
||||
files: CrawledFile[];
|
||||
totalFiles: number; // files matching filters
|
||||
skippedFiles: number; // filtered out or too large
|
||||
branch: string; // branch/tag that was crawled
|
||||
commitSha: string; // HEAD commit SHA
|
||||
}
|
||||
|
||||
export interface CrawlOptions {
|
||||
owner: string;
|
||||
repo: string;
|
||||
ref?: string; // branch, tag, or commit SHA; defaults to repo default branch
|
||||
token?: string; // GitHub PAT for private repos
|
||||
config?: RepoConfig; // parsed trueref.json
|
||||
onProgress?: (processed: number, total: number) => void;
|
||||
owner: string;
|
||||
repo: string;
|
||||
ref?: string; // branch, tag, or commit SHA; defaults to repo default branch
|
||||
token?: string; // GitHub PAT for private repos
|
||||
config?: RepoConfig; // parsed trueref.json
|
||||
onProgress?: (processed: number, total: number) => void;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -89,12 +118,14 @@ export interface CrawlOptions {
|
||||
## GitHub API Usage
|
||||
|
||||
### Step 1: Get default branch (if ref not specified)
|
||||
|
||||
```
|
||||
GET https://api.github.com/repos/{owner}/{repo}
|
||||
→ { default_branch: "main", stargazers_count: 12345 }
|
||||
```
|
||||
|
||||
### Step 2: Fetch file tree (recursive)
|
||||
|
||||
```
|
||||
GET https://api.github.com/repos/{owner}/{repo}/git/trees/{ref}?recursive=1
|
||||
→ {
|
||||
@@ -109,12 +140,14 @@ GET https://api.github.com/repos/{owner}/{repo}/git/trees/{ref}?recursive=1
|
||||
If `truncated: true`, the tree has >100k items. Use `--depth` pagination or filter top-level directories first.
|
||||
|
||||
### Step 3: Download file contents (parallel)
|
||||
|
||||
```
|
||||
GET https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={ref}
|
||||
→ { content: "<base64>", encoding: "base64", size: 1234, sha: "abc123" }
|
||||
```
|
||||
|
||||
Alternative for large repos: use raw content URL:
|
||||
|
||||
```
|
||||
GET https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}
|
||||
```
|
||||
@@ -124,48 +157,47 @@ GET https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}
|
||||
## Filtering Logic
|
||||
|
||||
```typescript
|
||||
function shouldIndexFile(
|
||||
filePath: string,
|
||||
fileSize: number,
|
||||
config?: RepoConfig
|
||||
): boolean {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const base = path.basename(filePath);
|
||||
function shouldIndexFile(filePath: string, fileSize: number, config?: RepoConfig): boolean {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const base = path.basename(filePath);
|
||||
|
||||
// 1. Must have indexable extension
|
||||
if (!INDEXABLE_EXTENSIONS.has(ext)) return false;
|
||||
// 1. Must have indexable extension
|
||||
if (!INDEXABLE_EXTENSIONS.has(ext)) return false;
|
||||
|
||||
// 2. Must not exceed size limit
|
||||
if (fileSize > MAX_FILE_SIZE_BYTES) return false;
|
||||
// 2. Must not exceed size limit
|
||||
if (fileSize > MAX_FILE_SIZE_BYTES) return false;
|
||||
|
||||
// 3. Exclude lockfiles and other non-source artifacts
|
||||
if (IGNORED_FILE_NAMES.has(base)) return false;
|
||||
// 3. Exclude lockfiles and other non-source artifacts
|
||||
if (IGNORED_FILE_NAMES.has(base)) return false;
|
||||
|
||||
// 4. Exclude minified and bundled assets
|
||||
if (base.includes('.min.') || base.endsWith('.bundle.js') || base.endsWith('.bundle.css')) {
|
||||
return false;
|
||||
}
|
||||
// 4. Exclude minified and bundled assets
|
||||
if (base.includes('.min.') || base.endsWith('.bundle.js') || base.endsWith('.bundle.css')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 5. Apply config excludeFiles (exact filename match)
|
||||
if (config?.excludeFiles?.includes(base)) return false;
|
||||
// 5. Apply config excludeFiles (exact filename match)
|
||||
if (config?.excludeFiles?.includes(base)) return false;
|
||||
|
||||
// 6. Exclude common dependency/build/cache directories at any depth
|
||||
if (isInIgnoredDirectory(filePath)) return false;
|
||||
// 6. Exclude common dependency/build/cache directories at any depth
|
||||
if (isInIgnoredDirectory(filePath)) return false;
|
||||
|
||||
// 7. Apply config excludeFolders (regex or prefix match)
|
||||
if (config?.excludeFolders?.some(folder =>
|
||||
filePath.startsWith(folder) || new RegExp(folder).test(filePath)
|
||||
)) return false;
|
||||
// 7. Apply config excludeFolders (regex or prefix match)
|
||||
if (
|
||||
config?.excludeFolders?.some(
|
||||
(folder) => filePath.startsWith(folder) || new RegExp(folder).test(filePath)
|
||||
)
|
||||
)
|
||||
return false;
|
||||
|
||||
// 8. Apply config folders allowlist (if specified, only index those paths)
|
||||
if (config?.folders?.length) {
|
||||
const inAllowedFolder = config.folders.some(folder =>
|
||||
filePath.startsWith(folder) || new RegExp(folder).test(filePath)
|
||||
);
|
||||
if (!inAllowedFolder) return false;
|
||||
}
|
||||
// 8. Apply config folders allowlist (if specified, only index those paths)
|
||||
if (config?.folders?.length) {
|
||||
const inAllowedFolder = config.folders.some(
|
||||
(folder) => filePath.startsWith(folder) || new RegExp(folder).test(filePath)
|
||||
);
|
||||
if (!inAllowedFolder) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
return true;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -177,20 +209,20 @@ The shared ignored-directory list is intentionally broader than the original bas
|
||||
|
||||
```typescript
|
||||
class GitHubRateLimiter {
|
||||
private remaining = 5000;
|
||||
private resetAt = Date.now();
|
||||
private remaining = 5000;
|
||||
private resetAt = Date.now();
|
||||
|
||||
updateFromHeaders(headers: Headers): void {
|
||||
this.remaining = parseInt(headers.get('X-RateLimit-Remaining') ?? '5000');
|
||||
this.resetAt = parseInt(headers.get('X-RateLimit-Reset') ?? '0') * 1000;
|
||||
}
|
||||
updateFromHeaders(headers: Headers): void {
|
||||
this.remaining = parseInt(headers.get('X-RateLimit-Remaining') ?? '5000');
|
||||
this.resetAt = parseInt(headers.get('X-RateLimit-Reset') ?? '0') * 1000;
|
||||
}
|
||||
|
||||
async waitIfNeeded(): Promise<void> {
|
||||
if (this.remaining <= 10) {
|
||||
const waitMs = Math.max(0, this.resetAt - Date.now()) + 1000;
|
||||
await sleep(waitMs);
|
||||
}
|
||||
}
|
||||
async waitIfNeeded(): Promise<void> {
|
||||
if (this.remaining <= 10) {
|
||||
const waitMs = Math.max(0, this.resetAt - Date.now()) + 1000;
|
||||
await sleep(waitMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -200,14 +232,14 @@ Requests are made with a concurrency limit of 10 parallel downloads using a sema
|
||||
|
||||
## Error Handling
|
||||
|
||||
| Scenario | Behavior |
|
||||
|----------|---------|
|
||||
| 404 Not Found | Throw `RepositoryNotFoundError` |
|
||||
| 401 Unauthorized | Throw `AuthenticationError` (invalid or missing token) |
|
||||
| 403 Forbidden | If `X-RateLimit-Remaining: 0`, wait and retry; else throw `PermissionError` |
|
||||
| 422 Unprocessable | Tree too large; switch to directory-by-directory traversal |
|
||||
| Network error | Retry up to 3 times with exponential backoff |
|
||||
| File content decode error | Skip file, log warning |
|
||||
| Scenario | Behavior |
|
||||
| ------------------------- | --------------------------------------------------------------------------- |
|
||||
| 404 Not Found | Throw `RepositoryNotFoundError` |
|
||||
| 401 Unauthorized | Throw `AuthenticationError` (invalid or missing token) |
|
||||
| 403 Forbidden | If `X-RateLimit-Remaining: 0`, wait and retry; else throw `PermissionError` |
|
||||
| 422 Unprocessable | Tree too large; switch to directory-by-directory traversal |
|
||||
| Network error | Retry up to 3 times with exponential backoff |
|
||||
| File content decode error | Skip file, log warning |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -38,9 +38,9 @@ Reuses `CrawledFile` and `CrawlResult` from TRUEREF-0003 crawler types:
|
||||
|
||||
```typescript
|
||||
export interface LocalCrawlOptions {
|
||||
rootPath: string; // absolute path to repository root
|
||||
config?: RepoConfig; // parsed trueref.json
|
||||
onProgress?: (processed: number, total: number) => void;
|
||||
rootPath: string; // absolute path to repository root
|
||||
config?: RepoConfig; // parsed trueref.json
|
||||
onProgress?: (processed: number, total: number) => void;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -50,75 +50,73 @@ export interface LocalCrawlOptions {
|
||||
|
||||
```typescript
|
||||
export class LocalCrawler {
|
||||
async crawl(options: LocalCrawlOptions): Promise<CrawlResult> {
|
||||
// 1. Load root .gitignore if present
|
||||
const gitignore = await this.loadGitignore(options.rootPath);
|
||||
async crawl(options: LocalCrawlOptions): Promise<CrawlResult> {
|
||||
// 1. Load root .gitignore if present
|
||||
const gitignore = await this.loadGitignore(options.rootPath);
|
||||
|
||||
// 2. Enumerate files recursively, pruning ignored directories early
|
||||
const allFiles = await this.walkDirectory(options.rootPath, '', gitignore);
|
||||
// 2. Enumerate files recursively, pruning ignored directories early
|
||||
const allFiles = await this.walkDirectory(options.rootPath, '', gitignore);
|
||||
|
||||
// 3. Look for trueref.json / context7.json first
|
||||
const configFile = allFiles.find(f =>
|
||||
f === 'trueref.json' || f === 'context7.json'
|
||||
);
|
||||
let config = options.config;
|
||||
if (configFile && !config) {
|
||||
config = await this.parseConfigFile(
|
||||
path.join(options.rootPath, configFile)
|
||||
);
|
||||
}
|
||||
// 3. Look for trueref.json / context7.json first
|
||||
const configFile = allFiles.find((f) => f === 'trueref.json' || f === 'context7.json');
|
||||
let config = options.config;
|
||||
if (configFile && !config) {
|
||||
config = await this.parseConfigFile(path.join(options.rootPath, configFile));
|
||||
}
|
||||
|
||||
// 4. Filter files
|
||||
const filteredFiles = allFiles.filter(relPath => {
|
||||
const stat = statCache.get(relPath);
|
||||
return shouldIndexFile(relPath, stat.size, config);
|
||||
});
|
||||
// 4. Filter files
|
||||
const filteredFiles = allFiles.filter((relPath) => {
|
||||
const stat = statCache.get(relPath);
|
||||
return shouldIndexFile(relPath, stat.size, config);
|
||||
});
|
||||
|
||||
// 5. Read and return file contents
|
||||
const crawledFiles: CrawledFile[] = [];
|
||||
for (const [i, relPath] of filteredFiles.entries()) {
|
||||
const absPath = path.join(options.rootPath, relPath);
|
||||
const content = await fs.readFile(absPath, 'utf-8');
|
||||
const sha = computeSHA256(content);
|
||||
crawledFiles.push({
|
||||
path: relPath,
|
||||
content,
|
||||
size: Buffer.byteLength(content, 'utf-8'),
|
||||
sha,
|
||||
language: detectLanguage(relPath),
|
||||
});
|
||||
options.onProgress?.(i + 1, filteredFiles.length);
|
||||
}
|
||||
// 5. Read and return file contents
|
||||
const crawledFiles: CrawledFile[] = [];
|
||||
for (const [i, relPath] of filteredFiles.entries()) {
|
||||
const absPath = path.join(options.rootPath, relPath);
|
||||
const content = await fs.readFile(absPath, 'utf-8');
|
||||
const sha = computeSHA256(content);
|
||||
crawledFiles.push({
|
||||
path: relPath,
|
||||
content,
|
||||
size: Buffer.byteLength(content, 'utf-8'),
|
||||
sha,
|
||||
language: detectLanguage(relPath)
|
||||
});
|
||||
options.onProgress?.(i + 1, filteredFiles.length);
|
||||
}
|
||||
|
||||
return {
|
||||
files: crawledFiles,
|
||||
totalFiles: filteredFiles.length,
|
||||
skippedFiles: allFiles.length - filteredFiles.length,
|
||||
branch: 'local',
|
||||
commitSha: computeSHA256(crawledFiles.map(f => f.sha).join('')),
|
||||
};
|
||||
}
|
||||
return {
|
||||
files: crawledFiles,
|
||||
totalFiles: filteredFiles.length,
|
||||
skippedFiles: allFiles.length - filteredFiles.length,
|
||||
branch: 'local',
|
||||
commitSha: computeSHA256(crawledFiles.map((f) => f.sha).join(''))
|
||||
};
|
||||
}
|
||||
|
||||
private async walkDirectory(dir: string, rel = '', gitignore?: GitignoreFilter): Promise<string[]> {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
const files: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() && !entry.isDirectory()) continue; // skip symlinks, devices
|
||||
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
if (shouldPruneDirectory(relPath) || gitignore?.isIgnored(relPath, true)) {
|
||||
continue;
|
||||
}
|
||||
files.push(...await this.walkDirectory(
|
||||
path.join(dir, entry.name), relPath, gitignore
|
||||
));
|
||||
} else {
|
||||
if (gitignore?.isIgnored(relPath, false)) continue;
|
||||
files.push(relPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
private async walkDirectory(
|
||||
dir: string,
|
||||
rel = '',
|
||||
gitignore?: GitignoreFilter
|
||||
): Promise<string[]> {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
const files: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() && !entry.isDirectory()) continue; // skip symlinks, devices
|
||||
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
if (shouldPruneDirectory(relPath) || gitignore?.isIgnored(relPath, true)) {
|
||||
continue;
|
||||
}
|
||||
files.push(...(await this.walkDirectory(path.join(dir, entry.name), relPath, gitignore)));
|
||||
} else {
|
||||
if (gitignore?.isIgnored(relPath, false)) continue;
|
||||
files.push(relPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -142,7 +140,7 @@ Directory pruning should happen during the walk so large dependency trees are ne
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
function computeSHA256(content: string): string {
|
||||
return createHash('sha256').update(content, 'utf-8').digest('hex');
|
||||
return createHash('sha256').update(content, 'utf-8').digest('hex');
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -30,19 +30,19 @@ Implement the document parsing and chunking pipeline that transforms raw file co
|
||||
|
||||
## Supported File Types
|
||||
|
||||
| Extension | Parser Strategy |
|
||||
|-----------|----------------|
|
||||
| `.md`, `.mdx` | Heading-based section splitting + code block extraction |
|
||||
| `.txt`, `.rst` | Paragraph-based splitting |
|
||||
| `.ts`, `.tsx`, `.js`, `.jsx` | AST-free: function/class boundary detection via regex |
|
||||
| `.py` | `def`/`class` boundary detection |
|
||||
| `.go` | `func`/`type` boundary detection |
|
||||
| `.rs` | `fn`/`impl`/`struct` boundary detection |
|
||||
| `.java`, `.cs`, `.kt`, `.swift` | Class/method boundary detection |
|
||||
| `.rb` | `def`/`class` boundary detection |
|
||||
| `.json`, `.yaml`, `.yml`, `.toml` | Structural chunking (top-level keys) |
|
||||
| `.html`, `.svelte`, `.vue` | Text content extraction + script block splitting |
|
||||
| Other code | Line-count-based sliding window (200 lines per chunk) |
|
||||
| Extension | Parser Strategy |
|
||||
| --------------------------------- | ------------------------------------------------------- |
|
||||
| `.md`, `.mdx` | Heading-based section splitting + code block extraction |
|
||||
| `.txt`, `.rst` | Paragraph-based splitting |
|
||||
| `.ts`, `.tsx`, `.js`, `.jsx` | AST-free: function/class boundary detection via regex |
|
||||
| `.py` | `def`/`class` boundary detection |
|
||||
| `.go` | `func`/`type` boundary detection |
|
||||
| `.rs` | `fn`/`impl`/`struct` boundary detection |
|
||||
| `.java`, `.cs`, `.kt`, `.swift` | Class/method boundary detection |
|
||||
| `.rb` | `def`/`class` boundary detection |
|
||||
| `.json`, `.yaml`, `.yml`, `.toml` | Structural chunking (top-level keys) |
|
||||
| `.html`, `.svelte`, `.vue` | Text content extraction + script block splitting |
|
||||
| Other code | Line-count-based sliding window (200 lines per chunk) |
|
||||
|
||||
---
|
||||
|
||||
@@ -52,9 +52,9 @@ Use a simple character-based approximation (no tokenizer library needed for v1):
|
||||
|
||||
```typescript
|
||||
function estimateTokens(text: string): number {
|
||||
// Empirically: ~4 chars per token for English prose
|
||||
// ~3 chars per token for code (more symbols)
|
||||
return Math.ceil(text.length / 3.5);
|
||||
// Empirically: ~4 chars per token for English prose
|
||||
// ~3 chars per token for code (more symbols)
|
||||
return Math.ceil(text.length / 3.5);
|
||||
}
|
||||
```
|
||||
|
||||
@@ -74,49 +74,49 @@ The Markdown parser is the most important parser as most documentation is Markdo
|
||||
|
||||
```typescript
|
||||
interface MarkdownSection {
|
||||
headings: string[]; // heading stack at this point
|
||||
content: string; // text content (sans code blocks)
|
||||
codeBlocks: { language: string; code: string }[];
|
||||
headings: string[]; // heading stack at this point
|
||||
content: string; // text content (sans code blocks)
|
||||
codeBlocks: { language: string; code: string }[];
|
||||
}
|
||||
|
||||
function parseMarkdown(content: string, filePath: string): Snippet[] {
|
||||
const sections = splitIntoSections(content);
|
||||
const snippets: Snippet[] = [];
|
||||
const sections = splitIntoSections(content);
|
||||
const snippets: Snippet[] = [];
|
||||
|
||||
for (const section of sections) {
|
||||
const breadcrumb = section.headings.join(' > ');
|
||||
const title = section.headings.at(-1) ?? path.basename(filePath);
|
||||
for (const section of sections) {
|
||||
const breadcrumb = section.headings.join(' > ');
|
||||
const title = section.headings.at(-1) ?? path.basename(filePath);
|
||||
|
||||
// Emit info snippet for text content
|
||||
if (section.content.trim().length >= 20) {
|
||||
const chunks = chunkText(section.content, MAX_TOKENS, OVERLAP_TOKENS);
|
||||
for (const chunk of chunks) {
|
||||
snippets.push({
|
||||
type: 'info',
|
||||
title,
|
||||
content: chunk,
|
||||
breadcrumb,
|
||||
tokenCount: estimateTokens(chunk),
|
||||
});
|
||||
}
|
||||
}
|
||||
// Emit info snippet for text content
|
||||
if (section.content.trim().length >= 20) {
|
||||
const chunks = chunkText(section.content, MAX_TOKENS, OVERLAP_TOKENS);
|
||||
for (const chunk of chunks) {
|
||||
snippets.push({
|
||||
type: 'info',
|
||||
title,
|
||||
content: chunk,
|
||||
breadcrumb,
|
||||
tokenCount: estimateTokens(chunk)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Emit code snippets for each code block
|
||||
for (const block of section.codeBlocks) {
|
||||
if (block.code.trim().length >= 20) {
|
||||
snippets.push({
|
||||
type: 'code',
|
||||
title,
|
||||
content: block.code,
|
||||
language: block.language || detectLanguage('.' + block.language),
|
||||
breadcrumb,
|
||||
tokenCount: estimateTokens(block.code),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// Emit code snippets for each code block
|
||||
for (const block of section.codeBlocks) {
|
||||
if (block.code.trim().length >= 20) {
|
||||
snippets.push({
|
||||
type: 'code',
|
||||
title,
|
||||
content: block.code,
|
||||
language: block.language || detectLanguage('.' + block.language),
|
||||
breadcrumb,
|
||||
tokenCount: estimateTokens(block.code)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return snippets;
|
||||
return snippets;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -135,43 +135,41 @@ For non-Markdown code files, use regex-based function/class boundary detection.
|
||||
|
||||
```typescript
|
||||
const BOUNDARY_PATTERNS: Record<string, RegExp> = {
|
||||
typescript: /^(export\s+)?(async\s+)?(function|class|interface|type|const|let|var)\s+\w+/m,
|
||||
python: /^(async\s+)?(def|class)\s+\w+/m,
|
||||
go: /^(func|type|var|const)\s+\w+/m,
|
||||
rust: /^(pub\s+)?(fn|impl|struct|enum|trait)\s+\w+/m,
|
||||
java: /^(public|private|protected|static).*?(class|interface|enum|void|\w+)\s+\w+\s*[({]/m,
|
||||
typescript: /^(export\s+)?(async\s+)?(function|class|interface|type|const|let|var)\s+\w+/m,
|
||||
python: /^(async\s+)?(def|class)\s+\w+/m,
|
||||
go: /^(func|type|var|const)\s+\w+/m,
|
||||
rust: /^(pub\s+)?(fn|impl|struct|enum|trait)\s+\w+/m,
|
||||
java: /^(public|private|protected|static).*?(class|interface|enum|void|\w+)\s+\w+\s*[({]/m
|
||||
};
|
||||
|
||||
function parseCodeFile(
|
||||
content: string,
|
||||
filePath: string,
|
||||
language: string
|
||||
): Snippet[] {
|
||||
const pattern = BOUNDARY_PATTERNS[language];
|
||||
const breadcrumb = filePath;
|
||||
const title = path.basename(filePath);
|
||||
function parseCodeFile(content: string, filePath: string, language: string): Snippet[] {
|
||||
const pattern = BOUNDARY_PATTERNS[language];
|
||||
const breadcrumb = filePath;
|
||||
const title = path.basename(filePath);
|
||||
|
||||
if (!pattern) {
|
||||
// Fallback: sliding window
|
||||
return slidingWindowChunks(content, filePath, language);
|
||||
}
|
||||
if (!pattern) {
|
||||
// Fallback: sliding window
|
||||
return slidingWindowChunks(content, filePath, language);
|
||||
}
|
||||
|
||||
const chunks = splitAtBoundaries(content, pattern);
|
||||
return chunks
|
||||
.filter(chunk => chunk.trim().length >= 20)
|
||||
.flatMap(chunk => {
|
||||
if (estimateTokens(chunk) <= MAX_TOKENS) {
|
||||
return [{
|
||||
type: 'code' as const,
|
||||
title,
|
||||
content: chunk,
|
||||
language,
|
||||
breadcrumb,
|
||||
tokenCount: estimateTokens(chunk),
|
||||
}];
|
||||
}
|
||||
return slidingWindowChunks(chunk, filePath, language);
|
||||
});
|
||||
const chunks = splitAtBoundaries(content, pattern);
|
||||
return chunks
|
||||
.filter((chunk) => chunk.trim().length >= 20)
|
||||
.flatMap((chunk) => {
|
||||
if (estimateTokens(chunk) <= MAX_TOKENS) {
|
||||
return [
|
||||
{
|
||||
type: 'code' as const,
|
||||
title,
|
||||
content: chunk,
|
||||
language,
|
||||
breadcrumb,
|
||||
tokenCount: estimateTokens(chunk)
|
||||
}
|
||||
];
|
||||
}
|
||||
return slidingWindowChunks(chunk, filePath, language);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
@@ -188,27 +186,23 @@ const MIN_CONTENT_LENGTH = 20; // characters
|
||||
### Sliding Window Chunker
|
||||
|
||||
```typescript
|
||||
function chunkText(
|
||||
text: string,
|
||||
maxTokens: number,
|
||||
overlapTokens: number
|
||||
): string[] {
|
||||
const words = text.split(/\s+/);
|
||||
const wordsPerToken = 0.75; // ~0.75 words per token
|
||||
const maxWords = Math.floor(maxTokens * wordsPerToken);
|
||||
const overlapWords = Math.floor(overlapTokens * wordsPerToken);
|
||||
function chunkText(text: string, maxTokens: number, overlapTokens: number): string[] {
|
||||
const words = text.split(/\s+/);
|
||||
const wordsPerToken = 0.75; // ~0.75 words per token
|
||||
const maxWords = Math.floor(maxTokens * wordsPerToken);
|
||||
const overlapWords = Math.floor(overlapTokens * wordsPerToken);
|
||||
|
||||
const chunks: string[] = [];
|
||||
let start = 0;
|
||||
const chunks: string[] = [];
|
||||
let start = 0;
|
||||
|
||||
while (start < words.length) {
|
||||
const end = Math.min(start + maxWords, words.length);
|
||||
chunks.push(words.slice(start, end).join(' '));
|
||||
if (end === words.length) break;
|
||||
start = end - overlapWords;
|
||||
}
|
||||
while (start < words.length) {
|
||||
const end = Math.min(start + maxWords, words.length);
|
||||
chunks.push(words.slice(start, end).join(' '));
|
||||
if (end === words.length) break;
|
||||
start = end - overlapWords;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
return chunks;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -218,34 +212,42 @@ function chunkText(
|
||||
|
||||
```typescript
|
||||
const LANGUAGE_MAP: Record<string, string> = {
|
||||
'.ts': 'typescript', '.tsx': 'typescript',
|
||||
'.js': 'javascript', '.jsx': 'javascript',
|
||||
'.py': 'python',
|
||||
'.rb': 'ruby',
|
||||
'.go': 'go',
|
||||
'.rs': 'rust',
|
||||
'.java': 'java',
|
||||
'.cs': 'csharp',
|
||||
'.cpp': 'cpp', '.c': 'c', '.h': 'c',
|
||||
'.swift': 'swift',
|
||||
'.kt': 'kotlin',
|
||||
'.php': 'php',
|
||||
'.scala': 'scala',
|
||||
'.sh': 'bash', '.bash': 'bash', '.zsh': 'bash',
|
||||
'.md': 'markdown', '.mdx': 'markdown',
|
||||
'.json': 'json',
|
||||
'.yaml': 'yaml', '.yml': 'yaml',
|
||||
'.toml': 'toml',
|
||||
'.html': 'html',
|
||||
'.css': 'css',
|
||||
'.svelte': 'svelte',
|
||||
'.vue': 'vue',
|
||||
'.sql': 'sql',
|
||||
'.ts': 'typescript',
|
||||
'.tsx': 'typescript',
|
||||
'.js': 'javascript',
|
||||
'.jsx': 'javascript',
|
||||
'.py': 'python',
|
||||
'.rb': 'ruby',
|
||||
'.go': 'go',
|
||||
'.rs': 'rust',
|
||||
'.java': 'java',
|
||||
'.cs': 'csharp',
|
||||
'.cpp': 'cpp',
|
||||
'.c': 'c',
|
||||
'.h': 'c',
|
||||
'.swift': 'swift',
|
||||
'.kt': 'kotlin',
|
||||
'.php': 'php',
|
||||
'.scala': 'scala',
|
||||
'.sh': 'bash',
|
||||
'.bash': 'bash',
|
||||
'.zsh': 'bash',
|
||||
'.md': 'markdown',
|
||||
'.mdx': 'markdown',
|
||||
'.json': 'json',
|
||||
'.yaml': 'yaml',
|
||||
'.yml': 'yaml',
|
||||
'.toml': 'toml',
|
||||
'.html': 'html',
|
||||
'.css': 'css',
|
||||
'.svelte': 'svelte',
|
||||
'.vue': 'vue',
|
||||
'.sql': 'sql'
|
||||
};
|
||||
|
||||
function detectLanguage(filePath: string): string {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
return LANGUAGE_MAP[ext] ?? 'text';
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
return LANGUAGE_MAP[ext] ?? 'text';
|
||||
}
|
||||
```
|
||||
|
||||
@@ -255,32 +257,32 @@ function detectLanguage(filePath: string): string {
|
||||
|
||||
```typescript
|
||||
export interface ParseOptions {
|
||||
repositoryId: string;
|
||||
documentId: string;
|
||||
versionId?: string;
|
||||
repositoryId: string;
|
||||
documentId: string;
|
||||
versionId?: string;
|
||||
}
|
||||
|
||||
export function parseFile(
|
||||
file: CrawledFile,
|
||||
options: ParseOptions
|
||||
): NewSnippet[] {
|
||||
const language = detectLanguage(file.path);
|
||||
let rawSnippets: Omit<NewSnippet, 'id' | 'repositoryId' | 'documentId' | 'versionId' | 'createdAt'>[];
|
||||
export function parseFile(file: CrawledFile, options: ParseOptions): NewSnippet[] {
|
||||
const language = detectLanguage(file.path);
|
||||
let rawSnippets: Omit<
|
||||
NewSnippet,
|
||||
'id' | 'repositoryId' | 'documentId' | 'versionId' | 'createdAt'
|
||||
>[];
|
||||
|
||||
if (language === 'markdown') {
|
||||
rawSnippets = parseMarkdown(file.content, file.path);
|
||||
} else {
|
||||
rawSnippets = parseCodeFile(file.content, file.path, language);
|
||||
}
|
||||
if (language === 'markdown') {
|
||||
rawSnippets = parseMarkdown(file.content, file.path);
|
||||
} else {
|
||||
rawSnippets = parseCodeFile(file.content, file.path, language);
|
||||
}
|
||||
|
||||
return rawSnippets.map(s => ({
|
||||
...s,
|
||||
id: crypto.randomUUID(),
|
||||
repositoryId: options.repositoryId,
|
||||
documentId: options.documentId,
|
||||
versionId: options.versionId ?? null,
|
||||
createdAt: new Date(),
|
||||
}));
|
||||
return rawSnippets.map((s) => ({
|
||||
...s,
|
||||
id: crypto.randomUUID(),
|
||||
repositoryId: options.repositoryId,
|
||||
documentId: options.documentId,
|
||||
versionId: options.versionId ?? null,
|
||||
createdAt: new Date()
|
||||
}));
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -33,42 +33,37 @@ Implement the full-text search engine using SQLite's built-in FTS5 extension. Th
|
||||
// src/lib/server/search/search.service.ts
|
||||
|
||||
export interface SnippetSearchOptions {
|
||||
repositoryId: string;
|
||||
versionId?: string;
|
||||
type?: 'code' | 'info';
|
||||
limit?: number; // default: 20
|
||||
offset?: number; // default: 0
|
||||
repositoryId: string;
|
||||
versionId?: string;
|
||||
type?: 'code' | 'info';
|
||||
limit?: number; // default: 20
|
||||
offset?: number; // default: 0
|
||||
}
|
||||
|
||||
export interface SnippetSearchResult {
|
||||
snippet: Snippet;
|
||||
score: number; // BM25 rank (negative, lower = better)
|
||||
repository: Pick<Repository, 'id' | 'title'>;
|
||||
snippet: Snippet;
|
||||
score: number; // BM25 rank (negative, lower = better)
|
||||
repository: Pick<Repository, 'id' | 'title'>;
|
||||
}
|
||||
|
||||
export interface LibrarySearchOptions {
|
||||
libraryName: string;
|
||||
query?: string; // semantic relevance hint
|
||||
limit?: number; // default: 10
|
||||
libraryName: string;
|
||||
query?: string; // semantic relevance hint
|
||||
limit?: number; // default: 10
|
||||
}
|
||||
|
||||
export interface LibrarySearchResult {
|
||||
repository: Repository;
|
||||
versions: RepositoryVersion[];
|
||||
score: number; // composite relevance score
|
||||
repository: Repository;
|
||||
versions: RepositoryVersion[];
|
||||
score: number; // composite relevance score
|
||||
}
|
||||
|
||||
export class SearchService {
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
|
||||
searchSnippets(
|
||||
query: string,
|
||||
options: SnippetSearchOptions
|
||||
): SnippetSearchResult[]
|
||||
searchSnippets(query: string, options: SnippetSearchOptions): SnippetSearchResult[];
|
||||
|
||||
searchRepositories(
|
||||
options: LibrarySearchOptions
|
||||
): LibrarySearchResult[]
|
||||
searchRepositories(options: LibrarySearchOptions): LibrarySearchResult[];
|
||||
}
|
||||
```
|
||||
|
||||
@@ -101,21 +96,21 @@ The FTS5 MATCH query uses the porter stemmer and unicode61 tokenizer (configured
|
||||
|
||||
```typescript
|
||||
function preprocessQuery(raw: string): string {
|
||||
// 1. Trim and normalize whitespace
|
||||
let q = raw.trim().replace(/\s+/g, ' ');
|
||||
// 1. Trim and normalize whitespace
|
||||
let q = raw.trim().replace(/\s+/g, ' ');
|
||||
|
||||
// 2. Escape FTS5 special characters that aren't intended as operators
|
||||
// Keep: * (prefix), " " (phrase), AND, OR, NOT
|
||||
q = q.replace(/[()]/g, ' ');
|
||||
// 2. Escape FTS5 special characters that aren't intended as operators
|
||||
// Keep: * (prefix), " " (phrase), AND, OR, NOT
|
||||
q = q.replace(/[()]/g, ' ');
|
||||
|
||||
// 3. Add prefix wildcard to last token for "typing as you go" feel
|
||||
const tokens = q.split(' ');
|
||||
const lastToken = tokens.at(-1) ?? '';
|
||||
if (lastToken.length >= 3 && !lastToken.endsWith('*')) {
|
||||
tokens[tokens.length - 1] = lastToken + '*';
|
||||
}
|
||||
// 3. Add prefix wildcard to last token for "typing as you go" feel
|
||||
const tokens = q.split(' ');
|
||||
const lastToken = tokens.at(-1) ?? '';
|
||||
if (lastToken.length >= 3 && !lastToken.endsWith('*')) {
|
||||
tokens[tokens.length - 1] = lastToken + '*';
|
||||
}
|
||||
|
||||
return tokens.join(' ');
|
||||
return tokens.join(' ');
|
||||
}
|
||||
```
|
||||
|
||||
@@ -174,56 +169,65 @@ searchRepositories(options: LibrarySearchOptions): LibrarySearchResult[] {
|
||||
The search results must be formatted for the REST API and MCP tool responses:
|
||||
|
||||
### Library search response (for `resolve-library-id`):
|
||||
|
||||
```typescript
|
||||
function formatLibraryResults(results: LibrarySearchResult[]): string {
|
||||
if (results.length === 0) {
|
||||
return 'No libraries found matching your search.';
|
||||
}
|
||||
if (results.length === 0) {
|
||||
return 'No libraries found matching your search.';
|
||||
}
|
||||
|
||||
return results.map((r, i) => {
|
||||
const repo = r.repository;
|
||||
const versions = r.versions.map(v => v.tag).join(', ') || 'default branch';
|
||||
return [
|
||||
`${i + 1}. ${repo.title}`,
|
||||
` Library ID: ${repo.id}`,
|
||||
` Description: ${repo.description ?? 'No description'}`,
|
||||
` Snippets: ${repo.totalSnippets} | Trust Score: ${repo.trustScore.toFixed(1)}/10`,
|
||||
` Available Versions: ${versions}`,
|
||||
].join('\n');
|
||||
}).join('\n\n');
|
||||
return results
|
||||
.map((r, i) => {
|
||||
const repo = r.repository;
|
||||
const versions = r.versions.map((v) => v.tag).join(', ') || 'default branch';
|
||||
return [
|
||||
`${i + 1}. ${repo.title}`,
|
||||
` Library ID: ${repo.id}`,
|
||||
` Description: ${repo.description ?? 'No description'}`,
|
||||
` Snippets: ${repo.totalSnippets} | Trust Score: ${repo.trustScore.toFixed(1)}/10`,
|
||||
` Available Versions: ${versions}`
|
||||
].join('\n');
|
||||
})
|
||||
.join('\n\n');
|
||||
}
|
||||
```
|
||||
|
||||
### Snippet search response (for `query-docs`):
|
||||
|
||||
```typescript
|
||||
function formatSnippetResults(
|
||||
results: SnippetSearchResult[],
|
||||
rules?: string[]
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
function formatSnippetResults(results: SnippetSearchResult[], rules?: string[]): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
// Prepend repository rules if present
|
||||
if (rules?.length) {
|
||||
parts.push('## Library Rules\n' + rules.map(r => `- ${r}`).join('\n'));
|
||||
}
|
||||
// Prepend repository rules if present
|
||||
if (rules?.length) {
|
||||
parts.push('## Library Rules\n' + rules.map((r) => `- ${r}`).join('\n'));
|
||||
}
|
||||
|
||||
for (const { snippet } of results) {
|
||||
if (snippet.type === 'code') {
|
||||
parts.push([
|
||||
snippet.title ? `### ${snippet.title}` : '',
|
||||
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
|
||||
`\`\`\`${snippet.language ?? ''}\n${snippet.content}\n\`\`\``,
|
||||
].filter(Boolean).join('\n'));
|
||||
} else {
|
||||
parts.push([
|
||||
snippet.title ? `### ${snippet.title}` : '',
|
||||
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
|
||||
snippet.content,
|
||||
].filter(Boolean).join('\n'));
|
||||
}
|
||||
}
|
||||
for (const { snippet } of results) {
|
||||
if (snippet.type === 'code') {
|
||||
parts.push(
|
||||
[
|
||||
snippet.title ? `### ${snippet.title}` : '',
|
||||
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
|
||||
`\`\`\`${snippet.language ?? ''}\n${snippet.content}\n\`\`\``
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
);
|
||||
} else {
|
||||
parts.push(
|
||||
[
|
||||
snippet.title ? `### ${snippet.title}` : '',
|
||||
snippet.breadcrumb ? `*${snippet.breadcrumb}*` : '',
|
||||
snippet.content
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n\n---\n\n');
|
||||
return parts.join('\n\n---\n\n');
|
||||
}
|
||||
```
|
||||
|
||||
@@ -235,26 +239,26 @@ Compute `trustScore` (0–10) when a repository is first indexed:
|
||||
|
||||
```typescript
|
||||
function computeTrustScore(repo: Repository): number {
|
||||
let score = 0;
|
||||
let score = 0;
|
||||
|
||||
// Stars (up to 4 points): log scale, 10k stars = 4 pts
|
||||
if (repo.stars) {
|
||||
score += Math.min(4, Math.log10(repo.stars + 1));
|
||||
}
|
||||
// Stars (up to 4 points): log scale, 10k stars = 4 pts
|
||||
if (repo.stars) {
|
||||
score += Math.min(4, Math.log10(repo.stars + 1));
|
||||
}
|
||||
|
||||
// Documentation coverage (up to 3 points)
|
||||
score += Math.min(3, repo.totalSnippets / 500);
|
||||
// Documentation coverage (up to 3 points)
|
||||
score += Math.min(3, repo.totalSnippets / 500);
|
||||
|
||||
// Source type (1 point for GitHub, 0 for local)
|
||||
if (repo.source === 'github') score += 1;
|
||||
// Source type (1 point for GitHub, 0 for local)
|
||||
if (repo.source === 'github') score += 1;
|
||||
|
||||
// Successful indexing (1 point)
|
||||
if (repo.state === 'indexed') score += 1;
|
||||
// Successful indexing (1 point)
|
||||
if (repo.state === 'indexed') score += 1;
|
||||
|
||||
// Has description (1 point)
|
||||
if (repo.description) score += 1;
|
||||
// Has description (1 point)
|
||||
if (repo.description) score += 1;
|
||||
|
||||
return Math.min(10, parseFloat(score.toFixed(1)));
|
||||
return Math.min(10, parseFloat(score.toFixed(1)));
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -34,18 +34,18 @@ Implement a pluggable embedding generation system that produces vector represent
|
||||
// src/lib/server/embeddings/provider.ts
|
||||
|
||||
export interface EmbeddingVector {
|
||||
values: Float32Array;
|
||||
dimensions: number;
|
||||
model: string;
|
||||
values: Float32Array;
|
||||
dimensions: number;
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingProvider {
|
||||
readonly name: string;
|
||||
readonly dimensions: number;
|
||||
readonly model: string;
|
||||
readonly name: string;
|
||||
readonly dimensions: number;
|
||||
readonly model: string;
|
||||
|
||||
embed(texts: string[]): Promise<EmbeddingVector[]>;
|
||||
isAvailable(): Promise<boolean>;
|
||||
embed(texts: string[]): Promise<EmbeddingVector[]>;
|
||||
isAvailable(): Promise<boolean>;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -55,51 +55,51 @@ export interface EmbeddingProvider {
|
||||
|
||||
```typescript
|
||||
export interface OpenAIProviderConfig {
|
||||
baseUrl: string; // e.g. "https://api.openai.com/v1" or "http://localhost:11434/v1"
|
||||
apiKey: string;
|
||||
model: string; // e.g. "text-embedding-3-small", "nomic-embed-text"
|
||||
dimensions?: number; // override for models that support it (e.g. text-embedding-3-small)
|
||||
maxBatchSize?: number; // default: 100
|
||||
baseUrl: string; // e.g. "https://api.openai.com/v1" or "http://localhost:11434/v1"
|
||||
apiKey: string;
|
||||
model: string; // e.g. "text-embedding-3-small", "nomic-embed-text"
|
||||
dimensions?: number; // override for models that support it (e.g. text-embedding-3-small)
|
||||
maxBatchSize?: number; // default: 100
|
||||
}
|
||||
|
||||
export class OpenAIEmbeddingProvider implements EmbeddingProvider {
|
||||
constructor(private config: OpenAIProviderConfig) {}
|
||||
constructor(private config: OpenAIProviderConfig) {}
|
||||
|
||||
async embed(texts: string[]): Promise<EmbeddingVector[]> {
|
||||
// Batch into groups of maxBatchSize
|
||||
const batches = chunk(texts, this.config.maxBatchSize ?? 100);
|
||||
const allEmbeddings: EmbeddingVector[] = [];
|
||||
async embed(texts: string[]): Promise<EmbeddingVector[]> {
|
||||
// Batch into groups of maxBatchSize
|
||||
const batches = chunk(texts, this.config.maxBatchSize ?? 100);
|
||||
const allEmbeddings: EmbeddingVector[] = [];
|
||||
|
||||
for (const batch of batches) {
|
||||
const response = await fetch(`${this.config.baseUrl}/embeddings`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.config.model,
|
||||
input: batch,
|
||||
dimensions: this.config.dimensions,
|
||||
}),
|
||||
});
|
||||
for (const batch of batches) {
|
||||
const response = await fetch(`${this.config.baseUrl}/embeddings`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.config.apiKey}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: this.config.model,
|
||||
input: batch,
|
||||
dimensions: this.config.dimensions
|
||||
})
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new EmbeddingError(`API error: ${response.status}`);
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new EmbeddingError(`API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
for (const item of data.data) {
|
||||
allEmbeddings.push({
|
||||
values: new Float32Array(item.embedding),
|
||||
dimensions: item.embedding.length,
|
||||
model: this.config.model,
|
||||
});
|
||||
}
|
||||
}
|
||||
const data = await response.json();
|
||||
for (const item of data.data) {
|
||||
allEmbeddings.push({
|
||||
values: new Float32Array(item.embedding),
|
||||
dimensions: item.embedding.length,
|
||||
model: this.config.model
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return allEmbeddings;
|
||||
}
|
||||
return allEmbeddings;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -110,41 +110,41 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
|
||||
```typescript
|
||||
// Uses @xenova/transformers — only loaded if installed
|
||||
export class LocalEmbeddingProvider implements EmbeddingProvider {
|
||||
private pipeline: unknown = null;
|
||||
private pipeline: unknown = null;
|
||||
|
||||
readonly name = 'local';
|
||||
readonly model = 'Xenova/all-MiniLM-L6-v2'; // 384-dim, fast, small
|
||||
readonly dimensions = 384;
|
||||
readonly name = 'local';
|
||||
readonly model = 'Xenova/all-MiniLM-L6-v2'; // 384-dim, fast, small
|
||||
readonly dimensions = 384;
|
||||
|
||||
async embed(texts: string[]): Promise<EmbeddingVector[]> {
|
||||
if (!this.pipeline) {
|
||||
const { pipeline } = await import('@xenova/transformers');
|
||||
this.pipeline = await pipeline('feature-extraction', this.model);
|
||||
}
|
||||
async embed(texts: string[]): Promise<EmbeddingVector[]> {
|
||||
if (!this.pipeline) {
|
||||
const { pipeline } = await import('@xenova/transformers');
|
||||
this.pipeline = await pipeline('feature-extraction', this.model);
|
||||
}
|
||||
|
||||
const results: EmbeddingVector[] = [];
|
||||
for (const text of texts) {
|
||||
const output = await (this.pipeline as Function)(text, {
|
||||
pooling: 'mean',
|
||||
normalize: true,
|
||||
});
|
||||
results.push({
|
||||
values: new Float32Array(output.data),
|
||||
dimensions: this.dimensions,
|
||||
model: this.model,
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
const results: EmbeddingVector[] = [];
|
||||
for (const text of texts) {
|
||||
const output = await (this.pipeline as Function)(text, {
|
||||
pooling: 'mean',
|
||||
normalize: true
|
||||
});
|
||||
results.push({
|
||||
values: new Float32Array(output.data),
|
||||
dimensions: this.dimensions,
|
||||
model: this.model
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
try {
|
||||
await import('@xenova/transformers');
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
async isAvailable(): Promise<boolean> {
|
||||
try {
|
||||
await import('@xenova/transformers');
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -154,53 +154,55 @@ export class LocalEmbeddingProvider implements EmbeddingProvider {
|
||||
|
||||
```typescript
|
||||
export class EmbeddingService {
|
||||
constructor(
|
||||
private db: BetterSQLite3.Database,
|
||||
private provider: EmbeddingProvider
|
||||
) {}
|
||||
constructor(
|
||||
private db: BetterSQLite3.Database,
|
||||
private provider: EmbeddingProvider
|
||||
) {}
|
||||
|
||||
async embedSnippets(
|
||||
snippetIds: string[],
|
||||
onProgress?: (done: number, total: number) => void
|
||||
): Promise<void> {
|
||||
const snippets = this.db.prepare(
|
||||
`SELECT id, content, type FROM snippets WHERE id IN (${snippetIds.map(() => '?').join(',')})`
|
||||
).all(...snippetIds) as Snippet[];
|
||||
async embedSnippets(
|
||||
snippetIds: string[],
|
||||
onProgress?: (done: number, total: number) => void
|
||||
): Promise<void> {
|
||||
const snippets = this.db
|
||||
.prepare(
|
||||
`SELECT id, content, type FROM snippets WHERE id IN (${snippetIds.map(() => '?').join(',')})`
|
||||
)
|
||||
.all(...snippetIds) as Snippet[];
|
||||
|
||||
// Prepare text for embedding: combine title + content
|
||||
const texts = snippets.map(s =>
|
||||
[s.title, s.breadcrumb, s.content].filter(Boolean).join('\n').slice(0, 2048)
|
||||
);
|
||||
// Prepare text for embedding: combine title + content
|
||||
const texts = snippets.map((s) =>
|
||||
[s.title, s.breadcrumb, s.content].filter(Boolean).join('\n').slice(0, 2048)
|
||||
);
|
||||
|
||||
const BATCH_SIZE = 50;
|
||||
const insert = this.db.prepare(`
|
||||
const BATCH_SIZE = 50;
|
||||
const insert = this.db.prepare(`
|
||||
INSERT OR REPLACE INTO snippet_embeddings (snippet_id, model, dimensions, embedding, created_at)
|
||||
VALUES (?, ?, ?, ?, unixepoch())
|
||||
`);
|
||||
|
||||
for (let i = 0; i < snippets.length; i += BATCH_SIZE) {
|
||||
const batch = snippets.slice(i, i + BATCH_SIZE);
|
||||
const batchTexts = texts.slice(i, i + BATCH_SIZE);
|
||||
for (let i = 0; i < snippets.length; i += BATCH_SIZE) {
|
||||
const batch = snippets.slice(i, i + BATCH_SIZE);
|
||||
const batchTexts = texts.slice(i, i + BATCH_SIZE);
|
||||
|
||||
const embeddings = await this.provider.embed(batchTexts);
|
||||
const embeddings = await this.provider.embed(batchTexts);
|
||||
|
||||
const insertMany = this.db.transaction(() => {
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const snippet = batch[j];
|
||||
const embedding = embeddings[j];
|
||||
insert.run(
|
||||
snippet.id,
|
||||
embedding.model,
|
||||
embedding.dimensions,
|
||||
Buffer.from(embedding.values.buffer)
|
||||
);
|
||||
}
|
||||
});
|
||||
insertMany();
|
||||
const insertMany = this.db.transaction(() => {
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const snippet = batch[j];
|
||||
const embedding = embeddings[j];
|
||||
insert.run(
|
||||
snippet.id,
|
||||
embedding.model,
|
||||
embedding.dimensions,
|
||||
Buffer.from(embedding.values.buffer)
|
||||
);
|
||||
}
|
||||
});
|
||||
insertMany();
|
||||
|
||||
onProgress?.(Math.min(i + BATCH_SIZE, snippets.length), snippets.length);
|
||||
}
|
||||
}
|
||||
onProgress?.(Math.min(i + BATCH_SIZE, snippets.length), snippets.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -212,13 +214,13 @@ Stored in the `settings` table as JSON:
|
||||
|
||||
```typescript
|
||||
export interface EmbeddingConfig {
|
||||
provider: 'openai' | 'local' | 'none';
|
||||
openai?: {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
dimensions?: number;
|
||||
};
|
||||
provider: 'openai' | 'local' | 'none';
|
||||
openai?: {
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
dimensions?: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Settings key: 'embedding_config'
|
||||
@@ -227,14 +229,15 @@ export interface EmbeddingConfig {
|
||||
### API Endpoints
|
||||
|
||||
`GET /api/v1/settings/embedding`
|
||||
|
||||
```json
|
||||
{
|
||||
"provider": "openai",
|
||||
"openai": {
|
||||
"baseUrl": "https://api.openai.com/v1",
|
||||
"model": "text-embedding-3-small",
|
||||
"dimensions": 1536
|
||||
}
|
||||
"provider": "openai",
|
||||
"openai": {
|
||||
"baseUrl": "https://api.openai.com/v1",
|
||||
"model": "text-embedding-3-small",
|
||||
"dimensions": 1536
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -251,11 +254,7 @@ Embeddings are stored as raw `Float32Array` binary blobs:
|
||||
const buffer = Buffer.from(float32Array.buffer);
|
||||
|
||||
// Retrieve
|
||||
const float32Array = new Float32Array(
|
||||
buffer.buffer,
|
||||
buffer.byteOffset,
|
||||
buffer.byteLength / 4
|
||||
);
|
||||
const float32Array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -102,21 +102,21 @@ async vectorSearch(
|
||||
|
||||
```typescript
|
||||
function reciprocalRankFusion(
|
||||
...rankings: Array<Array<{ id: string; score: number }>>
|
||||
...rankings: Array<Array<{ id: string; score: number }>>
|
||||
): Array<{ id: string; rrfScore: number }> {
|
||||
const K = 60; // RRF constant (standard value)
|
||||
const scores = new Map<string, number>();
|
||||
const K = 60; // RRF constant (standard value)
|
||||
const scores = new Map<string, number>();
|
||||
|
||||
for (const ranking of rankings) {
|
||||
ranking.forEach(({ id }, rank) => {
|
||||
const current = scores.get(id) ?? 0;
|
||||
scores.set(id, current + 1 / (K + rank + 1));
|
||||
});
|
||||
}
|
||||
for (const ranking of rankings) {
|
||||
ranking.forEach(({ id }, rank) => {
|
||||
const current = scores.get(id) ?? 0;
|
||||
scores.set(id, current + 1 / (K + rank + 1));
|
||||
});
|
||||
}
|
||||
|
||||
return Array.from(scores.entries())
|
||||
.map(([id, rrfScore]) => ({ id, rrfScore }))
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore);
|
||||
return Array.from(scores.entries())
|
||||
.map(([id, rrfScore]) => ({ id, rrfScore }))
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore);
|
||||
}
|
||||
```
|
||||
|
||||
@@ -126,65 +126,62 @@ function reciprocalRankFusion(
|
||||
|
||||
```typescript
|
||||
export interface HybridSearchOptions {
|
||||
repositoryId: string;
|
||||
versionId?: string;
|
||||
type?: 'code' | 'info';
|
||||
limit?: number;
|
||||
alpha?: number; // 0.0 = FTS5 only, 1.0 = vector only, 0.5 = balanced
|
||||
repositoryId: string;
|
||||
versionId?: string;
|
||||
type?: 'code' | 'info';
|
||||
limit?: number;
|
||||
alpha?: number; // 0.0 = FTS5 only, 1.0 = vector only, 0.5 = balanced
|
||||
}
|
||||
|
||||
export class HybridSearchService {
|
||||
constructor(
|
||||
private db: BetterSQLite3.Database,
|
||||
private searchService: SearchService,
|
||||
private embeddingProvider: EmbeddingProvider | null,
|
||||
) {}
|
||||
constructor(
|
||||
private db: BetterSQLite3.Database,
|
||||
private searchService: SearchService,
|
||||
private embeddingProvider: EmbeddingProvider | null
|
||||
) {}
|
||||
|
||||
async search(
|
||||
query: string,
|
||||
options: HybridSearchOptions
|
||||
): Promise<SnippetSearchResult[]> {
|
||||
const limit = options.limit ?? 20;
|
||||
const alpha = options.alpha ?? 0.5;
|
||||
async search(query: string, options: HybridSearchOptions): Promise<SnippetSearchResult[]> {
|
||||
const limit = options.limit ?? 20;
|
||||
const alpha = options.alpha ?? 0.5;
|
||||
|
||||
// Always run FTS5 search
|
||||
const ftsResults = this.searchService.searchSnippets(query, {
|
||||
repositoryId: options.repositoryId,
|
||||
versionId: options.versionId,
|
||||
type: options.type,
|
||||
limit: limit * 3, // get more candidates for fusion
|
||||
});
|
||||
// Always run FTS5 search
|
||||
const ftsResults = this.searchService.searchSnippets(query, {
|
||||
repositoryId: options.repositoryId,
|
||||
versionId: options.versionId,
|
||||
type: options.type,
|
||||
limit: limit * 3 // get more candidates for fusion
|
||||
});
|
||||
|
||||
// If no embedding provider or alpha = 0, return FTS5 results directly
|
||||
if (!this.embeddingProvider || alpha === 0) {
|
||||
return ftsResults.slice(0, limit);
|
||||
}
|
||||
// If no embedding provider or alpha = 0, return FTS5 results directly
|
||||
if (!this.embeddingProvider || alpha === 0) {
|
||||
return ftsResults.slice(0, limit);
|
||||
}
|
||||
|
||||
// Embed the query and run vector search
|
||||
const [queryEmbedding] = await this.embeddingProvider.embed([query]);
|
||||
const vectorResults = await this.vectorSearch(
|
||||
queryEmbedding.values,
|
||||
options.repositoryId,
|
||||
limit * 3
|
||||
);
|
||||
// Embed the query and run vector search
|
||||
const [queryEmbedding] = await this.embeddingProvider.embed([query]);
|
||||
const vectorResults = await this.vectorSearch(
|
||||
queryEmbedding.values,
|
||||
options.repositoryId,
|
||||
limit * 3
|
||||
);
|
||||
|
||||
// Normalize result lists for RRF
|
||||
const ftsRanked = ftsResults.map((r, i) => ({
|
||||
id: r.snippet.id,
|
||||
score: i,
|
||||
}));
|
||||
const vecRanked = vectorResults.map((r, i) => ({
|
||||
id: r.snippetId,
|
||||
score: i,
|
||||
}));
|
||||
// Normalize result lists for RRF
|
||||
const ftsRanked = ftsResults.map((r, i) => ({
|
||||
id: r.snippet.id,
|
||||
score: i
|
||||
}));
|
||||
const vecRanked = vectorResults.map((r, i) => ({
|
||||
id: r.snippetId,
|
||||
score: i
|
||||
}));
|
||||
|
||||
// Apply RRF
|
||||
const fused = reciprocalRankFusion(ftsRanked, vecRanked);
|
||||
// Apply RRF
|
||||
const fused = reciprocalRankFusion(ftsRanked, vecRanked);
|
||||
|
||||
// Fetch full snippet data for top results
|
||||
const topIds = fused.slice(0, limit).map(r => r.id);
|
||||
return this.fetchSnippetsByIds(topIds, options.repositoryId);
|
||||
}
|
||||
// Fetch full snippet data for top results
|
||||
const topIds = fused.slice(0, limit).map((r) => r.id);
|
||||
return this.fetchSnippetsByIds(topIds, options.repositoryId);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -197,9 +194,9 @@ The hybrid search alpha value can be set per-request or globally via settings:
|
||||
```typescript
|
||||
// Default config stored in settings table under key 'search_config'
|
||||
export interface SearchConfig {
|
||||
alpha: number; // 0.5 default
|
||||
maxResults: number; // 20 default
|
||||
enableHybrid: boolean; // true if embedding provider is configured
|
||||
alpha: number; // 0.5 default
|
||||
maxResults: number; // 20 default
|
||||
enableHybrid: boolean; // true if embedding provider is configured
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -56,75 +56,83 @@ Implement the end-to-end indexing pipeline that orchestrates crawling, parsing,
|
||||
// src/lib/server/pipeline/job-queue.ts
|
||||
|
||||
export class JobQueue {
|
||||
private isRunning = false;
|
||||
private isRunning = false;
|
||||
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
|
||||
enqueue(repositoryId: string, versionId?: string): IndexingJob {
|
||||
const job: NewIndexingJob = {
|
||||
id: crypto.randomUUID(),
|
||||
repositoryId,
|
||||
versionId: versionId ?? null,
|
||||
status: 'queued',
|
||||
progress: 0,
|
||||
totalFiles: 0,
|
||||
processedFiles: 0,
|
||||
error: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
enqueue(repositoryId: string, versionId?: string): IndexingJob {
|
||||
const job: NewIndexingJob = {
|
||||
id: crypto.randomUUID(),
|
||||
repositoryId,
|
||||
versionId: versionId ?? null,
|
||||
status: 'queued',
|
||||
progress: 0,
|
||||
totalFiles: 0,
|
||||
processedFiles: 0,
|
||||
error: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
createdAt: new Date()
|
||||
};
|
||||
|
||||
this.db.prepare(`
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO indexing_jobs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run(...Object.values(job));
|
||||
`
|
||||
)
|
||||
.run(...Object.values(job));
|
||||
|
||||
// Kick off processing if not already running
|
||||
if (!this.isRunning) {
|
||||
setImmediate(() => this.processNext());
|
||||
}
|
||||
// Kick off processing if not already running
|
||||
if (!this.isRunning) {
|
||||
setImmediate(() => this.processNext());
|
||||
}
|
||||
|
||||
return job;
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
private async processNext(): Promise<void> {
|
||||
if (this.isRunning) return;
|
||||
private async processNext(): Promise<void> {
|
||||
if (this.isRunning) return;
|
||||
|
||||
const job = this.db.prepare(`
|
||||
const job = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT * FROM indexing_jobs
|
||||
WHERE status = 'queued'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
`).get() as IndexingJob | undefined;
|
||||
`
|
||||
)
|
||||
.get() as IndexingJob | undefined;
|
||||
|
||||
if (!job) return;
|
||||
if (!job) return;
|
||||
|
||||
this.isRunning = true;
|
||||
try {
|
||||
await this.pipeline.run(job);
|
||||
} finally {
|
||||
this.isRunning = false;
|
||||
// Check for next queued job
|
||||
const nextJob = this.db.prepare(
|
||||
`SELECT id FROM indexing_jobs WHERE status = 'queued' LIMIT 1`
|
||||
).get();
|
||||
if (nextJob) setImmediate(() => this.processNext());
|
||||
}
|
||||
}
|
||||
this.isRunning = true;
|
||||
try {
|
||||
await this.pipeline.run(job);
|
||||
} finally {
|
||||
this.isRunning = false;
|
||||
// Check for next queued job
|
||||
const nextJob = this.db
|
||||
.prepare(`SELECT id FROM indexing_jobs WHERE status = 'queued' LIMIT 1`)
|
||||
.get();
|
||||
if (nextJob) setImmediate(() => this.processNext());
|
||||
}
|
||||
}
|
||||
|
||||
getJob(id: string): IndexingJob | null {
|
||||
return this.db.prepare(
|
||||
`SELECT * FROM indexing_jobs WHERE id = ?`
|
||||
).get(id) as IndexingJob | null;
|
||||
}
|
||||
getJob(id: string): IndexingJob | null {
|
||||
return this.db
|
||||
.prepare(`SELECT * FROM indexing_jobs WHERE id = ?`)
|
||||
.get(id) as IndexingJob | null;
|
||||
}
|
||||
|
||||
listJobs(repositoryId?: string, limit = 20): IndexingJob[] {
|
||||
const query = repositoryId
|
||||
? `SELECT * FROM indexing_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT ?`
|
||||
: `SELECT * FROM indexing_jobs ORDER BY created_at DESC LIMIT ?`;
|
||||
const params = repositoryId ? [repositoryId, limit] : [limit];
|
||||
return this.db.prepare(query).all(...params) as IndexingJob[];
|
||||
}
|
||||
listJobs(repositoryId?: string, limit = 20): IndexingJob[] {
|
||||
const query = repositoryId
|
||||
? `SELECT * FROM indexing_jobs WHERE repository_id = ? ORDER BY created_at DESC LIMIT ?`
|
||||
: `SELECT * FROM indexing_jobs ORDER BY created_at DESC LIMIT ?`;
|
||||
const params = repositoryId ? [repositoryId, limit] : [limit];
|
||||
return this.db.prepare(query).all(...params) as IndexingJob[];
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -136,94 +144,96 @@ export class JobQueue {
|
||||
// src/lib/server/pipeline/indexing.pipeline.ts
|
||||
|
||||
export class IndexingPipeline {
|
||||
constructor(
|
||||
private db: BetterSQLite3.Database,
|
||||
private githubCrawler: GitHubCrawler,
|
||||
private localCrawler: LocalCrawler,
|
||||
private embeddingService: EmbeddingService | null,
|
||||
) {}
|
||||
constructor(
|
||||
private db: BetterSQLite3.Database,
|
||||
private githubCrawler: GitHubCrawler,
|
||||
private localCrawler: LocalCrawler,
|
||||
private embeddingService: EmbeddingService | null
|
||||
) {}
|
||||
|
||||
async run(job: IndexingJob): Promise<void> {
|
||||
this.updateJob(job.id, { status: 'running', startedAt: new Date() });
|
||||
async run(job: IndexingJob): Promise<void> {
|
||||
this.updateJob(job.id, { status: 'running', startedAt: new Date() });
|
||||
|
||||
try {
|
||||
const repo = this.getRepository(job.repositoryId);
|
||||
if (!repo) throw new Error(`Repository ${job.repositoryId} not found`);
|
||||
try {
|
||||
const repo = this.getRepository(job.repositoryId);
|
||||
if (!repo) throw new Error(`Repository ${job.repositoryId} not found`);
|
||||
|
||||
// Update repo state
|
||||
this.updateRepo(repo.id, { state: 'indexing' });
|
||||
// Update repo state
|
||||
this.updateRepo(repo.id, { state: 'indexing' });
|
||||
|
||||
// Step 1: Crawl
|
||||
const crawlResult = await this.crawl(repo, job);
|
||||
// Step 1: Crawl
|
||||
const crawlResult = await this.crawl(repo, job);
|
||||
|
||||
// Step 2: Parse and diff
|
||||
const { newSnippets, changedDocIds, newDocuments } =
|
||||
await this.parseAndDiff(crawlResult, repo, job);
|
||||
// Step 2: Parse and diff
|
||||
const { newSnippets, changedDocIds, newDocuments } = await this.parseAndDiff(
|
||||
crawlResult,
|
||||
repo,
|
||||
job
|
||||
);
|
||||
|
||||
// Step 3: Atomic replacement
|
||||
this.replaceSnippets(repo.id, changedDocIds, newDocuments, newSnippets);
|
||||
// Step 3: Atomic replacement
|
||||
this.replaceSnippets(repo.id, changedDocIds, newDocuments, newSnippets);
|
||||
|
||||
// Step 4: Embeddings (async, non-blocking for job completion)
|
||||
if (this.embeddingService && newSnippets.length > 0) {
|
||||
await this.embeddingService.embedSnippets(
|
||||
newSnippets.map(s => s.id),
|
||||
(done, total) => {
|
||||
// Update job progress for embedding phase
|
||||
}
|
||||
);
|
||||
}
|
||||
// Step 4: Embeddings (async, non-blocking for job completion)
|
||||
if (this.embeddingService && newSnippets.length > 0) {
|
||||
await this.embeddingService.embedSnippets(
|
||||
newSnippets.map((s) => s.id),
|
||||
(done, total) => {
|
||||
// Update job progress for embedding phase
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Step 5: Update repo stats
|
||||
const stats = this.computeStats(repo.id);
|
||||
this.updateRepo(repo.id, {
|
||||
state: 'indexed',
|
||||
totalSnippets: stats.totalSnippets,
|
||||
totalTokens: stats.totalTokens,
|
||||
trustScore: computeTrustScore({ ...repo, ...stats }),
|
||||
lastIndexedAt: new Date(),
|
||||
});
|
||||
// Step 5: Update repo stats
|
||||
const stats = this.computeStats(repo.id);
|
||||
this.updateRepo(repo.id, {
|
||||
state: 'indexed',
|
||||
totalSnippets: stats.totalSnippets,
|
||||
totalTokens: stats.totalTokens,
|
||||
trustScore: computeTrustScore({ ...repo, ...stats }),
|
||||
lastIndexedAt: new Date()
|
||||
});
|
||||
|
||||
this.updateJob(job.id, {
|
||||
status: 'done',
|
||||
progress: 100,
|
||||
completedAt: new Date(),
|
||||
});
|
||||
this.updateJob(job.id, {
|
||||
status: 'done',
|
||||
progress: 100,
|
||||
completedAt: new Date()
|
||||
});
|
||||
} catch (error) {
|
||||
this.updateJob(job.id, {
|
||||
status: 'failed',
|
||||
error: (error as Error).message,
|
||||
completedAt: new Date()
|
||||
});
|
||||
this.updateRepo(job.repositoryId, { state: 'error' });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
this.updateJob(job.id, {
|
||||
status: 'failed',
|
||||
error: (error as Error).message,
|
||||
completedAt: new Date(),
|
||||
});
|
||||
this.updateRepo(job.repositoryId, { state: 'error' });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
private replaceSnippets(
|
||||
repositoryId: string,
|
||||
changedDocIds: string[],
|
||||
newDocuments: NewDocument[],
|
||||
newSnippets: NewSnippet[]
|
||||
): void {
|
||||
// Single transaction: delete old → insert new
|
||||
this.db.transaction(() => {
|
||||
if (changedDocIds.length > 0) {
|
||||
// Cascade deletes snippets via FK constraint
|
||||
this.db
|
||||
.prepare(`DELETE FROM documents WHERE id IN (${changedDocIds.map(() => '?').join(',')})`)
|
||||
.run(...changedDocIds);
|
||||
}
|
||||
|
||||
private replaceSnippets(
|
||||
repositoryId: string,
|
||||
changedDocIds: string[],
|
||||
newDocuments: NewDocument[],
|
||||
newSnippets: NewSnippet[]
|
||||
): void {
|
||||
// Single transaction: delete old → insert new
|
||||
this.db.transaction(() => {
|
||||
if (changedDocIds.length > 0) {
|
||||
// Cascade deletes snippets via FK constraint
|
||||
this.db.prepare(
|
||||
`DELETE FROM documents WHERE id IN (${changedDocIds.map(() => '?').join(',')})`
|
||||
).run(...changedDocIds);
|
||||
}
|
||||
for (const doc of newDocuments) {
|
||||
this.insertDocument(doc);
|
||||
}
|
||||
|
||||
for (const doc of newDocuments) {
|
||||
this.insertDocument(doc);
|
||||
}
|
||||
|
||||
for (const snippet of newSnippets) {
|
||||
this.insertSnippet(snippet);
|
||||
}
|
||||
})();
|
||||
}
|
||||
for (const snippet of newSnippets) {
|
||||
this.insertSnippet(snippet);
|
||||
}
|
||||
})();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -233,26 +243,24 @@ export class IndexingPipeline {
|
||||
|
||||
```typescript
|
||||
function calculateProgress(
|
||||
processedFiles: number,
|
||||
totalFiles: number,
|
||||
embeddingsDone: number,
|
||||
embeddingsTotal: number,
|
||||
hasEmbeddings: boolean
|
||||
processedFiles: number,
|
||||
totalFiles: number,
|
||||
embeddingsDone: number,
|
||||
embeddingsTotal: number,
|
||||
hasEmbeddings: boolean
|
||||
): number {
|
||||
if (totalFiles === 0) return 0;
|
||||
if (totalFiles === 0) return 0;
|
||||
|
||||
if (!hasEmbeddings) {
|
||||
// Crawl + parse = 100%
|
||||
return Math.round((processedFiles / totalFiles) * 100);
|
||||
}
|
||||
if (!hasEmbeddings) {
|
||||
// Crawl + parse = 100%
|
||||
return Math.round((processedFiles / totalFiles) * 100);
|
||||
}
|
||||
|
||||
// Crawl+parse = 80%, embeddings = 20%
|
||||
const parseProgress = (processedFiles / totalFiles) * 80;
|
||||
const embedProgress = embeddingsTotal > 0
|
||||
? (embeddingsDone / embeddingsTotal) * 20
|
||||
: 0;
|
||||
// Crawl+parse = 80%, embeddings = 20%
|
||||
const parseProgress = (processedFiles / totalFiles) * 80;
|
||||
const embedProgress = embeddingsTotal > 0 ? (embeddingsDone / embeddingsTotal) * 20 : 0;
|
||||
|
||||
return Math.round(parseProgress + embedProgress);
|
||||
return Math.round(parseProgress + embedProgress);
|
||||
}
|
||||
```
|
||||
|
||||
@@ -263,20 +271,21 @@ function calculateProgress(
|
||||
### `GET /api/v1/jobs/:id`
|
||||
|
||||
Response `200`:
|
||||
|
||||
```json
|
||||
{
|
||||
"job": {
|
||||
"id": "uuid",
|
||||
"repositoryId": "/facebook/react",
|
||||
"status": "running",
|
||||
"progress": 47,
|
||||
"totalFiles": 342,
|
||||
"processedFiles": 162,
|
||||
"error": null,
|
||||
"startedAt": "2026-03-22T10:00:00Z",
|
||||
"completedAt": null,
|
||||
"createdAt": "2026-03-22T09:59:55Z"
|
||||
}
|
||||
"job": {
|
||||
"id": "uuid",
|
||||
"repositoryId": "/facebook/react",
|
||||
"status": "running",
|
||||
"progress": 47,
|
||||
"totalFiles": 342,
|
||||
"processedFiles": 162,
|
||||
"error": null,
|
||||
"startedAt": "2026-03-22T10:00:00Z",
|
||||
"completedAt": null,
|
||||
"createdAt": "2026-03-22T09:59:55Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -285,6 +294,7 @@ Response `200`:
|
||||
Query params: `repositoryId` (optional), `status` (optional), `limit` (default 20).
|
||||
|
||||
Response `200`:
|
||||
|
||||
```json
|
||||
{
|
||||
"jobs": [...],
|
||||
@@ -300,20 +310,24 @@ On application start, mark any jobs in `running` state as `failed` (they were in
|
||||
|
||||
```typescript
|
||||
function recoverStaleJobs(db: BetterSQLite3.Database): void {
|
||||
db.prepare(`
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE indexing_jobs
|
||||
SET status = 'failed',
|
||||
error = 'Server restarted while job was running',
|
||||
completed_at = unixepoch()
|
||||
WHERE status = 'running'
|
||||
`).run();
|
||||
`
|
||||
).run();
|
||||
|
||||
// Also reset any repositories stuck in 'indexing' state
|
||||
db.prepare(`
|
||||
// Also reset any repositories stuck in 'indexing' state
|
||||
db.prepare(
|
||||
`
|
||||
UPDATE repositories
|
||||
SET state = 'error'
|
||||
WHERE state = 'indexing'
|
||||
`).run();
|
||||
`
|
||||
).run();
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -32,33 +32,33 @@ Implement the public-facing REST API endpoints that replicate context7's `/api/v
|
||||
|
||||
### Query Parameters
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `libraryName` | string | Yes | Library name to search for |
|
||||
| `query` | string | No | User's question for relevance ranking |
|
||||
| `limit` | integer | No | Max results (default: 10, max: 50) |
|
||||
| Parameter | Type | Required | Description |
|
||||
| ------------- | ------- | -------- | ------------------------------------- |
|
||||
| `libraryName` | string | Yes | Library name to search for |
|
||||
| `query` | string | No | User's question for relevance ranking |
|
||||
| `limit` | integer | No | Max results (default: 10, max: 50) |
|
||||
|
||||
### Response `200` (`type=json`, default):
|
||||
|
||||
```json
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"id": "/facebook/react",
|
||||
"title": "React",
|
||||
"description": "A JavaScript library for building user interfaces",
|
||||
"branch": "main",
|
||||
"lastUpdateDate": "2026-03-22T10:00:00Z",
|
||||
"state": "finalized",
|
||||
"totalTokens": 142000,
|
||||
"totalSnippets": 1247,
|
||||
"stars": 228000,
|
||||
"trustScore": 9.2,
|
||||
"benchmarkScore": 87,
|
||||
"versions": ["v18.3.0", "v17.0.2"],
|
||||
"source": "https://github.com/facebook/react"
|
||||
}
|
||||
]
|
||||
"results": [
|
||||
{
|
||||
"id": "/facebook/react",
|
||||
"title": "React",
|
||||
"description": "A JavaScript library for building user interfaces",
|
||||
"branch": "main",
|
||||
"lastUpdateDate": "2026-03-22T10:00:00Z",
|
||||
"state": "finalized",
|
||||
"totalTokens": 142000,
|
||||
"totalSnippets": 1247,
|
||||
"stars": 228000,
|
||||
"trustScore": 9.2,
|
||||
"benchmarkScore": 87,
|
||||
"versions": ["v18.3.0", "v17.0.2"],
|
||||
"source": "https://github.com/facebook/react"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
@@ -67,11 +67,11 @@ Note: `state: "finalized"` maps from TrueRef's `state: "indexed"` for compatibil
|
||||
### State Mapping
|
||||
|
||||
| TrueRef state | context7 state |
|
||||
|---------------|---------------|
|
||||
| `pending` | `initial` |
|
||||
| `indexing` | `initial` |
|
||||
| `indexed` | `finalized` |
|
||||
| `error` | `error` |
|
||||
| ------------- | -------------- |
|
||||
| `pending` | `initial` |
|
||||
| `indexing` | `initial` |
|
||||
| `indexed` | `finalized` |
|
||||
| `error` | `error` |
|
||||
|
||||
---
|
||||
|
||||
@@ -81,43 +81,43 @@ Note: `state: "finalized"` maps from TrueRef's `state: "indexed"` for compatibil
|
||||
|
||||
### Query Parameters
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `libraryId` | string | Yes | Library ID, e.g. `/facebook/react` or `/facebook/react/v18.3.0` |
|
||||
| `query` | string | Yes | Specific question about the library |
|
||||
| `type` | string | No | `json` (default) or `txt` (plain text for LLM injection) |
|
||||
| `tokens` | integer | No | Approximate max token count for response (default: 10000) |
|
||||
| Parameter | Type | Required | Description |
|
||||
| ----------- | ------- | -------- | --------------------------------------------------------------- |
|
||||
| `libraryId` | string | Yes | Library ID, e.g. `/facebook/react` or `/facebook/react/v18.3.0` |
|
||||
| `query` | string | Yes | Specific question about the library |
|
||||
| `type` | string | No | `json` (default) or `txt` (plain text for LLM injection) |
|
||||
| `tokens` | integer | No | Approximate max token count for response (default: 10000) |
|
||||
|
||||
### Response `200` (`type=json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"snippets": [
|
||||
{
|
||||
"type": "code",
|
||||
"title": "Basic Component",
|
||||
"description": "Getting Started > Components",
|
||||
"language": "tsx",
|
||||
"codeList": [
|
||||
{
|
||||
"language": "tsx",
|
||||
"code": "function MyComponent() {\n return <div>Hello</div>;\n}"
|
||||
}
|
||||
],
|
||||
"id": "uuid",
|
||||
"tokenCount": 45,
|
||||
"pageTitle": "Getting Started"
|
||||
},
|
||||
{
|
||||
"type": "info",
|
||||
"text": "React components let you split the UI into independent...",
|
||||
"breadcrumb": "Core Concepts > Components",
|
||||
"pageId": "uuid",
|
||||
"tokenCount": 120
|
||||
}
|
||||
],
|
||||
"rules": ["Always use functional components", "..."],
|
||||
"totalTokens": 2840
|
||||
"snippets": [
|
||||
{
|
||||
"type": "code",
|
||||
"title": "Basic Component",
|
||||
"description": "Getting Started > Components",
|
||||
"language": "tsx",
|
||||
"codeList": [
|
||||
{
|
||||
"language": "tsx",
|
||||
"code": "function MyComponent() {\n return <div>Hello</div>;\n}"
|
||||
}
|
||||
],
|
||||
"id": "uuid",
|
||||
"tokenCount": 45,
|
||||
"pageTitle": "Getting Started"
|
||||
},
|
||||
{
|
||||
"type": "info",
|
||||
"text": "React components let you split the UI into independent...",
|
||||
"breadcrumb": "Core Concepts > Components",
|
||||
"pageId": "uuid",
|
||||
"tokenCount": 120
|
||||
}
|
||||
],
|
||||
"rules": ["Always use functional components", "..."],
|
||||
"totalTokens": 2840
|
||||
}
|
||||
```
|
||||
|
||||
@@ -125,7 +125,7 @@ Note: `state: "finalized"` maps from TrueRef's `state: "indexed"` for compatibil
|
||||
|
||||
Plain text formatted for direct LLM context injection:
|
||||
|
||||
```
|
||||
````
|
||||
## Library Rules
|
||||
- Always use functional components
|
||||
- Use hooks for state management
|
||||
@@ -139,15 +139,17 @@ Plain text formatted for direct LLM context injection:
|
||||
function MyComponent() {
|
||||
return <div>Hello</div>;
|
||||
}
|
||||
```
|
||||
````
|
||||
|
||||
---
|
||||
|
||||
### React components let you split the UI...
|
||||
*Core Concepts > Components*
|
||||
|
||||
_Core Concepts > Components_
|
||||
|
||||
React components let you split the UI into independent, reusable pieces...
|
||||
```
|
||||
|
||||
````
|
||||
|
||||
---
|
||||
|
||||
@@ -167,7 +169,7 @@ function parseLibraryId(libraryId: string): {
|
||||
version: match[3],
|
||||
};
|
||||
}
|
||||
```
|
||||
````
|
||||
|
||||
---
|
||||
|
||||
@@ -176,20 +178,17 @@ function parseLibraryId(libraryId: string): {
|
||||
The `tokens` parameter limits the total response size. Snippets are added greedily until the budget is exhausted:
|
||||
|
||||
```typescript
|
||||
function selectSnippetsWithinBudget(
|
||||
snippets: Snippet[],
|
||||
maxTokens: number
|
||||
): Snippet[] {
|
||||
const selected: Snippet[] = [];
|
||||
let usedTokens = 0;
|
||||
function selectSnippetsWithinBudget(snippets: Snippet[], maxTokens: number): Snippet[] {
|
||||
const selected: Snippet[] = [];
|
||||
let usedTokens = 0;
|
||||
|
||||
for (const snippet of snippets) {
|
||||
if (usedTokens + (snippet.tokenCount ?? 0) > maxTokens) break;
|
||||
selected.push(snippet);
|
||||
usedTokens += snippet.tokenCount ?? 0;
|
||||
}
|
||||
for (const snippet of snippets) {
|
||||
if (usedTokens + (snippet.tokenCount ?? 0) > maxTokens) break;
|
||||
selected.push(snippet);
|
||||
usedTokens += snippet.tokenCount ?? 0;
|
||||
}
|
||||
|
||||
return selected;
|
||||
return selected;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -215,6 +214,7 @@ Default token budget: 10,000 tokens (~7,500 words) — enough for ~20 medium sni
|
||||
## CORS Configuration
|
||||
|
||||
All API routes include:
|
||||
|
||||
```
|
||||
Access-Control-Allow-Origin: *
|
||||
Access-Control-Allow-Methods: GET, POST, PATCH, DELETE, OPTIONS
|
||||
|
||||
@@ -32,8 +32,8 @@ Implement a Model Context Protocol (MCP) server that exposes `resolve-library-id
|
||||
|
||||
```json
|
||||
{
|
||||
"@modelcontextprotocol/sdk": "^1.25.1",
|
||||
"zod": "^4.3.4"
|
||||
"@modelcontextprotocol/sdk": "^1.25.1",
|
||||
"zod": "^4.3.4"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -46,189 +46,190 @@ Implement a Model Context Protocol (MCP) server that exposes `resolve-library-id
|
||||
|
||||
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import {
|
||||
CallToolRequestSchema,
|
||||
ListToolsRequestSchema,
|
||||
} from '@modelcontextprotocol/sdk/types.js';
|
||||
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
import { z } from 'zod';
|
||||
|
||||
const API_BASE = process.env.TRUEREF_API_URL ?? 'http://localhost:5173';
|
||||
|
||||
const server = new Server(
|
||||
{
|
||||
name: 'io.github.trueref/trueref',
|
||||
version: '1.0.0',
|
||||
},
|
||||
{
|
||||
capabilities: { tools: {} },
|
||||
}
|
||||
{
|
||||
name: 'io.github.trueref/trueref',
|
||||
version: '1.0.0'
|
||||
},
|
||||
{
|
||||
capabilities: { tools: {} }
|
||||
}
|
||||
);
|
||||
|
||||
// Tool schemas — identical to context7 for drop-in compatibility
|
||||
const ResolveLibraryIdSchema = z.object({
|
||||
libraryName: z.string().describe(
|
||||
'Library name to search for and resolve to a TrueRef library ID'
|
||||
),
|
||||
query: z.string().describe(
|
||||
"The user's question or context to help rank results"
|
||||
),
|
||||
libraryName: z
|
||||
.string()
|
||||
.describe('Library name to search for and resolve to a TrueRef library ID'),
|
||||
query: z.string().describe("The user's question or context to help rank results")
|
||||
});
|
||||
|
||||
const QueryDocsSchema = z.object({
|
||||
libraryId: z.string().describe(
|
||||
'The TrueRef library ID obtained from resolve-library-id, e.g. /facebook/react'
|
||||
),
|
||||
query: z.string().describe(
|
||||
'Specific question about the library to retrieve relevant documentation'
|
||||
),
|
||||
tokens: z.number().optional().describe(
|
||||
'Maximum token budget for the response (default: 10000)'
|
||||
),
|
||||
libraryId: z
|
||||
.string()
|
||||
.describe('The TrueRef library ID obtained from resolve-library-id, e.g. /facebook/react'),
|
||||
query: z
|
||||
.string()
|
||||
.describe('Specific question about the library to retrieve relevant documentation'),
|
||||
tokens: z.number().optional().describe('Maximum token budget for the response (default: 10000)')
|
||||
});
|
||||
|
||||
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
||||
tools: [
|
||||
{
|
||||
name: 'resolve-library-id',
|
||||
description: [
|
||||
'Searches TrueRef to find a library matching the given name.',
|
||||
'Returns a list of matching libraries with their IDs.',
|
||||
'ALWAYS call this tool before query-docs to get the correct library ID.',
|
||||
'Call at most 3 times per user question.',
|
||||
].join(' '),
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
libraryName: {
|
||||
type: 'string',
|
||||
description: 'Library name to search for',
|
||||
},
|
||||
query: {
|
||||
type: 'string',
|
||||
description: "User's question for relevance ranking",
|
||||
},
|
||||
},
|
||||
required: ['libraryName', 'query'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'query-docs',
|
||||
description: [
|
||||
'Fetches documentation and code examples from TrueRef for a specific library.',
|
||||
'Requires a library ID obtained from resolve-library-id.',
|
||||
'Returns relevant snippets formatted for LLM consumption.',
|
||||
'Call at most 3 times per user question.',
|
||||
].join(' '),
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
libraryId: {
|
||||
type: 'string',
|
||||
description: 'TrueRef library ID, e.g. /facebook/react',
|
||||
},
|
||||
query: {
|
||||
type: 'string',
|
||||
description: 'Specific question about the library',
|
||||
},
|
||||
tokens: {
|
||||
type: 'number',
|
||||
description: 'Max token budget (default: 10000)',
|
||||
},
|
||||
},
|
||||
required: ['libraryId', 'query'],
|
||||
},
|
||||
},
|
||||
],
|
||||
tools: [
|
||||
{
|
||||
name: 'resolve-library-id',
|
||||
description: [
|
||||
'Searches TrueRef to find a library matching the given name.',
|
||||
'Returns a list of matching libraries with their IDs.',
|
||||
'ALWAYS call this tool before query-docs to get the correct library ID.',
|
||||
'Call at most 3 times per user question.'
|
||||
].join(' '),
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
libraryName: {
|
||||
type: 'string',
|
||||
description: 'Library name to search for'
|
||||
},
|
||||
query: {
|
||||
type: 'string',
|
||||
description: "User's question for relevance ranking"
|
||||
}
|
||||
},
|
||||
required: ['libraryName', 'query']
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'query-docs',
|
||||
description: [
|
||||
'Fetches documentation and code examples from TrueRef for a specific library.',
|
||||
'Requires a library ID obtained from resolve-library-id.',
|
||||
'Returns relevant snippets formatted for LLM consumption.',
|
||||
'Call at most 3 times per user question.'
|
||||
].join(' '),
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
libraryId: {
|
||||
type: 'string',
|
||||
description: 'TrueRef library ID, e.g. /facebook/react'
|
||||
},
|
||||
query: {
|
||||
type: 'string',
|
||||
description: 'Specific question about the library'
|
||||
},
|
||||
tokens: {
|
||||
type: 'number',
|
||||
description: 'Max token budget (default: 10000)'
|
||||
}
|
||||
},
|
||||
required: ['libraryId', 'query']
|
||||
}
|
||||
}
|
||||
]
|
||||
}));
|
||||
|
||||
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
||||
const { name, arguments: args } = request.params;
|
||||
const { name, arguments: args } = request.params;
|
||||
|
||||
if (name === 'resolve-library-id') {
|
||||
const { libraryName, query } = ResolveLibraryIdSchema.parse(args);
|
||||
if (name === 'resolve-library-id') {
|
||||
const { libraryName, query } = ResolveLibraryIdSchema.parse(args);
|
||||
|
||||
const url = new URL(`${API_BASE}/api/v1/libs/search`);
|
||||
url.searchParams.set('libraryName', libraryName);
|
||||
url.searchParams.set('query', query);
|
||||
url.searchParams.set('type', 'txt');
|
||||
const url = new URL(`${API_BASE}/api/v1/libs/search`);
|
||||
url.searchParams.set('libraryName', libraryName);
|
||||
url.searchParams.set('query', query);
|
||||
url.searchParams.set('type', 'txt');
|
||||
|
||||
const response = await fetch(url.toString());
|
||||
if (!response.ok) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Error searching libraries: ${response.status} ${response.statusText}`,
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
const response = await fetch(url.toString());
|
||||
if (!response.ok) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error searching libraries: ${response.status} ${response.statusText}`
|
||||
}
|
||||
],
|
||||
isError: true
|
||||
};
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
return {
|
||||
content: [{ type: 'text', text }],
|
||||
};
|
||||
}
|
||||
const text = await response.text();
|
||||
return {
|
||||
content: [{ type: 'text', text }]
|
||||
};
|
||||
}
|
||||
|
||||
if (name === 'query-docs') {
|
||||
const { libraryId, query, tokens } = QueryDocsSchema.parse(args);
|
||||
if (name === 'query-docs') {
|
||||
const { libraryId, query, tokens } = QueryDocsSchema.parse(args);
|
||||
|
||||
const url = new URL(`${API_BASE}/api/v1/context`);
|
||||
url.searchParams.set('libraryId', libraryId);
|
||||
url.searchParams.set('query', query);
|
||||
url.searchParams.set('type', 'txt');
|
||||
if (tokens) url.searchParams.set('tokens', String(tokens));
|
||||
const url = new URL(`${API_BASE}/api/v1/context`);
|
||||
url.searchParams.set('libraryId', libraryId);
|
||||
url.searchParams.set('query', query);
|
||||
url.searchParams.set('type', 'txt');
|
||||
if (tokens) url.searchParams.set('tokens', String(tokens));
|
||||
|
||||
const response = await fetch(url.toString());
|
||||
if (!response.ok) {
|
||||
const status = response.status;
|
||||
if (status === 404) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Library "${libraryId}" not found. Please run resolve-library-id first.`,
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
if (status === 503) {
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Library "${libraryId}" is currently being indexed. Please try again in a moment.`,
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Error fetching documentation: ${response.status} ${response.statusText}`,
|
||||
}],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
const response = await fetch(url.toString());
|
||||
if (!response.ok) {
|
||||
const status = response.status;
|
||||
if (status === 404) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Library "${libraryId}" not found. Please run resolve-library-id first.`
|
||||
}
|
||||
],
|
||||
isError: true
|
||||
};
|
||||
}
|
||||
if (status === 503) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Library "${libraryId}" is currently being indexed. Please try again in a moment.`
|
||||
}
|
||||
],
|
||||
isError: true
|
||||
};
|
||||
}
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error fetching documentation: ${response.status} ${response.statusText}`
|
||||
}
|
||||
],
|
||||
isError: true
|
||||
};
|
||||
}
|
||||
|
||||
const text = await response.text();
|
||||
return {
|
||||
content: [{ type: 'text', text }],
|
||||
};
|
||||
}
|
||||
const text = await response.text();
|
||||
return {
|
||||
content: [{ type: 'text', text }]
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: 'text', text: `Unknown tool: ${name}` }],
|
||||
isError: true,
|
||||
};
|
||||
return {
|
||||
content: [{ type: 'text', text: `Unknown tool: ${name}` }],
|
||||
isError: true
|
||||
};
|
||||
});
|
||||
|
||||
async function main() {
|
||||
const transport = new StdioServerTransport();
|
||||
await server.connect(transport);
|
||||
// Server runs until process exits
|
||||
const transport = new StdioServerTransport();
|
||||
await server.connect(transport);
|
||||
// Server runs until process exits
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
process.stderr.write(`MCP server error: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
process.stderr.write(`MCP server error: ${err.message}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
```
|
||||
|
||||
@@ -238,18 +239,19 @@ main().catch((err) => {
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"mcp:start": "node --experimental-vm-modules src/mcp/index.ts"
|
||||
}
|
||||
"scripts": {
|
||||
"mcp:start": "node --experimental-vm-modules src/mcp/index.ts"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Or with `tsx` for TypeScript-direct execution:
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"mcp:start": "tsx src/mcp/index.ts"
|
||||
}
|
||||
"scripts": {
|
||||
"mcp:start": "tsx src/mcp/index.ts"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -261,30 +263,31 @@ Users add to `.mcp.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"trueref": {
|
||||
"command": "node",
|
||||
"args": ["/path/to/trueref/dist/mcp/index.js"],
|
||||
"env": {
|
||||
"TRUEREF_API_URL": "http://localhost:5173"
|
||||
}
|
||||
}
|
||||
}
|
||||
"mcpServers": {
|
||||
"trueref": {
|
||||
"command": "node",
|
||||
"args": ["/path/to/trueref/dist/mcp/index.js"],
|
||||
"env": {
|
||||
"TRUEREF_API_URL": "http://localhost:5173"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Or with tsx for development:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"trueref": {
|
||||
"command": "npx",
|
||||
"args": ["tsx", "/path/to/trueref/src/mcp/index.ts"],
|
||||
"env": {
|
||||
"TRUEREF_API_URL": "http://localhost:5173"
|
||||
}
|
||||
}
|
||||
}
|
||||
"mcpServers": {
|
||||
"trueref": {
|
||||
"command": "npx",
|
||||
"args": ["tsx", "/path/to/trueref/src/mcp/index.ts"],
|
||||
"env": {
|
||||
"TRUEREF_API_URL": "http://localhost:5173"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -295,13 +298,15 @@ Or with tsx for development:
|
||||
The MCP server should include a `resources` list item (optional) or the library responses themselves prepend rules. Additionally, users should add a Claude rule file:
|
||||
|
||||
```markdown
|
||||
<!-- .claude/rules/trueref.md -->
|
||||
---
|
||||
## <!-- .claude/rules/trueref.md -->
|
||||
|
||||
description: Use TrueRef to retrieve documentation for indexed libraries
|
||||
alwaysApply: true
|
||||
|
||||
---
|
||||
|
||||
When answering questions about indexed libraries, always use the TrueRef MCP tools:
|
||||
|
||||
1. Call `resolve-library-id` with the library name and the user's question to get the library ID
|
||||
2. Call `query-docs` with the library ID and question to retrieve relevant documentation
|
||||
3. Use the returned documentation to answer the question accurately
|
||||
|
||||
@@ -50,64 +50,64 @@ import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
|
||||
const { values: args } = parseArgs({
|
||||
options: {
|
||||
transport: { type: 'string', default: 'stdio' },
|
||||
port: { type: 'string', default: process.env.PORT ?? '3001' },
|
||||
},
|
||||
options: {
|
||||
transport: { type: 'string', default: 'stdio' },
|
||||
port: { type: 'string', default: process.env.PORT ?? '3001' }
|
||||
}
|
||||
});
|
||||
|
||||
async function startHttp(server: Server, port: number): Promise<void> {
|
||||
const httpServer = createServer(async (req, res) => {
|
||||
const url = new URL(req.url!, `http://localhost:${port}`);
|
||||
const httpServer = createServer(async (req, res) => {
|
||||
const url = new URL(req.url!, `http://localhost:${port}`);
|
||||
|
||||
// Health check
|
||||
if (url.pathname === '/ping') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: true }));
|
||||
return;
|
||||
}
|
||||
// Health check
|
||||
if (url.pathname === '/ping') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: true }));
|
||||
return;
|
||||
}
|
||||
|
||||
// MCP endpoint
|
||||
if (url.pathname === '/mcp') {
|
||||
// CORS preflight
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
res.setHeader('Access-Control-Allow-Methods', 'POST, GET, OPTIONS');
|
||||
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Accept');
|
||||
// MCP endpoint
|
||||
if (url.pathname === '/mcp') {
|
||||
// CORS preflight
|
||||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||||
res.setHeader('Access-Control-Allow-Methods', 'POST, GET, OPTIONS');
|
||||
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Accept');
|
||||
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const transport = new StreamableHTTPServerTransport({
|
||||
sessionIdGenerator: () => crypto.randomUUID(),
|
||||
});
|
||||
const transport = new StreamableHTTPServerTransport({
|
||||
sessionIdGenerator: () => crypto.randomUUID()
|
||||
});
|
||||
|
||||
await server.connect(transport);
|
||||
await transport.handleRequest(req, res);
|
||||
return;
|
||||
}
|
||||
await server.connect(transport);
|
||||
await transport.handleRequest(req, res);
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404);
|
||||
res.end('Not Found');
|
||||
});
|
||||
res.writeHead(404);
|
||||
res.end('Not Found');
|
||||
});
|
||||
|
||||
httpServer.listen(port, () => {
|
||||
process.stderr.write(`TrueRef MCP server listening on http://localhost:${port}/mcp\n`);
|
||||
});
|
||||
httpServer.listen(port, () => {
|
||||
process.stderr.write(`TrueRef MCP server listening on http://localhost:${port}/mcp\n`);
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const mcpServer = createMcpServer(); // shared server creation
|
||||
const mcpServer = createMcpServer(); // shared server creation
|
||||
|
||||
if (args.transport === 'http') {
|
||||
const port = parseInt(args.port!, 10);
|
||||
await startHttp(mcpServer, port);
|
||||
} else {
|
||||
const transport = new StdioServerTransport();
|
||||
await mcpServer.connect(transport);
|
||||
}
|
||||
if (args.transport === 'http') {
|
||||
const port = parseInt(args.port!, 10);
|
||||
await startHttp(mcpServer, port);
|
||||
} else {
|
||||
const transport = new StdioServerTransport();
|
||||
await mcpServer.connect(transport);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -117,10 +117,10 @@ async function main() {
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"mcp:start": "tsx src/mcp/index.ts",
|
||||
"mcp:http": "tsx src/mcp/index.ts --transport http --port 3001"
|
||||
}
|
||||
"scripts": {
|
||||
"mcp:start": "tsx src/mcp/index.ts",
|
||||
"mcp:http": "tsx src/mcp/index.ts --transport http --port 3001"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -132,12 +132,12 @@ For HTTP transport, users configure Claude Code with the remote URL:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"trueref": {
|
||||
"type": "http",
|
||||
"url": "http://localhost:3001/mcp"
|
||||
}
|
||||
}
|
||||
"mcpServers": {
|
||||
"trueref": {
|
||||
"type": "http",
|
||||
"url": "http://localhost:3001/mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -32,53 +32,53 @@ Support `trueref.json` configuration files placed in the root of a repository. T
|
||||
// src/lib/server/config/trueref-config.schema.ts
|
||||
|
||||
export interface TrueRefConfig {
|
||||
/**
|
||||
* Override the display name for this library.
|
||||
* 1–100 characters.
|
||||
*/
|
||||
projectTitle?: string;
|
||||
/**
|
||||
* Override the display name for this library.
|
||||
* 1–100 characters.
|
||||
*/
|
||||
projectTitle?: string;
|
||||
|
||||
/**
|
||||
* Description of the library for search ranking.
|
||||
* 10–500 characters.
|
||||
*/
|
||||
description?: string;
|
||||
/**
|
||||
* Description of the library for search ranking.
|
||||
* 10–500 characters.
|
||||
*/
|
||||
description?: string;
|
||||
|
||||
/**
|
||||
* Folders to include in indexing (allowlist).
|
||||
* Each entry is a path prefix or regex string.
|
||||
* If empty/absent, all folders are included.
|
||||
* Examples: ["src/", "docs/", "^packages/core"]
|
||||
*/
|
||||
folders?: string[];
|
||||
/**
|
||||
* Folders to include in indexing (allowlist).
|
||||
* Each entry is a path prefix or regex string.
|
||||
* If empty/absent, all folders are included.
|
||||
* Examples: ["src/", "docs/", "^packages/core"]
|
||||
*/
|
||||
folders?: string[];
|
||||
|
||||
/**
|
||||
* Folders to exclude from indexing.
|
||||
* Applied after `folders` allowlist.
|
||||
* Examples: ["test/", "fixtures/", "__mocks__"]
|
||||
*/
|
||||
excludeFolders?: string[];
|
||||
/**
|
||||
* Folders to exclude from indexing.
|
||||
* Applied after `folders` allowlist.
|
||||
* Examples: ["test/", "fixtures/", "__mocks__"]
|
||||
*/
|
||||
excludeFolders?: string[];
|
||||
|
||||
/**
|
||||
* Exact filenames to exclude (no path, no regex).
|
||||
* Examples: ["README.md", "CHANGELOG.md", "jest.config.ts"]
|
||||
*/
|
||||
excludeFiles?: string[];
|
||||
/**
|
||||
* Exact filenames to exclude (no path, no regex).
|
||||
* Examples: ["README.md", "CHANGELOG.md", "jest.config.ts"]
|
||||
*/
|
||||
excludeFiles?: string[];
|
||||
|
||||
/**
|
||||
* Best practices / rules to inject at the top of every query-docs response.
|
||||
* Each rule: 5–500 characters.
|
||||
* Maximum 20 rules.
|
||||
*/
|
||||
rules?: string[];
|
||||
/**
|
||||
* Best practices / rules to inject at the top of every query-docs response.
|
||||
* Each rule: 5–500 characters.
|
||||
* Maximum 20 rules.
|
||||
*/
|
||||
rules?: string[];
|
||||
|
||||
/**
|
||||
* Previously released versions to make available for versioned queries.
|
||||
*/
|
||||
previousVersions?: Array<{
|
||||
tag: string; // git tag (e.g. "v1.2.3")
|
||||
title: string; // human-readable (e.g. "Version 1.2.3")
|
||||
}>;
|
||||
/**
|
||||
* Previously released versions to make available for versioned queries.
|
||||
*/
|
||||
previousVersions?: Array<{
|
||||
tag: string; // git tag (e.g. "v1.2.3")
|
||||
title: string; // human-readable (e.g. "Version 1.2.3")
|
||||
}>;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -88,14 +88,14 @@ export interface TrueRefConfig {
|
||||
|
||||
```typescript
|
||||
const CONFIG_CONSTRAINTS = {
|
||||
projectTitle: { minLength: 1, maxLength: 100 },
|
||||
description: { minLength: 10, maxLength: 500 },
|
||||
folders: { maxItems: 50, maxLength: 200 }, // per entry
|
||||
excludeFolders: { maxItems: 50, maxLength: 200 },
|
||||
excludeFiles: { maxItems: 100, maxLength: 200 },
|
||||
rules: { maxItems: 20, minLength: 5, maxLength: 500 },
|
||||
previousVersions: { maxItems: 50 },
|
||||
versionTag: { pattern: /^v?\d+\.\d+(\.\d+)?(-.*)?$/ },
|
||||
projectTitle: { minLength: 1, maxLength: 100 },
|
||||
description: { minLength: 10, maxLength: 500 },
|
||||
folders: { maxItems: 50, maxLength: 200 }, // per entry
|
||||
excludeFolders: { maxItems: 50, maxLength: 200 },
|
||||
excludeFiles: { maxItems: 100, maxLength: 200 },
|
||||
rules: { maxItems: 20, minLength: 5, maxLength: 500 },
|
||||
previousVersions: { maxItems: 50 },
|
||||
versionTag: { pattern: /^v?\d+\.\d+(\.\d+)?(-.*)?$/ }
|
||||
};
|
||||
```
|
||||
|
||||
@@ -107,94 +107,96 @@ const CONFIG_CONSTRAINTS = {
|
||||
// src/lib/server/config/config-parser.ts
|
||||
|
||||
export interface ParsedConfig {
|
||||
config: TrueRefConfig;
|
||||
source: 'trueref.json' | 'context7.json';
|
||||
warnings: string[];
|
||||
config: TrueRefConfig;
|
||||
source: 'trueref.json' | 'context7.json';
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export function parseConfigFile(content: string, filename: string): ParsedConfig {
|
||||
let raw: unknown;
|
||||
let raw: unknown;
|
||||
|
||||
try {
|
||||
raw = JSON.parse(content);
|
||||
} catch (e) {
|
||||
throw new ConfigParseError(`${filename} is not valid JSON: ${(e as Error).message}`);
|
||||
}
|
||||
try {
|
||||
raw = JSON.parse(content);
|
||||
} catch (e) {
|
||||
throw new ConfigParseError(`${filename} is not valid JSON: ${(e as Error).message}`);
|
||||
}
|
||||
|
||||
if (typeof raw !== 'object' || raw === null) {
|
||||
throw new ConfigParseError(`${filename} must be a JSON object`);
|
||||
}
|
||||
if (typeof raw !== 'object' || raw === null) {
|
||||
throw new ConfigParseError(`${filename} must be a JSON object`);
|
||||
}
|
||||
|
||||
const config = raw as Record<string, unknown>;
|
||||
const validated: TrueRefConfig = {};
|
||||
const warnings: string[] = [];
|
||||
const config = raw as Record<string, unknown>;
|
||||
const validated: TrueRefConfig = {};
|
||||
const warnings: string[] = [];
|
||||
|
||||
// projectTitle
|
||||
if (config.projectTitle !== undefined) {
|
||||
if (typeof config.projectTitle !== 'string') {
|
||||
warnings.push('projectTitle must be a string, ignoring');
|
||||
} else if (config.projectTitle.length > 100) {
|
||||
validated.projectTitle = config.projectTitle.slice(0, 100);
|
||||
warnings.push('projectTitle truncated to 100 characters');
|
||||
} else {
|
||||
validated.projectTitle = config.projectTitle;
|
||||
}
|
||||
}
|
||||
// projectTitle
|
||||
if (config.projectTitle !== undefined) {
|
||||
if (typeof config.projectTitle !== 'string') {
|
||||
warnings.push('projectTitle must be a string, ignoring');
|
||||
} else if (config.projectTitle.length > 100) {
|
||||
validated.projectTitle = config.projectTitle.slice(0, 100);
|
||||
warnings.push('projectTitle truncated to 100 characters');
|
||||
} else {
|
||||
validated.projectTitle = config.projectTitle;
|
||||
}
|
||||
}
|
||||
|
||||
// description
|
||||
if (config.description !== undefined) {
|
||||
if (typeof config.description === 'string') {
|
||||
validated.description = config.description.slice(0, 500);
|
||||
}
|
||||
}
|
||||
// description
|
||||
if (config.description !== undefined) {
|
||||
if (typeof config.description === 'string') {
|
||||
validated.description = config.description.slice(0, 500);
|
||||
}
|
||||
}
|
||||
|
||||
// folders / excludeFolders / excludeFiles — validated as string arrays
|
||||
for (const field of ['folders', 'excludeFolders', 'excludeFiles'] as const) {
|
||||
if (config[field] !== undefined) {
|
||||
if (!Array.isArray(config[field])) {
|
||||
warnings.push(`${field} must be an array, ignoring`);
|
||||
} else {
|
||||
validated[field] = (config[field] as unknown[])
|
||||
.filter((item): item is string => {
|
||||
if (typeof item !== 'string') {
|
||||
warnings.push(`${field} entry must be a string, skipping: ${item}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.slice(0, field === 'excludeFiles' ? 100 : 50);
|
||||
}
|
||||
}
|
||||
}
|
||||
// folders / excludeFolders / excludeFiles — validated as string arrays
|
||||
for (const field of ['folders', 'excludeFolders', 'excludeFiles'] as const) {
|
||||
if (config[field] !== undefined) {
|
||||
if (!Array.isArray(config[field])) {
|
||||
warnings.push(`${field} must be an array, ignoring`);
|
||||
} else {
|
||||
validated[field] = (config[field] as unknown[])
|
||||
.filter((item): item is string => {
|
||||
if (typeof item !== 'string') {
|
||||
warnings.push(`${field} entry must be a string, skipping: ${item}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.slice(0, field === 'excludeFiles' ? 100 : 50);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rules
|
||||
if (config.rules !== undefined) {
|
||||
if (Array.isArray(config.rules)) {
|
||||
validated.rules = (config.rules as unknown[])
|
||||
.filter((r): r is string => typeof r === 'string' && r.length >= 5)
|
||||
.map(r => r.slice(0, 500))
|
||||
.slice(0, 20);
|
||||
}
|
||||
}
|
||||
// rules
|
||||
if (config.rules !== undefined) {
|
||||
if (Array.isArray(config.rules)) {
|
||||
validated.rules = (config.rules as unknown[])
|
||||
.filter((r): r is string => typeof r === 'string' && r.length >= 5)
|
||||
.map((r) => r.slice(0, 500))
|
||||
.slice(0, 20);
|
||||
}
|
||||
}
|
||||
|
||||
// previousVersions
|
||||
if (config.previousVersions !== undefined) {
|
||||
if (Array.isArray(config.previousVersions)) {
|
||||
validated.previousVersions = (config.previousVersions as unknown[])
|
||||
.filter((v): v is { tag: string; title: string } =>
|
||||
typeof v === 'object' && v !== null &&
|
||||
typeof (v as Record<string, unknown>).tag === 'string' &&
|
||||
typeof (v as Record<string, unknown>).title === 'string'
|
||||
)
|
||||
.slice(0, 50);
|
||||
}
|
||||
}
|
||||
// previousVersions
|
||||
if (config.previousVersions !== undefined) {
|
||||
if (Array.isArray(config.previousVersions)) {
|
||||
validated.previousVersions = (config.previousVersions as unknown[])
|
||||
.filter(
|
||||
(v): v is { tag: string; title: string } =>
|
||||
typeof v === 'object' &&
|
||||
v !== null &&
|
||||
typeof (v as Record<string, unknown>).tag === 'string' &&
|
||||
typeof (v as Record<string, unknown>).title === 'string'
|
||||
)
|
||||
.slice(0, 50);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
config: validated,
|
||||
source: filename.startsWith('trueref') ? 'trueref.json' : 'context7.json',
|
||||
warnings,
|
||||
};
|
||||
return {
|
||||
config: validated,
|
||||
source: filename.startsWith('trueref') ? 'trueref.json' : 'context7.json',
|
||||
warnings
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
@@ -219,21 +221,15 @@ When `query-docs` returns results, `rules` from `repository_configs` are prepend
|
||||
|
||||
```typescript
|
||||
// In formatters.ts
|
||||
function buildContextResponse(
|
||||
snippets: Snippet[],
|
||||
config: RepositoryConfig | null
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
function buildContextResponse(snippets: Snippet[], config: RepositoryConfig | null): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
if (config?.rules?.length) {
|
||||
parts.push(
|
||||
'## Library Best Practices\n' +
|
||||
config.rules.map(r => `- ${r}`).join('\n')
|
||||
);
|
||||
}
|
||||
if (config?.rules?.length) {
|
||||
parts.push('## Library Best Practices\n' + config.rules.map((r) => `- ${r}`).join('\n'));
|
||||
}
|
||||
|
||||
// ... append snippet content
|
||||
return parts.join('\n\n---\n\n');
|
||||
// ... append snippet content
|
||||
return parts.join('\n\n---\n\n');
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -45,34 +45,37 @@ Examples:
|
||||
### `GET /api/v1/libs/:id/versions`
|
||||
|
||||
Response `200`:
|
||||
|
||||
```json
|
||||
{
|
||||
"versions": [
|
||||
{
|
||||
"id": "/facebook/react/v18.3.0",
|
||||
"repositoryId": "/facebook/react",
|
||||
"tag": "v18.3.0",
|
||||
"title": "React v18.3.0",
|
||||
"state": "indexed",
|
||||
"totalSnippets": 892,
|
||||
"indexedAt": "2026-03-22T10:00:00Z"
|
||||
}
|
||||
]
|
||||
"versions": [
|
||||
{
|
||||
"id": "/facebook/react/v18.3.0",
|
||||
"repositoryId": "/facebook/react",
|
||||
"tag": "v18.3.0",
|
||||
"title": "React v18.3.0",
|
||||
"state": "indexed",
|
||||
"totalSnippets": 892,
|
||||
"indexedAt": "2026-03-22T10:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### `POST /api/v1/libs/:id/versions`
|
||||
|
||||
Request body:
|
||||
|
||||
```json
|
||||
{
|
||||
"tag": "v18.3.0",
|
||||
"title": "React v18.3.0",
|
||||
"autoIndex": true
|
||||
"tag": "v18.3.0",
|
||||
"title": "React v18.3.0",
|
||||
"autoIndex": true
|
||||
}
|
||||
```
|
||||
|
||||
Response `201`:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": { ...RepositoryVersion },
|
||||
@@ -96,23 +99,22 @@ Response `202` with job details.
|
||||
|
||||
```typescript
|
||||
async function listGitHubTags(
|
||||
owner: string,
|
||||
repo: string,
|
||||
token?: string
|
||||
owner: string,
|
||||
repo: string,
|
||||
token?: string
|
||||
): Promise<Array<{ name: string; commit: { sha: string } }>> {
|
||||
const headers: Record<string, string> = {
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
'User-Agent': 'TrueRef/1.0',
|
||||
};
|
||||
if (token) headers['Authorization'] = `Bearer ${token}`;
|
||||
const headers: Record<string, string> = {
|
||||
Accept: 'application/vnd.github.v3+json',
|
||||
'User-Agent': 'TrueRef/1.0'
|
||||
};
|
||||
if (token) headers['Authorization'] = `Bearer ${token}`;
|
||||
|
||||
const response = await fetch(
|
||||
`https://api.github.com/repos/${owner}/${repo}/tags?per_page=100`,
|
||||
{ headers }
|
||||
);
|
||||
const response = await fetch(`https://api.github.com/repos/${owner}/${repo}/tags?per_page=100`, {
|
||||
headers
|
||||
});
|
||||
|
||||
if (!response.ok) throw new GitHubApiError(response.status);
|
||||
return response.json();
|
||||
if (!response.ok) throw new GitHubApiError(response.status);
|
||||
return response.json();
|
||||
}
|
||||
```
|
||||
|
||||
@@ -124,28 +126,26 @@ In the search/context endpoints, the `libraryId` is parsed to extract the option
|
||||
|
||||
```typescript
|
||||
function resolveSearchTarget(libraryId: string): {
|
||||
repositoryId: string;
|
||||
versionId?: string;
|
||||
repositoryId: string;
|
||||
versionId?: string;
|
||||
} {
|
||||
const { repositoryId, version } = parseLibraryId(libraryId);
|
||||
const { repositoryId, version } = parseLibraryId(libraryId);
|
||||
|
||||
if (!version) {
|
||||
// Query default branch: versionId = NULL
|
||||
return { repositoryId };
|
||||
}
|
||||
if (!version) {
|
||||
// Query default branch: versionId = NULL
|
||||
return { repositoryId };
|
||||
}
|
||||
|
||||
// Look up versionId from tag
|
||||
const versionRecord = db.prepare(
|
||||
`SELECT id FROM repository_versions WHERE repository_id = ? AND tag = ?`
|
||||
).get(repositoryId, version) as { id: string } | undefined;
|
||||
// Look up versionId from tag
|
||||
const versionRecord = db
|
||||
.prepare(`SELECT id FROM repository_versions WHERE repository_id = ? AND tag = ?`)
|
||||
.get(repositoryId, version) as { id: string } | undefined;
|
||||
|
||||
if (!versionRecord) {
|
||||
throw new NotFoundError(
|
||||
`Version "${version}" not found for library "${repositoryId}"`
|
||||
);
|
||||
}
|
||||
if (!versionRecord) {
|
||||
throw new NotFoundError(`Version "${version}" not found for library "${repositoryId}"`);
|
||||
}
|
||||
|
||||
return { repositoryId, versionId: versionRecord.id };
|
||||
return { repositoryId, versionId: versionRecord.id };
|
||||
}
|
||||
```
|
||||
|
||||
@@ -157,20 +157,20 @@ Snippets with `version_id IS NULL` belong to the default branch; snippets with a
|
||||
|
||||
```typescript
|
||||
export class VersionService {
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
constructor(private db: BetterSQLite3.Database) {}
|
||||
|
||||
list(repositoryId: string): RepositoryVersion[]
|
||||
list(repositoryId: string): RepositoryVersion[];
|
||||
|
||||
add(repositoryId: string, tag: string, title?: string): RepositoryVersion
|
||||
add(repositoryId: string, tag: string, title?: string): RepositoryVersion;
|
||||
|
||||
remove(repositoryId: string, tag: string): void
|
||||
remove(repositoryId: string, tag: string): void;
|
||||
|
||||
getByTag(repositoryId: string, tag: string): RepositoryVersion | null
|
||||
getByTag(repositoryId: string, tag: string): RepositoryVersion | null;
|
||||
|
||||
registerFromConfig(
|
||||
repositoryId: string,
|
||||
previousVersions: { tag: string; title: string }[]
|
||||
): RepositoryVersion[]
|
||||
registerFromConfig(
|
||||
repositoryId: string,
|
||||
previousVersions: { tag: string; title: string }[]
|
||||
): RepositoryVersion[];
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -49,79 +49,79 @@ Implement the main web interface for managing repositories. Built with SvelteKit
|
||||
```svelte
|
||||
<!-- src/lib/components/RepositoryCard.svelte -->
|
||||
<script lang="ts">
|
||||
import type { Repository } from '$lib/types';
|
||||
import type { Repository } from '$lib/types';
|
||||
|
||||
let { repo, onReindex, onDelete } = $props<{
|
||||
repo: Repository;
|
||||
onReindex: (id: string) => void;
|
||||
onDelete: (id: string) => void;
|
||||
}>();
|
||||
let { repo, onReindex, onDelete } = $props<{
|
||||
repo: Repository;
|
||||
onReindex: (id: string) => void;
|
||||
onDelete: (id: string) => void;
|
||||
}>();
|
||||
|
||||
const stateColors = {
|
||||
pending: 'bg-gray-100 text-gray-600',
|
||||
indexing: 'bg-blue-100 text-blue-700',
|
||||
indexed: 'bg-green-100 text-green-700',
|
||||
error: 'bg-red-100 text-red-700',
|
||||
};
|
||||
const stateColors = {
|
||||
pending: 'bg-gray-100 text-gray-600',
|
||||
indexing: 'bg-blue-100 text-blue-700',
|
||||
indexed: 'bg-green-100 text-green-700',
|
||||
error: 'bg-red-100 text-red-700'
|
||||
};
|
||||
|
||||
const stateLabels = {
|
||||
pending: 'Pending',
|
||||
indexing: 'Indexing...',
|
||||
indexed: 'Indexed',
|
||||
error: 'Error',
|
||||
};
|
||||
const stateLabels = {
|
||||
pending: 'Pending',
|
||||
indexing: 'Indexing...',
|
||||
indexed: 'Indexed',
|
||||
error: 'Error'
|
||||
};
|
||||
</script>
|
||||
|
||||
<div class="rounded-xl border border-gray-200 bg-white p-5 shadow-sm">
|
||||
<div class="flex items-start justify-between">
|
||||
<div>
|
||||
<h3 class="font-semibold text-gray-900">{repo.title}</h3>
|
||||
<p class="mt-0.5 font-mono text-sm text-gray-500">{repo.id}</p>
|
||||
</div>
|
||||
<span class="rounded-full px-2.5 py-0.5 text-xs font-medium {stateColors[repo.state]}">
|
||||
{stateLabels[repo.state]}
|
||||
</span>
|
||||
</div>
|
||||
<div class="flex items-start justify-between">
|
||||
<div>
|
||||
<h3 class="font-semibold text-gray-900">{repo.title}</h3>
|
||||
<p class="mt-0.5 font-mono text-sm text-gray-500">{repo.id}</p>
|
||||
</div>
|
||||
<span class="rounded-full px-2.5 py-0.5 text-xs font-medium {stateColors[repo.state]}">
|
||||
{stateLabels[repo.state]}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{#if repo.description}
|
||||
<p class="mt-2 line-clamp-2 text-sm text-gray-600">{repo.description}</p>
|
||||
{/if}
|
||||
{#if repo.description}
|
||||
<p class="mt-2 line-clamp-2 text-sm text-gray-600">{repo.description}</p>
|
||||
{/if}
|
||||
|
||||
<div class="mt-4 flex gap-4 text-sm text-gray-500">
|
||||
<span>{repo.totalSnippets.toLocaleString()} snippets</span>
|
||||
<span>·</span>
|
||||
<span>Trust: {repo.trustScore.toFixed(1)}/10</span>
|
||||
{#if repo.stars}
|
||||
<span>·</span>
|
||||
<span>★ {repo.stars.toLocaleString()}</span>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="mt-4 flex gap-4 text-sm text-gray-500">
|
||||
<span>{repo.totalSnippets.toLocaleString()} snippets</span>
|
||||
<span>·</span>
|
||||
<span>Trust: {repo.trustScore.toFixed(1)}/10</span>
|
||||
{#if repo.stars}
|
||||
<span>·</span>
|
||||
<span>★ {repo.stars.toLocaleString()}</span>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if repo.state === 'error'}
|
||||
<p class="mt-2 text-xs text-red-600">Indexing failed. Check jobs for details.</p>
|
||||
{/if}
|
||||
{#if repo.state === 'error'}
|
||||
<p class="mt-2 text-xs text-red-600">Indexing failed. Check jobs for details.</p>
|
||||
{/if}
|
||||
|
||||
<div class="mt-4 flex gap-2">
|
||||
<button
|
||||
onclick={() => onReindex(repo.id)}
|
||||
class="rounded-lg bg-blue-600 px-3 py-1.5 text-sm text-white hover:bg-blue-700"
|
||||
disabled={repo.state === 'indexing'}
|
||||
>
|
||||
{repo.state === 'indexing' ? 'Indexing...' : 'Re-index'}
|
||||
</button>
|
||||
<a
|
||||
href="/repos/{encodeURIComponent(repo.id)}"
|
||||
class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm text-gray-700 hover:bg-gray-50"
|
||||
>
|
||||
Details
|
||||
</a>
|
||||
<button
|
||||
onclick={() => onDelete(repo.id)}
|
||||
class="ml-auto rounded-lg px-3 py-1.5 text-sm text-red-600 hover:bg-red-50"
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
<div class="mt-4 flex gap-2">
|
||||
<button
|
||||
onclick={() => onReindex(repo.id)}
|
||||
class="rounded-lg bg-blue-600 px-3 py-1.5 text-sm text-white hover:bg-blue-700"
|
||||
disabled={repo.state === 'indexing'}
|
||||
>
|
||||
{repo.state === 'indexing' ? 'Indexing...' : 'Re-index'}
|
||||
</button>
|
||||
<a
|
||||
href="/repos/{encodeURIComponent(repo.id)}"
|
||||
class="rounded-lg border border-gray-200 px-3 py-1.5 text-sm text-gray-700 hover:bg-gray-50"
|
||||
>
|
||||
Details
|
||||
</a>
|
||||
<button
|
||||
onclick={() => onDelete(repo.id)}
|
||||
class="ml-auto rounded-lg px-3 py-1.5 text-sm text-red-600 hover:bg-red-50"
|
||||
>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
```
|
||||
|
||||
@@ -132,98 +132,104 @@ Implement the main web interface for managing repositories. Built with SvelteKit
|
||||
```svelte
|
||||
<!-- src/lib/components/AddRepositoryModal.svelte -->
|
||||
<script lang="ts">
|
||||
let { onClose, onAdded } = $props<{
|
||||
onClose: () => void;
|
||||
onAdded: () => void;
|
||||
}>();
|
||||
let { onClose, onAdded } = $props<{
|
||||
onClose: () => void;
|
||||
onAdded: () => void;
|
||||
}>();
|
||||
|
||||
let source = $state<'github' | 'local'>('github');
|
||||
let sourceUrl = $state('');
|
||||
let githubToken = $state('');
|
||||
let loading = $state(false);
|
||||
let error = $state<string | null>(null);
|
||||
let source = $state<'github' | 'local'>('github');
|
||||
let sourceUrl = $state('');
|
||||
let githubToken = $state('');
|
||||
let loading = $state(false);
|
||||
let error = $state<string | null>(null);
|
||||
|
||||
async function handleSubmit() {
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
const res = await fetch('/api/v1/libs', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ source, sourceUrl, githubToken: githubToken || undefined }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const data = await res.json();
|
||||
throw new Error(data.error ?? 'Failed to add repository');
|
||||
}
|
||||
onAdded();
|
||||
onClose();
|
||||
} catch (e) {
|
||||
error = (e as Error).message;
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
async function handleSubmit() {
|
||||
loading = true;
|
||||
error = null;
|
||||
try {
|
||||
const res = await fetch('/api/v1/libs', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ source, sourceUrl, githubToken: githubToken || undefined })
|
||||
});
|
||||
if (!res.ok) {
|
||||
const data = await res.json();
|
||||
throw new Error(data.error ?? 'Failed to add repository');
|
||||
}
|
||||
onAdded();
|
||||
onClose();
|
||||
} catch (e) {
|
||||
error = (e as Error).message;
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<dialog class="modal" open>
|
||||
<div class="modal-box max-w-md">
|
||||
<h2 class="mb-4 text-lg font-semibold">Add Repository</h2>
|
||||
<div class="modal-box max-w-md">
|
||||
<h2 class="mb-4 text-lg font-semibold">Add Repository</h2>
|
||||
|
||||
<div class="mb-4 flex gap-2">
|
||||
<button
|
||||
class="flex-1 rounded-lg py-2 text-sm {source === 'github' ? 'bg-blue-600 text-white' : 'border border-gray-200 text-gray-700'}"
|
||||
onclick={() => source = 'github'}
|
||||
>GitHub</button>
|
||||
<button
|
||||
class="flex-1 rounded-lg py-2 text-sm {source === 'local' ? 'bg-blue-600 text-white' : 'border border-gray-200 text-gray-700'}"
|
||||
onclick={() => source = 'local'}
|
||||
>Local Path</button>
|
||||
</div>
|
||||
<div class="mb-4 flex gap-2">
|
||||
<button
|
||||
class="flex-1 rounded-lg py-2 text-sm {source === 'github'
|
||||
? 'bg-blue-600 text-white'
|
||||
: 'border border-gray-200 text-gray-700'}"
|
||||
onclick={() => (source = 'github')}>GitHub</button
|
||||
>
|
||||
<button
|
||||
class="flex-1 rounded-lg py-2 text-sm {source === 'local'
|
||||
? 'bg-blue-600 text-white'
|
||||
: 'border border-gray-200 text-gray-700'}"
|
||||
onclick={() => (source = 'local')}>Local Path</button
|
||||
>
|
||||
</div>
|
||||
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium text-gray-700">
|
||||
{source === 'github' ? 'GitHub URL' : 'Absolute Path'}
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={sourceUrl}
|
||||
placeholder={source === 'github'
|
||||
? 'https://github.com/facebook/react'
|
||||
: '/home/user/projects/my-sdk'}
|
||||
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
|
||||
/>
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium text-gray-700">
|
||||
{source === 'github' ? 'GitHub URL' : 'Absolute Path'}
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={sourceUrl}
|
||||
placeholder={source === 'github'
|
||||
? 'https://github.com/facebook/react'
|
||||
: '/home/user/projects/my-sdk'}
|
||||
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
|
||||
/>
|
||||
</label>
|
||||
|
||||
{#if source === 'github'}
|
||||
<label class="mt-3 block">
|
||||
<span class="text-sm font-medium text-gray-700">GitHub Token (optional, for private repos)</span>
|
||||
<input
|
||||
type="password"
|
||||
bind:value={githubToken}
|
||||
placeholder="ghp_..."
|
||||
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
|
||||
/>
|
||||
</label>
|
||||
{/if}
|
||||
{#if source === 'github'}
|
||||
<label class="mt-3 block">
|
||||
<span class="text-sm font-medium text-gray-700"
|
||||
>GitHub Token (optional, for private repos)</span
|
||||
>
|
||||
<input
|
||||
type="password"
|
||||
bind:value={githubToken}
|
||||
placeholder="ghp_..."
|
||||
class="mt-1 w-full rounded-lg border border-gray-300 px-3 py-2 text-sm"
|
||||
/>
|
||||
</label>
|
||||
{/if}
|
||||
|
||||
{#if error}
|
||||
<p class="mt-3 text-sm text-red-600">{error}</p>
|
||||
{/if}
|
||||
{#if error}
|
||||
<p class="mt-3 text-sm text-red-600">{error}</p>
|
||||
{/if}
|
||||
|
||||
<div class="mt-6 flex justify-end gap-3">
|
||||
<button onclick={onClose} class="rounded-lg border border-gray-200 px-4 py-2 text-sm">
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
onclick={handleSubmit}
|
||||
disabled={loading || !sourceUrl}
|
||||
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
|
||||
>
|
||||
{loading ? 'Adding...' : 'Add & Index'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-6 flex justify-end gap-3">
|
||||
<button onclick={onClose} class="rounded-lg border border-gray-200 px-4 py-2 text-sm">
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
onclick={handleSubmit}
|
||||
disabled={loading || !sourceUrl}
|
||||
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
|
||||
>
|
||||
{loading ? 'Adding...' : 'Add & Index'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</dialog>
|
||||
```
|
||||
|
||||
@@ -234,48 +240,48 @@ Implement the main web interface for managing repositories. Built with SvelteKit
|
||||
```svelte
|
||||
<!-- src/lib/components/IndexingProgress.svelte -->
|
||||
<script lang="ts">
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import type { IndexingJob } from '$lib/types';
|
||||
import { onMount, onDestroy } from 'svelte';
|
||||
import type { IndexingJob } from '$lib/types';
|
||||
|
||||
let { jobId } = $props<{ jobId: string }>();
|
||||
let job = $state<IndexingJob | null>(null);
|
||||
let interval: ReturnType<typeof setInterval>;
|
||||
let { jobId } = $props<{ jobId: string }>();
|
||||
let job = $state<IndexingJob | null>(null);
|
||||
let interval: ReturnType<typeof setInterval>;
|
||||
|
||||
async function pollJob() {
|
||||
const res = await fetch(`/api/v1/jobs/${jobId}`);
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
job = data.job;
|
||||
if (job?.status === 'done' || job?.status === 'failed') {
|
||||
clearInterval(interval);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function pollJob() {
|
||||
const res = await fetch(`/api/v1/jobs/${jobId}`);
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
job = data.job;
|
||||
if (job?.status === 'done' || job?.status === 'failed') {
|
||||
clearInterval(interval);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
onMount(() => {
|
||||
pollJob();
|
||||
interval = setInterval(pollJob, 2000);
|
||||
});
|
||||
onMount(() => {
|
||||
pollJob();
|
||||
interval = setInterval(pollJob, 2000);
|
||||
});
|
||||
|
||||
onDestroy(() => clearInterval(interval));
|
||||
onDestroy(() => clearInterval(interval));
|
||||
</script>
|
||||
|
||||
{#if job}
|
||||
<div class="mt-2">
|
||||
<div class="flex justify-between text-xs text-gray-500">
|
||||
<span>{job.processedFiles} / {job.totalFiles} files</span>
|
||||
<span>{job.progress}%</span>
|
||||
</div>
|
||||
<div class="mt-1 h-1.5 w-full rounded-full bg-gray-200">
|
||||
<div
|
||||
class="h-1.5 rounded-full bg-blue-600 transition-all"
|
||||
style="width: {job.progress}%"
|
||||
></div>
|
||||
</div>
|
||||
{#if job.status === 'failed'}
|
||||
<p class="mt-1 text-xs text-red-600">{job.error}</p>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="mt-2">
|
||||
<div class="flex justify-between text-xs text-gray-500">
|
||||
<span>{job.processedFiles} / {job.totalFiles} files</span>
|
||||
<span>{job.progress}%</span>
|
||||
</div>
|
||||
<div class="mt-1 h-1.5 w-full rounded-full bg-gray-200">
|
||||
<div
|
||||
class="h-1.5 rounded-full bg-blue-600 transition-all"
|
||||
style="width: {job.progress}%"
|
||||
></div>
|
||||
</div>
|
||||
{#if job.status === 'failed'}
|
||||
<p class="mt-1 text-xs text-red-600">{job.error}</p>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
```
|
||||
|
||||
@@ -288,9 +294,9 @@ Implement the main web interface for managing repositories. Built with SvelteKit
|
||||
import type { PageServerLoad } from './$types';
|
||||
|
||||
export const load: PageServerLoad = async ({ fetch }) => {
|
||||
const res = await fetch('/api/v1/libs');
|
||||
const data = await res.json();
|
||||
return { repositories: data.libraries };
|
||||
const res = await fetch('/api/v1/libs');
|
||||
const data = await res.json();
|
||||
return { repositories: data.libraries };
|
||||
};
|
||||
```
|
||||
|
||||
|
||||
@@ -57,25 +57,31 @@ An interactive search interface within the web UI that lets users test the docum
|
||||
```svelte
|
||||
<!-- src/lib/components/search/LibraryResult.svelte -->
|
||||
<script lang="ts">
|
||||
let { result, onSelect } = $props<{
|
||||
result: { id: string; title: string; description: string; totalSnippets: number; trustScore: number };
|
||||
onSelect: (id: string) => void;
|
||||
}>();
|
||||
let { result, onSelect } = $props<{
|
||||
result: {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string;
|
||||
totalSnippets: number;
|
||||
trustScore: number;
|
||||
};
|
||||
onSelect: (id: string) => void;
|
||||
}>();
|
||||
</script>
|
||||
|
||||
<button
|
||||
onclick={() => onSelect(result.id)}
|
||||
class="w-full rounded-xl border border-gray-200 bg-white p-4 text-left shadow-sm hover:border-blue-300 hover:shadow-md transition-all"
|
||||
onclick={() => onSelect(result.id)}
|
||||
class="w-full rounded-xl border border-gray-200 bg-white p-4 text-left shadow-sm transition-all hover:border-blue-300 hover:shadow-md"
|
||||
>
|
||||
<div class="flex items-center justify-between">
|
||||
<span class="font-semibold text-gray-900">{result.title}</span>
|
||||
<span class="text-xs text-gray-400">Trust {result.trustScore.toFixed(1)}/10</span>
|
||||
</div>
|
||||
<p class="font-mono text-xs text-gray-400">{result.id}</p>
|
||||
{#if result.description}
|
||||
<p class="mt-1.5 text-sm text-gray-600 line-clamp-2">{result.description}</p>
|
||||
{/if}
|
||||
<p class="mt-2 text-xs text-gray-400">{result.totalSnippets.toLocaleString()} snippets</p>
|
||||
<div class="flex items-center justify-between">
|
||||
<span class="font-semibold text-gray-900">{result.title}</span>
|
||||
<span class="text-xs text-gray-400">Trust {result.trustScore.toFixed(1)}/10</span>
|
||||
</div>
|
||||
<p class="font-mono text-xs text-gray-400">{result.id}</p>
|
||||
{#if result.description}
|
||||
<p class="mt-1.5 line-clamp-2 text-sm text-gray-600">{result.description}</p>
|
||||
{/if}
|
||||
<p class="mt-2 text-xs text-gray-400">{result.totalSnippets.toLocaleString()} snippets</p>
|
||||
</button>
|
||||
```
|
||||
|
||||
@@ -86,37 +92,39 @@ An interactive search interface within the web UI that lets users test the docum
|
||||
```svelte
|
||||
<!-- src/lib/components/search/SnippetCard.svelte -->
|
||||
<script lang="ts">
|
||||
import type { Snippet } from '$lib/types';
|
||||
import type { Snippet } from '$lib/types';
|
||||
|
||||
let { snippet } = $props<{ snippet: Snippet }>();
|
||||
let { snippet } = $props<{ snippet: Snippet }>();
|
||||
</script>
|
||||
|
||||
<div class="rounded-xl border border-gray-200 bg-white overflow-hidden">
|
||||
<div class="flex items-center justify-between border-b border-gray-100 px-4 py-2.5">
|
||||
<div class="flex items-center gap-2">
|
||||
{#if snippet.type === 'code'}
|
||||
<span class="rounded bg-purple-100 px-1.5 py-0.5 text-xs text-purple-700">code</span>
|
||||
{:else}
|
||||
<span class="rounded bg-blue-100 px-1.5 py-0.5 text-xs text-blue-700">info</span>
|
||||
{/if}
|
||||
{#if snippet.title}
|
||||
<span class="text-sm font-medium text-gray-800">{snippet.title}</span>
|
||||
{/if}
|
||||
</div>
|
||||
<span class="text-xs text-gray-400">{snippet.tokenCount} tokens</span>
|
||||
</div>
|
||||
<div class="overflow-hidden rounded-xl border border-gray-200 bg-white">
|
||||
<div class="flex items-center justify-between border-b border-gray-100 px-4 py-2.5">
|
||||
<div class="flex items-center gap-2">
|
||||
{#if snippet.type === 'code'}
|
||||
<span class="rounded bg-purple-100 px-1.5 py-0.5 text-xs text-purple-700">code</span>
|
||||
{:else}
|
||||
<span class="rounded bg-blue-100 px-1.5 py-0.5 text-xs text-blue-700">info</span>
|
||||
{/if}
|
||||
{#if snippet.title}
|
||||
<span class="text-sm font-medium text-gray-800">{snippet.title}</span>
|
||||
{/if}
|
||||
</div>
|
||||
<span class="text-xs text-gray-400">{snippet.tokenCount} tokens</span>
|
||||
</div>
|
||||
|
||||
{#if snippet.breadcrumb}
|
||||
<p class="bg-gray-50 px-4 py-1.5 text-xs text-gray-500 italic">{snippet.breadcrumb}</p>
|
||||
{/if}
|
||||
{#if snippet.breadcrumb}
|
||||
<p class="bg-gray-50 px-4 py-1.5 text-xs text-gray-500 italic">{snippet.breadcrumb}</p>
|
||||
{/if}
|
||||
|
||||
<div class="p-4">
|
||||
{#if snippet.type === 'code'}
|
||||
<pre class="overflow-x-auto rounded bg-gray-950 p-4 text-sm text-gray-100"><code>{snippet.content}</code></pre>
|
||||
{:else}
|
||||
<div class="prose prose-sm max-w-none text-gray-700">{snippet.content}</div>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="p-4">
|
||||
{#if snippet.type === 'code'}
|
||||
<pre class="overflow-x-auto rounded bg-gray-950 p-4 text-sm text-gray-100"><code
|
||||
>{snippet.content}</code
|
||||
></pre>
|
||||
{:else}
|
||||
<div class="prose prose-sm max-w-none text-gray-700">{snippet.content}</div>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
```
|
||||
|
||||
@@ -127,44 +135,44 @@ An interactive search interface within the web UI that lets users test the docum
|
||||
```svelte
|
||||
<!-- src/routes/search/+page.svelte -->
|
||||
<script lang="ts">
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
|
||||
let libraryName = $state('');
|
||||
let selectedLibraryId = $state<string | null>(null);
|
||||
let query = $state('');
|
||||
let libraryResults = $state<LibrarySearchResult[]>([]);
|
||||
let snippets = $state<Snippet[]>([]);
|
||||
let loadingLibraries = $state(false);
|
||||
let loadingSnippets = $state(false);
|
||||
let libraryName = $state('');
|
||||
let selectedLibraryId = $state<string | null>(null);
|
||||
let query = $state('');
|
||||
let libraryResults = $state<LibrarySearchResult[]>([]);
|
||||
let snippets = $state<Snippet[]>([]);
|
||||
let loadingLibraries = $state(false);
|
||||
let loadingSnippets = $state(false);
|
||||
|
||||
async function searchLibraries() {
|
||||
loadingLibraries = true;
|
||||
const res = await fetch(
|
||||
`/api/v1/libs/search?libraryName=${encodeURIComponent(libraryName)}&query=${encodeURIComponent(query)}`
|
||||
);
|
||||
const data = await res.json();
|
||||
libraryResults = data.results;
|
||||
loadingLibraries = false;
|
||||
}
|
||||
async function searchLibraries() {
|
||||
loadingLibraries = true;
|
||||
const res = await fetch(
|
||||
`/api/v1/libs/search?libraryName=${encodeURIComponent(libraryName)}&query=${encodeURIComponent(query)}`
|
||||
);
|
||||
const data = await res.json();
|
||||
libraryResults = data.results;
|
||||
loadingLibraries = false;
|
||||
}
|
||||
|
||||
async function searchDocs() {
|
||||
if (!selectedLibraryId) return;
|
||||
loadingSnippets = true;
|
||||
const url = new URL('/api/v1/context', window.location.origin);
|
||||
url.searchParams.set('libraryId', selectedLibraryId);
|
||||
url.searchParams.set('query', query);
|
||||
const res = await fetch(url);
|
||||
const data = await res.json();
|
||||
snippets = data.snippets;
|
||||
loadingSnippets = false;
|
||||
async function searchDocs() {
|
||||
if (!selectedLibraryId) return;
|
||||
loadingSnippets = true;
|
||||
const url = new URL('/api/v1/context', window.location.origin);
|
||||
url.searchParams.set('libraryId', selectedLibraryId);
|
||||
url.searchParams.set('query', query);
|
||||
const res = await fetch(url);
|
||||
const data = await res.json();
|
||||
snippets = data.snippets;
|
||||
loadingSnippets = false;
|
||||
|
||||
// Update URL
|
||||
goto(`/search?lib=${encodeURIComponent(selectedLibraryId)}&q=${encodeURIComponent(query)}`, {
|
||||
replaceState: true,
|
||||
keepFocus: true,
|
||||
});
|
||||
}
|
||||
// Update URL
|
||||
goto(`/search?lib=${encodeURIComponent(selectedLibraryId)}&q=${encodeURIComponent(query)}`, {
|
||||
replaceState: true,
|
||||
keepFocus: true
|
||||
});
|
||||
}
|
||||
</script>
|
||||
```
|
||||
|
||||
@@ -177,9 +185,9 @@ Use a minimal, zero-dependency approach for v1 — wrap code blocks in `<pre><co
|
||||
```typescript
|
||||
// Optional: lazy-load highlight.js only when code snippets are present
|
||||
async function highlightCode(code: string, language: string): Promise<string> {
|
||||
const hljs = await import('highlight.js/lib/core');
|
||||
// Register only needed languages
|
||||
return hljs.highlight(code, { language }).value;
|
||||
const hljs = await import('highlight.js/lib/core');
|
||||
// Register only needed languages
|
||||
return hljs.highlight(code, { language }).value;
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -30,39 +30,39 @@ Optimize re-indexing by skipping files that haven't changed since the last index
|
||||
|
||||
```typescript
|
||||
interface FileDiff {
|
||||
added: CrawledFile[]; // new files not in DB
|
||||
modified: CrawledFile[]; // files with changed checksum
|
||||
deleted: string[]; // file paths in DB but not in crawl
|
||||
unchanged: string[]; // file paths with matching checksum
|
||||
added: CrawledFile[]; // new files not in DB
|
||||
modified: CrawledFile[]; // files with changed checksum
|
||||
deleted: string[]; // file paths in DB but not in crawl
|
||||
unchanged: string[]; // file paths with matching checksum
|
||||
}
|
||||
|
||||
function computeDiff(
|
||||
crawledFiles: CrawledFile[],
|
||||
existingDocs: Document[] // documents currently in DB for this repo
|
||||
crawledFiles: CrawledFile[],
|
||||
existingDocs: Document[] // documents currently in DB for this repo
|
||||
): FileDiff {
|
||||
const existingMap = new Map(existingDocs.map(d => [d.filePath, d]));
|
||||
const crawledMap = new Map(crawledFiles.map(f => [f.path, f]));
|
||||
const existingMap = new Map(existingDocs.map((d) => [d.filePath, d]));
|
||||
const crawledMap = new Map(crawledFiles.map((f) => [f.path, f]));
|
||||
|
||||
const added: CrawledFile[] = [];
|
||||
const modified: CrawledFile[] = [];
|
||||
const unchanged: string[] = [];
|
||||
const added: CrawledFile[] = [];
|
||||
const modified: CrawledFile[] = [];
|
||||
const unchanged: string[] = [];
|
||||
|
||||
for (const file of crawledFiles) {
|
||||
const existing = existingMap.get(file.path);
|
||||
if (!existing) {
|
||||
added.push(file);
|
||||
} else if (existing.checksum !== file.sha) {
|
||||
modified.push(file);
|
||||
} else {
|
||||
unchanged.push(file.path);
|
||||
}
|
||||
}
|
||||
for (const file of crawledFiles) {
|
||||
const existing = existingMap.get(file.path);
|
||||
if (!existing) {
|
||||
added.push(file);
|
||||
} else if (existing.checksum !== file.sha) {
|
||||
modified.push(file);
|
||||
} else {
|
||||
unchanged.push(file.path);
|
||||
}
|
||||
}
|
||||
|
||||
const deleted = existingDocs
|
||||
.filter(doc => !crawledMap.has(doc.filePath))
|
||||
.map(doc => doc.filePath);
|
||||
const deleted = existingDocs
|
||||
.filter((doc) => !crawledMap.has(doc.filePath))
|
||||
.map((doc) => doc.filePath);
|
||||
|
||||
return { added, modified, deleted, unchanged };
|
||||
return { added, modified, deleted, unchanged };
|
||||
}
|
||||
```
|
||||
|
||||
@@ -78,7 +78,7 @@ const diff = computeDiff(crawledResult.files, existingDocs);
|
||||
|
||||
// Log diff summary
|
||||
this.updateJob(job.id, {
|
||||
totalFiles: crawledResult.files.length,
|
||||
totalFiles: crawledResult.files.length
|
||||
});
|
||||
|
||||
// Process only changed/new files
|
||||
@@ -89,29 +89,29 @@ const docIdsToDelete: string[] = [];
|
||||
|
||||
// Map modified files to their existing document IDs for deletion
|
||||
for (const file of diff.modified) {
|
||||
const existing = existingDocs.find(d => d.filePath === file.path);
|
||||
if (existing) docIdsToDelete.push(existing.id);
|
||||
const existing = existingDocs.find((d) => d.filePath === file.path);
|
||||
if (existing) docIdsToDelete.push(existing.id);
|
||||
}
|
||||
|
||||
// Map deleted file paths to document IDs
|
||||
for (const filePath of diff.deleted) {
|
||||
const existing = existingDocs.find(d => d.filePath === filePath);
|
||||
if (existing) docIdsToDelete.push(existing.id);
|
||||
const existing = existingDocs.find((d) => d.filePath === filePath);
|
||||
if (existing) docIdsToDelete.push(existing.id);
|
||||
}
|
||||
|
||||
// Parse new/modified files
|
||||
for (const [i, file] of filesToProcess.entries()) {
|
||||
const docId = crypto.randomUUID();
|
||||
newDocuments.push({ id: docId, ...buildDocument(file, repo.id, job.versionId) });
|
||||
newSnippets.push(...parseFile(file, { repositoryId: repo.id, documentId: docId }));
|
||||
const docId = crypto.randomUUID();
|
||||
newDocuments.push({ id: docId, ...buildDocument(file, repo.id, job.versionId) });
|
||||
newSnippets.push(...parseFile(file, { repositoryId: repo.id, documentId: docId }));
|
||||
|
||||
// Count ALL files (including skipped) in progress
|
||||
const totalProcessed = diff.unchanged.length + i + 1;
|
||||
const progress = Math.round((totalProcessed / crawledResult.files.length) * 80);
|
||||
this.updateJob(job.id, {
|
||||
processedFiles: totalProcessed,
|
||||
progress,
|
||||
});
|
||||
// Count ALL files (including skipped) in progress
|
||||
const totalProcessed = diff.unchanged.length + i + 1;
|
||||
const progress = Math.round((totalProcessed / crawledResult.files.length) * 80);
|
||||
this.updateJob(job.id, {
|
||||
processedFiles: totalProcessed,
|
||||
progress
|
||||
});
|
||||
}
|
||||
|
||||
// Atomic replacement of only changed documents
|
||||
@@ -123,6 +123,7 @@ this.replaceSnippets(repo.id, docIdsToDelete, newDocuments, newSnippets);
|
||||
## Performance Impact
|
||||
|
||||
For a typical repository with 1,000 files where 50 changed:
|
||||
|
||||
- **Without incremental**: 1,000 files parsed + 1,000 embed batches
|
||||
- **With incremental**: 50 files parsed + 50 embed batches
|
||||
- Estimated speedup: ~20x for re-indexing
|
||||
|
||||
@@ -32,24 +32,24 @@ A settings page within the web UI that allows users to configure the embedding p
|
||||
|
||||
```typescript
|
||||
const PROVIDER_PRESETS = [
|
||||
{
|
||||
name: 'OpenAI',
|
||||
baseUrl: 'https://api.openai.com/v1',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536,
|
||||
},
|
||||
{
|
||||
name: 'Ollama (local)',
|
||||
baseUrl: 'http://localhost:11434/v1',
|
||||
model: 'nomic-embed-text',
|
||||
dimensions: 768,
|
||||
},
|
||||
{
|
||||
name: 'Azure OpenAI',
|
||||
baseUrl: 'https://{resource}.openai.azure.com/openai/deployments/{deployment}/v1',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536,
|
||||
},
|
||||
{
|
||||
name: 'OpenAI',
|
||||
baseUrl: 'https://api.openai.com/v1',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536
|
||||
},
|
||||
{
|
||||
name: 'Ollama (local)',
|
||||
baseUrl: 'http://localhost:11434/v1',
|
||||
model: 'nomic-embed-text',
|
||||
dimensions: 768
|
||||
},
|
||||
{
|
||||
name: 'Azure OpenAI',
|
||||
baseUrl: 'https://{resource}.openai.azure.com/openai/deployments/{deployment}/v1',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536
|
||||
}
|
||||
];
|
||||
```
|
||||
|
||||
@@ -60,133 +60,157 @@ const PROVIDER_PRESETS = [
|
||||
```svelte
|
||||
<!-- src/routes/settings/+page.svelte -->
|
||||
<script lang="ts">
|
||||
let provider = $state<'none' | 'openai' | 'local'>('none');
|
||||
let baseUrl = $state('https://api.openai.com/v1');
|
||||
let apiKey = $state('');
|
||||
let model = $state('text-embedding-3-small');
|
||||
let dimensions = $state<number | undefined>(1536);
|
||||
let testStatus = $state<'idle' | 'testing' | 'ok' | 'error'>('idle');
|
||||
let testError = $state<string | null>(null);
|
||||
let saving = $state(false);
|
||||
let provider = $state<'none' | 'openai' | 'local'>('none');
|
||||
let baseUrl = $state('https://api.openai.com/v1');
|
||||
let apiKey = $state('');
|
||||
let model = $state('text-embedding-3-small');
|
||||
let dimensions = $state<number | undefined>(1536);
|
||||
let testStatus = $state<'idle' | 'testing' | 'ok' | 'error'>('idle');
|
||||
let testError = $state<string | null>(null);
|
||||
let saving = $state(false);
|
||||
|
||||
async function testConnection() {
|
||||
testStatus = 'testing';
|
||||
testError = null;
|
||||
try {
|
||||
const res = await fetch('/api/v1/settings/embedding/test', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } }),
|
||||
});
|
||||
if (res.ok) {
|
||||
testStatus = 'ok';
|
||||
} else {
|
||||
const data = await res.json();
|
||||
testStatus = 'error';
|
||||
testError = data.error;
|
||||
}
|
||||
} catch (e) {
|
||||
testStatus = 'error';
|
||||
testError = (e as Error).message;
|
||||
}
|
||||
}
|
||||
async function testConnection() {
|
||||
testStatus = 'testing';
|
||||
testError = null;
|
||||
try {
|
||||
const res = await fetch('/api/v1/settings/embedding/test', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } })
|
||||
});
|
||||
if (res.ok) {
|
||||
testStatus = 'ok';
|
||||
} else {
|
||||
const data = await res.json();
|
||||
testStatus = 'error';
|
||||
testError = data.error;
|
||||
}
|
||||
} catch (e) {
|
||||
testStatus = 'error';
|
||||
testError = (e as Error).message;
|
||||
}
|
||||
}
|
||||
|
||||
async function save() {
|
||||
saving = true;
|
||||
await fetch('/api/v1/settings/embedding', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } }),
|
||||
});
|
||||
saving = false;
|
||||
}
|
||||
async function save() {
|
||||
saving = true;
|
||||
await fetch('/api/v1/settings/embedding', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ provider, openai: { baseUrl, apiKey, model, dimensions } })
|
||||
});
|
||||
saving = false;
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="mx-auto max-w-2xl py-8">
|
||||
<h1 class="mb-6 text-2xl font-bold text-gray-900">Settings</h1>
|
||||
<h1 class="mb-6 text-2xl font-bold text-gray-900">Settings</h1>
|
||||
|
||||
<section class="rounded-xl border border-gray-200 bg-white p-6">
|
||||
<h2 class="mb-1 text-lg font-semibold">Embedding Provider</h2>
|
||||
<p class="mb-4 text-sm text-gray-500">
|
||||
Embeddings enable semantic search. Without them, only keyword search (FTS5) is used.
|
||||
</p>
|
||||
<section class="rounded-xl border border-gray-200 bg-white p-6">
|
||||
<h2 class="mb-1 text-lg font-semibold">Embedding Provider</h2>
|
||||
<p class="mb-4 text-sm text-gray-500">
|
||||
Embeddings enable semantic search. Without them, only keyword search (FTS5) is used.
|
||||
</p>
|
||||
|
||||
<div class="mb-4 flex gap-2">
|
||||
{#each ['none', 'openai', 'local'] as p}
|
||||
<button
|
||||
onclick={() => provider = p}
|
||||
class="rounded-lg px-4 py-2 text-sm {provider === p
|
||||
? 'bg-blue-600 text-white'
|
||||
: 'border border-gray-200 text-gray-700 hover:bg-gray-50'}"
|
||||
>
|
||||
{p === 'none' ? 'None (FTS5 only)' : p === 'openai' ? 'OpenAI-compatible' : 'Local Model'}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
<div class="mb-4 flex gap-2">
|
||||
{#each ['none', 'openai', 'local'] as p}
|
||||
<button
|
||||
onclick={() => (provider = p)}
|
||||
class="rounded-lg px-4 py-2 text-sm {provider === p
|
||||
? 'bg-blue-600 text-white'
|
||||
: 'border border-gray-200 text-gray-700 hover:bg-gray-50'}"
|
||||
>
|
||||
{p === 'none' ? 'None (FTS5 only)' : p === 'openai' ? 'OpenAI-compatible' : 'Local Model'}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
|
||||
{#if provider === 'none'}
|
||||
<div class="rounded-lg bg-amber-50 border border-amber-200 p-3 text-sm text-amber-700">
|
||||
Search will use keyword matching only. Results may be less relevant for complex questions.
|
||||
</div>
|
||||
{/if}
|
||||
{#if provider === 'none'}
|
||||
<div class="rounded-lg border border-amber-200 bg-amber-50 p-3 text-sm text-amber-700">
|
||||
Search will use keyword matching only. Results may be less relevant for complex questions.
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if provider === 'openai'}
|
||||
<div class="space-y-3">
|
||||
<!-- Preset buttons -->
|
||||
<div class="flex gap-2 flex-wrap">
|
||||
{#each PROVIDER_PRESETS as preset}
|
||||
<button
|
||||
onclick={() => { baseUrl = preset.baseUrl; model = preset.model; dimensions = preset.dimensions; }}
|
||||
class="rounded border border-gray-200 px-2.5 py-1 text-xs text-gray-600 hover:bg-gray-50"
|
||||
>
|
||||
{preset.name}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
{#if provider === 'openai'}
|
||||
<div class="space-y-3">
|
||||
<!-- Preset buttons -->
|
||||
<div class="flex flex-wrap gap-2">
|
||||
{#each PROVIDER_PRESETS as preset}
|
||||
<button
|
||||
onclick={() => {
|
||||
baseUrl = preset.baseUrl;
|
||||
model = preset.model;
|
||||
dimensions = preset.dimensions;
|
||||
}}
|
||||
class="rounded border border-gray-200 px-2.5 py-1 text-xs text-gray-600 hover:bg-gray-50"
|
||||
>
|
||||
{preset.name}
|
||||
</button>
|
||||
{/each}
|
||||
</div>
|
||||
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">Base URL</span>
|
||||
<input type="text" bind:value={baseUrl} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">Base URL</span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={baseUrl}
|
||||
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
|
||||
/>
|
||||
</label>
|
||||
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">API Key</span>
|
||||
<input type="password" bind:value={apiKey} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" placeholder="sk-..." />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">API Key</span>
|
||||
<input
|
||||
type="password"
|
||||
bind:value={apiKey}
|
||||
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
|
||||
placeholder="sk-..."
|
||||
/>
|
||||
</label>
|
||||
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">Model</span>
|
||||
<input type="text" bind:value={model} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">Model</span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={model}
|
||||
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
|
||||
/>
|
||||
</label>
|
||||
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">Dimensions (optional override)</span>
|
||||
<input type="number" bind:value={dimensions} class="mt-1 w-full rounded-lg border px-3 py-2 text-sm" />
|
||||
</label>
|
||||
<label class="block">
|
||||
<span class="text-sm font-medium">Dimensions (optional override)</span>
|
||||
<input
|
||||
type="number"
|
||||
bind:value={dimensions}
|
||||
class="mt-1 w-full rounded-lg border px-3 py-2 text-sm"
|
||||
/>
|
||||
</label>
|
||||
|
||||
<div class="flex items-center gap-3">
|
||||
<button onclick={testConnection} class="rounded-lg border border-gray-300 px-3 py-1.5 text-sm">
|
||||
{testStatus === 'testing' ? 'Testing...' : 'Test Connection'}
|
||||
</button>
|
||||
{#if testStatus === 'ok'}
|
||||
<span class="text-sm text-green-600">✓ Connection successful</span>
|
||||
{:else if testStatus === 'error'}
|
||||
<span class="text-sm text-red-600">✗ {testError}</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
<div class="flex items-center gap-3">
|
||||
<button
|
||||
onclick={testConnection}
|
||||
class="rounded-lg border border-gray-300 px-3 py-1.5 text-sm"
|
||||
>
|
||||
{testStatus === 'testing' ? 'Testing...' : 'Test Connection'}
|
||||
</button>
|
||||
{#if testStatus === 'ok'}
|
||||
<span class="text-sm text-green-600">✓ Connection successful</span>
|
||||
{:else if testStatus === 'error'}
|
||||
<span class="text-sm text-red-600">✗ {testError}</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="mt-6 flex justify-end">
|
||||
<button
|
||||
onclick={save}
|
||||
disabled={saving}
|
||||
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
|
||||
>
|
||||
{saving ? 'Saving...' : 'Save Settings'}
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
<div class="mt-6 flex justify-end">
|
||||
<button
|
||||
onclick={save}
|
||||
disabled={saving}
|
||||
class="rounded-lg bg-blue-600 px-4 py-2 text-sm text-white disabled:opacity-50"
|
||||
>
|
||||
{saving ? 'Saving...' : 'Save Settings'}
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
```
|
||||
|
||||
|
||||
@@ -80,6 +80,7 @@ git -C /path/to/repo archive <commit-hash> | tar -x -C /tmp/trueref-idx/<repo>-<
|
||||
```
|
||||
|
||||
Advantages over `git checkout` or worktrees:
|
||||
|
||||
- Working directory is completely untouched
|
||||
- No `.git` directory in the output (cleaner for parsing)
|
||||
- Temp directory deleted after indexing with no git state to clean up
|
||||
@@ -102,26 +103,26 @@ Allow commit hashes to be pinned explicitly per version, overriding tag resoluti
|
||||
|
||||
```json
|
||||
{
|
||||
"previousVersions": [
|
||||
{
|
||||
"tag": "v2.0.0",
|
||||
"title": "Version 2.0.0",
|
||||
"commitHash": "a3f9c12abc..."
|
||||
}
|
||||
]
|
||||
"previousVersions": [
|
||||
{
|
||||
"tag": "v2.0.0",
|
||||
"title": "Version 2.0.0",
|
||||
"commitHash": "a3f9c12abc..."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Edge Cases
|
||||
|
||||
| Case | Handling |
|
||||
|------|----------|
|
||||
| Annotated tags | `rev-parse <tag>^{commit}` peels to commit automatically |
|
||||
| Mutable tags (e.g. `latest`) | Re-resolve on re-index; warn in UI if hash has changed |
|
||||
| Branch as version | `rev-parse origin/<branch>^{commit}` gives tip; re-resolves on re-index |
|
||||
| Shallow clone | Run `git fetch --unshallow` before `git archive` if commit is unavailable |
|
||||
| Submodules | `git archive --recurse-submodules` or document as a known limitation |
|
||||
| Git LFS | `git lfs pull` required after archive if LFS-tracked files are needed for indexing |
|
||||
| Case | Handling |
|
||||
| ---------------------------- | ---------------------------------------------------------------------------------- |
|
||||
| Annotated tags | `rev-parse <tag>^{commit}` peels to commit automatically |
|
||||
| Mutable tags (e.g. `latest`) | Re-resolve on re-index; warn in UI if hash has changed |
|
||||
| Branch as version | `rev-parse origin/<branch>^{commit}` gives tip; re-resolves on re-index |
|
||||
| Shallow clone | Run `git fetch --unshallow` before `git archive` if commit is unavailable |
|
||||
| Submodules | `git archive --recurse-submodules` or document as a known limitation |
|
||||
| Git LFS | `git lfs pull` required after archive if LFS-tracked files are needed for indexing |
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
@@ -169,12 +170,12 @@ fi
|
||||
|
||||
Username conventions by server type:
|
||||
|
||||
| Server | HTTPS username | Password |
|
||||
|--------|---------------|----------|
|
||||
| Bitbucket Server / Data Center | `x-token-auth` | HTTP access token |
|
||||
| Bitbucket Cloud | account username | App password |
|
||||
| GitLab (self-hosted or cloud) | `oauth2` | Personal access token |
|
||||
| GitLab deploy token | `gitlab-deploy-token` | Deploy token secret |
|
||||
| Server | HTTPS username | Password |
|
||||
| ------------------------------ | --------------------- | --------------------- |
|
||||
| Bitbucket Server / Data Center | `x-token-auth` | HTTP access token |
|
||||
| Bitbucket Cloud | account username | App password |
|
||||
| GitLab (self-hosted or cloud) | `oauth2` | Personal access token |
|
||||
| GitLab deploy token | `gitlab-deploy-token` | Deploy token secret |
|
||||
|
||||
SSH authentication is also supported and preferred for long-lived deployments. The host SSH configuration (`~/.ssh/config`) handles per-host key selection and travels into the container via volume mount.
|
||||
|
||||
@@ -220,7 +221,7 @@ services:
|
||||
web:
|
||||
build: .
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- '3000:3000'
|
||||
volumes:
|
||||
- trueref-data:/data
|
||||
- ${USERPROFILE}/.ssh:/root/.ssh:ro
|
||||
@@ -228,20 +229,20 @@ services:
|
||||
- ${CORP_CA_CERT}:/certs/corp-ca.crt:ro
|
||||
environment:
|
||||
DATABASE_URL: /data/trueref.db
|
||||
GIT_TOKEN_BITBUCKET: "${BITBUCKET_TOKEN}"
|
||||
GIT_TOKEN_GITLAB: "${GITLAB_TOKEN}"
|
||||
BITBUCKET_HOST: "${BITBUCKET_HOST}"
|
||||
GITLAB_HOST: "${GITLAB_HOST}"
|
||||
GIT_TOKEN_BITBUCKET: '${BITBUCKET_TOKEN}'
|
||||
GIT_TOKEN_GITLAB: '${GITLAB_TOKEN}'
|
||||
BITBUCKET_HOST: '${BITBUCKET_HOST}'
|
||||
GITLAB_HOST: '${GITLAB_HOST}'
|
||||
restart: unless-stopped
|
||||
|
||||
mcp:
|
||||
build: .
|
||||
command: mcp
|
||||
ports:
|
||||
- "3001:3001"
|
||||
- '3001:3001'
|
||||
environment:
|
||||
TRUEREF_API_URL: http://web:3000
|
||||
MCP_PORT: "3001"
|
||||
MCP_PORT: '3001'
|
||||
depends_on:
|
||||
- web
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -107,16 +107,16 @@ An embedding profile is persisted configuration selecting one provider adapter p
|
||||
|
||||
```typescript
|
||||
interface EmbeddingProfile {
|
||||
id: string;
|
||||
providerKind: string;
|
||||
title: string;
|
||||
enabled: boolean;
|
||||
isDefault: boolean;
|
||||
config: Record<string, unknown>;
|
||||
model: string;
|
||||
dimensions: number;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
id: string;
|
||||
providerKind: string;
|
||||
title: string;
|
||||
enabled: boolean;
|
||||
isDefault: boolean;
|
||||
config: Record<string, unknown>;
|
||||
model: string;
|
||||
dimensions: number;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -317,4 +317,4 @@ Indexing must embed snippets against the default active profile, and profile cha
|
||||
|
||||
## Notes
|
||||
|
||||
This ticket intentionally leaves `libs/search` as keyword-only. The caller is expected to identify the target library and, when needed, pass a version-qualified library ID such as `/owner/repo/v1.2.3` before requesting semantic retrieval.
|
||||
This ticket intentionally leaves `libs/search` as keyword-only. The caller is expected to identify the target library and, when needed, pass a version-qualified library ID such as `/owner/repo/v1.2.3` before requesting semantic retrieval.
|
||||
|
||||
Reference in New Issue
Block a user