feat(TRUEREF-0009-0010): implement indexing pipeline job queue and public REST API
- SQLite-backed job queue with sequential processing and startup recovery - Atomic snippet replacement in single transaction - context7-compatible GET /api/v1/libs/search and GET /api/v1/context - Token budget limiting and JSON/txt response format support - CORS headers on all API routes via SvelteKit handle hook - Library ID parser supporting /owner/repo and /owner/repo/version Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -64,9 +64,38 @@ try {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Request handler (pass-through)
|
||||
// CORS headers applied to all /api/* responses
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const CORS_HEADERS = {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, POST, PATCH, DELETE, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, Authorization'
|
||||
} as const;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Request handler — CORS + pass-through
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const handle: Handle = async ({ event, resolve }) => {
|
||||
return resolve(event);
|
||||
const { pathname } = event.url;
|
||||
|
||||
// Handle CORS pre-flight for all API routes.
|
||||
if (event.request.method === 'OPTIONS' && pathname.startsWith('/api/')) {
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: CORS_HEADERS
|
||||
});
|
||||
}
|
||||
|
||||
const response = await resolve(event);
|
||||
|
||||
// Attach CORS headers to all API responses.
|
||||
if (pathname.startsWith('/api/')) {
|
||||
for (const [key, value] of Object.entries(CORS_HEADERS)) {
|
||||
response.headers.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
return response;
|
||||
};
|
||||
|
||||
277
src/lib/server/api/formatters.test.ts
Normal file
277
src/lib/server/api/formatters.test.ts
Normal file
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* Unit tests for API formatters (TRUEREF-0010).
|
||||
*
|
||||
* Covers state mapping, library search JSON formatting, and context
|
||||
* JSON/txt response formatting.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
mapState,
|
||||
formatLibrarySearchJson,
|
||||
formatContextJson,
|
||||
formatContextTxt
|
||||
} from './formatters';
|
||||
import type { LibrarySearchResult } from '$lib/server/search/search.service';
|
||||
import type { SnippetSearchResult } from '$lib/server/search/search.service';
|
||||
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeRepo(overrides: Partial<Repository> = {}): Repository {
|
||||
return {
|
||||
id: '/facebook/react',
|
||||
title: 'React',
|
||||
description: 'A JavaScript library for building user interfaces',
|
||||
source: 'github',
|
||||
sourceUrl: 'https://github.com/facebook/react',
|
||||
branch: 'main',
|
||||
state: 'indexed',
|
||||
totalSnippets: 1247,
|
||||
totalTokens: 142000,
|
||||
trustScore: 9.2,
|
||||
benchmarkScore: 87,
|
||||
stars: 228000,
|
||||
githubToken: null,
|
||||
lastIndexedAt: new Date('2026-03-22T10:00:00Z'),
|
||||
createdAt: new Date('2024-01-01T00:00:00Z'),
|
||||
updatedAt: new Date('2026-03-22T10:00:00Z'),
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function makeVersion(tag: string): RepositoryVersion {
|
||||
return {
|
||||
id: `/facebook/react/${tag}`,
|
||||
repositoryId: '/facebook/react',
|
||||
tag,
|
||||
title: null,
|
||||
state: 'indexed',
|
||||
totalSnippets: 100,
|
||||
indexedAt: new Date(),
|
||||
createdAt: new Date()
|
||||
};
|
||||
}
|
||||
|
||||
function makeSnippet(overrides: Partial<Snippet> = {}): Snippet {
|
||||
return {
|
||||
id: 'snippet-1',
|
||||
documentId: 'doc-1',
|
||||
repositoryId: '/facebook/react',
|
||||
versionId: null,
|
||||
type: 'code',
|
||||
title: 'Basic Component',
|
||||
content: 'function MyComponent() {\n return <div>Hello</div>;\n}',
|
||||
language: 'tsx',
|
||||
breadcrumb: 'Getting Started > Components',
|
||||
tokenCount: 45,
|
||||
createdAt: new Date(),
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function makeSnippetResult(snippet: Snippet): SnippetSearchResult {
|
||||
return {
|
||||
snippet,
|
||||
score: -1.5,
|
||||
repository: { id: snippet.repositoryId, title: 'React' }
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mapState
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('mapState', () => {
|
||||
it('maps indexed → finalized', () => {
|
||||
expect(mapState('indexed')).toBe('finalized');
|
||||
});
|
||||
|
||||
it('maps pending → initial', () => {
|
||||
expect(mapState('pending')).toBe('initial');
|
||||
});
|
||||
|
||||
it('maps indexing → initial', () => {
|
||||
expect(mapState('indexing')).toBe('initial');
|
||||
});
|
||||
|
||||
it('maps error → error', () => {
|
||||
expect(mapState('error')).toBe('error');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// formatLibrarySearchJson
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('formatLibrarySearchJson', () => {
|
||||
it('returns results array with correct shape', () => {
|
||||
const results: LibrarySearchResult[] = [
|
||||
{
|
||||
repository: makeRepo(),
|
||||
versions: [makeVersion('v18.3.0'), makeVersion('v17.0.2')],
|
||||
score: 150
|
||||
}
|
||||
];
|
||||
|
||||
const response = formatLibrarySearchJson(results);
|
||||
|
||||
expect(response.results).toHaveLength(1);
|
||||
const r = response.results[0];
|
||||
expect(r.id).toBe('/facebook/react');
|
||||
expect(r.title).toBe('React');
|
||||
expect(r.state).toBe('finalized');
|
||||
expect(r.totalTokens).toBe(142000);
|
||||
expect(r.totalSnippets).toBe(1247);
|
||||
expect(r.versions).toEqual(['v18.3.0', 'v17.0.2']);
|
||||
expect(r.stars).toBe(228000);
|
||||
expect(r.lastUpdateDate).toBe('2026-03-22T10:00:00.000Z');
|
||||
expect(r.source).toBe('https://github.com/facebook/react');
|
||||
});
|
||||
|
||||
it('returns empty results array when no results', () => {
|
||||
const response = formatLibrarySearchJson([]);
|
||||
expect(response.results).toEqual([]);
|
||||
});
|
||||
|
||||
it('maps non-indexed state to initial', () => {
|
||||
const results: LibrarySearchResult[] = [
|
||||
{ repository: makeRepo({ state: 'pending' }), versions: [], score: 0 }
|
||||
];
|
||||
const response = formatLibrarySearchJson(results);
|
||||
expect(response.results[0].state).toBe('initial');
|
||||
});
|
||||
|
||||
it('handles null lastIndexedAt', () => {
|
||||
const results: LibrarySearchResult[] = [
|
||||
{ repository: makeRepo({ lastIndexedAt: null }), versions: [], score: 0 }
|
||||
];
|
||||
const response = formatLibrarySearchJson(results);
|
||||
expect(response.results[0].lastUpdateDate).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// formatContextJson
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('formatContextJson', () => {
|
||||
it('formats code snippets correctly', () => {
|
||||
const snippet = makeSnippet({ type: 'code' });
|
||||
const results = [makeSnippetResult(snippet)];
|
||||
|
||||
const response = formatContextJson(results, []);
|
||||
|
||||
expect(response.snippets).toHaveLength(1);
|
||||
const s = response.snippets[0];
|
||||
expect(s.type).toBe('code');
|
||||
if (s.type === 'code') {
|
||||
expect(s.title).toBe('Basic Component');
|
||||
expect(s.language).toBe('tsx');
|
||||
expect(s.codeList).toHaveLength(1);
|
||||
expect(s.codeList[0].code).toContain('MyComponent');
|
||||
expect(s.pageTitle).toBe('Getting Started');
|
||||
}
|
||||
});
|
||||
|
||||
it('formats info snippets correctly', () => {
|
||||
const snippet = makeSnippet({
|
||||
id: 'info-1',
|
||||
type: 'info',
|
||||
title: null,
|
||||
content: 'React components let you split the UI...',
|
||||
language: null,
|
||||
breadcrumb: 'Core Concepts > Components'
|
||||
});
|
||||
const results = [makeSnippetResult(snippet)];
|
||||
|
||||
const response = formatContextJson(results, []);
|
||||
|
||||
const s = response.snippets[0];
|
||||
expect(s.type).toBe('info');
|
||||
if (s.type === 'info') {
|
||||
expect(s.text).toContain('React components');
|
||||
expect(s.breadcrumb).toBe('Core Concepts > Components');
|
||||
expect(s.pageId).toBe('info-1');
|
||||
}
|
||||
});
|
||||
|
||||
it('includes rules in response', () => {
|
||||
const rules = ['Always use functional components', 'Use hooks for state management'];
|
||||
const response = formatContextJson([], rules);
|
||||
expect(response.rules).toEqual(rules);
|
||||
});
|
||||
|
||||
it('computes totalTokens correctly', () => {
|
||||
const snippets = [
|
||||
makeSnippetResult(makeSnippet({ id: 'a', tokenCount: 100 })),
|
||||
makeSnippetResult(makeSnippet({ id: 'b', tokenCount: 200 }))
|
||||
];
|
||||
const response = formatContextJson(snippets, []);
|
||||
expect(response.totalTokens).toBe(300);
|
||||
});
|
||||
|
||||
it('handles null tokenCount in totalTokens sum', () => {
|
||||
const snippets = [makeSnippetResult(makeSnippet({ tokenCount: null }))];
|
||||
const response = formatContextJson(snippets, []);
|
||||
expect(response.totalTokens).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// formatContextTxt
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('formatContextTxt', () => {
|
||||
it('prepends rules section when rules are present', () => {
|
||||
const rules = ['Always use functional components'];
|
||||
const txt = formatContextTxt([], rules);
|
||||
expect(txt).toContain('## Library Rules');
|
||||
expect(txt).toContain('- Always use functional components');
|
||||
});
|
||||
|
||||
it('omits rules section when no rules', () => {
|
||||
const snippet = makeSnippet();
|
||||
const txt = formatContextTxt([makeSnippetResult(snippet)], []);
|
||||
expect(txt).not.toContain('## Library Rules');
|
||||
});
|
||||
|
||||
it('formats code snippets with fenced code block', () => {
|
||||
const snippet = makeSnippet({ type: 'code', language: 'tsx' });
|
||||
const txt = formatContextTxt([makeSnippetResult(snippet)], []);
|
||||
expect(txt).toContain('```tsx');
|
||||
expect(txt).toContain('MyComponent');
|
||||
expect(txt).toContain('```');
|
||||
});
|
||||
|
||||
it('formats info snippets as plain text', () => {
|
||||
const snippet = makeSnippet({
|
||||
type: 'info',
|
||||
content: 'React is a UI library.',
|
||||
language: null
|
||||
});
|
||||
const txt = formatContextTxt([makeSnippetResult(snippet)], []);
|
||||
expect(txt).toContain('React is a UI library.');
|
||||
expect(txt).not.toContain('```');
|
||||
});
|
||||
|
||||
it('includes breadcrumb as italic line', () => {
|
||||
const snippet = makeSnippet({ breadcrumb: 'Getting Started > Components' });
|
||||
const txt = formatContextTxt([makeSnippetResult(snippet)], []);
|
||||
expect(txt).toContain('*Getting Started > Components*');
|
||||
});
|
||||
|
||||
it('separates snippets with ---', () => {
|
||||
const s1 = makeSnippetResult(makeSnippet({ id: 'a' }));
|
||||
const s2 = makeSnippetResult(makeSnippet({ id: 'b', type: 'info', content: 'hello' }));
|
||||
const txt = formatContextTxt([s1, s2], []);
|
||||
expect(txt).toContain('---');
|
||||
});
|
||||
|
||||
it('returns empty string for empty inputs with no rules', () => {
|
||||
const txt = formatContextTxt([], []);
|
||||
expect(txt).toBe('');
|
||||
});
|
||||
});
|
||||
237
src/lib/server/api/formatters.ts
Normal file
237
src/lib/server/api/formatters.ts
Normal file
@@ -0,0 +1,237 @@
|
||||
/**
|
||||
* Response formatters for the context7-compatible REST API.
|
||||
*
|
||||
* Provides two output shapes for each endpoint:
|
||||
* - JSON (`type=json`, default): structured data for programmatic consumers.
|
||||
* - Text (`type=txt`): plain Markdown formatted for direct LLM injection.
|
||||
*
|
||||
* State mapping (TrueRef → context7):
|
||||
* pending → initial
|
||||
* indexing → initial
|
||||
* indexed → finalized
|
||||
* error → error
|
||||
*/
|
||||
|
||||
import type { Repository, RepositoryVersion, Snippet } from '$lib/types';
|
||||
import type { LibrarySearchResult } from '$lib/server/search/search.service';
|
||||
import type { SnippetSearchResult } from '$lib/server/search/search.service';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// State mapping
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type TrueRefState = 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
type Context7State = 'initial' | 'finalized' | 'error';
|
||||
|
||||
export function mapState(state: TrueRefState): Context7State {
|
||||
switch (state) {
|
||||
case 'indexed':
|
||||
return 'finalized';
|
||||
case 'error':
|
||||
return 'error';
|
||||
default:
|
||||
return 'initial';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CORS headers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const CORS_HEADERS = {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, POST, PATCH, DELETE, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, Authorization'
|
||||
} as const;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /api/v1/libs/search — JSON response shape
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface LibrarySearchJsonResult {
|
||||
id: string;
|
||||
title: string;
|
||||
description: string | null;
|
||||
branch: string | null;
|
||||
lastUpdateDate: string | null;
|
||||
state: Context7State;
|
||||
totalTokens: number | null;
|
||||
totalSnippets: number | null;
|
||||
stars: number | null;
|
||||
trustScore: number | null;
|
||||
benchmarkScore: number | null;
|
||||
versions: string[];
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface LibrarySearchJsonResponse {
|
||||
results: LibrarySearchJsonResult[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert internal LibrarySearchResult[] to the context7-compatible JSON body.
|
||||
*/
|
||||
export function formatLibrarySearchJson(results: LibrarySearchResult[]): LibrarySearchJsonResponse {
|
||||
return {
|
||||
results: results.map(({ repository, versions }) =>
|
||||
formatSingleLibraryJson(repository, versions)
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
export function formatSingleLibraryJson(
|
||||
repository: Repository,
|
||||
versions: RepositoryVersion[]
|
||||
): LibrarySearchJsonResult {
|
||||
return {
|
||||
id: repository.id,
|
||||
title: repository.title,
|
||||
description: repository.description ?? null,
|
||||
branch: repository.branch ?? null,
|
||||
lastUpdateDate: repository.lastIndexedAt ? repository.lastIndexedAt.toISOString() : null,
|
||||
state: mapState(repository.state as TrueRefState),
|
||||
totalTokens: repository.totalTokens ?? null,
|
||||
totalSnippets: repository.totalSnippets ?? null,
|
||||
stars: repository.stars ?? null,
|
||||
trustScore: repository.trustScore ?? null,
|
||||
benchmarkScore: repository.benchmarkScore ?? null,
|
||||
versions: versions.map((v) => v.tag),
|
||||
source: repository.sourceUrl
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /api/v1/context — JSON response shapes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface CodeListItem {
|
||||
language: string;
|
||||
code: string;
|
||||
}
|
||||
|
||||
export interface CodeSnippetJson {
|
||||
type: 'code';
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
language: string | null;
|
||||
codeList: CodeListItem[];
|
||||
id: string;
|
||||
tokenCount: number | null;
|
||||
pageTitle: string | null;
|
||||
}
|
||||
|
||||
export interface InfoSnippetJson {
|
||||
type: 'info';
|
||||
text: string;
|
||||
breadcrumb: string | null;
|
||||
pageId: string;
|
||||
tokenCount: number | null;
|
||||
}
|
||||
|
||||
export type SnippetJson = CodeSnippetJson | InfoSnippetJson;
|
||||
|
||||
export interface ContextJsonResponse {
|
||||
snippets: SnippetJson[];
|
||||
rules: string[];
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a ranked list of snippets to the context7-compatible JSON body.
|
||||
*
|
||||
* @param snippets - Ranked snippet search results (already token-budget trimmed).
|
||||
* @param rules - Rules from `trueref.json` / `repository_configs`.
|
||||
*/
|
||||
export function formatContextJson(
|
||||
snippets: SnippetSearchResult[],
|
||||
rules: string[]
|
||||
): ContextJsonResponse {
|
||||
const mapped: SnippetJson[] = snippets.map(({ snippet }) => {
|
||||
if (snippet.type === 'code') {
|
||||
const codeSnippet: CodeSnippetJson = {
|
||||
type: 'code',
|
||||
title: snippet.title ?? null,
|
||||
description: snippet.breadcrumb ?? null,
|
||||
language: snippet.language ?? null,
|
||||
codeList: [
|
||||
{
|
||||
language: snippet.language ?? '',
|
||||
code: snippet.content
|
||||
}
|
||||
],
|
||||
id: snippet.id,
|
||||
tokenCount: snippet.tokenCount ?? null,
|
||||
pageTitle: extractPageTitle(snippet.breadcrumb)
|
||||
};
|
||||
return codeSnippet;
|
||||
} else {
|
||||
const infoSnippet: InfoSnippetJson = {
|
||||
type: 'info',
|
||||
text: snippet.content,
|
||||
breadcrumb: snippet.breadcrumb ?? null,
|
||||
pageId: snippet.id,
|
||||
tokenCount: snippet.tokenCount ?? null
|
||||
};
|
||||
return infoSnippet;
|
||||
}
|
||||
});
|
||||
|
||||
const totalTokens = snippets.reduce((sum, { snippet }) => sum + (snippet.tokenCount ?? 0), 0);
|
||||
|
||||
return {
|
||||
snippets: mapped,
|
||||
rules,
|
||||
totalTokens
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the top-level page title from a breadcrumb string.
|
||||
* e.g. "Getting Started > Components" → "Getting Started"
|
||||
*/
|
||||
function extractPageTitle(breadcrumb: string | null | undefined): string | null {
|
||||
if (!breadcrumb) return null;
|
||||
const parts = breadcrumb.split('>');
|
||||
return parts[0].trim() || null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /api/v1/context — txt response
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Format snippets as plain Markdown text suitable for direct LLM injection.
|
||||
*
|
||||
* @param snippets - Ranked snippet search results (already token-budget trimmed).
|
||||
* @param rules - Rules from `trueref.json` / `repository_configs`.
|
||||
*/
|
||||
export function formatContextTxt(snippets: SnippetSearchResult[], rules: string[]): string {
|
||||
const parts: string[] = [];
|
||||
|
||||
if (rules.length > 0) {
|
||||
parts.push('## Library Rules\n' + rules.map((r) => `- ${r}`).join('\n'));
|
||||
parts.push('---');
|
||||
}
|
||||
|
||||
for (const { snippet } of snippets) {
|
||||
const section: string[] = [];
|
||||
|
||||
if (snippet.type === 'code') {
|
||||
if (snippet.title) section.push(`### ${snippet.title}`);
|
||||
if (snippet.breadcrumb) section.push(`*${snippet.breadcrumb}*`);
|
||||
section.push(`\`\`\`${snippet.language ?? ''}\n${snippet.content}\n\`\`\``);
|
||||
} else {
|
||||
if (snippet.title) section.push(`### ${snippet.title}`);
|
||||
if (snippet.breadcrumb) section.push(`*${snippet.breadcrumb}*`);
|
||||
section.push(snippet.content);
|
||||
}
|
||||
|
||||
parts.push(section.filter(Boolean).join('\n'));
|
||||
parts.push('---');
|
||||
}
|
||||
|
||||
// Remove trailing separator
|
||||
if (parts.at(-1) === '---') parts.pop();
|
||||
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
48
src/lib/server/api/library-id.test.ts
Normal file
48
src/lib/server/api/library-id.test.ts
Normal file
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Unit tests for parseLibraryId (TRUEREF-0010).
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { parseLibraryId } from './library-id';
|
||||
|
||||
describe('parseLibraryId', () => {
|
||||
it('parses /owner/repo (default branch)', () => {
|
||||
const result = parseLibraryId('/facebook/react');
|
||||
expect(result.repositoryId).toBe('/facebook/react');
|
||||
expect(result.version).toBeUndefined();
|
||||
});
|
||||
|
||||
it('parses /owner/repo/version', () => {
|
||||
const result = parseLibraryId('/facebook/react/v18.3.0');
|
||||
expect(result.repositoryId).toBe('/facebook/react');
|
||||
expect(result.version).toBe('v18.3.0');
|
||||
});
|
||||
|
||||
it('parses /owner/repo/version with dot-separated version', () => {
|
||||
const result = parseLibraryId('/sveltejs/svelte/4.0.0');
|
||||
expect(result.repositoryId).toBe('/sveltejs/svelte');
|
||||
expect(result.version).toBe('4.0.0');
|
||||
});
|
||||
|
||||
it('parses /owner/repo/branch-name as version', () => {
|
||||
const result = parseLibraryId('/vercel/next.js/canary');
|
||||
expect(result.repositoryId).toBe('/vercel/next.js');
|
||||
expect(result.version).toBe('canary');
|
||||
});
|
||||
|
||||
it('throws on missing leading slash', () => {
|
||||
expect(() => parseLibraryId('facebook/react')).toThrow('Invalid libraryId');
|
||||
});
|
||||
|
||||
it('throws on empty string', () => {
|
||||
expect(() => parseLibraryId('')).toThrow('Invalid libraryId');
|
||||
});
|
||||
|
||||
it('throws on single-segment path', () => {
|
||||
expect(() => parseLibraryId('/react')).toThrow('Invalid libraryId');
|
||||
});
|
||||
|
||||
it('throws on path with only a slash', () => {
|
||||
expect(() => parseLibraryId('/')).toThrow('Invalid libraryId');
|
||||
});
|
||||
});
|
||||
32
src/lib/server/api/library-id.ts
Normal file
32
src/lib/server/api/library-id.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* Library ID parsing utilities.
|
||||
*
|
||||
* Parses the `libraryId` query parameter used by the context7-compatible API.
|
||||
* Supports two formats:
|
||||
* - /owner/repo (default branch)
|
||||
* - /owner/repo/version (specific version tag)
|
||||
*/
|
||||
|
||||
export interface ParsedLibraryId {
|
||||
/** The canonical repository ID, e.g. "/facebook/react" */
|
||||
repositoryId: string;
|
||||
/** The version tag, e.g. "v18.3.0" — absent for default branch queries */
|
||||
version?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a libraryId string into its constituent parts.
|
||||
*
|
||||
* @throws Error when the string does not match the expected pattern.
|
||||
*/
|
||||
export function parseLibraryId(libraryId: string): ParsedLibraryId {
|
||||
const match = libraryId.match(/^(\/[^/]+\/[^/]+)(\/(.+))?$/);
|
||||
if (!match) {
|
||||
throw new Error(`Invalid libraryId: ${libraryId}`);
|
||||
}
|
||||
|
||||
return {
|
||||
repositoryId: match[1],
|
||||
version: match[3]
|
||||
};
|
||||
}
|
||||
75
src/lib/server/api/token-budget.test.ts
Normal file
75
src/lib/server/api/token-budget.test.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Unit tests for selectSnippetsWithinBudget (TRUEREF-0010).
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { selectSnippetsWithinBudget, DEFAULT_TOKEN_BUDGET } from './token-budget';
|
||||
import type { Snippet } from '$lib/types';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeSnippet(id: string, tokenCount: number | null): Snippet {
|
||||
return {
|
||||
id,
|
||||
documentId: 'doc-1',
|
||||
repositoryId: '/test/repo',
|
||||
versionId: null,
|
||||
type: 'info',
|
||||
title: null,
|
||||
content: 'content',
|
||||
language: null,
|
||||
breadcrumb: null,
|
||||
tokenCount,
|
||||
createdAt: new Date()
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('selectSnippetsWithinBudget', () => {
|
||||
it('returns all snippets when total tokens are within budget', () => {
|
||||
const snippets = [makeSnippet('a', 100), makeSnippet('b', 200), makeSnippet('c', 300)];
|
||||
const result = selectSnippetsWithinBudget(snippets, 1000);
|
||||
expect(result.map((s) => s.id)).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
it('stops adding when next snippet exceeds the budget', () => {
|
||||
const snippets = [makeSnippet('a', 100), makeSnippet('b', 500), makeSnippet('c', 200)];
|
||||
// budget = 550 → a (100) + b (500) = 600 exceeds; only a fits then b would push over
|
||||
const result = selectSnippetsWithinBudget(snippets, 550);
|
||||
// a(100) fits; a+b=600 > 550, stop
|
||||
expect(result.map((s) => s.id)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('includes exactly one snippet when it fits the budget precisely', () => {
|
||||
const snippets = [makeSnippet('a', 100)];
|
||||
const result = selectSnippetsWithinBudget(snippets, 100);
|
||||
expect(result.map((s) => s.id)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('returns empty array when first snippet already exceeds budget', () => {
|
||||
const snippets = [makeSnippet('a', 200), makeSnippet('b', 50)];
|
||||
const result = selectSnippetsWithinBudget(snippets, 100);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('treats null tokenCount as 0', () => {
|
||||
const snippets = [makeSnippet('a', null), makeSnippet('b', null), makeSnippet('c', null)];
|
||||
const result = selectSnippetsWithinBudget(snippets, 0);
|
||||
// 0 + 0 = 0 which does NOT exceed 0, so all three pass
|
||||
expect(result.map((s) => s.id)).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
it('returns empty array for empty input', () => {
|
||||
const result = selectSnippetsWithinBudget([], 10_000);
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('DEFAULT_TOKEN_BUDGET is 10000', () => {
|
||||
expect(DEFAULT_TOKEN_BUDGET).toBe(10_000);
|
||||
});
|
||||
});
|
||||
36
src/lib/server/api/token-budget.ts
Normal file
36
src/lib/server/api/token-budget.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Token budget selection for context responses.
|
||||
*
|
||||
* Implements a greedy selection algorithm: snippets are added in ranked order
|
||||
* until adding the next snippet would exceed the token budget.
|
||||
*/
|
||||
|
||||
import type { Snippet } from '$lib/types';
|
||||
|
||||
/**
|
||||
* Select snippets from a ranked list up to a maximum token budget.
|
||||
*
|
||||
* Snippets are evaluated in order. A snippet is included when its token count
|
||||
* does not push the running total past `maxTokens`. The loop halts at the
|
||||
* first snippet that would exceed the budget.
|
||||
*
|
||||
* @param snippets - Ranked list of snippets (best first).
|
||||
* @param maxTokens - Inclusive upper bound on total token count.
|
||||
* @returns The largest prefix of `snippets` whose combined token count
|
||||
* does not exceed `maxTokens`.
|
||||
*/
|
||||
export function selectSnippetsWithinBudget(snippets: Snippet[], maxTokens: number): Snippet[] {
|
||||
const selected: Snippet[] = [];
|
||||
let usedTokens = 0;
|
||||
|
||||
for (const snippet of snippets) {
|
||||
if (usedTokens + (snippet.tokenCount ?? 0) > maxTokens) break;
|
||||
selected.push(snippet);
|
||||
usedTokens += snippet.tokenCount ?? 0;
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
/** Default token budget when the caller does not specify `tokens`. */
|
||||
export const DEFAULT_TOKEN_BUDGET = 10_000;
|
||||
207
src/routes/api/v1/context/+server.ts
Normal file
207
src/routes/api/v1/context/+server.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* GET /api/v1/context
|
||||
*
|
||||
* Fetch documentation snippets for a library. Compatible with context7's
|
||||
* GET /api/v2/context interface.
|
||||
*
|
||||
* Query parameters:
|
||||
* libraryId (required) — e.g. "/facebook/react" or "/facebook/react/v18.3.0"
|
||||
* query (required) — specific question about the library
|
||||
* type (optional) — "json" (default) or "txt"
|
||||
* tokens (optional) — approximate max token count (default 10000)
|
||||
*/
|
||||
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getClient } from '$lib/server/db/client';
|
||||
import { SearchService } from '$lib/server/search/search.service';
|
||||
import { HybridSearchService } from '$lib/server/search/hybrid.search.service';
|
||||
import { parseLibraryId } from '$lib/server/api/library-id';
|
||||
import { selectSnippetsWithinBudget, DEFAULT_TOKEN_BUDGET } from '$lib/server/api/token-budget';
|
||||
import {
|
||||
formatContextJson,
|
||||
formatContextTxt,
|
||||
CORS_HEADERS
|
||||
} from '$lib/server/api/formatters';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getServices() {
|
||||
const db = getClient();
|
||||
const searchService = new SearchService(db);
|
||||
// No embedding provider — pure FTS5 mode (alpha=0 equivalent).
|
||||
const hybridService = new HybridSearchService(db, searchService, null);
|
||||
return { db, searchService, hybridService };
|
||||
}
|
||||
|
||||
interface RawRepoConfig {
|
||||
rules: string | null;
|
||||
}
|
||||
|
||||
function getRules(db: ReturnType<typeof getClient>, repositoryId: string): string[] {
|
||||
const row = db
|
||||
.prepare<[string], RawRepoConfig>(`SELECT rules FROM repository_configs WHERE repository_id = ?`)
|
||||
.get(repositoryId);
|
||||
|
||||
if (!row?.rules) return [];
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(row.rules);
|
||||
return Array.isArray(parsed) ? (parsed as string[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
interface RawRepoState {
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
title: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Route handler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const GET: RequestHandler = async ({ url }) => {
|
||||
const libraryId = url.searchParams.get('libraryId');
|
||||
|
||||
if (!libraryId || !libraryId.trim()) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'libraryId is required', code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const query = url.searchParams.get('query');
|
||||
|
||||
if (!query || !query.trim()) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'query is required', code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const responseType = url.searchParams.get('type') ?? 'json';
|
||||
const tokensRaw = parseInt(url.searchParams.get('tokens') ?? String(DEFAULT_TOKEN_BUDGET), 10);
|
||||
const maxTokens = isNaN(tokensRaw) || tokensRaw < 1 ? DEFAULT_TOKEN_BUDGET : tokensRaw;
|
||||
|
||||
// Parse the libraryId
|
||||
let parsed: ReturnType<typeof parseLibraryId>;
|
||||
try {
|
||||
parsed = parseLibraryId(libraryId);
|
||||
} catch {
|
||||
return new Response(
|
||||
JSON.stringify({ error: `Invalid libraryId: ${libraryId}`, code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const { db, hybridService } = getServices();
|
||||
|
||||
// Verify the repository exists and check its state.
|
||||
const repo = db
|
||||
.prepare<[string], RawRepoState>(`SELECT state, title FROM repositories WHERE id = ?`)
|
||||
.get(parsed.repositoryId);
|
||||
|
||||
if (!repo) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: `Library ${parsed.repositoryId} not found or not yet indexed`,
|
||||
code: 'LIBRARY_NOT_FOUND'
|
||||
}),
|
||||
{
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
if (repo.state === 'indexing' || repo.state === 'pending') {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: 'Library is currently being indexed, please try again shortly',
|
||||
code: 'INDEXING_IN_PROGRESS'
|
||||
}),
|
||||
{
|
||||
status: 503,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve version ID if a specific version was requested.
|
||||
let versionId: string | undefined;
|
||||
if (parsed.version) {
|
||||
const versionRow = db
|
||||
.prepare<[string, string], { id: string }>(
|
||||
`SELECT id FROM repository_versions WHERE repository_id = ? AND tag = ?`
|
||||
)
|
||||
.get(parsed.repositoryId, parsed.version);
|
||||
|
||||
// Version not found is not fatal — fall back to default branch.
|
||||
versionId = versionRow?.id;
|
||||
}
|
||||
|
||||
// Execute hybrid search (falls back to FTS5 when no embedding provider is set).
|
||||
const searchResults = await hybridService.search(query, {
|
||||
repositoryId: parsed.repositoryId,
|
||||
versionId,
|
||||
limit: 50 // fetch more than needed; token budget will trim
|
||||
});
|
||||
|
||||
// Apply token budget.
|
||||
const snippets = searchResults.map((r) => r.snippet);
|
||||
const selected = selectSnippetsWithinBudget(snippets, maxTokens);
|
||||
|
||||
// Re-wrap selected snippets as SnippetSearchResult for formatters.
|
||||
const selectedResults = selected.map((snippet) => {
|
||||
const found = searchResults.find((r) => r.snippet.id === snippet.id)!;
|
||||
return found;
|
||||
});
|
||||
|
||||
// Load rules from repository_configs.
|
||||
const rules = getRules(db, parsed.repositoryId);
|
||||
|
||||
if (responseType === 'txt') {
|
||||
const text = formatContextTxt(selectedResults, rules);
|
||||
return new Response(text, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain; charset=utf-8',
|
||||
...CORS_HEADERS
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Default: JSON
|
||||
const body = formatContextJson(selectedResults, rules);
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Internal server error';
|
||||
return new Response(JSON.stringify({ error: message, code: 'INTERNAL_ERROR' }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const OPTIONS: RequestHandler = () => {
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: CORS_HEADERS
|
||||
});
|
||||
};
|
||||
64
src/routes/api/v1/libs/search/+server.ts
Normal file
64
src/routes/api/v1/libs/search/+server.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* GET /api/v1/libs/search
|
||||
*
|
||||
* Search libraries by name. Compatible with context7's
|
||||
* GET /api/v2/libs/search interface.
|
||||
*
|
||||
* Query parameters:
|
||||
* libraryName (required) — library name to search for
|
||||
* query (optional) — user's question for relevance ranking
|
||||
* limit (optional) — max results, default 10, max 50
|
||||
* type (optional) — "json" (default) or "txt"
|
||||
*/
|
||||
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getClient } from '$lib/server/db/client';
|
||||
import { SearchService } from '$lib/server/search/search.service';
|
||||
import { formatLibrarySearchJson } from '$lib/server/api/formatters';
|
||||
import { CORS_HEADERS } from '$lib/server/api/formatters';
|
||||
|
||||
function getService(): SearchService {
|
||||
return new SearchService(getClient());
|
||||
}
|
||||
|
||||
export const GET: RequestHandler = ({ url }) => {
|
||||
const libraryName = url.searchParams.get('libraryName');
|
||||
|
||||
if (!libraryName || !libraryName.trim()) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'libraryName is required', code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const query = url.searchParams.get('query') ?? undefined;
|
||||
const limitRaw = parseInt(url.searchParams.get('limit') ?? '10', 10);
|
||||
const limit = Math.min(isNaN(limitRaw) || limitRaw < 1 ? 10 : limitRaw, 50);
|
||||
|
||||
try {
|
||||
const service = getService();
|
||||
const results = service.searchRepositories({ libraryName, query, limit });
|
||||
const body = formatLibrarySearchJson(results);
|
||||
|
||||
return json(body, {
|
||||
headers: CORS_HEADERS
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Internal server error';
|
||||
return new Response(JSON.stringify({ error: message, code: 'INTERNAL_ERROR' }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const OPTIONS: RequestHandler = () => {
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: CORS_HEADERS
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user