feat(TRUEREF-0009-0010): implement indexing pipeline job queue and public REST API
- SQLite-backed job queue with sequential processing and startup recovery - Atomic snippet replacement in single transaction - context7-compatible GET /api/v1/libs/search and GET /api/v1/context - Token budget limiting and JSON/txt response format support - CORS headers on all API routes via SvelteKit handle hook - Library ID parser supporting /owner/repo and /owner/repo/version Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Giancarmine Salucci
207
src/routes/api/v1/context/+server.ts
Normal file
207
src/routes/api/v1/context/+server.ts
Normal file
@@ -0,0 +1,207 @@
|
||||
/**
|
||||
* GET /api/v1/context
|
||||
*
|
||||
* Fetch documentation snippets for a library. Compatible with context7's
|
||||
* GET /api/v2/context interface.
|
||||
*
|
||||
* Query parameters:
|
||||
* libraryId (required) — e.g. "/facebook/react" or "/facebook/react/v18.3.0"
|
||||
* query (required) — specific question about the library
|
||||
* type (optional) — "json" (default) or "txt"
|
||||
* tokens (optional) — approximate max token count (default 10000)
|
||||
*/
|
||||
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getClient } from '$lib/server/db/client';
|
||||
import { SearchService } from '$lib/server/search/search.service';
|
||||
import { HybridSearchService } from '$lib/server/search/hybrid.search.service';
|
||||
import { parseLibraryId } from '$lib/server/api/library-id';
|
||||
import { selectSnippetsWithinBudget, DEFAULT_TOKEN_BUDGET } from '$lib/server/api/token-budget';
|
||||
import {
|
||||
formatContextJson,
|
||||
formatContextTxt,
|
||||
CORS_HEADERS
|
||||
} from '$lib/server/api/formatters';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function getServices() {
|
||||
const db = getClient();
|
||||
const searchService = new SearchService(db);
|
||||
// No embedding provider — pure FTS5 mode (alpha=0 equivalent).
|
||||
const hybridService = new HybridSearchService(db, searchService, null);
|
||||
return { db, searchService, hybridService };
|
||||
}
|
||||
|
||||
interface RawRepoConfig {
|
||||
rules: string | null;
|
||||
}
|
||||
|
||||
function getRules(db: ReturnType<typeof getClient>, repositoryId: string): string[] {
|
||||
const row = db
|
||||
.prepare<[string], RawRepoConfig>(`SELECT rules FROM repository_configs WHERE repository_id = ?`)
|
||||
.get(repositoryId);
|
||||
|
||||
if (!row?.rules) return [];
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(row.rules);
|
||||
return Array.isArray(parsed) ? (parsed as string[]) : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
interface RawRepoState {
|
||||
state: 'pending' | 'indexing' | 'indexed' | 'error';
|
||||
title: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Route handler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const GET: RequestHandler = async ({ url }) => {
|
||||
const libraryId = url.searchParams.get('libraryId');
|
||||
|
||||
if (!libraryId || !libraryId.trim()) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'libraryId is required', code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const query = url.searchParams.get('query');
|
||||
|
||||
if (!query || !query.trim()) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'query is required', code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const responseType = url.searchParams.get('type') ?? 'json';
|
||||
const tokensRaw = parseInt(url.searchParams.get('tokens') ?? String(DEFAULT_TOKEN_BUDGET), 10);
|
||||
const maxTokens = isNaN(tokensRaw) || tokensRaw < 1 ? DEFAULT_TOKEN_BUDGET : tokensRaw;
|
||||
|
||||
// Parse the libraryId
|
||||
let parsed: ReturnType<typeof parseLibraryId>;
|
||||
try {
|
||||
parsed = parseLibraryId(libraryId);
|
||||
} catch {
|
||||
return new Response(
|
||||
JSON.stringify({ error: `Invalid libraryId: ${libraryId}`, code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const { db, hybridService } = getServices();
|
||||
|
||||
// Verify the repository exists and check its state.
|
||||
const repo = db
|
||||
.prepare<[string], RawRepoState>(`SELECT state, title FROM repositories WHERE id = ?`)
|
||||
.get(parsed.repositoryId);
|
||||
|
||||
if (!repo) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: `Library ${parsed.repositoryId} not found or not yet indexed`,
|
||||
code: 'LIBRARY_NOT_FOUND'
|
||||
}),
|
||||
{
|
||||
status: 404,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
if (repo.state === 'indexing' || repo.state === 'pending') {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
error: 'Library is currently being indexed, please try again shortly',
|
||||
code: 'INDEXING_IN_PROGRESS'
|
||||
}),
|
||||
{
|
||||
status: 503,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve version ID if a specific version was requested.
|
||||
let versionId: string | undefined;
|
||||
if (parsed.version) {
|
||||
const versionRow = db
|
||||
.prepare<[string, string], { id: string }>(
|
||||
`SELECT id FROM repository_versions WHERE repository_id = ? AND tag = ?`
|
||||
)
|
||||
.get(parsed.repositoryId, parsed.version);
|
||||
|
||||
// Version not found is not fatal — fall back to default branch.
|
||||
versionId = versionRow?.id;
|
||||
}
|
||||
|
||||
// Execute hybrid search (falls back to FTS5 when no embedding provider is set).
|
||||
const searchResults = await hybridService.search(query, {
|
||||
repositoryId: parsed.repositoryId,
|
||||
versionId,
|
||||
limit: 50 // fetch more than needed; token budget will trim
|
||||
});
|
||||
|
||||
// Apply token budget.
|
||||
const snippets = searchResults.map((r) => r.snippet);
|
||||
const selected = selectSnippetsWithinBudget(snippets, maxTokens);
|
||||
|
||||
// Re-wrap selected snippets as SnippetSearchResult for formatters.
|
||||
const selectedResults = selected.map((snippet) => {
|
||||
const found = searchResults.find((r) => r.snippet.id === snippet.id)!;
|
||||
return found;
|
||||
});
|
||||
|
||||
// Load rules from repository_configs.
|
||||
const rules = getRules(db, parsed.repositoryId);
|
||||
|
||||
if (responseType === 'txt') {
|
||||
const text = formatContextTxt(selectedResults, rules);
|
||||
return new Response(text, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain; charset=utf-8',
|
||||
...CORS_HEADERS
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Default: JSON
|
||||
const body = formatContextJson(selectedResults, rules);
|
||||
return new Response(JSON.stringify(body), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Internal server error';
|
||||
return new Response(JSON.stringify({ error: message, code: 'INTERNAL_ERROR' }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const OPTIONS: RequestHandler = () => {
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: CORS_HEADERS
|
||||
});
|
||||
};
|
||||
64
src/routes/api/v1/libs/search/+server.ts
Normal file
64
src/routes/api/v1/libs/search/+server.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* GET /api/v1/libs/search
|
||||
*
|
||||
* Search libraries by name. Compatible with context7's
|
||||
* GET /api/v2/libs/search interface.
|
||||
*
|
||||
* Query parameters:
|
||||
* libraryName (required) — library name to search for
|
||||
* query (optional) — user's question for relevance ranking
|
||||
* limit (optional) — max results, default 10, max 50
|
||||
* type (optional) — "json" (default) or "txt"
|
||||
*/
|
||||
|
||||
import { json } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { getClient } from '$lib/server/db/client';
|
||||
import { SearchService } from '$lib/server/search/search.service';
|
||||
import { formatLibrarySearchJson } from '$lib/server/api/formatters';
|
||||
import { CORS_HEADERS } from '$lib/server/api/formatters';
|
||||
|
||||
function getService(): SearchService {
|
||||
return new SearchService(getClient());
|
||||
}
|
||||
|
||||
export const GET: RequestHandler = ({ url }) => {
|
||||
const libraryName = url.searchParams.get('libraryName');
|
||||
|
||||
if (!libraryName || !libraryName.trim()) {
|
||||
return new Response(
|
||||
JSON.stringify({ error: 'libraryName is required', code: 'MISSING_PARAMETER' }),
|
||||
{
|
||||
status: 400,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const query = url.searchParams.get('query') ?? undefined;
|
||||
const limitRaw = parseInt(url.searchParams.get('limit') ?? '10', 10);
|
||||
const limit = Math.min(isNaN(limitRaw) || limitRaw < 1 ? 10 : limitRaw, 50);
|
||||
|
||||
try {
|
||||
const service = getService();
|
||||
const results = service.searchRepositories({ libraryName, query, limit });
|
||||
const body = formatLibrarySearchJson(results);
|
||||
|
||||
return json(body, {
|
||||
headers: CORS_HEADERS
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : 'Internal server error';
|
||||
return new Response(JSON.stringify({ error: message, code: 'INTERNAL_ERROR' }), {
|
||||
status: 500,
|
||||
headers: { 'Content-Type': 'application/json', ...CORS_HEADERS }
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
export const OPTIONS: RequestHandler = () => {
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: CORS_HEADERS
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user