feat(TRUEREF-0022): complete iteration 0 — worker-thread indexing, parallel jobs, SSE progress
- Move IndexingPipeline.run() into Worker Threads via WorkerPool - Add dedicated embedding worker thread with single model instance - Add stage/stageDetail columns to indexing_jobs schema - Create ProgressBroadcaster for SSE channel management - Add SSE endpoints: GET /api/v1/jobs/:id/stream, GET /api/v1/jobs/stream - Replace UI polling with EventSource on repo detail and admin pages - Add concurrency settings UI and API endpoint - Build worker entries separately via esbuild
This commit is contained in:
38
scripts/build-workers.mjs
Normal file
38
scripts/build-workers.mjs
Normal file
@@ -0,0 +1,38 @@
|
||||
import * as esbuild from 'esbuild';
|
||||
import { existsSync } from 'node:fs';
|
||||
|
||||
const entries = [
|
||||
'src/lib/server/pipeline/worker-entry.ts',
|
||||
'src/lib/server/pipeline/embed-worker-entry.ts'
|
||||
];
|
||||
|
||||
try {
|
||||
const existing = entries.filter(e => existsSync(e));
|
||||
if (existing.length === 0) {
|
||||
console.log('[build-workers] No worker entry files found yet, skipping.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
await esbuild.build({
|
||||
entryPoints: existing,
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
target: 'node20',
|
||||
format: 'esm',
|
||||
outdir: 'build/workers',
|
||||
outExtension: { '.js': '.mjs' },
|
||||
alias: {
|
||||
'$lib': './src/lib',
|
||||
'$lib/server': './src/lib/server'
|
||||
},
|
||||
external: ['better-sqlite3', '@xenova/transformers'],
|
||||
banner: {
|
||||
js: "import { createRequire } from 'module'; const require = createRequire(import.meta.url);"
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[build-workers] Compiled ${existing.length} worker(s) to build/workers/`);
|
||||
} catch (err) {
|
||||
console.error('[build-workers] Error:', err);
|
||||
process.exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user