mozempk/trueref
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(TRUEREF-0021): reduce event loop blocking, add busy_timeout, and add TRUEREF-0022 PRD

Browse Source
This commit is contained in:
U811073
2026-03-30 15:06:54 +02:00
committed by Giancarmine Salucci
parent f4fe8c6043
commit 6f3f4db19b
6 changed files with 577 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/lib/server/pipeline/indexing.pipeline.ts
View File

@@ -215,7 +215,19 @@ export class IndexingPipeline {
this.updateJob(job.id, { processedFiles, progress: initialProgress });
}
// Yield the event loop and flush progress every N files.
// Lower = more responsive UI; higher = less overhead.
const YIELD_EVERY = 20;
for (const [i, file] of filesToProcess.entries()) {
// Yield the Node.js event loop periodically so the HTTP server can
// handle incoming requests (navigation, polling) between file parses.
// Without this, the synchronous parse + SQLite work blocks the thread
// entirely and the UI becomes unresponsive during indexing.
if (i > 0 && i % YIELD_EVERY === 0) {
await new Promise<void>((resolve) => setImmediate(resolve));
}
const checksum = file.sha || sha256(file.content);
// Create new document record.
@@ -247,16 +259,20 @@ export class IndexingPipeline {
newDocuments.push(newDoc);
newSnippets.push(...snippets);
// Count ALL files (including skipped unchanged ones) in progress.
// Write progress to the DB only on yield boundaries or the final file.
// Avoids a synchronous SQLite UPDATE on every single iteration.
const totalProcessed = diff.unchanged.length + i + 1;
const progress = calculateProgress(
totalProcessed,
totalFiles,
0,
0,
this.embeddingService !== null
);
this.updateJob(job.id, { processedFiles: totalProcessed, progress });
const isLast = i === filesToProcess.length - 1;
if (isLast || i % YIELD_EVERY === YIELD_EVERY - 1) {
const progress = calculateProgress(
totalProcessed,
totalFiles,
0,
0,
this.embeddingService !== null
);
this.updateJob(job.id, { processedFiles: totalProcessed, progress });
}
}
// After the loop processedFiles should reflect the full count.