fix(TRUEREF-0021): reduce event loop blocking, add busy_timeout, and add TRUEREF-0022 PRD

src/lib/server/db/client.ts

@@ -13,6 +13,7 @@ export function getClient(): Database.Database {
 		_client = new Database(env.DATABASE_URL);
 		_client.pragma('journal_mode = WAL');
 		_client.pragma('foreign_keys = ON');
+		_client.pragma('busy_timeout = 5000');
 	}
 	return _client;
 }

src/lib/server/db/index.ts

@@ -15,6 +15,10 @@ const client = new Database(env.DATABASE_URL);
 client.pragma('journal_mode = WAL');
 // Enforce foreign key constraints.
 client.pragma('foreign_keys = ON');
+// Wait up to 5 s when the DB is locked instead of failing immediately.
+// Prevents SQLITE_BUSY errors when the indexing pipeline holds the write lock
+// and an HTTP request arrives simultaneously.
+client.pragma('busy_timeout = 5000');
 
 export const db = drizzle(client, { schema });
 

src/lib/server/pipeline/indexing.pipeline.ts

@@ -215,7 +215,19 @@ export class IndexingPipeline {
 				this.updateJob(job.id, { processedFiles, progress: initialProgress });
 			}
 
+			// Yield the event loop and flush progress every N files.
+			// Lower = more responsive UI; higher = less overhead.
+			const YIELD_EVERY = 20;
+
 			for (const [i, file] of filesToProcess.entries()) {
+				// Yield the Node.js event loop periodically so the HTTP server can
+				// handle incoming requests (navigation, polling) between file parses.
+				// Without this, the synchronous parse + SQLite work blocks the thread
+				// entirely and the UI becomes unresponsive during indexing.
+				if (i > 0 && i % YIELD_EVERY === 0) {
+					await new Promise<void>((resolve) => setImmediate(resolve));
+				}
+
 				const checksum = file.sha || sha256(file.content);
 
 				// Create new document record.
@@ -247,16 +259,20 @@ export class IndexingPipeline {
 				newDocuments.push(newDoc);
 				newSnippets.push(...snippets);
 
-				// Count ALL files (including skipped unchanged ones) in progress.
+				// Write progress to the DB only on yield boundaries or the final file.
+				// Avoids a synchronous SQLite UPDATE on every single iteration.
 				const totalProcessed = diff.unchanged.length + i + 1;
-				const progress = calculateProgress(
-					totalProcessed,
-					totalFiles,
-					0,
-					0,
-					this.embeddingService !== null
-				);
-				this.updateJob(job.id, { processedFiles: totalProcessed, progress });
+				const isLast = i === filesToProcess.length - 1;
+				if (isLast || i % YIELD_EVERY === YIELD_EVERY - 1) {
+					const progress = calculateProgress(
+						totalProcessed,
+						totalFiles,
+						0,
+						0,
+						this.embeddingService !== null
+					);
+					this.updateJob(job.id, { processedFiles: totalProcessed, progress });
+				}
 			}
 
 			// After the loop processedFiles should reflect the full count.