mozempk/trueref
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(TRUEREF-0017): implement incremental re-indexing with checksum diff

Browse Source 
- computeDiff classifies files into added/modified/deleted/unchanged buckets
- Only changed and new files are parsed and re-embedded on re-runs
- Deleted files removed atomically from DB
- Progress counts all files including unchanged for accurate reporting
- ~20x speedup for re-indexing large repositories with few changes

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Giancarmine Salucci
2026-03-23 09:07:20 +01:00
parent 22bf4c1014
commit 9e3f62e329
4 changed files with 365 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/lib/server/pipeline/indexing.pipeline.ts
View File

@@ -15,12 +15,13 @@
import { createHash } from 'node:crypto';
import type Database from 'better-sqlite3';
import type { IndexingJob, NewDocument, NewSnippet, Repository } from '$lib/types';
import type { Document, IndexingJob, NewDocument, NewSnippet, Repository } from '$lib/types';
import type { crawl as GithubCrawlFn } from '$lib/server/crawler/github.crawler.js';
import type { LocalCrawler } from '$lib/server/crawler/local.crawler.js';
import type { EmbeddingService } from '$lib/server/embeddings/embedding.service.js';
import { parseFile } from '$lib/server/parser/index.js';
import { computeTrustScore } from '$lib/server/search/trust-score.js';
import { computeDiff } from './diff.js';
// ---------------------------------------------------------------------------
// Progress calculation
@@ -94,43 +95,33 @@ export class IndexingPipeline {
this.updateJob(job.id, { totalFiles });
// ---- Stage 2: Parse & diff ------------------------------------------
// Load all existing documents for this repo so computeDiff can
// classify every crawled file and detect deletions.
const existingDocs = this.getExistingDocuments(repo.id, normJob.versionId);
const diff = computeDiff(crawlResult.files, existingDocs);
// Accumulate new documents/snippets; skip unchanged files.
const newDocuments: NewDocument[] = [];
const newSnippets: NewSnippet[] = [];
const changedDocIds: string[] = [];
let processedFiles = 0;
// Schedule stale documents (modified + deleted) for deletion.
for (const file of diff.modified) {
const existing = existingDocs.find((d) => d.filePath === file.path);
if (existing) changedDocIds.push(existing.id);
}
for (const filePath of diff.deleted) {
const existing = existingDocs.find((d) => d.filePath === filePath);
if (existing) changedDocIds.push(existing.id);
}
for (const file of crawlResult.files) {
// Only parse and embed files that are new or have changed.
const filesToProcess = [...diff.added, ...diff.modified];
let processedFiles = diff.unchanged.length; // unchanged files count as processed
for (const [i, file] of filesToProcess.entries()) {
const checksum = file.sha || sha256(file.content);
// Check whether an identical document already exists.
const existingDoc = this.db
.prepare<[string, string], { id: string; checksum: string }>(
`SELECT id, checksum FROM documents
WHERE repository_id = ? AND file_path = ? LIMIT 1`
)
.get(repo.id, file.path);
if (existingDoc && existingDoc.checksum === checksum) {
// File unchanged — reuse existing snippets, nothing to do.
processedFiles++;
const progress = calculateProgress(
processedFiles,
totalFiles,
0,
0,
this.embeddingService !== null
);
this.updateJob(job.id, { processedFiles, progress });
continue;
}
// File is new or changed — schedule old doc for deletion.
if (existingDoc) {
changedDocIds.push(existingDoc.id);
}
// Create new document record.
const documentId = crypto.randomUUID();
const now = new Date();
@@ -160,17 +151,21 @@ export class IndexingPipeline {
newDocuments.push(newDoc);
newSnippets.push(...snippets);
processedFiles++;
// Count ALL files (including skipped unchanged ones) in progress.
const totalProcessed = diff.unchanged.length + i + 1;
const progress = calculateProgress(
processedFiles,
totalProcessed,
totalFiles,
0,
0,
this.embeddingService !== null
);
this.updateJob(job.id, { processedFiles, progress });
this.updateJob(job.id, { processedFiles: totalProcessed, progress });
}
// After the loop processedFiles should reflect the full count.
processedFiles = diff.unchanged.length + filesToProcess.length;
// ---- Stage 3: Atomic replacement ------------------------------------
this.replaceSnippets(repo.id, changedDocIds, newDocuments, newSnippets);
@@ -368,6 +363,27 @@ export class IndexingPipeline {
// Private — DB helpers
// -------------------------------------------------------------------------
private getExistingDocuments(repositoryId: string, versionId: string | null): Document[] {
if (versionId) {
return this.db
.prepare<[string, string], Document>(
`SELECT id, repository_id as repositoryId, version_id as versionId,
file_path as filePath, title, language, token_count as tokenCount,
checksum, indexed_at as indexedAt
FROM documents WHERE repository_id = ? AND version_id = ?`
)
.all(repositoryId, versionId) as Document[];
}
return this.db
.prepare<[string], Document>(
`SELECT id, repository_id as repositoryId, version_id as versionId,
file_path as filePath, title, language, token_count as tokenCount,
checksum, indexed_at as indexedAt
FROM documents WHERE repository_id = ? AND version_id IS NULL`
)
.all(repositoryId) as Document[];
}
private getRepository(id: string): Repository | null {
return (
(this.db