fix(MULTIVERSION-0001): fix multi-version indexing — jobs never created or triggered for secondary versions
Two bugs prevented secondary versions from ever being indexed: 1. JobQueue.enqueue() and RepositoryService.createIndexingJob() deduplication only checked repository_id, so a queued default-branch job blocked all version-specific jobs for the same repo. Fix: include version_id in the WHERE clause so only exact (repository_id, version_id) pairs are deduped. 2. POST /api/v1/libs/:id/versions used repoService.createIndexingJob() which inserts a job record but never triggers queue processing. Fix: use queue.enqueue() (same fallback pattern as the libs endpoint) so setImmediate fires processNext() after the job is inserted. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -36,14 +36,17 @@ export class JobQueue {
|
|||||||
* existing job instead of creating a duplicate.
|
* existing job instead of creating a duplicate.
|
||||||
*/
|
*/
|
||||||
enqueue(repositoryId: string, versionId?: string): IndexingJob {
|
enqueue(repositoryId: string, versionId?: string): IndexingJob {
|
||||||
// Return early if there's already an active job for this repo.
|
// Return early if there's already an active job for this exact (repo, version) pair.
|
||||||
|
const resolvedVersionId = versionId ?? null;
|
||||||
const activeRaw = this.db
|
const activeRaw = this.db
|
||||||
.prepare<[string], IndexingJobEntity>(
|
.prepare<[string, string | null, string | null], IndexingJobEntity>(
|
||||||
`${JOB_SELECT}
|
`${JOB_SELECT}
|
||||||
WHERE repository_id = ? AND status IN ('queued', 'running')
|
WHERE repository_id = ?
|
||||||
|
AND (version_id = ? OR (version_id IS NULL AND ? IS NULL))
|
||||||
|
AND status IN ('queued', 'running')
|
||||||
ORDER BY created_at DESC LIMIT 1`
|
ORDER BY created_at DESC LIMIT 1`
|
||||||
)
|
)
|
||||||
.get(repositoryId);
|
.get(repositoryId, resolvedVersionId, resolvedVersionId);
|
||||||
|
|
||||||
if (activeRaw) {
|
if (activeRaw) {
|
||||||
// Ensure the queue is draining even if enqueue was called concurrently.
|
// Ensure the queue is draining even if enqueue was called concurrently.
|
||||||
|
|||||||
@@ -529,4 +529,24 @@ describe('RepositoryService.createIndexingJob()', () => {
|
|||||||
const job = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
|
const job = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
|
||||||
expect(job.versionId).toBe('/facebook/react/v18.3.0');
|
expect(job.versionId).toBe('/facebook/react/v18.3.0');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('allows separate jobs for the same repo but different versions', () => {
|
||||||
|
const defaultJob = service.createIndexingJob('/facebook/react');
|
||||||
|
const versionJob = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
|
||||||
|
expect(versionJob.id).not.toBe(defaultJob.id);
|
||||||
|
expect(defaultJob.versionId).toBeNull();
|
||||||
|
expect(versionJob.versionId).toBe('/facebook/react/v18.3.0');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns the existing job when the same (repo, version) pair is already queued', () => {
|
||||||
|
const job1 = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
|
||||||
|
const job2 = service.createIndexingJob('/facebook/react', '/facebook/react/v18.3.0');
|
||||||
|
expect(job2.id).toBe(job1.id);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns the existing default-branch job when called again without a versionId', () => {
|
||||||
|
const job1 = service.createIndexingJob('/facebook/react');
|
||||||
|
const job2 = service.createIndexingJob('/facebook/react');
|
||||||
|
expect(job2.id).toBe(job1.id);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -319,14 +319,17 @@ export class RepositoryService {
|
|||||||
* If a job is already running, returns the existing job.
|
* If a job is already running, returns the existing job.
|
||||||
*/
|
*/
|
||||||
createIndexingJob(repositoryId: string, versionId?: string): IndexingJob {
|
createIndexingJob(repositoryId: string, versionId?: string): IndexingJob {
|
||||||
// Check for running job
|
// Check for an existing queued/running job for this exact (repo, version) pair.
|
||||||
|
const resolvedVersionId = versionId ?? null;
|
||||||
const runningJob = this.db
|
const runningJob = this.db
|
||||||
.prepare(
|
.prepare(
|
||||||
`SELECT * FROM indexing_jobs
|
`SELECT * FROM indexing_jobs
|
||||||
WHERE repository_id = ? AND status IN ('queued', 'running')
|
WHERE repository_id = ?
|
||||||
|
AND (version_id = ? OR (version_id IS NULL AND ? IS NULL))
|
||||||
|
AND status IN ('queued', 'running')
|
||||||
ORDER BY created_at DESC LIMIT 1`
|
ORDER BY created_at DESC LIMIT 1`
|
||||||
)
|
)
|
||||||
.get(repositoryId) as IndexingJobEntity | undefined;
|
.get(repositoryId, resolvedVersionId, resolvedVersionId) as IndexingJobEntity | undefined;
|
||||||
|
|
||||||
if (runningJob) return IndexingJobMapper.fromEntity(new IndexingJobEntity(runningJob));
|
if (runningJob) return IndexingJobMapper.fromEntity(new IndexingJobEntity(runningJob));
|
||||||
|
|
||||||
|
|||||||
@@ -348,6 +348,37 @@ describe('API contract integration', () => {
|
|||||||
expect(getBody.versions[0]).not.toHaveProperty('total_snippets');
|
expect(getBody.versions[0]).not.toHaveProperty('total_snippets');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('POST /api/v1/libs/:id/versions creates distinct jobs for different versions of the same repo', async () => {
|
||||||
|
const repoService = new RepositoryService(db);
|
||||||
|
repoService.add({ source: 'github', sourceUrl: 'https://github.com/facebook/react' });
|
||||||
|
|
||||||
|
const postV1 = await postVersions({
|
||||||
|
params: { id: encodeURIComponent('/facebook/react') },
|
||||||
|
request: new Request('http://test/api/v1/libs/%2Ffacebook%2Freact/versions', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify({ tag: 'v18.3.0', autoIndex: true })
|
||||||
|
})
|
||||||
|
} as never);
|
||||||
|
const bodyV1 = await postV1.json();
|
||||||
|
|
||||||
|
const postV2 = await postVersions({
|
||||||
|
params: { id: encodeURIComponent('/facebook/react') },
|
||||||
|
request: new Request('http://test/api/v1/libs/%2Ffacebook%2Freact/versions', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json' },
|
||||||
|
body: JSON.stringify({ tag: 'v17.0.2', autoIndex: true })
|
||||||
|
})
|
||||||
|
} as never);
|
||||||
|
const bodyV2 = await postV2.json();
|
||||||
|
|
||||||
|
expect(postV1.status).toBe(201);
|
||||||
|
expect(postV2.status).toBe(201);
|
||||||
|
expect(bodyV1.job.id).not.toBe(bodyV2.job.id);
|
||||||
|
expect(bodyV1.job.versionId).toBe('/facebook/react/v18.3.0');
|
||||||
|
expect(bodyV2.job.versionId).toBe('/facebook/react/v17.0.2');
|
||||||
|
});
|
||||||
|
|
||||||
it('GET /api/v1/context returns informative txt output for empty results', async () => {
|
it('GET /api/v1/context returns informative txt output for empty results', async () => {
|
||||||
const repositoryId = seedRepo(db);
|
const repositoryId = seedRepo(db);
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import { RepositoryVersionMapper } from '$lib/server/mappers/repository-version.
|
|||||||
import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js';
|
import { IndexingJobMapper } from '$lib/server/mappers/indexing-job.mapper.js';
|
||||||
import { RepositoryService } from '$lib/server/services/repository.service';
|
import { RepositoryService } from '$lib/server/services/repository.service';
|
||||||
import { VersionService } from '$lib/server/services/version.service';
|
import { VersionService } from '$lib/server/services/version.service';
|
||||||
|
import { getQueue } from '$lib/server/pipeline/startup';
|
||||||
import { handleServiceError, NotFoundError, InvalidInputError } from '$lib/server/utils/validation';
|
import { handleServiceError, NotFoundError, InvalidInputError } from '$lib/server/utils/validation';
|
||||||
|
|
||||||
function getServices() {
|
function getServices() {
|
||||||
@@ -78,7 +79,10 @@ export const POST: RequestHandler = async ({ params, request }) => {
|
|||||||
|
|
||||||
let job: ReturnType<typeof IndexingJobMapper.toDto> | undefined;
|
let job: ReturnType<typeof IndexingJobMapper.toDto> | undefined;
|
||||||
if (autoIndex) {
|
if (autoIndex) {
|
||||||
const indexingJob = repoService.createIndexingJob(repositoryId, version.id);
|
const queue = getQueue();
|
||||||
|
const indexingJob = queue
|
||||||
|
? queue.enqueue(repositoryId, version.id)
|
||||||
|
: repoService.createIndexingJob(repositoryId, version.id);
|
||||||
job = IndexingJobMapper.toDto(indexingJob);
|
job = IndexingJobMapper.toDto(indexingJob);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user