TRUEREF-0022 fix, more tests
This commit is contained in:
Giancarmine Salucci
330
src/lib/server/pipeline/worker-pool.test.ts
Normal file
330
src/lib/server/pipeline/worker-pool.test.ts
Normal file
@@ -0,0 +1,330 @@
|
||||
/**
|
||||
* Tests for WorkerPool (TRUEREF-0022).
|
||||
*
|
||||
* Real node:worker_threads Workers are replaced by FakeWorker (an EventEmitter)
|
||||
* so no subprocess is ever spawned. We maintain our own registry of created
|
||||
* FakeWorker instances so we can inspect postMessage calls and emit events.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { writeFileSync, unlinkSync, existsSync } from 'node:fs';
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hoist FakeWorker + registry so vi.mock can reference them.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const { createdWorkers, FakeWorker } = vi.hoisted(() => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const { EventEmitter } = require('node:events') as typeof import('node:events');
|
||||
|
||||
const createdWorkers: InstanceType<typeof FakeWorkerClass>[] = [];
|
||||
|
||||
class FakeWorkerClass extends EventEmitter {
|
||||
threadId = Math.floor(Math.random() * 100_000);
|
||||
// Auto-emit 'exit' with code 0 when a shutdown message is received
|
||||
postMessage = vi.fn((msg: { type: string }) => {
|
||||
if (msg.type === 'shutdown') {
|
||||
// Emit exit asynchronously so shutdown() loop can process it
|
||||
setImmediate(() => {
|
||||
this.emit('exit', 0);
|
||||
this.threadId = 0; // signal exited
|
||||
});
|
||||
}
|
||||
});
|
||||
terminate = vi.fn(() => {
|
||||
this.threadId = 0;
|
||||
});
|
||||
|
||||
constructor(_script: string, _opts?: unknown) {
|
||||
super();
|
||||
createdWorkers.push(this);
|
||||
}
|
||||
}
|
||||
|
||||
return { createdWorkers, FakeWorker: FakeWorkerClass };
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mock node:worker_threads BEFORE importing WorkerPool.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
vi.mock('node:worker_threads', () => {
|
||||
return { Worker: FakeWorker };
|
||||
});
|
||||
|
||||
import { WorkerPool, type WorkerPoolOptions } from './worker-pool.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FAKE_SCRIPT = '/tmp/fake-worker-pool-test.mjs';
|
||||
const MISSING_SCRIPT = '/tmp/this-file-does-not-exist-worker-pool.mjs';
|
||||
|
||||
function makeOpts(overrides: Partial<WorkerPoolOptions> = {}): WorkerPoolOptions {
|
||||
return {
|
||||
concurrency: 2,
|
||||
workerScript: FAKE_SCRIPT,
|
||||
embedWorkerScript: MISSING_SCRIPT,
|
||||
dbPath: ':memory:',
|
||||
onProgress: vi.fn(),
|
||||
onJobDone: vi.fn(),
|
||||
onJobFailed: vi.fn(),
|
||||
onEmbedDone: vi.fn(),
|
||||
onEmbedFailed: vi.fn(),
|
||||
...overrides
|
||||
} as unknown as WorkerPoolOptions;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Setup / teardown
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
beforeEach(() => {
|
||||
// Create the fake script so existsSync returns true
|
||||
writeFileSync(FAKE_SCRIPT, '// placeholder\n');
|
||||
// Clear registry and reset all mocks
|
||||
createdWorkers.length = 0;
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(FAKE_SCRIPT)) unlinkSync(FAKE_SCRIPT);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fallback mode (no real workers)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('WorkerPool fallback mode', () => {
|
||||
it('enters fallback mode when workerScript does not exist', () => {
|
||||
const pool = new WorkerPool(makeOpts({ workerScript: MISSING_SCRIPT }));
|
||||
|
||||
expect(pool.isFallbackMode).toBe(true);
|
||||
});
|
||||
|
||||
it('does not throw when constructed in fallback mode', () => {
|
||||
expect(() => new WorkerPool(makeOpts({ workerScript: MISSING_SCRIPT }))).not.toThrow();
|
||||
});
|
||||
|
||||
it('enqueue is a no-op in fallback mode — callbacks are never called', () => {
|
||||
const opts = makeOpts({ workerScript: MISSING_SCRIPT });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
pool.enqueue('job-1', '/repo/1');
|
||||
|
||||
expect(opts.onJobDone).not.toHaveBeenCalled();
|
||||
expect(opts.onProgress).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('spawns no workers in fallback mode', () => {
|
||||
new WorkerPool(makeOpts({ workerScript: MISSING_SCRIPT }));
|
||||
|
||||
expect(createdWorkers).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Normal mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('WorkerPool normal mode', () => {
|
||||
it('isFallbackMode is false when workerScript exists', () => {
|
||||
const pool = new WorkerPool(makeOpts({ concurrency: 1 }));
|
||||
|
||||
expect(pool.isFallbackMode).toBe(false);
|
||||
});
|
||||
|
||||
it('spawns `concurrency` parse workers on construction', () => {
|
||||
new WorkerPool(makeOpts({ concurrency: 3 }));
|
||||
|
||||
expect(createdWorkers).toHaveLength(3);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// enqueue dispatches to an idle worker
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('enqueue sends { type: "run", jobId } to an idle worker', () => {
|
||||
const pool = new WorkerPool(makeOpts({ concurrency: 1 }));
|
||||
|
||||
pool.enqueue('job-42', '/repo/1');
|
||||
|
||||
expect(createdWorkers).toHaveLength(1);
|
||||
expect(createdWorkers[0].postMessage).toHaveBeenCalledWith({ type: 'run', jobId: 'job-42' });
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// "done" message — onJobDone called, next queued job dispatched
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('calls onJobDone and dispatches the next queued job when a worker emits "done"', () => {
|
||||
const opts = makeOpts({ concurrency: 1 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
// Enqueue two jobs — second must wait because concurrency=1
|
||||
pool.enqueue('job-A', '/repo/1');
|
||||
pool.enqueue('job-B', '/repo/2');
|
||||
|
||||
const worker = createdWorkers[0];
|
||||
|
||||
// Simulate job-A completing
|
||||
worker.emit('message', { type: 'done', jobId: 'job-A' });
|
||||
|
||||
expect(opts.onJobDone).toHaveBeenCalledWith('job-A');
|
||||
// The same worker should now run job-B
|
||||
expect(worker.postMessage).toHaveBeenCalledWith({ type: 'run', jobId: 'job-B' });
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// "failed" message — onJobFailed called
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('calls onJobFailed when a worker emits a "failed" message', () => {
|
||||
const opts = makeOpts({ concurrency: 1 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
pool.enqueue('job-fail', '/repo/1');
|
||||
|
||||
const worker = createdWorkers[0];
|
||||
worker.emit('message', { type: 'failed', jobId: 'job-fail', error: 'parse error' });
|
||||
|
||||
expect(opts.onJobFailed).toHaveBeenCalledWith('job-fail', 'parse error');
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Per-repo serialization
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('does not dispatch a second job for the same repo while first is running', () => {
|
||||
const opts = makeOpts({ concurrency: 2 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
pool.enqueue('job-1', '/repo/same');
|
||||
pool.enqueue('job-2', '/repo/same');
|
||||
|
||||
// Only job-1 should have been dispatched (run message sent)
|
||||
const runCalls = createdWorkers.flatMap((w) =>
|
||||
w.postMessage.mock.calls.filter((c) => (c[0] as { type: string })?.type === 'run')
|
||||
);
|
||||
expect(runCalls.filter((c) => (c[0] as unknown as { jobId: string }).jobId === 'job-1')).toHaveLength(1);
|
||||
expect(runCalls.filter((c) => (c[0] as unknown as { jobId: string }).jobId === 'job-2')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('starts jobs for different repos concurrently', () => {
|
||||
const opts = makeOpts({ concurrency: 2 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
pool.enqueue('job-alpha', '/repo/alpha');
|
||||
pool.enqueue('job-beta', '/repo/beta');
|
||||
|
||||
const runCalls = createdWorkers.flatMap((w) =>
|
||||
w.postMessage.mock.calls.filter((c) => (c[0] as { type: string })?.type === 'run')
|
||||
);
|
||||
const dispatchedIds = runCalls.map((c) => (c[0] as unknown as { jobId: string }).jobId);
|
||||
expect(dispatchedIds).toContain('job-alpha');
|
||||
expect(dispatchedIds).toContain('job-beta');
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Worker crash (exit code != 0)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('calls onJobFailed and spawns a replacement worker when a worker exits with code 1', () => {
|
||||
const opts = makeOpts({ concurrency: 1 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
pool.enqueue('job-crash', '/repo/1');
|
||||
|
||||
const originalWorker = createdWorkers[0];
|
||||
// Simulate crash while the job is running
|
||||
originalWorker.emit('exit', 1);
|
||||
|
||||
expect(opts.onJobFailed).toHaveBeenCalledWith('job-crash', expect.stringContaining('1'));
|
||||
// A replacement worker must have been spawned
|
||||
expect(createdWorkers.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('does NOT call onJobFailed when a worker exits cleanly (code 0)', () => {
|
||||
const opts = makeOpts({ concurrency: 1 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
// Exit without any running job
|
||||
const worker = createdWorkers[0];
|
||||
worker.emit('exit', 0);
|
||||
|
||||
expect(opts.onJobFailed).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// setMaxConcurrency — scale up
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('spawns additional workers when setMaxConcurrency is increased', () => {
|
||||
const pool = new WorkerPool(makeOpts({ concurrency: 1 }));
|
||||
const before = createdWorkers.length; // 1
|
||||
|
||||
pool.setMaxConcurrency(3);
|
||||
|
||||
expect(createdWorkers.length).toBe(before + 2);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// setMaxConcurrency — scale down
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('sends "shutdown" to idle workers when setMaxConcurrency is decreased', () => {
|
||||
const opts = makeOpts({ concurrency: 3 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
pool.setMaxConcurrency(1);
|
||||
|
||||
const shutdownWorkers = createdWorkers.filter((w) =>
|
||||
w.postMessage.mock.calls.some((c) => (c[0] as { type: string })?.type === 'shutdown')
|
||||
);
|
||||
// Two workers should have received shutdown
|
||||
expect(shutdownWorkers.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// shutdown
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('sends "shutdown" to all workers on pool.shutdown()', () => {
|
||||
const opts = makeOpts({ concurrency: 2 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
// Don't await — shutdown() is async but the postMessage calls happen synchronously
|
||||
void pool.shutdown();
|
||||
|
||||
for (const worker of createdWorkers) {
|
||||
const hasShutdown = worker.postMessage.mock.calls.some(
|
||||
(c) => (c[0] as { type: string })?.type === 'shutdown'
|
||||
);
|
||||
expect(hasShutdown).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Enqueue after shutdown is a no-op
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
it('ignores enqueue calls after shutdown is initiated', () => {
|
||||
const opts = makeOpts({ concurrency: 1 });
|
||||
const pool = new WorkerPool(opts);
|
||||
|
||||
// Don't await — shutdown() sets shuttingDown=true synchronously
|
||||
void pool.shutdown();
|
||||
|
||||
// Reset postMessage mocks to isolate post-shutdown calls
|
||||
for (const w of createdWorkers) w.postMessage.mockClear();
|
||||
|
||||
pool.enqueue('job-late', '/repo/1');
|
||||
|
||||
const runCalls = createdWorkers.flatMap((w) =>
|
||||
w.postMessage.mock.calls.filter((c) => (c[0] as { type: string })?.type === 'run')
|
||||
);
|
||||
expect(runCalls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user