fix: harden queue lifecycle and publish gate
- Preserve phase results on partial retry and keep interrupted phase context after restart. - Avoid webhook bookkeeping crashes when retention deletes stale jobs. - Add deeper unit, integration, and e2e coverage around queue seams. - Require verify job to pass before publish runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
107
tests/JobQueue.e2e.test.ts
Normal file
107
tests/JobQueue.e2e.test.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import { waitFor } from './helpers.js';
|
||||
import { JobQueueHarness } from './e2e/JobQueueHarness.js';
|
||||
|
||||
describe('JobQueue e2e', () => {
|
||||
it('runs a multi-phase workflow with retry, SSE, and webhooks', async () => {
|
||||
const harness = new JobQueueHarness<{ url: string }>();
|
||||
let processAttempts = 0;
|
||||
|
||||
try {
|
||||
const queue = await harness.start(
|
||||
{
|
||||
phases: ['download', 'process', 'upload'],
|
||||
concurrency: 2,
|
||||
retry: {
|
||||
maxAttempts: 2,
|
||||
baseDelayMs: 5,
|
||||
classifyError: async (error) =>
|
||||
error instanceof Error && error.message === 'recoverable' ? 'recoverable' : 'fatal',
|
||||
},
|
||||
webhook: {
|
||||
events: ['job:retrying', 'job:completed'],
|
||||
},
|
||||
},
|
||||
{ webhookDelayMs: 5 },
|
||||
);
|
||||
const stream = await harness.createStream({ includeSnapshot: false });
|
||||
|
||||
queue.handle('download', async (_job, ctx) => {
|
||||
await ctx.progress(100, 'downloaded');
|
||||
return { filePath: '/tmp/video.mp4' };
|
||||
});
|
||||
queue.handle('process', async (_job, ctx) => {
|
||||
processAttempts += 1;
|
||||
const filePath = ctx.phaseResult<{ filePath: string }>('download')?.filePath;
|
||||
|
||||
if (processAttempts === 1) {
|
||||
await ctx.progress(25, 'processing');
|
||||
throw new Error('recoverable');
|
||||
}
|
||||
|
||||
await ctx.progress(100, 'processed');
|
||||
return { outputPath: `${filePath}.json` };
|
||||
});
|
||||
queue.handle('upload', async (_job, ctx) => {
|
||||
const outputPath = ctx.phaseResult<{ outputPath: string }>('process')?.outputPath;
|
||||
await ctx.progress(100, 'uploaded');
|
||||
return { uploaded: Boolean(outputPath) };
|
||||
});
|
||||
|
||||
const jobId = await queue.enqueue({ url: 'https://example.com/video' });
|
||||
await harness.waitForJobStatus(jobId, 'completed');
|
||||
await waitFor(() => harness.webhooks.length >= 2);
|
||||
await stream.stop();
|
||||
|
||||
const job = queue.getJob(jobId);
|
||||
expect(job?.phaseResults.download).toEqual({ filePath: '/tmp/video.mp4' });
|
||||
expect(job?.phaseResults.process).toEqual({ outputPath: '/tmp/video.mp4.json' });
|
||||
expect(job?.phaseResults.upload).toEqual({ uploaded: true });
|
||||
expect(harness.webhooks.map((entry) => entry.event)).toEqual(['job:retrying', 'job:completed']);
|
||||
expect(stream.eventNames).toContain('job:retrying');
|
||||
expect(stream.eventNames).toContain('job:completed');
|
||||
expect(stream.eventNames.filter((event) => event === 'job:phase:completed')).toHaveLength(3);
|
||||
} finally {
|
||||
await harness.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it('survives stale-job webhook completion after retention deletes the job', async () => {
|
||||
const harness = new JobQueueHarness<{ url: string }>();
|
||||
|
||||
try {
|
||||
const queue = await harness.start(
|
||||
{
|
||||
phases: ['run'],
|
||||
concurrency: 1,
|
||||
webhook: {
|
||||
events: ['job:stale'],
|
||||
},
|
||||
retention: {
|
||||
staleAfterMs: 20,
|
||||
deleteAfterMs: 40,
|
||||
intervalMs: 10,
|
||||
},
|
||||
},
|
||||
{ webhookDelayMs: 80 },
|
||||
);
|
||||
const stream = await harness.createStream({ includeSnapshot: false });
|
||||
|
||||
queue.handle('run', async () => ({ ok: true }));
|
||||
|
||||
const firstJob = await queue.enqueue({ url: 'https://example.com/one' });
|
||||
await harness.waitForJobStatus(firstJob, 'completed');
|
||||
await harness.waitForJobDeletion(firstJob);
|
||||
await waitFor(() => harness.webhooks.length >= 1, { timeoutMs: 4_000 });
|
||||
|
||||
const secondJob = await queue.enqueue({ url: 'https://example.com/two' });
|
||||
await harness.waitForJobStatus(secondJob, 'completed');
|
||||
await stream.stop();
|
||||
|
||||
expect(harness.webhooks[0]?.event).toBe('job:stale');
|
||||
expect(stream.eventNames).toContain('job:stale');
|
||||
expect(stream.eventNames).toContain('job:deleted');
|
||||
} finally {
|
||||
await harness.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,6 @@
|
||||
import { JobQueue } from '../src/index.js';
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
import { JobQueue, SqliteStorage } from '../src/index.js';
|
||||
import { cleanupDir, createDbPath, createTempDir, waitFor } from './helpers.js';
|
||||
|
||||
describe('JobQueue', () => {
|
||||
@@ -132,4 +134,315 @@ describe('JobQueue', () => {
|
||||
expect(chunks.join('\n')).toContain('event: job:completed');
|
||||
expect(chunks.join('\n')).toContain('event: job:progress');
|
||||
});
|
||||
|
||||
it('marks unfinished phases as cancelled when cancelling a pending job', async () => {
|
||||
const dir = createTempDir();
|
||||
const queue = new JobQueue<{ url: string }>({
|
||||
dbPath: createDbPath(dir),
|
||||
phases: ['download', 'process'],
|
||||
concurrency: 1,
|
||||
});
|
||||
|
||||
try {
|
||||
const jobId = await queue.enqueue(
|
||||
{ url: 'https://example.com/video' },
|
||||
{ scheduledAt: new Date(Date.now() + 60_000) },
|
||||
);
|
||||
const cancelled = await queue.cancel(jobId);
|
||||
|
||||
expect(cancelled.status).toBe('cancelled');
|
||||
expect(cancelled.phases.map((phase) => phase.status)).toEqual(['cancelled', 'cancelled']);
|
||||
} finally {
|
||||
await queue.shutdown();
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('emits job:cancelled once when cancelling after a phase completes', async () => {
|
||||
const dir = createTempDir();
|
||||
const queue = new JobQueue<{ url: string }>({
|
||||
dbPath: createDbPath(dir),
|
||||
phases: ['download', 'process'],
|
||||
concurrency: 1,
|
||||
});
|
||||
let processStarted = false;
|
||||
let cancelledEvents = 0;
|
||||
|
||||
queue.on('job:phase:completed', (job, phase) => {
|
||||
if (phase.name === 'download') {
|
||||
void queue.cancel(job.id);
|
||||
}
|
||||
});
|
||||
queue.on('job:cancelled', () => {
|
||||
cancelledEvents += 1;
|
||||
});
|
||||
|
||||
queue.handle('download', async (_job, ctx) => {
|
||||
await ctx.progress(100, 'downloaded');
|
||||
return { filePath: '/tmp/video.mp4' };
|
||||
});
|
||||
queue.handle('process', async () => {
|
||||
processStarted = true;
|
||||
return { outputPath: '/tmp/video.txt' };
|
||||
});
|
||||
|
||||
try {
|
||||
const jobId = await queue.enqueue({ url: 'https://example.com/video' });
|
||||
await waitFor(() => queue.getJob(jobId)?.status === 'cancelled');
|
||||
|
||||
expect(cancelledEvents).toBe(1);
|
||||
expect(processStarted).toBe(false);
|
||||
expect(queue.getJob(jobId)?.phases.map((phase) => phase.status)).toEqual([
|
||||
'completed',
|
||||
'cancelled',
|
||||
]);
|
||||
} finally {
|
||||
await queue.shutdown();
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for in-flight webhooks during shutdown', async () => {
|
||||
const dir = createTempDir();
|
||||
const deliveries: string[] = [];
|
||||
const server = createServer((request, response) => {
|
||||
let body = '';
|
||||
request.on('data', (chunk) => {
|
||||
body += chunk.toString();
|
||||
});
|
||||
request.on('end', () => {
|
||||
setTimeout(() => {
|
||||
deliveries.push(body);
|
||||
response.writeHead(202).end();
|
||||
}, 50);
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, '127.0.0.1', () => resolve());
|
||||
});
|
||||
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('Server address unavailable');
|
||||
}
|
||||
|
||||
const queue = new JobQueue<{ url: string }>({
|
||||
dbPath: createDbPath(dir),
|
||||
phases: ['run'],
|
||||
concurrency: 1,
|
||||
webhook: {
|
||||
url: `http://127.0.0.1:${address.port}/hook`,
|
||||
events: ['job:completed'],
|
||||
},
|
||||
});
|
||||
let deliveredEvents = 0;
|
||||
|
||||
queue.on('job:webhook:delivered', () => {
|
||||
deliveredEvents += 1;
|
||||
});
|
||||
queue.handle('run', async () => ({ ok: true }));
|
||||
|
||||
try {
|
||||
const jobId = await queue.enqueue({ url: 'https://example.com/video' });
|
||||
await waitFor(() => queue.getJob(jobId)?.status === 'completed');
|
||||
|
||||
const startedAt = Date.now();
|
||||
await queue.shutdown();
|
||||
|
||||
expect(Date.now() - startedAt).toBeGreaterThanOrEqual(40);
|
||||
expect(deliveredEvents).toBe(1);
|
||||
expect(deliveries).toHaveLength(1);
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('cleans up listeners and storage when shutdown times out', async () => {
|
||||
const dir = createTempDir();
|
||||
const queue = new JobQueue<{ url: string }>({
|
||||
dbPath: createDbPath(dir),
|
||||
phases: ['run'],
|
||||
concurrency: 1,
|
||||
});
|
||||
const internalQueue = queue as JobQueue<{ url: string }> & {
|
||||
storage: { close: () => void };
|
||||
events: { removeAllListeners: () => void };
|
||||
};
|
||||
const closeSpy = vi.spyOn(internalQueue.storage, 'close');
|
||||
const removeAllListenersSpy = vi.spyOn(internalQueue.events, 'removeAllListeners');
|
||||
|
||||
queue.handle(
|
||||
'run',
|
||||
async (_job, ctx) =>
|
||||
await new Promise((resolve) => {
|
||||
ctx.signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
resolve({ ok: false });
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
const jobId = await queue.enqueue({ url: 'https://example.com/video' });
|
||||
await waitFor(() => queue.getJob(jobId)?.status === 'active');
|
||||
|
||||
await expect(queue.shutdown(10)).rejects.toThrow(/Timed out waiting for workers to drain/);
|
||||
expect(removeAllListenersSpy).toHaveBeenCalledTimes(1);
|
||||
expect(closeSpy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves completed phase results when retrying from partial progress', async () => {
|
||||
const dir = createTempDir();
|
||||
const queue = new JobQueue<{ url: string }>({
|
||||
dbPath: createDbPath(dir),
|
||||
phases: ['download', 'process'],
|
||||
concurrency: 1,
|
||||
});
|
||||
let downloadRuns = 0;
|
||||
let processRuns = 0;
|
||||
let resumedFilePath: string | undefined;
|
||||
|
||||
queue.handle('download', async () => {
|
||||
downloadRuns += 1;
|
||||
return { filePath: '/tmp/video.mp4' };
|
||||
});
|
||||
queue.handle('process', async (_job, ctx) => {
|
||||
processRuns += 1;
|
||||
const filePath = ctx.phaseResult<{ filePath: string }>('download')?.filePath;
|
||||
|
||||
if (processRuns === 1) {
|
||||
expect(filePath).toBe('/tmp/video.mp4');
|
||||
throw new Error('fatal');
|
||||
}
|
||||
|
||||
resumedFilePath = filePath;
|
||||
return { outputPath: `${filePath}.json` };
|
||||
});
|
||||
|
||||
try {
|
||||
const jobId = await queue.enqueue({ url: 'https://example.com/video' });
|
||||
await waitFor(() => queue.getJob(jobId)?.status === 'failed');
|
||||
|
||||
await queue.retry(jobId, { fromStart: false });
|
||||
await waitFor(() => queue.getJob(jobId)?.status === 'completed');
|
||||
|
||||
expect(downloadRuns).toBe(1);
|
||||
expect(processRuns).toBe(2);
|
||||
expect(resumedFilePath).toBe('/tmp/video.mp4');
|
||||
expect(queue.getJob(jobId)?.phaseResults.download).toEqual({ filePath: '/tmp/video.mp4' });
|
||||
} finally {
|
||||
await queue.shutdown();
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('restores interrupted active jobs as failed with phase context on restart', async () => {
|
||||
const dir = createTempDir();
|
||||
const dbPath = createDbPath(dir);
|
||||
const storage = new SqliteStorage<{ url: string }>(dbPath);
|
||||
|
||||
try {
|
||||
const job = storage.createJob(
|
||||
'job-1',
|
||||
{ url: 'https://example.com/video' },
|
||||
[
|
||||
{
|
||||
name: 'run',
|
||||
status: 'pending',
|
||||
progress: 0,
|
||||
message: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
{},
|
||||
1,
|
||||
);
|
||||
|
||||
expect(storage.claimPendingJob(job.id)).toBe(true);
|
||||
storage.saveProgress(
|
||||
job.id,
|
||||
'run',
|
||||
[
|
||||
{
|
||||
name: 'run',
|
||||
status: 'active',
|
||||
progress: 25,
|
||||
message: 'working',
|
||||
startedAt: new Date().toISOString(),
|
||||
completedAt: null,
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
25,
|
||||
'working',
|
||||
);
|
||||
storage.close();
|
||||
|
||||
const restarted = new JobQueue<{ url: string }>({
|
||||
dbPath,
|
||||
phases: ['run'],
|
||||
concurrency: 1,
|
||||
});
|
||||
|
||||
try {
|
||||
const restartedJob = restarted.getJob(job.id);
|
||||
expect(restartedJob?.status).toBe('failed');
|
||||
expect(restartedJob?.error?.phase).toBe('run');
|
||||
expect(restartedJob?.error?.attempt).toBe(1);
|
||||
} finally {
|
||||
await restarted.shutdown();
|
||||
}
|
||||
} finally {
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('executes scheduled jobs when their wakeup time arrives', async () => {
|
||||
const dir = createTempDir();
|
||||
const queue = new JobQueue<{ url: string }>({
|
||||
dbPath: createDbPath(dir),
|
||||
phases: ['run'],
|
||||
concurrency: 1,
|
||||
});
|
||||
const startedAt: number[] = [];
|
||||
const scheduledAt = Date.now() + 50;
|
||||
|
||||
queue.handle('run', async () => {
|
||||
startedAt.push(Date.now());
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
try {
|
||||
const jobId = await queue.enqueue(
|
||||
{ url: 'https://example.com/video' },
|
||||
{ scheduledAt: new Date(scheduledAt) },
|
||||
);
|
||||
await waitFor(() => queue.getJob(jobId)?.status === 'completed', { timeoutMs: 4_000 });
|
||||
|
||||
expect(startedAt).toHaveLength(1);
|
||||
expect(startedAt[0]).toBeGreaterThanOrEqual(scheduledAt - 10);
|
||||
} finally {
|
||||
await queue.shutdown();
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,29 @@
|
||||
import { RetryStrategy } from '../src/index.js';
|
||||
|
||||
function createJob(retryCount = 0, maxAttempts = 4) {
|
||||
return {
|
||||
id: 'job-1',
|
||||
status: 'active' as const,
|
||||
data: {},
|
||||
currentPhase: 'run',
|
||||
phases: [],
|
||||
phaseResults: {},
|
||||
progress: 0,
|
||||
progressMessage: null,
|
||||
error: null,
|
||||
retryCount,
|
||||
maxAttempts,
|
||||
webhookUrl: null,
|
||||
webhookSent: false,
|
||||
createdAt: new Date().toISOString(),
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
updatedAt: new Date().toISOString(),
|
||||
scheduledAt: null,
|
||||
cancelledAt: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe('RetryStrategy', () => {
|
||||
it('uses exponential backoff for recoverable errors', async () => {
|
||||
const strategy = new RetryStrategy({
|
||||
@@ -8,60 +32,64 @@ describe('RetryStrategy', () => {
|
||||
classifyError: async () => 'recoverable',
|
||||
});
|
||||
|
||||
const decision = await strategy.shouldRetry(new Error('boom'), {
|
||||
id: 'job-1',
|
||||
status: 'active',
|
||||
data: {},
|
||||
currentPhase: 'run',
|
||||
phases: [],
|
||||
phaseResults: {},
|
||||
progress: 0,
|
||||
progressMessage: null,
|
||||
error: null,
|
||||
retryCount: 1,
|
||||
maxAttempts: 4,
|
||||
webhookUrl: null,
|
||||
webhookSent: false,
|
||||
createdAt: new Date().toISOString(),
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
updatedAt: new Date().toISOString(),
|
||||
scheduledAt: null,
|
||||
cancelledAt: null,
|
||||
});
|
||||
const decision = await strategy.shouldRetry(new Error('boom'), createJob(1));
|
||||
|
||||
expect(decision.retry).toBe(true);
|
||||
expect(decision.delayMs).toBe(200);
|
||||
expect(decision.disposition).toBe('recoverable');
|
||||
});
|
||||
|
||||
it('uses fixed backoff when configured', async () => {
|
||||
const strategy = new RetryStrategy({
|
||||
maxAttempts: 4,
|
||||
strategy: 'fixed',
|
||||
baseDelayMs: 75,
|
||||
classifyError: async () => 'recoverable',
|
||||
});
|
||||
|
||||
const decision = await strategy.shouldRetry(new Error('boom'), createJob(2));
|
||||
|
||||
expect(decision.retry).toBe(true);
|
||||
expect(decision.delayMs).toBe(75);
|
||||
});
|
||||
|
||||
it('uses linear backoff and caps to maxDelayMs', async () => {
|
||||
const strategy = new RetryStrategy({
|
||||
maxAttempts: 6,
|
||||
strategy: 'linear',
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 250,
|
||||
classifyError: async () => 'recoverable',
|
||||
});
|
||||
|
||||
const decision = await strategy.shouldRetry(new Error('boom'), createJob(4, 6));
|
||||
|
||||
expect(decision.retry).toBe(true);
|
||||
expect(decision.delayMs).toBe(250);
|
||||
});
|
||||
|
||||
it('stops retrying when max attempts are exhausted', async () => {
|
||||
const strategy = new RetryStrategy({
|
||||
maxAttempts: 3,
|
||||
classifyError: async () => 'recoverable',
|
||||
});
|
||||
|
||||
const decision = await strategy.shouldRetry(new Error('boom'), createJob(2, 3));
|
||||
|
||||
expect(decision).toEqual({
|
||||
retry: false,
|
||||
delayMs: 0,
|
||||
disposition: 'recoverable',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not retry fatal errors', async () => {
|
||||
const strategy = new RetryStrategy({
|
||||
maxAttempts: 4,
|
||||
classifyError: async () => 'fatal',
|
||||
});
|
||||
|
||||
const decision = await strategy.shouldRetry(new Error('fatal'), {
|
||||
id: 'job-1',
|
||||
status: 'active',
|
||||
data: {},
|
||||
currentPhase: 'run',
|
||||
phases: [],
|
||||
phaseResults: {},
|
||||
progress: 0,
|
||||
progressMessage: null,
|
||||
error: null,
|
||||
retryCount: 0,
|
||||
maxAttempts: 4,
|
||||
webhookUrl: null,
|
||||
webhookSent: false,
|
||||
createdAt: new Date().toISOString(),
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
updatedAt: new Date().toISOString(),
|
||||
scheduledAt: null,
|
||||
cancelledAt: null,
|
||||
});
|
||||
const decision = await strategy.shouldRetry(new Error('fatal'), createJob());
|
||||
|
||||
expect(decision).toEqual({
|
||||
retry: false,
|
||||
|
||||
@@ -73,4 +73,126 @@ describe('SqliteStorage', () => {
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves phase results when resetting for partial retry', () => {
|
||||
const dir = createTempDir();
|
||||
const storage = new SqliteStorage<{ url: string }>(createDbPath(dir));
|
||||
|
||||
try {
|
||||
const job = storage.createJob(
|
||||
'job-1',
|
||||
{ url: 'https://example.com' },
|
||||
[
|
||||
{
|
||||
name: 'download',
|
||||
status: 'pending',
|
||||
progress: 0,
|
||||
message: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
},
|
||||
{
|
||||
name: 'process',
|
||||
status: 'pending',
|
||||
progress: 0,
|
||||
message: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
{},
|
||||
3,
|
||||
);
|
||||
|
||||
const retried = storage.resetForRetry(
|
||||
job.id,
|
||||
[
|
||||
{
|
||||
name: 'download',
|
||||
status: 'completed',
|
||||
progress: 100,
|
||||
message: null,
|
||||
startedAt: new Date().toISOString(),
|
||||
completedAt: new Date().toISOString(),
|
||||
error: null,
|
||||
},
|
||||
{
|
||||
name: 'process',
|
||||
status: 'pending',
|
||||
progress: 0,
|
||||
message: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
{ download: { filePath: '/tmp/file' } },
|
||||
3,
|
||||
null,
|
||||
);
|
||||
|
||||
expect(retried.status).toBe('pending');
|
||||
expect(retried.progress).toBe(0);
|
||||
expect(retried.phaseResults.download).toEqual({ filePath: '/tmp/file' });
|
||||
} finally {
|
||||
storage.close();
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves interrupted phase when resetting active jobs on restart', () => {
|
||||
const dir = createTempDir();
|
||||
const storage = new SqliteStorage<{ url: string }>(createDbPath(dir));
|
||||
|
||||
try {
|
||||
const job = storage.createJob(
|
||||
'job-1',
|
||||
{ url: 'https://example.com' },
|
||||
[
|
||||
{
|
||||
name: 'download',
|
||||
status: 'pending',
|
||||
progress: 0,
|
||||
message: null,
|
||||
startedAt: null,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
{},
|
||||
3,
|
||||
);
|
||||
|
||||
expect(storage.claimPendingJob(job.id)).toBe(true);
|
||||
|
||||
storage.saveProgress(
|
||||
job.id,
|
||||
'download',
|
||||
[
|
||||
{
|
||||
name: 'download',
|
||||
status: 'active',
|
||||
progress: 25,
|
||||
message: 'working',
|
||||
startedAt: new Date().toISOString(),
|
||||
completedAt: null,
|
||||
error: null,
|
||||
},
|
||||
],
|
||||
25,
|
||||
'working',
|
||||
);
|
||||
|
||||
const reset = storage.resetActiveJobs('Interrupted by process restart');
|
||||
expect(reset).toHaveLength(1);
|
||||
expect(reset[0]?.status).toBe('failed');
|
||||
expect(reset[0]?.error?.phase).toBe('download');
|
||||
expect(reset[0]?.error?.attempt).toBe(1);
|
||||
} finally {
|
||||
storage.close();
|
||||
cleanupDir(dir);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -72,4 +72,128 @@ describe('WebhookDispatcher', () => {
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('retries retryable webhook responses until success', async () => {
|
||||
let attempts = 0;
|
||||
const server = createServer((_request, response) => {
|
||||
attempts += 1;
|
||||
response.writeHead(attempts < 3 ? 503 : 202).end();
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, '127.0.0.1', () => resolve());
|
||||
});
|
||||
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('Server address unavailable');
|
||||
}
|
||||
|
||||
const dispatcher = new WebhookDispatcher({
|
||||
url: `http://127.0.0.1:${address.port}/hook`,
|
||||
events: ['job:completed'],
|
||||
maxAttempts: 4,
|
||||
baseDelayMs: 1,
|
||||
maxDelayMs: 1,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await dispatcher.dispatch('job:completed', {
|
||||
id: 'job-1',
|
||||
status: 'completed',
|
||||
data: { url: 'https://example.com' },
|
||||
currentPhase: null,
|
||||
phases: [],
|
||||
phaseResults: {},
|
||||
progress: 100,
|
||||
progressMessage: null,
|
||||
error: null,
|
||||
retryCount: 0,
|
||||
maxAttempts: 1,
|
||||
webhookUrl: null,
|
||||
webhookSent: false,
|
||||
createdAt: new Date().toISOString(),
|
||||
startedAt: null,
|
||||
completedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
scheduledAt: null,
|
||||
cancelledAt: null,
|
||||
});
|
||||
|
||||
expect(result.status).toBe(202);
|
||||
expect(attempts).toBe(3);
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it('does not retry non-retryable webhook responses', async () => {
|
||||
let attempts = 0;
|
||||
const server = createServer((_request, response) => {
|
||||
attempts += 1;
|
||||
response.writeHead(400).end();
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, '127.0.0.1', () => resolve());
|
||||
});
|
||||
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('Server address unavailable');
|
||||
}
|
||||
|
||||
const dispatcher = new WebhookDispatcher({
|
||||
url: `http://127.0.0.1:${address.port}/hook`,
|
||||
events: ['job:completed'],
|
||||
maxAttempts: 4,
|
||||
baseDelayMs: 1,
|
||||
maxDelayMs: 1,
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
dispatcher.dispatch('job:completed', {
|
||||
id: 'job-1',
|
||||
status: 'completed',
|
||||
data: { url: 'https://example.com' },
|
||||
currentPhase: null,
|
||||
phases: [],
|
||||
phaseResults: {},
|
||||
progress: 100,
|
||||
progressMessage: null,
|
||||
error: null,
|
||||
retryCount: 0,
|
||||
maxAttempts: 1,
|
||||
webhookUrl: null,
|
||||
webhookSent: false,
|
||||
createdAt: new Date().toISOString(),
|
||||
startedAt: null,
|
||||
completedAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
scheduledAt: null,
|
||||
cancelledAt: null,
|
||||
}),
|
||||
).rejects.toThrow(/non-retryable status 400/);
|
||||
expect(attempts).toBe(1);
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,4 +23,22 @@ describe('WorkerPool', () => {
|
||||
|
||||
expect(maxActive).toBe(2);
|
||||
});
|
||||
|
||||
it('times out while draining slow tasks', async () => {
|
||||
const pool = new WorkerPool(1);
|
||||
let release!: () => void;
|
||||
const blocked = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
|
||||
void pool.run(async () => {
|
||||
await blocked;
|
||||
});
|
||||
|
||||
await waitFor(() => pool.activeCount === 1);
|
||||
await expect(pool.drain(10)).rejects.toThrow(/Timed out waiting for workers to drain/);
|
||||
|
||||
release();
|
||||
await pool.drain();
|
||||
});
|
||||
});
|
||||
|
||||
187
tests/e2e/JobQueueHarness.ts
Normal file
187
tests/e2e/JobQueueHarness.ts
Normal file
@@ -0,0 +1,187 @@
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
import { JobQueue, type JobData, type QueueConfig, type QueueStreamEvent, type StreamOptions } from '../../src/index.js';
|
||||
import { cleanupDir, createDbPath, createTempDir, waitFor } from '../helpers.js';
|
||||
|
||||
interface HarnessStream<TData extends JobData> {
|
||||
events: QueueStreamEvent<TData>[];
|
||||
eventNames: string[];
|
||||
stop: () => Promise<void>;
|
||||
}
|
||||
|
||||
interface WebhookCapture {
|
||||
event: string;
|
||||
body: string;
|
||||
payload: unknown;
|
||||
}
|
||||
|
||||
function parseSseChunk<TData extends JobData>(chunk: string): Array<{ event: string; payload: QueueStreamEvent<TData> }> {
|
||||
return chunk
|
||||
.split('\n\n')
|
||||
.map((block) => block.trim())
|
||||
.filter(Boolean)
|
||||
.map((block) => {
|
||||
const lines = block.split('\n');
|
||||
const event = lines.find((line) => line.startsWith('event: '))?.slice(7);
|
||||
const data = lines.find((line) => line.startsWith('data: '))?.slice(6);
|
||||
|
||||
if (!event || !data) {
|
||||
throw new Error(`Unable to parse SSE block: ${block}`);
|
||||
}
|
||||
|
||||
return {
|
||||
event,
|
||||
payload: JSON.parse(data) as QueueStreamEvent<TData>,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export class JobQueueHarness<TData extends JobData = JobData> {
|
||||
public readonly dir = createTempDir('jobqueue-e2e-');
|
||||
public readonly webhooks: WebhookCapture[] = [];
|
||||
|
||||
private queue: JobQueue<TData> | null = null;
|
||||
private server:
|
||||
| ReturnType<typeof createServer>
|
||||
| null = null;
|
||||
private webhookUrl: string | null = null;
|
||||
private readonly streams: Array<{ stop: () => Promise<void> }> = [];
|
||||
|
||||
public async start(
|
||||
config: Omit<QueueConfig<TData>, 'dbPath'>,
|
||||
options: { webhookDelayMs?: number } = {},
|
||||
): Promise<JobQueue<TData>> {
|
||||
if (config.webhook) {
|
||||
await this.startWebhookServer(options.webhookDelayMs ?? 0);
|
||||
}
|
||||
|
||||
this.queue = new JobQueue<TData>({
|
||||
...config,
|
||||
dbPath: createDbPath(this.dir),
|
||||
webhook:
|
||||
config.webhook && this.webhookUrl
|
||||
? {
|
||||
...config.webhook,
|
||||
url: config.webhook.url ?? this.webhookUrl,
|
||||
}
|
||||
: config.webhook,
|
||||
});
|
||||
|
||||
return this.queue;
|
||||
}
|
||||
|
||||
public getQueue(): JobQueue<TData> {
|
||||
if (!this.queue) {
|
||||
throw new Error('Harness queue has not been started');
|
||||
}
|
||||
|
||||
return this.queue;
|
||||
}
|
||||
|
||||
public async createStream(options: StreamOptions = {}): Promise<HarnessStream<TData>> {
|
||||
const events: QueueStreamEvent<TData>[] = [];
|
||||
const eventNames: string[] = [];
|
||||
const reader = this.getQueue().createEventStream(options).getReader();
|
||||
let active = true;
|
||||
|
||||
const readLoop = (async () => {
|
||||
while (active) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!value) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const chunk = new TextDecoder().decode(value);
|
||||
for (const parsed of parseSseChunk<TData>(chunk)) {
|
||||
eventNames.push(parsed.event);
|
||||
events.push(parsed.payload);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
const stop = async () => {
|
||||
active = false;
|
||||
await reader.cancel();
|
||||
await readLoop;
|
||||
};
|
||||
|
||||
this.streams.push({ stop });
|
||||
return { events, eventNames, stop };
|
||||
}
|
||||
|
||||
public async waitForJobStatus(id: string, status: string, timeoutMs = 4_000): Promise<void> {
|
||||
await waitFor(() => this.getQueue().getJob(id)?.status === status, { timeoutMs });
|
||||
}
|
||||
|
||||
public async waitForJobDeletion(id: string, timeoutMs = 4_000): Promise<void> {
|
||||
await waitFor(() => this.getQueue().getJob(id) === null, { timeoutMs });
|
||||
}
|
||||
|
||||
public async cleanup(): Promise<void> {
|
||||
for (const stream of this.streams.splice(0)) {
|
||||
await stream.stop();
|
||||
}
|
||||
|
||||
if (this.queue) {
|
||||
await this.queue.shutdown();
|
||||
this.queue = null;
|
||||
}
|
||||
|
||||
if (this.server) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
this.server?.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
this.server = null;
|
||||
}
|
||||
|
||||
cleanupDir(this.dir);
|
||||
}
|
||||
|
||||
private async startWebhookServer(delayMs: number): Promise<void> {
|
||||
this.server = createServer((request, response) => {
|
||||
let body = '';
|
||||
request.on('data', (chunk) => {
|
||||
body += chunk.toString();
|
||||
});
|
||||
request.on('end', () => {
|
||||
setTimeout(() => {
|
||||
let payload: unknown = null;
|
||||
try {
|
||||
payload = JSON.parse(body);
|
||||
} catch {
|
||||
payload = body;
|
||||
}
|
||||
|
||||
this.webhooks.push({
|
||||
event: String(request.headers['x-jobqueue-event'] ?? ''),
|
||||
body,
|
||||
payload,
|
||||
});
|
||||
response.writeHead(202).end();
|
||||
}, delayMs);
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
this.server?.listen(0, '127.0.0.1', () => resolve());
|
||||
});
|
||||
|
||||
const address = this.server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('Webhook server address unavailable');
|
||||
}
|
||||
|
||||
this.webhookUrl = `http://127.0.0.1:${address.port}/hook`;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user