feat: add reusable jobqueue library

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

tests/JobQueue.test.ts

@@ -0,0 +1,135 @@
+import { JobQueue } from '../src/index.js';
+import { cleanupDir, createDbPath, createTempDir, waitFor } from './helpers.js';
+
+describe('JobQueue', () => {
+  it('runs multi-phase jobs to completion', async () => {
+    const dir = createTempDir();
+    const queue = new JobQueue<{ url: string }>({
+      dbPath: createDbPath(dir),
+      phases: ['download', 'process'],
+      concurrency: 1,
+    });
+    const events: string[] = [];
+
+    queue.on('job:started', () => events.push('started'));
+    queue.on('job:phase:completed', (_, phase) => events.push(`phase:${phase.name}`));
+    queue.on('job:completed', () => events.push('completed'));
+
+    queue.handle('download', async (_job, ctx) => {
+      await ctx.progress(50, 'downloading');
+      return { filePath: '/tmp/video.mp4' };
+    });
+
+    queue.handle('process', async (_job, ctx) => {
+      expect(ctx.phaseResult<{ filePath: string }>('download')?.filePath).toBe('/tmp/video.mp4');
+      await ctx.progress(25, 'processing');
+      return { outputPath: '/tmp/video.txt' };
+    });
+
+    try {
+      const jobId = await queue.enqueue({ url: 'https://example.com/video' });
+      await waitFor(() => queue.getJob(jobId)?.status === 'completed');
+
+      const job = queue.getJob(jobId);
+      expect(job?.status).toBe('completed');
+      expect(job?.phaseResults.download).toEqual({ filePath: '/tmp/video.mp4' });
+      expect(job?.phaseResults.process).toEqual({ outputPath: '/tmp/video.txt' });
+      expect(events).toEqual(['started', 'phase:download', 'phase:process', 'completed']);
+    } finally {
+      await queue.shutdown();
+      cleanupDir(dir);
+    }
+  });
+
+  it('retries recoverable failures and eventually completes', async () => {
+    const dir = createTempDir();
+    const queue = new JobQueue<{ url: string }>({
+      dbPath: createDbPath(dir),
+      phases: ['run'],
+      concurrency: 1,
+      retry: {
+        maxAttempts: 3,
+        baseDelayMs: 10,
+        classifyError: async (error) =>
+          error instanceof Error && error.message === 'recoverable' ? 'recoverable' : 'fatal',
+      },
+    });
+    let attempts = 0;
+    let retries = 0;
+
+    queue.on('job:retrying', () => {
+      retries += 1;
+    });
+
+    queue.handle('run', async () => {
+      attempts += 1;
+      if (attempts === 1) {
+        throw new Error('recoverable');
+      }
+
+      return { ok: true };
+    });
+
+    try {
+      const jobId = await queue.enqueue({ url: 'https://example.com/video' });
+      await waitFor(() => queue.getJob(jobId)?.status === 'completed', { timeoutMs: 4_000 });
+
+      const job = queue.getJob(jobId);
+      expect(job?.status).toBe('completed');
+      expect(job?.retryCount).toBe(1);
+      expect(retries).toBe(1);
+    } finally {
+      await queue.shutdown();
+      cleanupDir(dir);
+    }
+  });
+
+  it('streams queue events as SSE', async () => {
+    const dir = createTempDir();
+    const queue = new JobQueue<{ url: string }>({
+      dbPath: createDbPath(dir),
+      phases: ['run'],
+      concurrency: 1,
+    });
+    const stream = queue.createEventStream({ includeSnapshot: false });
+    const reader = stream.getReader();
+    const chunks: string[] = [];
+    const readPromise = (async () => {
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) {
+          return;
+        }
+
+        if (value) {
+          chunks.push(new TextDecoder().decode(value));
+          if (chunks.some((chunk) => chunk.includes('event: job:completed'))) {
+            return;
+          }
+        }
+      }
+    })();
+
+    queue.handle('run', async (_job, ctx) => {
+      await ctx.progress(100, 'done');
+      return { ok: true };
+    });
+
+    try {
+      await queue.enqueue({ url: 'https://example.com/video' });
+      await Promise.race([
+        readPromise,
+        waitFor(() => chunks.some((chunk) => chunk.includes('event: job:completed')), {
+          timeoutMs: 4_000,
+        }),
+      ]);
+    } finally {
+      await reader.cancel();
+      await queue.shutdown();
+      cleanupDir(dir);
+    }
+
+    expect(chunks.join('\n')).toContain('event: job:completed');
+    expect(chunks.join('\n')).toContain('event: job:progress');
+  });
+});

tests/RetentionScheduler.test.ts

@@ -0,0 +1,28 @@
+import { RetentionScheduler } from '../src/index.js';
+
+describe('RetentionScheduler', () => {
+  it('runs stale and delete handlers', async () => {
+    const calls: string[] = [];
+    const scheduler = new RetentionScheduler(
+      {
+        staleAfterMs: 1_000,
+        deleteAfterMs: 2_000,
+      },
+      {
+        markStale: async () => {
+          calls.push('stale');
+          return [];
+        },
+        deleteStale: async () => {
+          calls.push('delete');
+          return [];
+        },
+      },
+    );
+
+    await scheduler.runCycle();
+    scheduler.stop();
+
+    expect(calls).toEqual(['stale', 'delete']);
+  });
+});

tests/RetryStrategy.test.ts

@@ -0,0 +1,72 @@
+import { RetryStrategy } from '../src/index.js';
+
+describe('RetryStrategy', () => {
+  it('uses exponential backoff for recoverable errors', async () => {
+    const strategy = new RetryStrategy({
+      maxAttempts: 4,
+      baseDelayMs: 100,
+      classifyError: async () => 'recoverable',
+    });
+
+    const decision = await strategy.shouldRetry(new Error('boom'), {
+      id: 'job-1',
+      status: 'active',
+      data: {},
+      currentPhase: 'run',
+      phases: [],
+      phaseResults: {},
+      progress: 0,
+      progressMessage: null,
+      error: null,
+      retryCount: 1,
+      maxAttempts: 4,
+      webhookUrl: null,
+      webhookSent: false,
+      createdAt: new Date().toISOString(),
+      startedAt: null,
+      completedAt: null,
+      updatedAt: new Date().toISOString(),
+      scheduledAt: null,
+      cancelledAt: null,
+    });
+
+    expect(decision.retry).toBe(true);
+    expect(decision.delayMs).toBe(200);
+    expect(decision.disposition).toBe('recoverable');
+  });
+
+  it('does not retry fatal errors', async () => {
+    const strategy = new RetryStrategy({
+      maxAttempts: 4,
+      classifyError: async () => 'fatal',
+    });
+
+    const decision = await strategy.shouldRetry(new Error('fatal'), {
+      id: 'job-1',
+      status: 'active',
+      data: {},
+      currentPhase: 'run',
+      phases: [],
+      phaseResults: {},
+      progress: 0,
+      progressMessage: null,
+      error: null,
+      retryCount: 0,
+      maxAttempts: 4,
+      webhookUrl: null,
+      webhookSent: false,
+      createdAt: new Date().toISOString(),
+      startedAt: null,
+      completedAt: null,
+      updatedAt: new Date().toISOString(),
+      scheduledAt: null,
+      cancelledAt: null,
+    });
+
+    expect(decision).toEqual({
+      retry: false,
+      delayMs: 0,
+      disposition: 'fatal',
+    });
+  });
+});

tests/SqliteStorage.test.ts

@@ -0,0 +1,76 @@
+import { SqliteStorage } from '../src/index.js';
+import { cleanupDir, createDbPath, createTempDir } from './helpers.js';
+
+describe('SqliteStorage', () => {
+  it('creates, updates, and completes jobs', () => {
+    const dir = createTempDir();
+    const storage = new SqliteStorage<{ url: string }>(createDbPath(dir));
+
+    try {
+      const job = storage.createJob(
+        'job-1',
+        { url: 'https://example.com' },
+        [
+          {
+            name: 'download',
+            status: 'pending',
+            progress: 0,
+            message: null,
+            startedAt: null,
+            completedAt: null,
+            error: null,
+          },
+        ],
+        {},
+        3,
+      );
+
+      expect(job.status).toBe('pending');
+      expect(storage.claimPendingJob(job.id)).toBe(true);
+
+      const inProgress = storage.saveProgress(
+        job.id,
+        'download',
+        [
+          {
+            name: 'download',
+            status: 'active',
+            progress: 50,
+            message: 'halfway',
+            startedAt: new Date().toISOString(),
+            completedAt: null,
+            error: null,
+          },
+        ],
+        50,
+        'halfway',
+      );
+
+      expect(inProgress.status).toBe('active');
+      expect(inProgress.progress).toBe(50);
+
+      const completed = storage.completeJob(
+        job.id,
+        [
+          {
+            name: 'download',
+            status: 'completed',
+            progress: 100,
+            message: null,
+            startedAt: new Date().toISOString(),
+            completedAt: new Date().toISOString(),
+            error: null,
+          },
+        ],
+        { download: { filePath: '/tmp/file' } },
+      );
+
+      expect(completed.status).toBe('completed');
+      expect(completed.progress).toBe(100);
+      expect(completed.phaseResults.download).toEqual({ filePath: '/tmp/file' });
+    } finally {
+      storage.close();
+      cleanupDir(dir);
+    }
+  });
+});

tests/SseSerializer.test.ts

@@ -0,0 +1,25 @@
+import { SseSerializer } from '../src/index.js';
+
+describe('SseSerializer', () => {
+  it('formats SSE events', () => {
+    const serializer = new SseSerializer();
+    const event = serializer.event('job:completed', {
+      type: 'job:completed',
+      jobId: 'job-1',
+      timestamp: '2026-01-01T00:00:00.000Z',
+    });
+
+    expect(event).toContain('event: job:completed');
+    expect(event).toContain('"jobId":"job-1"');
+    expect(event.endsWith('\n\n')).toBe(true);
+  });
+
+  it('returns SSE headers', () => {
+    expect(SseSerializer.headers()).toEqual({
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      Connection: 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    });
+  });
+});

tests/WebhookDispatcher.test.ts

@@ -0,0 +1,75 @@
+import { createServer } from 'node:http';
+
+import { WebhookDispatcher } from '../src/index.js';
+
+describe('WebhookDispatcher', () => {
+  it('delivers signed webhook payloads', async () => {
+    const payloads: string[] = [];
+    const headers: string[] = [];
+    const server = createServer((request, response) => {
+      let body = '';
+      request.on('data', (chunk) => {
+        body += chunk.toString();
+      });
+      request.on('end', () => {
+        payloads.push(body);
+        headers.push(String(request.headers['x-jobqueue-signature']));
+        response.writeHead(202).end();
+      });
+    });
+
+    await new Promise<void>((resolve) => {
+      server.listen(0, '127.0.0.1', () => resolve());
+    });
+
+    const address = server.address();
+    if (!address || typeof address === 'string') {
+      throw new Error('Server address unavailable');
+    }
+
+    const dispatcher = new WebhookDispatcher({
+      url: `http://127.0.0.1:${address.port}/hook`,
+      secret: 'top-secret',
+      events: ['job:completed'],
+    });
+
+    try {
+      const result = await dispatcher.dispatch('job:completed', {
+        id: 'job-1',
+        status: 'completed',
+        data: { url: 'https://example.com' },
+        currentPhase: null,
+        phases: [],
+        phaseResults: {},
+        progress: 100,
+        progressMessage: null,
+        error: null,
+        retryCount: 0,
+        maxAttempts: 1,
+        webhookUrl: null,
+        webhookSent: false,
+        createdAt: new Date().toISOString(),
+        startedAt: null,
+        completedAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+        scheduledAt: null,
+        cancelledAt: null,
+      });
+
+      expect(result.status).toBe(202);
+      expect(payloads).toHaveLength(1);
+      expect(payloads[0]).toContain('"event":"job:completed"');
+      expect(headers[0]).toMatch(/^[a-f0-9]{64}$/);
+    } finally {
+      await new Promise<void>((resolve, reject) => {
+        server.close((error) => {
+          if (error) {
+            reject(error);
+            return;
+          }
+          resolve();
+        });
+      });
+    }
+  });
+});

tests/WorkerPool.test.ts

@@ -0,0 +1,26 @@
+import { WorkerPool } from '../src/index.js';
+import { waitFor } from './helpers.js';
+
+describe('WorkerPool', () => {
+  it('respects concurrency limits', async () => {
+    const pool = new WorkerPool(2);
+    let active = 0;
+    let maxActive = 0;
+
+    const tasks = Array.from({ length: 5 }, (_, index) =>
+      pool.run(async () => {
+        active += 1;
+        maxActive = Math.max(maxActive, active);
+        await new Promise((resolve) => {
+          setTimeout(resolve, 20 + index);
+        });
+        active -= 1;
+      }),
+    );
+
+    await Promise.all(tasks);
+    await waitFor(() => pool.activeCount === 0 && pool.pendingCount === 0);
+
+    expect(maxActive).toBe(2);
+  });
+});

tests/helpers.ts

@@ -0,0 +1,34 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+export function createTempDir(prefix = 'jobqueue-'): string {
+  return mkdtempSync(join(tmpdir(), prefix));
+}
+
+export function createDbPath(dir: string): string {
+  return join(dir, 'jobs.db');
+}
+
+export function cleanupDir(dir: string): void {
+  rmSync(dir, { recursive: true, force: true });
+}
+
+export async function waitFor(
+  predicate: () => boolean,
+  options: { timeoutMs?: number; intervalMs?: number } = {},
+): Promise<void> {
+  const timeoutMs = options.timeoutMs ?? 2_000;
+  const intervalMs = options.intervalMs ?? 10;
+  const start = Date.now();
+
+  while (!predicate()) {
+    if (Date.now() - start > timeoutMs) {
+      throw new Error(`waitFor timed out after ${timeoutMs}ms`);
+    }
+
+    await new Promise((resolve) => {
+      setTimeout(resolve, intervalMs);
+    });
+  }
+}