Files
insta-recipe/src/tests/queue-processor.spec.ts
Giancarmine Salucci 5b5bb947ef feat: replace Playwright extractor with yt-dlp subprocess
- Add instagram-extractor.ts: yt-dlp subprocess backend for Instagram
  caption extraction. No in-process browser state, maintained against
  Instagram frontend churn, supports cookies.txt for auth-walled reels.
- Add feature flag EXTRACTOR_BACKEND (ytdlp|playwright) in QueueProcessor
  so the old Playwright path remains available as fallback.
- Add 9 unit tests and 2 live-network integration tests for the new extractor.
- Dockerfile: install yt-dlp via pip3 alongside existing Chromium deps.
- docker-compose: expose EXTRACTOR_BACKEND env var (default: ytdlp).

Also in this commit:
- LLM: configurable per-request timeout via LLM_REQUEST_TIMEOUT_MS (default 120s);
  set maxRetries=0 to surface errors immediately; llama-swap /running health probe.
- QueueProcessor: thread progress callback through parser phase.
- LlmHealthIndicator: surface llama-swap loaded-model name.
- Logging: improve error serialization in queue-processor tests.
- .env.example: document llama-swap endpoint and model options.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-12 20:46:31 +02:00

284 lines
8.6 KiB
TypeScript

/**
* Integration tests for QueueProcessor
*
* Tests the processor's ability to handle queue items through mocked dependencies.
* The QueueProcessor auto-starts, so these tests verify actual processing behavior.
*/
import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
import { queueManager } from '$lib/server/queue/QueueManager';
// Mock web-push module BEFORE importing modules that depend on it
vi.mock('web-push', () => ({
default: {
setVapidDetails: vi.fn(),
sendNotification: vi.fn().mockResolvedValue(undefined)
}
}));
// Mock queueConfig BEFORE importing QueueProcessor
vi.mock('$lib/server/queue/config', () => ({
queueConfig: {
concurrency: 2,
maxRetries: 3,
tandoor: {
enabled: true,
token: 'test-token',
serverUrl: 'http://localhost:8080'
},
push: {
vapidPublicKey:
'BNextdcB_fQ0BVvyGioM5L8Tf9vKQjs-WnF-rUbnU8MdWIZQYfggIHxBnW21I-lq_0HykLCdMpYj8d5joavWdxQ',
vapidPrivateKey: 'JwxI_KcsBcehYcTOufMcbVWJjCq1QbH5FJmSyQuG680',
vapidEmail: 'mailto:test@example.com'
}
}
}));
// Mock external dependencies BEFORE importing QueueProcessor.
// QueueProcessor.extractionPhase picks between two extractor modules based on
// EXTRACTOR_BACKEND; mock both so behavior is identical regardless of default.
vi.mock('$lib/server/extraction', () => ({
extractTextAndThumbnail: vi.fn().mockResolvedValue({
bodyText: 'Default recipe text',
thumbnail: null
})
}));
vi.mock('$lib/server/instagram-extractor', () => ({
extractTextAndThumbnail: vi.fn().mockResolvedValue({
bodyText: 'Default recipe text',
thumbnail: null
})
}));
vi.mock('$lib/server/parser', () => ({
extractRecipe: vi.fn().mockResolvedValue({
name: 'Default Recipe',
ingredients: ['ingredient 1'],
steps: ['step 1'],
description: 'A default recipe'
})
}));
vi.mock('$lib/server/tandoor', () => ({
uploadRecipeWithIngredientsDTO: vi.fn().mockResolvedValue({
success: true,
recipeId: 999
}),
uploadRecipeImage: vi.fn().mockResolvedValue({
success: true
})
}));
import { extractTextAndThumbnail as extractFromExtraction } from '$lib/server/extraction';
import { extractTextAndThumbnail as extractFromYtDlp } from '$lib/server/instagram-extractor';
import { extractRecipe } from '$lib/server/parser';
import { uploadRecipeWithIngredientsDTO, uploadRecipeImage } from '$lib/server/tandoor';
import * as configModule from '$lib/server/queue/config';
// Alias used by existing assertions; default backend is ytdlp so the new
// instagram-extractor mock is what the processor actually invokes.
const extractTextAndThumbnail = extractFromYtDlp;
// Import processor AFTER mocks - it will auto-start (imported for side effects)
import '$lib/server/queue/QueueProcessor';
describe('QueueProcessor Integration Tests', () => {
beforeEach(async () => {
// Clear queue
queueManager.getAll().forEach((item) => queueManager.remove(item.id));
// Reset mocks and their implementations
vi.resetAllMocks();
// Set default mock implementations on BOTH backend modules so the test
// behavior is invariant to EXTRACTOR_BACKEND.
vi.mocked(extractFromExtraction).mockResolvedValue({
bodyText: 'Default recipe text',
thumbnail: null
});
vi.mocked(extractFromYtDlp).mockResolvedValue({
bodyText: 'Default recipe text',
thumbnail: null
});
vi.mocked(extractRecipe).mockResolvedValue({
name: 'Default Recipe',
servings: 2,
ingredients: [{ item: 'ingredient 1', amount: '1', unit: 'piece' }],
steps: ['step 1'],
description: 'A default recipe'
});
vi.mocked(uploadRecipeWithIngredientsDTO).mockResolvedValue({
success: true,
recipeId: 999
});
vi.mocked(uploadRecipeImage).mockResolvedValue({
success: true
});
});
afterEach(async () => {
// Wait for any pending processing to complete
await new Promise((resolve) => setTimeout(resolve, 100));
});
it('should process item through all phases when Tandoor is configured', async () => {
// Set up successful mocks
vi.mocked(extractTextAndThumbnail).mockResolvedValue({
bodyText: 'Recipe instructions here',
thumbnail: 'https://example.com/thumb.jpg'
});
vi.mocked(extractRecipe).mockResolvedValue({
name: 'Test Recipe',
servings: 4,
ingredients: [
{ item: 'flour', amount: '2', unit: 'cups' },
{ item: 'eggs', amount: '2', unit: 'pieces' }
],
steps: ['mix', 'bake'],
description: 'test'
});
vi.mocked(uploadRecipeWithIngredientsDTO).mockResolvedValue({
success: true,
recipeId: 123
});
// Enqueue (processor is already running from auto-start)
// Note: Tandoor is enabled in the mocked config
const item = queueManager.enqueue('https://instagram.com/p/test-tandoor');
// Wait for processing to complete - increased timeout
await new Promise((resolve) => setTimeout(resolve, 1000));
const updated = queueManager.get(item.id);
// Verify success
expect(updated?.status).toBe('success');
expect(updated?.extractedText).toBe('Recipe instructions here');
expect(updated?.recipe?.name).toBe('Test Recipe');
expect(updated?.tandoorRecipeId).toBe(123);
// Verify all functions were called
expect(extractTextAndThumbnail).toHaveBeenCalled();
expect(extractRecipe).toHaveBeenCalled();
expect(uploadRecipeWithIngredientsDTO).toHaveBeenCalled();
}, 10000); // Increase timeout for processing
it('should skip Tandoor upload when not configured', async () => {
// Temporarily disable Tandoor for this test
const originalConfig = { ...configModule.queueConfig };
vi.spyOn(configModule, 'queueConfig', 'get').mockReturnValue({
...originalConfig,
tandoor: {
enabled: false,
token: null,
serverUrl: null
}
});
vi.mocked(extractTextAndThumbnail).mockResolvedValue({
bodyText: 'Recipe text',
thumbnail: null
});
vi.mocked(extractRecipe).mockResolvedValue({
name: 'No Tandoor Recipe',
servings: null,
ingredients: [],
steps: [],
description: ''
});
const item = queueManager.enqueue('https://instagram.com/p/no-tandoor');
await new Promise((resolve) => setTimeout(resolve, 800));
const updated = queueManager.get(item.id);
// Should still succeed without Tandoor
expect(updated?.status).toBe('success');
expect(updated?.recipe?.name).toBe('No Tandoor Recipe');
expect(uploadRecipeWithIngredientsDTO).not.toHaveBeenCalled();
// Restore mock
vi.restoreAllMocks();
}, 10000);
it('should handle extraction errors', async () => {
vi.mocked(extractTextAndThumbnail).mockRejectedValue(new Error('Network timeout'));
const item = queueManager.enqueue('https://instagram.com/p/error');
await new Promise((resolve) => setTimeout(resolve, 800));
const updated = queueManager.get(item.id);
// Should mark as unhealthy (recoverable)
expect(updated?.status).toBe('unhealthy');
expect(updated?.error?.message).toContain('timeout');
}, 10000);
it('should handle parsing failure', async () => {
vi.mocked(extractTextAndThumbnail).mockResolvedValue({
bodyText: 'Not a recipe',
thumbnail: null
});
vi.mocked(extractRecipe).mockResolvedValue(null);
const item = queueManager.enqueue('https://instagram.com/p/not-recipe');
await new Promise((resolve) => setTimeout(resolve, 800));
const updated = queueManager.get(item.id);
// Should mark as error (non-recoverable - no recipe found)
expect(updated?.status).toBe('error');
expect(updated?.error?.message).toContain('recipe');
}, 10000);
it('should process multiple items respecting concurrency', async () => {
// Set up mocks with delay to observe concurrency
vi.mocked(extractTextAndThumbnail).mockImplementation(async () => {
await new Promise((resolve) => setTimeout(resolve, 300));
return { bodyText: 'text', thumbnail: null };
});
vi.mocked(extractRecipe).mockResolvedValue({
name: 'Concurrent Recipe',
servings: null,
ingredients: [],
steps: [],
description: ''
});
// Enqueue 3 items (Tandoor enabled by default in config mock)
queueManager.enqueue('https://instagram.com/p/item1');
queueManager.enqueue('https://instagram.com/p/item2');
queueManager.enqueue('https://instagram.com/p/item3');
// Wait a bit for processor to start working
await new Promise((resolve) => setTimeout(resolve, 150));
const items = queueManager.getAll();
const inProgress = items.filter((i) => i.status === 'in_progress');
// With concurrency=2, should have max 2 in progress at once
expect(inProgress.length).toBeLessThanOrEqual(2);
// Wait for all to complete
await new Promise((resolve) => setTimeout(resolve, 2000));
const final = queueManager.getAll();
const completed = final.filter((i) => i.status === 'success');
// All 3 should eventually complete
expect(completed.length).toBe(3);
}, 15000);
});