- Add instagram-extractor.ts: yt-dlp subprocess backend for Instagram caption extraction. No in-process browser state, maintained against Instagram frontend churn, supports cookies.txt for auth-walled reels. - Add feature flag EXTRACTOR_BACKEND (ytdlp|playwright) in QueueProcessor so the old Playwright path remains available as fallback. - Add 9 unit tests and 2 live-network integration tests for the new extractor. - Dockerfile: install yt-dlp via pip3 alongside existing Chromium deps. - docker-compose: expose EXTRACTOR_BACKEND env var (default: ytdlp). Also in this commit: - LLM: configurable per-request timeout via LLM_REQUEST_TIMEOUT_MS (default 120s); set maxRetries=0 to surface errors immediately; llama-swap /running health probe. - QueueProcessor: thread progress callback through parser phase. - LlmHealthIndicator: surface llama-swap loaded-model name. - Logging: improve error serialization in queue-processor tests. - .env.example: document llama-swap endpoint and model options. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
284 lines
8.6 KiB
TypeScript
284 lines
8.6 KiB
TypeScript
/**
|
|
* Integration tests for QueueProcessor
|
|
*
|
|
* Tests the processor's ability to handle queue items through mocked dependencies.
|
|
* The QueueProcessor auto-starts, so these tests verify actual processing behavior.
|
|
*/
|
|
|
|
import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
|
|
import { queueManager } from '$lib/server/queue/QueueManager';
|
|
|
|
// Mock web-push module BEFORE importing modules that depend on it
|
|
vi.mock('web-push', () => ({
|
|
default: {
|
|
setVapidDetails: vi.fn(),
|
|
sendNotification: vi.fn().mockResolvedValue(undefined)
|
|
}
|
|
}));
|
|
|
|
// Mock queueConfig BEFORE importing QueueProcessor
|
|
vi.mock('$lib/server/queue/config', () => ({
|
|
queueConfig: {
|
|
concurrency: 2,
|
|
maxRetries: 3,
|
|
tandoor: {
|
|
enabled: true,
|
|
token: 'test-token',
|
|
serverUrl: 'http://localhost:8080'
|
|
},
|
|
push: {
|
|
vapidPublicKey:
|
|
'BNextdcB_fQ0BVvyGioM5L8Tf9vKQjs-WnF-rUbnU8MdWIZQYfggIHxBnW21I-lq_0HykLCdMpYj8d5joavWdxQ',
|
|
vapidPrivateKey: 'JwxI_KcsBcehYcTOufMcbVWJjCq1QbH5FJmSyQuG680',
|
|
vapidEmail: 'mailto:test@example.com'
|
|
}
|
|
}
|
|
}));
|
|
|
|
// Mock external dependencies BEFORE importing QueueProcessor.
|
|
// QueueProcessor.extractionPhase picks between two extractor modules based on
|
|
// EXTRACTOR_BACKEND; mock both so behavior is identical regardless of default.
|
|
vi.mock('$lib/server/extraction', () => ({
|
|
extractTextAndThumbnail: vi.fn().mockResolvedValue({
|
|
bodyText: 'Default recipe text',
|
|
thumbnail: null
|
|
})
|
|
}));
|
|
vi.mock('$lib/server/instagram-extractor', () => ({
|
|
extractTextAndThumbnail: vi.fn().mockResolvedValue({
|
|
bodyText: 'Default recipe text',
|
|
thumbnail: null
|
|
})
|
|
}));
|
|
|
|
vi.mock('$lib/server/parser', () => ({
|
|
extractRecipe: vi.fn().mockResolvedValue({
|
|
name: 'Default Recipe',
|
|
ingredients: ['ingredient 1'],
|
|
steps: ['step 1'],
|
|
description: 'A default recipe'
|
|
})
|
|
}));
|
|
|
|
vi.mock('$lib/server/tandoor', () => ({
|
|
uploadRecipeWithIngredientsDTO: vi.fn().mockResolvedValue({
|
|
success: true,
|
|
recipeId: 999
|
|
}),
|
|
uploadRecipeImage: vi.fn().mockResolvedValue({
|
|
success: true
|
|
})
|
|
}));
|
|
|
|
import { extractTextAndThumbnail as extractFromExtraction } from '$lib/server/extraction';
|
|
import { extractTextAndThumbnail as extractFromYtDlp } from '$lib/server/instagram-extractor';
|
|
import { extractRecipe } from '$lib/server/parser';
|
|
import { uploadRecipeWithIngredientsDTO, uploadRecipeImage } from '$lib/server/tandoor';
|
|
import * as configModule from '$lib/server/queue/config';
|
|
|
|
// Alias used by existing assertions; default backend is ytdlp so the new
|
|
// instagram-extractor mock is what the processor actually invokes.
|
|
const extractTextAndThumbnail = extractFromYtDlp;
|
|
|
|
// Import processor AFTER mocks - it will auto-start (imported for side effects)
|
|
import '$lib/server/queue/QueueProcessor';
|
|
|
|
describe('QueueProcessor Integration Tests', () => {
|
|
beforeEach(async () => {
|
|
// Clear queue
|
|
queueManager.getAll().forEach((item) => queueManager.remove(item.id));
|
|
|
|
// Reset mocks and their implementations
|
|
vi.resetAllMocks();
|
|
|
|
// Set default mock implementations on BOTH backend modules so the test
|
|
// behavior is invariant to EXTRACTOR_BACKEND.
|
|
vi.mocked(extractFromExtraction).mockResolvedValue({
|
|
bodyText: 'Default recipe text',
|
|
thumbnail: null
|
|
});
|
|
vi.mocked(extractFromYtDlp).mockResolvedValue({
|
|
bodyText: 'Default recipe text',
|
|
thumbnail: null
|
|
});
|
|
|
|
vi.mocked(extractRecipe).mockResolvedValue({
|
|
name: 'Default Recipe',
|
|
servings: 2,
|
|
ingredients: [{ item: 'ingredient 1', amount: '1', unit: 'piece' }],
|
|
steps: ['step 1'],
|
|
description: 'A default recipe'
|
|
});
|
|
|
|
vi.mocked(uploadRecipeWithIngredientsDTO).mockResolvedValue({
|
|
success: true,
|
|
recipeId: 999
|
|
});
|
|
|
|
vi.mocked(uploadRecipeImage).mockResolvedValue({
|
|
success: true
|
|
});
|
|
});
|
|
|
|
afterEach(async () => {
|
|
// Wait for any pending processing to complete
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
});
|
|
|
|
it('should process item through all phases when Tandoor is configured', async () => {
|
|
// Set up successful mocks
|
|
vi.mocked(extractTextAndThumbnail).mockResolvedValue({
|
|
bodyText: 'Recipe instructions here',
|
|
thumbnail: 'https://example.com/thumb.jpg'
|
|
});
|
|
|
|
vi.mocked(extractRecipe).mockResolvedValue({
|
|
name: 'Test Recipe',
|
|
servings: 4,
|
|
ingredients: [
|
|
{ item: 'flour', amount: '2', unit: 'cups' },
|
|
{ item: 'eggs', amount: '2', unit: 'pieces' }
|
|
],
|
|
steps: ['mix', 'bake'],
|
|
description: 'test'
|
|
});
|
|
|
|
vi.mocked(uploadRecipeWithIngredientsDTO).mockResolvedValue({
|
|
success: true,
|
|
recipeId: 123
|
|
});
|
|
|
|
// Enqueue (processor is already running from auto-start)
|
|
// Note: Tandoor is enabled in the mocked config
|
|
const item = queueManager.enqueue('https://instagram.com/p/test-tandoor');
|
|
|
|
// Wait for processing to complete - increased timeout
|
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
|
|
const updated = queueManager.get(item.id);
|
|
|
|
// Verify success
|
|
expect(updated?.status).toBe('success');
|
|
expect(updated?.extractedText).toBe('Recipe instructions here');
|
|
expect(updated?.recipe?.name).toBe('Test Recipe');
|
|
expect(updated?.tandoorRecipeId).toBe(123);
|
|
|
|
// Verify all functions were called
|
|
expect(extractTextAndThumbnail).toHaveBeenCalled();
|
|
expect(extractRecipe).toHaveBeenCalled();
|
|
expect(uploadRecipeWithIngredientsDTO).toHaveBeenCalled();
|
|
}, 10000); // Increase timeout for processing
|
|
|
|
it('should skip Tandoor upload when not configured', async () => {
|
|
// Temporarily disable Tandoor for this test
|
|
const originalConfig = { ...configModule.queueConfig };
|
|
vi.spyOn(configModule, 'queueConfig', 'get').mockReturnValue({
|
|
...originalConfig,
|
|
tandoor: {
|
|
enabled: false,
|
|
token: null,
|
|
serverUrl: null
|
|
}
|
|
});
|
|
|
|
vi.mocked(extractTextAndThumbnail).mockResolvedValue({
|
|
bodyText: 'Recipe text',
|
|
thumbnail: null
|
|
});
|
|
|
|
vi.mocked(extractRecipe).mockResolvedValue({
|
|
name: 'No Tandoor Recipe',
|
|
servings: null,
|
|
ingredients: [],
|
|
steps: [],
|
|
description: ''
|
|
});
|
|
|
|
const item = queueManager.enqueue('https://instagram.com/p/no-tandoor');
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 800));
|
|
|
|
const updated = queueManager.get(item.id);
|
|
|
|
// Should still succeed without Tandoor
|
|
expect(updated?.status).toBe('success');
|
|
expect(updated?.recipe?.name).toBe('No Tandoor Recipe');
|
|
expect(uploadRecipeWithIngredientsDTO).not.toHaveBeenCalled();
|
|
|
|
// Restore mock
|
|
vi.restoreAllMocks();
|
|
}, 10000);
|
|
|
|
it('should handle extraction errors', async () => {
|
|
vi.mocked(extractTextAndThumbnail).mockRejectedValue(new Error('Network timeout'));
|
|
|
|
const item = queueManager.enqueue('https://instagram.com/p/error');
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 800));
|
|
|
|
const updated = queueManager.get(item.id);
|
|
|
|
// Should mark as unhealthy (recoverable)
|
|
expect(updated?.status).toBe('unhealthy');
|
|
expect(updated?.error?.message).toContain('timeout');
|
|
}, 10000);
|
|
|
|
it('should handle parsing failure', async () => {
|
|
vi.mocked(extractTextAndThumbnail).mockResolvedValue({
|
|
bodyText: 'Not a recipe',
|
|
thumbnail: null
|
|
});
|
|
|
|
vi.mocked(extractRecipe).mockResolvedValue(null);
|
|
|
|
const item = queueManager.enqueue('https://instagram.com/p/not-recipe');
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 800));
|
|
|
|
const updated = queueManager.get(item.id);
|
|
|
|
// Should mark as error (non-recoverable - no recipe found)
|
|
expect(updated?.status).toBe('error');
|
|
expect(updated?.error?.message).toContain('recipe');
|
|
}, 10000);
|
|
|
|
it('should process multiple items respecting concurrency', async () => {
|
|
// Set up mocks with delay to observe concurrency
|
|
vi.mocked(extractTextAndThumbnail).mockImplementation(async () => {
|
|
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
return { bodyText: 'text', thumbnail: null };
|
|
});
|
|
|
|
vi.mocked(extractRecipe).mockResolvedValue({
|
|
name: 'Concurrent Recipe',
|
|
servings: null,
|
|
ingredients: [],
|
|
steps: [],
|
|
description: ''
|
|
});
|
|
|
|
// Enqueue 3 items (Tandoor enabled by default in config mock)
|
|
queueManager.enqueue('https://instagram.com/p/item1');
|
|
queueManager.enqueue('https://instagram.com/p/item2');
|
|
queueManager.enqueue('https://instagram.com/p/item3');
|
|
|
|
// Wait a bit for processor to start working
|
|
await new Promise((resolve) => setTimeout(resolve, 150));
|
|
|
|
const items = queueManager.getAll();
|
|
const inProgress = items.filter((i) => i.status === 'in_progress');
|
|
|
|
// With concurrency=2, should have max 2 in progress at once
|
|
expect(inProgress.length).toBeLessThanOrEqual(2);
|
|
|
|
// Wait for all to complete
|
|
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
|
|
const final = queueManager.getAll();
|
|
const completed = final.filter((i) => i.status === 'success');
|
|
|
|
// All 3 should eventually complete
|
|
expect(completed.length).toBe(3);
|
|
}, 15000);
|
|
});
|