mozempk/insta-recipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify

Browse Source
This commit is contained in:
Giancarmine Salucci
2026-02-18 01:21:44 +01:00
parent 54321fd7c9
commit 49bccf8f15
84 changed files with 14474 additions and 13925 deletions
Download Patch File Download Diff File Expand all files Collapse all files

486
docs/API.md
View File

@@ -5,6 +5,7 @@ This document describes the InstaRecipe API endpoints for the async queue-based
## Base URL
All API endpoints are relative to your InstaRecipe instance:
```
https://your-instarecipe-instance.com/api
```
@@ -23,13 +24,16 @@ All endpoints return standardized error responses:
```json
{
"error": "Error type",
"message": "Human-readable error message",
"details": { /* Additional error context */ }
"error": "Error type",
"message": "Human-readable error message",
"details": {
/* Additional error context */
}
}
```
HTTP status codes follow REST conventions:
- `200` - Success
- `201` - Created
- `400` - Bad Request (invalid input)
@@ -45,13 +49,15 @@ HTTP status codes follow REST conventions:
Enqueue an Instagram URL for async processing.
**Request:**
```json
{
"url": "https://instagram.com/p/abc123"
"url": "https://instagram.com/p/abc123"
}
```
**Supported URL Formats:**
- Posts: `https://instagram.com/p/{post-id}`
- Posts (www): `https://www.instagram.com/p/{post-id}`
- Reels: `https://instagram.com/reel/{reel-id}`
@@ -59,12 +65,14 @@ Enqueue an Instagram URL for async processing.
- With query parameters: `https://instagram.com/reel/{reel-id}?utm_source=share`
**URL Requirements:**
- Must use HTTPS protocol
- Hostname must be `instagram.com` or `www.instagram.com`
- Any Instagram path is accepted (posts, reels, IGTV, etc.)
- Query parameters and hash fragments are allowed
**Examples:**
```json
// Post URL
{ "url": "https://instagram.com/p/ABC123" }
@@ -77,34 +85,36 @@ Enqueue an Instagram URL for async processing.
```
**Response (201 Created):**
```json
{
"id": "550e8400-e29b-41d4-a716-446655440000",
"url": "https://instagram.com/reel/DSevV5CDcNm/?utm_source=ig_web_copy_link",
"status": "pending",
"phases": [
{
"name": "extraction",
"status": "pending",
"progress": 0
},
{
"name": "parsing",
"status": "pending",
"progress": 0
},
{
"name": "uploading",
"status": "pending",
"progress": 0
}
],
"createdAt": "2024-12-21T10:30:00Z",
"updatedAt": "2024-12-21T10:30:00Z"
"id": "550e8400-e29b-41d4-a716-446655440000",
"url": "https://instagram.com/reel/DSevV5CDcNm/?utm_source=ig_web_copy_link",
"status": "pending",
"phases": [
{
"name": "extraction",
"status": "pending",
"progress": 0
},
{
"name": "parsing",
"status": "pending",
"progress": 0
},
{
"name": "uploading",
"status": "pending",
"progress": 0
}
],
"createdAt": "2024-12-21T10:30:00Z",
"updatedAt": "2024-12-21T10:30:00Z"
}
```
**Errors:**
- `400` - Invalid URL format (not a valid URL)
- `400` - URL must use HTTPS protocol
- `400` - URL must be from instagram.com domain
@@ -115,6 +125,7 @@ Enqueue an Instagram URL for async processing.
List queue items with optional filtering, pagination, and sorting.
**Query Parameters:**
- `status` (optional): Filter by status (`pending`, `in_progress`, `success`, `error`, `unhealthy`)
- `limit` (optional): Number of items to return (default: 50, max: 100)
- `offset` (optional): Number of items to skip (default: 0)
@@ -122,6 +133,7 @@ List queue items with optional filtering, pagination, and sorting.
- `order` (optional): Sort order (`asc`, `desc`) (default: `desc`)
**Examples:**
```bash
GET /api/queue # All items
GET /api/queue?status=error # Failed items only
@@ -130,67 +142,68 @@ GET /api/queue?sort=status&order=asc # Sort by status
```
**Response (200 OK):**
```json
{
"items": [
{
"id": "550e8400-e29b-41d4-a716-446655440000",
"url": "https://instagram.com/p/abc123",
"status": "success",
"phases": [
{
"name": "extraction",
"status": "completed",
"startedAt": "2024-12-21T10:30:01Z",
"completedAt": "2024-12-21T10:30:15Z",
"progress": 100
},
{
"name": "parsing",
"status": "completed",
"startedAt": "2024-12-21T10:30:15Z",
"completedAt": "2024-12-21T10:30:25Z",
"progress": 100
},
{
"name": "uploading",
"status": "completed",
"startedAt": "2024-12-21T10:30:25Z",
"completedAt": "2024-12-21T10:30:30Z",
"progress": 100
}
],
"results": {
"recipe": {
"name": "Chocolate Chip Cookies",
"description": "Delicious homemade cookies",
"servings": 24,
"ingredients": [
{
"food": "flour",
"amount": 2.25,
"unit": "cups"
}
],
"steps": [
{
"instruction": "Preheat oven to 375°F",
"time": 5
}
],
"keywords": ["cookies", "dessert", "chocolate"],
"image": "https://instagram.com/image.jpg"
},
"tandoorUrl": "https://tandoor.example.com/recipe/123",
"extractedText": "Raw extracted text...",
"thumbnail": "https://instagram.com/thumbnail.jpg"
},
"createdAt": "2024-12-21T10:30:00Z",
"updatedAt": "2024-12-21T10:30:30Z"
}
],
"total": 42,
"hasMore": true
"items": [
{
"id": "550e8400-e29b-41d4-a716-446655440000",
"url": "https://instagram.com/p/abc123",
"status": "success",
"phases": [
{
"name": "extraction",
"status": "completed",
"startedAt": "2024-12-21T10:30:01Z",
"completedAt": "2024-12-21T10:30:15Z",
"progress": 100
},
{
"name": "parsing",
"status": "completed",
"startedAt": "2024-12-21T10:30:15Z",
"completedAt": "2024-12-21T10:30:25Z",
"progress": 100
},
{
"name": "uploading",
"status": "completed",
"startedAt": "2024-12-21T10:30:25Z",
"completedAt": "2024-12-21T10:30:30Z",
"progress": 100
}
],
"results": {
"recipe": {
"name": "Chocolate Chip Cookies",
"description": "Delicious homemade cookies",
"servings": 24,
"ingredients": [
{
"food": "flour",
"amount": 2.25,
"unit": "cups"
}
],
"steps": [
{
"instruction": "Preheat oven to 375°F",
"time": 5
}
],
"keywords": ["cookies", "dessert", "chocolate"],
"image": "https://instagram.com/image.jpg"
},
"tandoorUrl": "https://tandoor.example.com/recipe/123",
"extractedText": "Raw extracted text...",
"thumbnail": "https://instagram.com/thumbnail.jpg"
},
"createdAt": "2024-12-21T10:30:00Z",
"updatedAt": "2024-12-21T10:30:30Z"
}
],
"total": 42,
"hasMore": true
}
```
@@ -199,12 +212,14 @@ GET /api/queue?sort=status&order=asc # Sort by status
Get details for a specific queue item.
**Path Parameters:**
- `id`: Queue item UUID
**Response (200 OK):**
Returns the same queue item structure as in the list response.
**Errors:**
- `400` - Invalid UUID format
- `404` - Queue item not found
@@ -213,22 +228,25 @@ Returns the same queue item structure as in the list response.
Retry a failed queue item. Only items with `error` or `unhealthy` status can be retried.
**Path Parameters:**
- `id`: Queue item UUID
**Response (200 OK):**
```json
{
"success": true,
"message": "Item queued for retry",
"item": {
"id": "550e8400-e29b-41d4-a716-446655440000",
"status": "pending",
"updatedAt": "2024-12-21T11:00:00Z"
}
"success": true,
"message": "Item queued for retry",
"item": {
"id": "550e8400-e29b-41d4-a716-446655440000",
"status": "pending",
"updatedAt": "2024-12-21T11:00:00Z"
}
}
```
**Errors:**
- `400` - Invalid UUID format
- `404` - Queue item not found
- `409` - Cannot retry item with current status (only `error` and `unhealthy` can be retried)
@@ -240,10 +258,12 @@ Retry a failed queue item. Only items with `error` or `unhealthy` status can be
Server-Sent Events (SSE) endpoint for real-time queue updates.
**Query Parameters:**
- `itemId` (optional): Filter updates for specific item
- `status` (optional): Filter updates by status
**Headers:**
```
Accept: text/event-stream
Cache-Control: no-cache
@@ -253,19 +273,23 @@ Cache-Control: no-cache
SSE stream with the following event types:
#### connection
Sent when connection is established:
```
event: connection
data: {"message": "Connected to queue stream", "timestamp": "2024-12-21T10:30:00Z"}
```
#### queue-update
#### queue-update
Sent when queue item status changes:
```
event: queue-update
data: {
"itemId": "550e8400-e29b-41d4-a716-446655440000",
"status": "in_progress",
"status": "in_progress",
"timestamp": "2024-12-21T10:30:01Z",
"progress": [
{
@@ -279,7 +303,9 @@ data: {
```
#### ping
Keep-alive ping sent every 30 seconds:
```
event: ping
data: {"timestamp": "2024-12-21T10:30:30Z"}
@@ -288,30 +314,32 @@ data: {"timestamp": "2024-12-21T10:30:30Z"}
**Usage Examples:**
**JavaScript:**
```javascript
const eventSource = new EventSource('/api/queue/stream');
eventSource.addEventListener('connection', (event) => {
console.log('Connected:', JSON.parse(event.data));
console.log('Connected:', JSON.parse(event.data));
});
eventSource.addEventListener('queue-update', (event) => {
const update = JSON.parse(event.data);
console.log('Queue update:', update);
updateUI(update);
const update = JSON.parse(event.data);
console.log('Queue update:', update);
updateUI(update);
});
eventSource.addEventListener('ping', (event) => {
console.log('Keep-alive ping');
console.log('Keep-alive ping');
});
eventSource.onerror = (error) => {
console.error('SSE error:', error);
// Reconnect logic here
console.error('SSE error:', error);
// Reconnect logic here
};
```
**curl:**
```bash
curl -N -H "Accept: text/event-stream" \
"https://your-instance.com/api/queue/stream?itemId=550e8400-e29b-41d4-a716-446655440000"
@@ -324,10 +352,11 @@ curl -N -H "Accept: text/event-stream" \
Get the VAPID public key required for push notification subscriptions.
**Response (200 OK):**
```json
{
"publicKey": "BDummyPublicKeyForDevelopment...",
"applicationServerKey": "BDummyPublicKeyForDevelopment..."
"publicKey": "BDummyPublicKeyForDevelopment...",
"applicationServerKey": "BDummyPublicKeyForDevelopment..."
}
```
@@ -336,29 +365,32 @@ Get the VAPID public key required for push notification subscriptions.
Subscribe to push notifications for queue processing updates.
**Request:**
```json
{
"subscription": {
"endpoint": "https://fcm.googleapis.com/fcm/send/...",
"keys": {
"p256dh": "BIBn3E_YUVpW5f6_Eq_GH...",
"auth": "tBiH_Y1nPSuVh7TRMhcf..."
}
},
"clientId": "unique-client-identifier"
"subscription": {
"endpoint": "https://fcm.googleapis.com/fcm/send/...",
"keys": {
"p256dh": "BIBn3E_YUVpW5f6_Eq_GH...",
"auth": "tBiH_Y1nPSuVh7TRMhcf..."
}
},
"clientId": "unique-client-identifier"
}
```
**Response (200 OK):**
```json
{
"success": true,
"message": "Successfully subscribed to push notifications",
"subscriptionCount": 5
"success": true,
"message": "Successfully subscribed to push notifications",
"subscriptionCount": 5
}
```
**Errors:**
- `400` - Invalid subscription object or missing clientId
### DELETE /api/notifications/subscribe
@@ -366,18 +398,20 @@ Subscribe to push notifications for queue processing updates.
Unsubscribe from push notifications.
**Request:**
```json
{
"clientId": "unique-client-identifier"
"clientId": "unique-client-identifier"
}
```
**Response (200 OK):**
```json
{
"success": true,
"message": "Successfully unsubscribed from push notifications",
"subscriptionCount": 4
"success": true,
"message": "Successfully unsubscribed from push notifications",
"subscriptionCount": 4
}
```
@@ -390,18 +424,19 @@ Unsubscribe from push notifications.
This synchronous extraction endpoint is deprecated. Use `POST /api/queue` instead.
**Migration:**
```javascript
// ❌ Old synchronous approach
const response = await fetch('/api/extract', {
method: 'POST',
body: JSON.stringify({ url })
method: 'POST',
body: JSON.stringify({ url })
});
const result = await response.json(); // Wait 30-60 seconds
// ✅ New async queue approach
// ✅ New async queue approach
const response = await fetch('/api/queue', {
method: 'POST',
body: JSON.stringify({ url })
method: 'POST',
body: JSON.stringify({ url })
});
const queueItem = await response.json(); // Immediate response
```
@@ -413,17 +448,18 @@ const queueItem = await response.json(); // Immediate response
This SSE endpoint is deprecated. Use `POST /api/queue` + `GET /api/queue/stream` instead.
**Migration:**
```javascript
// ❌ Old approach
const response = await fetch('/api/extract-stream', {
method: 'POST',
body: JSON.stringify({ url })
method: 'POST',
body: JSON.stringify({ url })
});
// ✅ New approach
const queueResponse = await fetch('/api/queue', {
method: 'POST',
body: JSON.stringify({ url })
method: 'POST',
body: JSON.stringify({ url })
});
const item = await queueResponse.json();
@@ -436,28 +472,28 @@ const eventSource = new EventSource(`/api/queue/stream?itemId=${item.id}`);
```typescript
interface QueueItem {
id: string; // UUID v4
url: string; // Instagram URL
status: 'pending' | 'in_progress' | 'success' | 'error' | 'unhealthy';
phases: Array<{
name: 'extraction' | 'parsing' | 'uploading';
status: 'pending' | 'in_progress' | 'completed' | 'error';
startedAt?: string; // ISO 8601 timestamp
completedAt?: string; // ISO 8601 timestamp
progress?: number; // 0-100
}>;
results?: {
recipe?: Recipe; // Structured recipe data
tandoorUrl?: string; // Tandoor recipe URL
extractedText?: string; // Raw extracted text
thumbnail?: string; // Image URL
};
error?: string; // Error message
createdAt: string; // ISO 8601 timestamp
updatedAt: string; // ISO 8601 timestamp
id: string; // UUID v4
url: string; // Instagram URL
status: 'pending' | 'in_progress' | 'success' | 'error' | 'unhealthy';
phases: Array<{
name: 'extraction' | 'parsing' | 'uploading';
status: 'pending' | 'in_progress' | 'completed' | 'error';
startedAt?: string; // ISO 8601 timestamp
completedAt?: string; // ISO 8601 timestamp
progress?: number; // 0-100
}>;
results?: {
recipe?: Recipe; // Structured recipe data
tandoorUrl?: string; // Tandoor recipe URL
extractedText?: string; // Raw extracted text
thumbnail?: string; // Image URL
};
error?: string; // Error message
createdAt: string; // ISO 8601 timestamp
updatedAt: string; // ISO 8601 timestamp
}
```
@@ -465,32 +501,33 @@ interface QueueItem {
```typescript
interface Recipe {
name: string;
description?: string;
servings?: number;
prepTime?: number; // Minutes
cookTime?: number; // Minutes
totalTime?: number; // Minutes
ingredients: Array<{
food: string;
amount?: number;
unit?: string;
}>;
steps: Array<{
instruction: string;
time?: number; // Minutes
}>;
keywords?: string[]; // Recipe tags
image?: string; // Image URL
nutrition?: { // Nutritional information
calories?: number;
protein?: number;
carbs?: number;
fat?: number;
};
name: string;
description?: string;
servings?: number;
prepTime?: number; // Minutes
cookTime?: number; // Minutes
totalTime?: number; // Minutes
ingredients: Array<{
food: string;
amount?: number;
unit?: string;
}>;
steps: Array<{
instruction: string;
time?: number; // Minutes
}>;
keywords?: string[]; // Recipe tags
image?: string; // Image URL
nutrition?: {
// Nutritional information
calories?: number;
protein?: number;
carbs?: number;
fat?: number;
};
}
```
@@ -517,59 +554,58 @@ When implementing clients, consider these error recovery strategies:
```javascript
async function processInstagramUrl(url) {
try {
// 1. Enqueue URL
const queueResponse = await fetch('/api/queue', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url })
});
const queueItem = await queueResponse.json();
console.log('Enqueued:', queueItem.id);
// 2. Listen for real-time updates
const eventSource = new EventSource(`/api/queue/stream?itemId=${queueItem.id}`);
return new Promise((resolve, reject) => {
eventSource.addEventListener('queue-update', (event) => {
const update = JSON.parse(event.data);
if (update.status === 'success') {
eventSource.close();
resolve(update.results);
} else if (update.status === 'error') {
eventSource.close();
reject(new Error(update.error));
}
// Handle progress updates
console.log('Progress:', update.progress);
});
eventSource.onerror = (error) => {
eventSource.close();
reject(error);
};
});
} catch (error) {
console.error('Processing failed:', error);
throw error;
}
try {
// 1. Enqueue URL
const queueResponse = await fetch('/api/queue', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url })
});
const queueItem = await queueResponse.json();
console.log('Enqueued:', queueItem.id);
// 2. Listen for real-time updates
const eventSource = new EventSource(`/api/queue/stream?itemId=${queueItem.id}`);
return new Promise((resolve, reject) => {
eventSource.addEventListener('queue-update', (event) => {
const update = JSON.parse(event.data);
if (update.status === 'success') {
eventSource.close();
resolve(update.results);
} else if (update.status === 'error') {
eventSource.close();
reject(new Error(update.error));
}
// Handle progress updates
console.log('Progress:', update.progress);
});
eventSource.onerror = (error) => {
eventSource.close();
reject(error);
};
});
} catch (error) {
console.error('Processing failed:', error);
throw error;
}
}
// Usage
processInstagramUrl('https://instagram.com/p/abc123')
.then(results => {
console.log('Recipe extracted:', results.recipe);
if (results.tandoorUrl) {
console.log('Uploaded to Tandoor:', results.tandoorUrl);
}
})
.catch(error => {
console.error('Extraction failed:', error.message);
});
.then((results) => {
console.log('Recipe extracted:', results.recipe);
if (results.tandoorUrl) {
console.log('Uploaded to Tandoor:', results.tandoorUrl);
}
})
.catch((error) => {
console.error('Extraction failed:', error.message);
});
```
For more examples and integration guides, see the [Migration Documentation](/docs/MIGRATION.md).
For more examples and integration guides, see the [Migration Documentation](/docs/MIGRATION.md).

51
docs/ARCHITECTURE.md
View File

@@ -91,21 +91,27 @@ insta-recipe/
## Key Directories
### `/src/lib/server/`
Server-side business logic following Hexagonal Architecture principles. Contains domain logic, adapters for external systems (Instagram, Tandoor, LLM), and port definitions.
### `/src/lib/client/`
Client-side utilities for PWA features (push notifications, install prompts, service worker messaging).
### `/src/routes/api/`
RESTful API endpoints implemented as SvelteKit server routes. Each directory contains `+server.ts` files exporting HTTP verb handlers.
### `/src/routes/share/`
Share target page allowing users to share Instagram URLs directly from their browser or mobile apps.
### `/src/lib/server/queue/`
Queue management system with in-memory storage, processor workers, and type definitions.
### `/docs/`
Comprehensive documentation including plans, outcomes, API specs, and migration guides.
---
@@ -113,33 +119,43 @@ Comprehensive documentation including plans, outcomes, API specs, and migration
## Design Patterns
### Singleton Pattern
Used for shared service instances:
- `QueueManager` (`queueManager` exported instance)
- `QueueProcessor` (`queueProcessor` exported instance)
- `PushNotificationService` (`pushNotificationService` exported instance)
- `ServiceWorkerMessageHandler` (`serviceWorkerMessageHandler` exported instance)
### Factory Pattern
Used for creating configured instances:
- `createLLM()` - Creates OpenAI client with environment configuration
- `createBrowserContext()` - Creates Playwright browser context with options
- `initializeBrowser()` - Initializes Chromium browser instance
### Observer Pattern
Implemented in QueueManager for real-time updates:
- Subscribers receive notifications on queue item changes
- Server-Sent Events (SSE) stream queue updates to clients
- Push notifications notify users of completion events
### Adapter Pattern (Hexagonal Architecture)
External systems accessed via adapters:
- **Instagram Adapter**: `extraction.ts` - Extracts content via Playwright
- **LLM Adapter**: `llm.ts`, `parser.ts` - Recipe parsing via OpenAI
- **Tandoor Adapter**: `tandoor.ts` - Recipe management system integration
- **Browser Adapter**: `browser.ts` - Playwright browser automation
### Strategy Pattern
Multiple extraction strategies with fallback:
1. Embedded JSON extraction
2. DOM selector extraction
3. GraphQL API extraction
@@ -150,28 +166,34 @@ Multiple extraction strategies with fallback:
## Key Components
### Queue Management System
**Location**: `src/lib/server/queue/`
Three-phase processing pipeline:
1. **Extraction Phase**: Extract text and thumbnail from Instagram
2. **Parsing Phase**: Parse recipe using LLM
3. **Uploading Phase**: Upload to Tandoor (if enabled)
**Components**:
- `QueueManager`: In-memory FIFO queue with CRUD operations
- `QueueProcessor`: Worker that processes items with configurable concurrency
- `types.ts`: Comprehensive type definitions for queue items and updates
### API Layer
**Location**: `src/routes/api/`
RESTful endpoints for:
- Queue operations (`POST /api/queue`, `GET /api/queue`, `GET /api/queue/[id]`)
- Real-time updates (`GET /api/queue/stream` - SSE)
- Push notifications (`POST /api/notifications/subscribe`)
- Health checks (`GET /api/health`, `GET /api/llm-health`)
### Client-Side Services
**Location**: `src/lib/client/`
- **PushNotificationManager**: Manages Web Push API subscriptions
@@ -179,14 +201,17 @@ RESTful endpoints for:
- **ServiceWorkerMessageHandler**: Processes service worker messages
### Instagram Extraction
**Location**: `src/lib/server/extraction.ts`
Multi-method extraction with intelligent fallback:
- Progress callbacks for real-time feedback
- Retry logic with configurable attempts
- Thumbnail extraction and validation
### LLM Integration
**Location**: `src/lib/server/parser.ts`, `src/lib/server/llm.ts`
- Recipe detection endpoint
@@ -198,6 +223,7 @@ Multi-method extraction with intelligent fallback:
## Dependencies
### Production Dependencies
- **@types/uuid** (^10.0.0) - UUID type definitions
- **date-fns** (^4.1.0) - Date utility library
- **openai** (^4.20.0) - OpenAI API client
@@ -206,6 +232,7 @@ Multi-method extraction with intelligent fallback:
- **zod** (^3.23.0) - Schema validation
### Development Dependencies
- **@sveltejs/kit** (^2.48.5) - SvelteKit framework
- **@sveltejs/adapter-node** (^5.4.0) - Node.js adapter
- **svelte** (^5.43.8) - Svelte 5 framework
@@ -223,12 +250,14 @@ Multi-method extraction with intelligent fallback:
## Module Organization
### SvelteKit Path Aliases
- `$lib` → `src/lib/`
- `$lib/*` → `src/lib/*`
- `$app/*` → SvelteKit app imports
- `$env/dynamic/private` → Environment variables (server-side)
### Directory Structure Conventions
- **Server-only code**: `src/lib/server/` (not bundled to client)
- **Client-only code**: `src/lib/client/` (not executed on server)
- **Shared code**: `src/lib/` (available to both)
@@ -240,6 +269,7 @@ Multi-method extraction with intelligent fallback:
## Data Flow
### Recipe Extraction Flow
```
User submits URL
↓
@@ -261,6 +291,7 @@ SSE updates notify client
```
### Real-time Updates Flow
```
Client connects to GET /api/queue/stream (SSE)
↓
@@ -274,6 +305,7 @@ Client updates UI reactively
```
### Push Notification Flow
```
Client requests permission
↓
@@ -295,37 +327,44 @@ Notification displayed to user
## Build System
### Build Command
```bash
npm run build
```
Generates production-ready build in `build/` directory using:
- Vite for bundling
- `@sveltejs/adapter-node` for Node.js deployment
- TypeScript compilation
- SvelteKit prerendering and optimization
### Test Command
```bash
npm test
```
Runs test suite using Vitest with two projects:
1. **Server tests**: Node environment for server-side code
2. **Client tests**: Playwright browser for Svelte components
### Development Server
```bash
npm run dev
```
Starts Vite dev server with:
- HTTPS enabled (certificates in `.ssl/`)
- Hot module replacement
- TypeScript checking
- File watching
### Linting & Formatting
```bash
npm run lint # ESLint + Prettier check
npm run format # Prettier write
@@ -336,19 +375,24 @@ npm run format # Prettier write
## Deployment
### Docker Deployment
Dockerfile includes:
- Node.js 22 Alpine base image
- Playwright Chromium installation
- Production build
- Port 3000 exposure
Run with:
```bash
docker-compose up
```
### Environment Variables
Required configuration:
- `OPENAI_API_KEY` - LLM API access
- `TANDOOR_URL` - Tandoor instance URL (optional)
- `TANDOOR_TOKEN` - Tandoor API token (optional)
@@ -360,13 +404,16 @@ Required configuration:
## Testing Architecture
### Test Categories
1. **Unit Tests**: Individual function testing
2. **Integration Tests**: Multi-component workflows
3. **API Tests**: Endpoint behavior validation
4. **Browser Tests**: Svelte component rendering
### Test Coverage
138 tests covering:
- Queue management operations
- Instagram URL validation
- SSE streaming
@@ -375,6 +422,7 @@ Required configuration:
- Notification service
### Test Configuration
- **Server tests**: Node environment with mocked dependencies
- **Client tests**: Playwright Chromium browser with Svelte testing library
@@ -383,15 +431,18 @@ Required configuration:
## Security Considerations
### SSL/TLS
- Development uses local SSL certificates signed by external Caddy CA
- Certificates stored in `.ssl/` (git-ignored)
- Required for PWA features (Service Worker, Push API)
### Authentication
- Basic auth for scheduled tasks (username/password from environment)
- Tandoor integration uses bearer token authentication
### Input Validation
- Instagram URL validation with regex patterns
- Zod schema validation for API payloads
- Error handling with custom error classes

346
docs/CODE_STYLE.md
View File

@@ -19,12 +19,14 @@
### Files & Directories
#### SvelteKit Route Files
- Route pages: `+page.svelte`
- Route servers: `+server.ts`
- Route layouts: `+layout.svelte`
- Type definitions: `$types.ts` (auto-generated)
**Example:**
```
src/routes/api/queue/
├── [id]/
@@ -37,19 +39,23 @@ src/routes/api/queue/
```
#### Library Files
- **PascalCase** for classes and managers: `QueueManager.ts`, `PushNotificationService.ts`
- **kebab-case** for utilities and configs: `tandoor-config.ts`, `instagram-url.ts`
- **lowercase** for general modules: `browser.ts`, `extraction.ts`, `parser.ts`
**Examples from codebase:**
- `src/lib/server/queue/QueueManager.ts`
- `src/lib/server/tandoor-config.ts`
- `src/lib/client/PushNotificationManager.ts`
#### Test Files
Pattern: `<name>.spec.ts` or `<name>.test.ts`
**Examples:**
- `queue-manager.spec.ts`
- `instagram-url-validation.spec.ts`
- `page.svelte.spec.ts`
@@ -57,10 +63,12 @@ Pattern: `<name>.spec.ts` or `<name>.test.ts`
### Variables & Functions
#### Variables
- **camelCase** for local variables and parameters
- **SCREAMING_SNAKE_CASE** for constants
**Examples:**
```typescript
// From QueueManager.ts
private items: Map<string, QueueItem> = new Map();
@@ -76,10 +84,12 @@ const unsubscribe = queueManager.subscribe(callback);
```
#### Functions
- **camelCase** for function names
- **Descriptive action verbs**: `enqueue`, `extractRecipe`, `uploadRecipeImage`
**Examples:**
```typescript
// From QueueManager.ts
enqueue(url: string): QueueItem { ... }
@@ -99,62 +109,62 @@ export async function extractRecipe(text: string): Promise<Recipe> { ... }
### Types & Interfaces
#### Interfaces & Types
- **PascalCase** for interface names
- Prefix with `I` is **NOT** used
- Exported types use `export type` or `export interface`
**Examples:**
```typescript
// From queue/types.ts
export interface QueueItem {
id: string;
url: string;
status: QueueItemStatus;
enqueuedAt: string;
// ...
id: string;
url: string;
status: QueueItemStatus;
enqueuedAt: string;
// ...
}
export interface QueueStatusUpdate {
type: string;
itemId: string;
status: QueueItemStatus;
// ...
type: string;
itemId: string;
status: QueueItemStatus;
// ...
}
export type QueueItemStatus =
| 'pending'
| 'in_progress'
| 'completed'
| 'failed';
export type QueueItemStatus = 'pending' | 'in_progress' | 'completed' | 'failed';
// From extraction.ts
export interface ExtractedContent {
text: string;
thumbnailUrl?: string;
text: string;
thumbnailUrl?: string;
}
export type ProgressCallback = (event: ProgressEvent) => void;
```
#### Zod Schemas
- **PascalCase** with `Schema` suffix
- Inferred types without suffix
**Examples:**
```typescript
// From parser.ts
const RecipeSchema = z.object({
name: z.string(),
description: z.string(),
servings: z.number(),
// ...
name: z.string(),
description: z.string(),
servings: z.number()
// ...
});
export type Recipe = z.infer<typeof RecipeSchema>;
// From tandoor.ts
const TandoorRecipeSchema = z.object({
// ...
// ...
});
export type TandoorRecipe = z.infer<typeof TandoorRecipeSchema>;
@@ -163,35 +173,38 @@ export type TandoorRecipe = z.infer<typeof TandoorRecipeSchema>;
### Classes
#### Class Names
- **PascalCase** for class names
- Descriptive suffixes: `Manager`, `Service`, `Processor`, `Handler`
**Examples:**
```typescript
// From QueueManager.ts
export class QueueManager {
private items: Map<string, QueueItem> = new Map();
// ...
private items: Map<string, QueueItem> = new Map();
// ...
}
// From QueueProcessor.ts
export class QueueProcessor {
private processing: Set<string> = new Set();
// ...
private processing: Set<string> = new Set();
// ...
}
// From PushNotificationService.ts
class PushNotificationService {
private subscriptions: Map<string, PushSubscription> = new Map();
// ...
private subscriptions: Map<string, PushSubscription> = new Map();
// ...
}
```
#### Singleton Export Pattern
```typescript
// Class definition
export class QueueManager {
// Implementation
// Implementation
}
// Singleton instance export
@@ -203,6 +216,7 @@ export const queueManager = new QueueManager();
## Indentation & Formatting
### General Rules
- **Indentation:** 2 spaces (enforced by Prettier)
- **No tabs**
- **Max line length:** 100 characters (soft limit, not enforced)
@@ -213,6 +227,7 @@ export const queueManager = new QueueManager();
### Code Examples
#### Function Declarations
```typescript
// From QueueManager.ts
enqueue(url: string): QueueItem {
@@ -234,43 +249,45 @@ enqueue(url: string): QueueItem {
retryCount: 0,
maxRetries: 3
};
this.items.set(item.id, item);
return item;
}
```
#### Async Functions
```typescript
// From extraction.ts
export async function extractTextAndThumbnail(
url: string,
onProgress?: ProgressCallback
url: string,
onProgress?: ProgressCallback
): Promise<ExtractedContent> {
const browser = await getBrowser();
const context = await createBrowserContext(browser);
const page = await context.newPage();
try {
await page.goto(url, { waitUntil: 'networkidle' });
// ...
} finally {
await context.close();
}
const browser = await getBrowser();
const context = await createBrowserContext(browser);
const page = await context.newPage();
try {
await page.goto(url, { waitUntil: 'networkidle' });
// ...
} finally {
await context.close();
}
}
```
#### Object Destructuring
```typescript
// From route handlers
export const POST: RequestHandler = async ({ request }) => {
const { url } = await request.json();
// ...
const { url } = await request.json();
// ...
};
export const GET: RequestHandler = async ({ params }) => {
const { id } = params;
// ...
const { id } = params;
// ...
};
```
@@ -279,12 +296,14 @@ export const GET: RequestHandler = async ({ params }) => {
## Import Patterns
### Import Order
1. External dependencies (Node.js built-ins, npm packages)
2. SvelteKit imports (`$lib`, `$app`, `$env`)
3. Relative imports (`./ `, `../`)
4. Type imports (separate from value imports when beneficial)
**Example:**
```typescript
// From QueueProcessor.ts
@@ -307,6 +326,7 @@ import type { QueueItem } from './types';
### Import Styles
#### Named Imports (Preferred)
```typescript
import { json } from '@sveltejs/kit';
import { queueManager } from '$lib/server/queue/QueueManager';
@@ -314,12 +334,14 @@ import { validateInstagramUrl } from '$lib/server/validation/instagram-url';
```
#### Type-Only Imports
```typescript
import type { RequestHandler } from './$types';
import type { QueueItem, QueueItemStatus } from './types';
```
#### Default Imports
```typescript
import OpenAI from 'openai';
import fs from 'fs';
@@ -329,6 +351,7 @@ import path from 'path';
### Export Patterns
#### Named Exports (Preferred)
```typescript
// Export functions
export async function extractRecipe(text: string): Promise<Recipe> { ... }
@@ -345,6 +368,7 @@ export interface QueueItem { ... }
```
#### Singleton Pattern Export
```typescript
// Define class
export class QueueManager { ... }
@@ -358,16 +382,18 @@ export const queueManager = new QueueManager();
## Comments & Documentation
### JSDoc Style
Used extensively for public APIs and exported functions.
**Function Documentation:**
```typescript
````typescript
/**
* Add URL to processing queue
*
*
* @param url - Instagram URL to process
* @returns Newly created queue item
*
*
* @example
* ```typescript
* const item = queueManager.enqueue('https://instagram.com/p/abc123');
@@ -377,41 +403,43 @@ Used extensively for public APIs and exported functions.
enqueue(url: string): QueueItem {
// Implementation
}
```
````
**Class Documentation:**
```typescript
````typescript
/**
* Singleton queue manager for processing Instagram URLs
*
*
* Features:
* - FIFO queue with unique IDs
* - Status tracking and updates
* - Progress event accumulation
* - Retry support for failed items
* - Pub/sub for real-time updates
*
*
* @example
* ```typescript
* import { queueManager } from './QueueManager';
*
*
* // Add item to queue
* const item = queueManager.enqueue('https://instagram.com/p/abc123');
* ```
*/
export class QueueManager {
// Implementation
// Implementation
}
```
````
**Module-Level Documentation:**
```typescript
/**
* Queue Manager - Core queue operations and event management
*
*
* Manages an in-memory queue of Instagram URL processing jobs.
* Provides CRUD operations and pub/sub mechanism for queue updates.
*
*
* Architecture: Domain Layer (Hexagonal Architecture)
* - Port: Defines queue operations interface
* - Implementation: In-memory Map-based storage
@@ -421,19 +449,21 @@ export class QueueManager {
### Inline Comments
#### Single-line Comments
```typescript
// Set restrictive permissions
fs.chmodSync(authFile, 0o600);
// FIFO order - get oldest pending item
const pendingItems = Array.from(this.items.values())
.filter(item => item.status === 'pending');
const pendingItems = Array.from(this.items.values()).filter((item) => item.status === 'pending');
```
#### Block Comments (Avoided)
Single-line comments preferred. Block comments used only for large comment blocks or temporarily disabling code during development.
### TODO Comments
```typescript
// TODO: Add retry logic with exponential backoff
// FIXME: Handle race condition when multiple workers dequeue
@@ -446,17 +476,19 @@ Single-line comments preferred. Block comments used only for large comment block
### Type Safety
#### Strict Mode Enabled
```json
// tsconfig.json
{
"compilerOptions": {
"strict": true,
"forceConsistentCasingInFileNames": true
}
"compilerOptions": {
"strict": true,
"forceConsistentCasingInFileNames": true
}
}
```
#### Type Annotations
```typescript
// Explicit return types for public functions
export async function extractRecipe(text: string): Promise<Recipe> { ... }
@@ -469,35 +501,24 @@ const items = queueManager.getAll(); // Type inferred
```
### Union Types
```typescript
export type QueueItemStatus =
| 'pending'
| 'in_progress'
| 'completed'
| 'failed';
export type QueueItemStatus = 'pending' | 'in_progress' | 'completed' | 'failed';
export type ProcessingPhase =
| 'extraction'
| 'parsing'
| 'uploading';
export type ProcessingPhase = 'extraction' | 'parsing' | 'uploading';
export type ProgressEventType =
| 'status'
| 'method'
| 'retry'
| 'error'
| 'thumbnail'
| 'complete';
export type ProgressEventType = 'status' | 'method' | 'retry' | 'error' | 'thumbnail' | 'complete';
```
### Generics
```typescript
// Generic function
async function fetchFromTandoor<T>(
url: string,
options: Partial<RequestInit> = { method: 'GET' }
url: string,
options: Partial<RequestInit> = { method: 'GET' }
): Promise<{ ok: boolean; data?: T; error?: string }> {
// Implementation
// Implementation
}
```
@@ -508,6 +529,7 @@ async function fetchFromTandoor<T>(
### Runes (Reactivity)
#### $state (Reactive Variables)
```svelte
<script lang="ts">
let count = $state(0);
@@ -516,13 +538,14 @@ async function fetchFromTandoor<T>(
```
#### $props (Component Props)
```svelte
<script lang="ts">
let {
let {
recipe = null,
tandoorEnabled = false,
onRetry,
onImportToTandoor
onImportToTandoor
} = $props<{
recipe: Recipe | null;
tandoorEnabled: boolean;
@@ -533,6 +556,7 @@ async function fetchFromTandoor<T>(
```
#### $derived (Computed Values)
```svelte
<script lang="ts">
let count = $state(0);
@@ -541,10 +565,11 @@ async function fetchFromTandoor<T>(
```
#### $effect (Side Effects)
```svelte
<script lang="ts">
let url = $state('');
$effect(() => {
console.log('URL changed:', url);
});
@@ -552,25 +577,26 @@ async function fetchFromTandoor<T>(
```
### Component Structure
```svelte
<script lang="ts">
// Imports
import { onMount } from 'svelte';
// Props
let { items } = $props<{ items: Item[] }>();
// State
let loading = $state(false);
// Derived state
let count = $derived(items.length);
// Functions
function handleClick() {
// ...
}
// Effects
$effect(() => {
// Side effects
@@ -593,46 +619,47 @@ async function fetchFromTandoor<T>(
## Error Handling
### Custom Error Classes
```typescript
// From api/errors.ts
export class ValidationError extends Error {
constructor(message: string) {
super(message);
this.name = 'ValidationError';
}
constructor(message: string) {
super(message);
this.name = 'ValidationError';
}
}
export class NotFoundError extends Error {
constructor(resource: string) {
super(`${resource} not found`);
this.name = 'NotFoundError';
}
constructor(resource: string) {
super(`${resource} not found`);
this.name = 'NotFoundError';
}
}
export class ConflictError extends Error {
constructor(message: string) {
super(message);
this.name = 'ConflictError';
}
constructor(message: string) {
super(message);
this.name = 'ConflictError';
}
}
```
### Try-Catch Pattern
```typescript
export const POST: RequestHandler = async ({ request }) => {
try {
const { url } = await request.json();
if (!url) {
throw new ValidationError('URL is required');
}
const item = queueManager.enqueue(url);
return json(item, { status: 201 });
} catch (error) {
return handleApiError(error);
}
try {
const { url } = await request.json();
if (!url) {
throw new ValidationError('URL is required');
}
const item = queueManager.enqueue(url);
return json(item, { status: 201 });
} catch (error) {
return handleApiError(error);
}
};
```
@@ -641,14 +668,16 @@ export const POST: RequestHandler = async ({ request }) => {
## Linting Configuration
### ESLint
**Config:** `eslint.config.js`
- Base: `@eslint/js` recommended
- TypeScript: `typescript-eslint` recommended
- Svelte: `eslint-plugin-svelte` recommended
- Svelte: `eslint-plugin-svelte` recommended
- Formatting: `eslint-config-prettier`
**Rules:**
```javascript
{
rules: {
@@ -658,15 +687,16 @@ export const POST: RequestHandler = async ({ request }) => {
```
### Prettier
**Config:** `.prettierrc`
```json
{
"useTabs": false,
"singleQuote": true,
"trailingComma": "es5",
"printWidth": 100,
"plugins": ["prettier-plugin-svelte", "prettier-plugin-tailwindcss"]
"useTabs": false,
"singleQuote": true,
"trailingComma": "es5",
"printWidth": 100,
"plugins": ["prettier-plugin-svelte", "prettier-plugin-tailwindcss"]
}
```
@@ -675,38 +705,40 @@ export const POST: RequestHandler = async ({ request }) => {
## Testing Conventions
### Test Structure
```typescript
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
describe('QueueManager', () => {
let manager: QueueManager;
beforeEach(() => {
manager = new QueueManager();
});
it('should enqueue items', () => {
const item = manager.enqueue('https://instagram.com/p/test');
expect(item.status).toBe('pending');
});
it('should dequeue items in FIFO order', () => {
manager.enqueue('url1');
manager.enqueue('url2');
const first = manager.dequeue();
expect(first?.url).toBe('url1');
});
let manager: QueueManager;
beforeEach(() => {
manager = new QueueManager();
});
it('should enqueue items', () => {
const item = manager.enqueue('https://instagram.com/p/test');
expect(item.status).toBe('pending');
});
it('should dequeue items in FIFO order', () => {
manager.enqueue('url1');
manager.enqueue('url2');
const first = manager.dequeue();
expect(first?.url).toBe('url1');
});
});
```
### Mock Pattern
```typescript
vi.mock('$lib/server/extraction', () => ({
extractTextAndThumbnail: vi.fn().mockResolvedValue({
text: 'Mock text',
thumbnailUrl: 'https://example.com/thumb.jpg'
})
extractTextAndThumbnail: vi.fn().mockResolvedValue({
text: 'Mock text',
thumbnailUrl: 'https://example.com/thumb.jpg'
})
}));
```
@@ -715,14 +747,15 @@ vi.mock('$lib/server/extraction', () => ({
## File Headers
### Module Documentation Pattern
Every major module includes a header comment:
```typescript
/**
* Module Name - Brief Description
*
*
* Detailed description of the module's purpose and functionality.
*
*
* Architecture: Layer Name (Hexagonal Architecture)
* - Port: Description of port interface
* - Implementation: Description of concrete implementation
@@ -730,13 +763,14 @@ Every major module includes a header comment:
```
**Example:**
```typescript
/**
* Queue Manager - Core queue operations and event management
*
*
* Manages an in-memory queue of Instagram URL processing jobs.
* Provides CRUD operations and pub/sub mechanism for queue updates.
*
*
* Architecture: Domain Layer (Hexagonal Architecture)
* - Port: Defines queue operations interface
* - Implementation: In-memory Map-based storage
@@ -748,6 +782,7 @@ Every major module includes a header comment:
## Additional Conventions
### Environment Variables
```typescript
import { env } from '$env/dynamic/private';
@@ -756,32 +791,37 @@ const tandoorUrl = env.TANDOOR_URL || null;
```
### Date Handling
ISO8601 strings throughout the application:
```typescript
const now = new Date().toISOString();
// Output: "2026-02-15T12:30:45.123Z"
```
### Null vs Undefined
- `null`: Intentional absence of value
- `undefined`: Not yet initialized or optional parameters
- Prefer `null` for API responses and data structures
### Async/Await
Always preferred over Promise chains:
```typescript
// Preferred
async function fetchData() {
const response = await fetch(url);
const data = await response.json();
return data;
const response = await fetch(url);
const data = await response.json();
return data;
}
// Avoid
function fetchData() {
return fetch(url)
.then(response => response.json())
.then(data => data);
return fetch(url)
.then((response) => response.json())
.then((data) => data);
}
```

611
docs/FINDINGS.md
View File

File diff suppressed because it is too large Load Diff

135
docs/MIGRATION.md
View File

@@ -11,6 +11,7 @@ The migration transformed InstaRecipe from a blocking, synchronous extraction sy
### Architecture Transformation
**Before: Synchronous System**
```
User Request → Direct Processing → Response (wait 30-60s)
↓ ↓ ↓
@@ -18,6 +19,7 @@ User Request → Direct Processing → Response (wait 30-60s)
```
**After: Async Queue System**
```
User Request → Queue Item Created → Immediate Response
↓ ↓ ↓
@@ -60,6 +62,7 @@ User Request → Queue Item Created → Immediate Response
### New Endpoints
#### Queue Management
```typescript
// Enqueue URL for processing
POST /api/queue
@@ -84,6 +87,7 @@ Events: connection, queue-update, ping
```
#### Push Notifications
```typescript
// Subscribe to push notifications
POST /api/notifications/subscribe
@@ -101,11 +105,11 @@ These endpoints are marked for removal and should not be used in new code:
```typescript
// ❌ DEPRECATED: Synchronous extraction
POST /api/extract
POST / api / extract;
// 👉 Use: POST /api/queue
// ❌ DEPRECATED: Long-polling progress
GET /api/extract-stream
GET / api / extract - stream;
// 👉 Use: GET /api/queue/stream
```
@@ -117,33 +121,33 @@ New queue items follow this structure:
```typescript
interface QueueItem {
id: string; // UUID v4
url: string; // Instagram URL
status: 'pending' | 'in_progress' | 'success' | 'error' | 'unhealthy';
// Processing phases with individual progress
phases: Array<{
name: 'extraction' | 'parsing' | 'uploading';
status: 'pending' | 'in_progress' | 'completed' | 'error';
startedAt?: string;
completedAt?: string;
progress?: number; // 0-100
}>;
// Results (populated on success)
results?: {
recipe?: Recipe; // Extracted recipe data
tandoorUrl?: string; // Link to uploaded recipe
extractedText?: string; // Raw extracted text
thumbnail?: string; // Image URL
};
// Error information
error?: string;
// Timestamps
createdAt: string;
updatedAt: string;
id: string; // UUID v4
url: string; // Instagram URL
status: 'pending' | 'in_progress' | 'success' | 'error' | 'unhealthy';
// Processing phases with individual progress
phases: Array<{
name: 'extraction' | 'parsing' | 'uploading';
status: 'pending' | 'in_progress' | 'completed' | 'error';
startedAt?: string;
completedAt?: string;
progress?: number; // 0-100
}>;
// Results (populated on success)
results?: {
recipe?: Recipe; // Extracted recipe data
tandoorUrl?: string; // Link to uploaded recipe
extractedText?: string; // Raw extracted text
thumbnail?: string; // Image URL
};
// Error information
error?: string;
// Timestamps
createdAt: string;
updatedAt: string;
}
```
@@ -167,33 +171,35 @@ interface QueueStatusUpdate {
### For Frontend Applications
1. **Replace Synchronous Calls**
```typescript
// ❌ Old synchronous approach
const response = await fetch('/api/extract', {
method: 'POST',
body: JSON.stringify({ url })
method: 'POST',
body: JSON.stringify({ url })
});
const result = await response.json(); // Wait 30-60 seconds
// ✅ New async queue approach
const response = await fetch('/api/queue', {
method: 'POST',
body: JSON.stringify({ url })
method: 'POST',
body: JSON.stringify({ url })
});
const queueItem = await response.json(); // Immediate response
// Navigate to dashboard for real-time updates
window.location.href = `/?highlight=${queueItem.id}`;
```
2. **Add Real-time Updates**
```typescript
// Setup Server-Sent Events for progress tracking
const eventSource = new EventSource(`/api/queue/stream?itemId=${itemId}`);
eventSource.addEventListener('queue-update', (event) => {
const update = JSON.parse(event.data);
updateUI(update);
const update = JSON.parse(event.data);
updateUI(update);
});
```
@@ -201,36 +207,37 @@ interface QueueStatusUpdate {
```typescript
// Handle different queue statuses
switch (item.status) {
case 'pending':
showPendingState();
break;
case 'in_progress':
showProgressBar(item.phases);
break;
case 'success':
showResults(item.results);
break;
case 'error':
showErrorWithRetry(item.error, item.id);
break;
case 'unhealthy':
showRetryableError(item.error, item.id);
break;
case 'pending':
showPendingState();
break;
case 'in_progress':
showProgressBar(item.phases);
break;
case 'success':
showResults(item.results);
break;
case 'error':
showErrorWithRetry(item.error, item.id);
break;
case 'unhealthy':
showRetryableError(item.error, item.id);
break;
}
```
### For Backend Integrations
1. **Update API Calls**
```python
# ❌ Old synchronous API
response = requests.post('/api/extract', json={'url': url})
# This would block for 30-60 seconds
# ✅ New async queue API
response = requests.post('/api/queue', json={'url': url})
queue_item = response.json()
# Poll or use SSE for updates
while True:
item = requests.get(f'/api/queue/{queue_item["id"]}').json()
@@ -240,9 +247,10 @@ interface QueueStatusUpdate {
```
2. **Implement SSE Client** (Python example)
```python
import sseclient
def listen_to_queue_updates(item_id):
messages = sseclient.SSEClient(f'/api/queue/stream?itemId={item_id}')
for msg in messages:
@@ -266,7 +274,7 @@ QUEUE_TIMEOUT_MS=30000 # Processing timeout
QUEUE_RETRY_ATTEMPTS=3 # Maximum retry attempts
# Push notification settings (optional)
VAPID_PUBLIC_KEY=BDummyPublicKey...
VAPID_PUBLIC_KEY=BDummyPublicKey...
VAPID_PRIVATE_KEY=DummyPrivateKey...
# Existing LLM and Tandoor settings remain the same
@@ -306,7 +314,7 @@ npm test
# Test specific components
npm test queue-manager
npm test queue-processor
npm test queue-processor
npm test queue-api
npm test queue-sse
```
@@ -314,18 +322,21 @@ npm test queue-sse
## Performance Considerations
### Before Migration
- **Blocking Operations**: Each request blocked a server thread
- **Single Processing**: One extraction at a time
- **No Progress**: Users waited without feedback
- **Memory Usage**: High memory usage during long operations
### After Migration
### After Migration
- **Non-blocking**: Requests return immediately
- **Concurrent Processing**: Multiple extractions in parallel
- **Real-time Feedback**: Live progress updates
- **Efficient Memory**: Event-driven, minimal memory footprint
### Performance Metrics
- **Response Time**: 50ms (queue) vs 30-60s (synchronous)
- **Throughput**: 2x concurrent processing vs 1x sequential
- **User Experience**: Immediate feedback vs long waiting
@@ -336,11 +347,13 @@ npm test queue-sse
If issues arise, the system can be rolled back by:
1. **Disable Queue Processing**
```env
QUEUE_PROCESSING_ENABLED=false
```
2. **Re-enable Legacy Endpoints** (if preserved)
```typescript
// Temporary fallback to synchronous processing
app.post('/api/extract', legacyExtractHandler);
@@ -389,10 +402,10 @@ curl -X POST https://localhost:5173/api/notifications/vapid-key
The migration to an async queue system represents a significant architectural improvement that provides:
- **Better User Experience**: Immediate responses and real-time progress
- **Improved Reliability**: Error recovery and retry mechanisms
- **Improved Reliability**: Error recovery and retry mechanisms
- **Enhanced Performance**: Concurrent processing and resource efficiency
- **Modern Features**: Push notifications and PWA capabilities
The new system maintains backward compatibility during the transition period while providing a clear migration path for all integrations.
For questions or issues during migration, refer to the comprehensive test suite and documentation, or open an issue in the project repository.
For questions or issues during migration, refer to the comprehensive test suite and documentation, or open an issue in the project repository.

103
docs/SVELTEKIT_SSR_GUIDE.md
View File

@@ -3,6 +3,7 @@
This guide documents SSR-safe patterns and anti-patterns for InstaRecipe development. All developers should follow these guidelines to prevent SSR errors.
## Table of Contents
- [Core Principle](#core-principle)
- [Browser API Detection](#browser-api-detection)
- [Lifecycle Hooks](#lifecycle-hooks)
@@ -18,6 +19,7 @@ This guide documents SSR-safe patterns and anti-patterns for InstaRecipe develop
**SvelteKit renders components on both the server and client.** Any browser-only APIs must be guarded or used in browser-only contexts.
### Browser-Only APIs (Require Guards)
- `window.*`
- `document.*`
- `localStorage`, `sessionStorage`
@@ -36,8 +38,8 @@ This guide documents SSR-safe patterns and anti-patterns for InstaRecipe develop
import { browser } from '$app/environment';
if (browser) {
// Safe: only runs in browser
const data = localStorage.getItem('key');
// Safe: only runs in browser
const data = localStorage.getItem('key');
}
```
@@ -49,14 +51,14 @@ if (browser) {
<script lang="ts">
import { browser } from '$app/environment';
import { onMount } from 'svelte';
let eventSource = $state<EventSource | null>(null);
function startSSEConnection() {
if (!browser) return; // ✅ Guard
eventSource = new EventSource('/api/stream');
}
onMount(() => {
if (browser) { // ✅ Explicit guard
startSSEConnection();
@@ -72,6 +74,7 @@ if (browser) {
### `onMount` - Browser-Only Lifecycle
**Use `onMount` for:**
- Browser API initialization
- Timer setup (`setInterval`, `setTimeout`)
- Event listener registration
@@ -81,12 +84,12 @@ if (browser) {
import { onMount } from 'svelte';
onMount(() => {
// ✅ Only runs in browser (built-in SSR guard)
const interval = setInterval(() => {
// Polling logic
}, 1000);
return () => clearInterval(interval); // Cleanup
// ✅ Only runs in browser (built-in SSR guard)
const interval = setInterval(() => {
// Polling logic
}, 1000);
return () => clearInterval(interval); // Cleanup
});
```
@@ -96,8 +99,8 @@ onMount(() => {
import { onDestroy } from 'svelte';
onDestroy(() => {
// ✅ Safe for cleanup
eventSource?.close();
// ✅ Safe for cleanup
eventSource?.close();
});
```
@@ -117,7 +120,7 @@ let stored = $state(localStorage.getItem('key')); // SSR crash!
// ✅ DO: Load in onMount
let stored = $state<string | null>(null);
onMount(() => {
stored = localStorage.getItem('key');
stored = localStorage.getItem('key');
});
```
@@ -142,29 +145,31 @@ let userAgent = $derived(navigator.userAgent); // SSR crash!
```typescript
// ❌ BAD: No browser guard
$effect(() => {
setInterval(() => checkHealth(), 1000); // SSR crash!
setInterval(() => checkHealth(), 1000); // SSR crash!
});
// ✅ GOOD: With browser guard
$effect(() => {
if (!browser) return;
const interval = setInterval(() => checkHealth(), 1000);
return () => clearInterval(interval);
if (!browser) return;
const interval = setInterval(() => checkHealth(), 1000);
return () => clearInterval(interval);
});
// ✅ BETTER: Use onMount for initialization instead
onMount(() => {
const interval = setInterval(() => checkHealth(), 1000);
return () => clearInterval(interval);
const interval = setInterval(() => checkHealth(), 1000);
return () => clearInterval(interval);
});
```
**When to use `$effect`:**
- Synchronizing derived state
- DOM manipulation (with browser guard)
- Reactive cleanup
**When NOT to use `$effect`:**
- Initialization (use `onMount`)
- API calls on mount (use `onMount`)
- Timer setup (use `onMount`)
@@ -189,11 +194,13 @@ if (browser && eventSource?.readyState === EventSource.OPEN)
```
**EventSource States:**
- `EventSource.CONNECTING = 0`
- `EventSource.OPEN = 1`
- `EventSource.CLOSED = 2`
**WebSocket States:**
- `WebSocket.CONNECTING = 0`
- `WebSocket.OPEN = 1`
- `WebSocket.CLOSING = 2`
@@ -220,8 +227,8 @@ const interval = setInterval(() => {}, 1000); // SSR crash!
// ✅ GOOD: In onMount
onMount(() => {
const interval = setInterval(() => {}, 1000);
return () => clearInterval(interval);
const interval = setInterval(() => {}, 1000);
return () => clearInterval(interval);
});
```
@@ -260,22 +267,23 @@ onMount(() => {
import { browser } from '$app/environment';
export class PushNotificationManager {
private static instance: PushNotificationManager | null = null;
static getInstance() {
if (!browser) return null; // ✅ Early return for SSR
// ... rest of implementation
}
private loadStoredSubscription() {
if (!browser) return null; // ✅ Guard localStorage
const stored = localStorage.getItem('pushSubscription');
return stored ? JSON.parse(stored) : null;
}
private static instance: PushNotificationManager | null = null;
static getInstance() {
if (!browser) return null; // ✅ Early return for SSR
// ... rest of implementation
}
private loadStoredSubscription() {
if (!browser) return null; // ✅ Guard localStorage
const stored = localStorage.getItem('pushSubscription');
return stored ? JSON.parse(stored) : null;
}
}
```
**Why it's good:**
- Guards all browser API access
- Early returns prevent unnecessary code execution during SSR
- Defensive programming with null checks
@@ -288,16 +296,16 @@ export class PushNotificationManager {
<script lang="ts">
import { browser } from '$app/environment';
import { onMount } from 'svelte';
let eventSource = $state<EventSource | null>(null);
onMount(async () => {
await loadQueueItems();
if (browser) { // ✅ Guard
startSSEConnection();
}
});
function startSSEConnection() {
if (!browser) return; // ✅ Double guard for safety
eventSource = new EventSource('/api/queue/stream');
@@ -316,7 +324,7 @@ export class PushNotificationManager {
```svelte
<script lang="ts">
import { onMount } from 'svelte';
onMount(() => {
// ✅ onMount only runs in browser
checkHealth(); // Initial check
@@ -327,6 +335,7 @@ export class PushNotificationManager {
```
**Why it's good:**
- Uses `onMount` instead of `$effect` for initialization
- Timer setup in browser-only context
- Proper cleanup with return function
@@ -344,7 +353,7 @@ let theme = $derived(localStorage.getItem('theme'));
// ✅ DO
let theme = $state<string | null>(null);
onMount(() => {
theme = localStorage.getItem('theme');
theme = localStorage.getItem('theme');
});
```
@@ -353,19 +362,19 @@ onMount(() => {
```typescript
// ❌ DON'T
$effect(() => {
// Runs during SSR!
fetch('/api/data');
// Runs during SSR!
fetch('/api/data');
});
// ✅ DO: Guard browser-specific side effects
$effect(() => {
if (!browser) return;
fetch('/api/data');
if (!browser) return;
fetch('/api/data');
});
// ✅ BETTER: Use onMount for initialization
onMount(() => {
fetch('/api/data');
fetch('/api/data');
});
```
@@ -387,8 +396,8 @@ const interval = setInterval(() => {}, 1000);
// ✅ DO
onMount(() => {
const interval = setInterval(() => {}, 1000);
return () => clearInterval(interval);
const interval = setInterval(() => {}, 1000);
return () => clearInterval(interval);
});
```
@@ -408,6 +417,7 @@ Watch server console for errors during build and preview.
### 2. Check for Hydration Warnings
Open browser DevTools console and look for:
- "Hydration failed"
- "The server response doesn't match the client content"
@@ -422,6 +432,7 @@ grep -r "navigator\." src/routes --include="*.svelte"
```
Then verify each usage is either:
- In an event handler (safe)
- In `onMount` (safe)
- Guarded with `if (browser)` (safe)

158
docs/TESTING.md
View File

@@ -7,7 +7,7 @@ This guide explains how to properly mock dependencies when testing SvelteKit app
SvelteKit has a unique architecture where code can run on both server and client. This affects how we mock:
1. **Server-only modules** (`$lib/server/*`, `*.server.ts`) - Only run on server
2. **Universal modules** - Can run on both server and client
2. **Universal modules** - Can run on both server and client
3. **Environment variables** - Different modules for static vs dynamic access
## Key Principles
@@ -32,12 +32,12 @@ SvelteKit has a unique architecture where code can run on both server and client
import { env } from '$env/dynamic/private';
export const queueConfig = {
concurrency: parseInt(env.QUEUE_CONCURRENCY || '2', 10),
maxRetries: parseInt(env.QUEUE_MAX_RETRIES || '3', 10),
tandoor: {
enabled: !!env.TANDOOR_TOKEN,
token: env.TANDOOR_TOKEN || null
}
concurrency: parseInt(env.QUEUE_CONCURRENCY || '2', 10),
maxRetries: parseInt(env.QUEUE_MAX_RETRIES || '3', 10),
tandoor: {
enabled: !!env.TANDOOR_TOKEN,
token: env.TANDOOR_TOKEN || null
}
};
```
@@ -49,21 +49,21 @@ import * as queueConfigModule from '$lib/server/queue/config';
// Mock the config module
vi.mock('$lib/server/queue/config', () => ({
queueConfig: {
concurrency: 2,
maxRetries: 3,
tandoor: { enabled: true, token: 'test-token' }
}
queueConfig: {
concurrency: 2,
maxRetries: 3,
tandoor: { enabled: true, token: 'test-token' }
}
}));
describe('QueueProcessor', () => {
beforeEach(() => {
vi.clearAllMocks();
});
afterEach(() => {
vi.restoreAllMocks();
});
beforeEach(() => {
vi.clearAllMocks();
});
afterEach(() => {
vi.restoreAllMocks();
});
});
```
@@ -78,10 +78,10 @@ import { vi } from 'vitest';
// IMPORTANT: Mock BEFORE importing the module that uses it
vi.mock('$lib/server/extraction', () => ({
extractTextAndThumbnail: vi.fn().mockResolvedValue({
bodyText: 'Mock recipe text',
thumbnail: 'https://mock.com/image.jpg'
})
extractTextAndThumbnail: vi.fn().mockResolvedValue({
bodyText: 'Mock recipe text',
thumbnail: 'https://mock.com/image.jpg'
})
}));
// NOW import the module that depends on these
@@ -89,15 +89,15 @@ import { queueProcessor } from '$lib/server/queue/QueueProcessor';
import { extractTextAndThumbnail } from '$lib/server/extraction';
describe('QueueProcessor', () => {
it('should use mocked services', async () => {
const item = queueManager.enqueue('https://instagram.com/p/test');
// Verify mock was called
expect(extractTextAndThumbnail).toHaveBeenCalledWith(
'https://instagram.com/p/test',
expect.any(Function)
);
});
it('should use mocked services', async () => {
const item = queueManager.enqueue('https://instagram.com/p/test');
// Verify mock was called
expect(extractTextAndThumbnail).toHaveBeenCalledWith(
'https://instagram.com/p/test',
expect.any(Function)
);
});
});
```
@@ -112,22 +112,22 @@ import { describe, it, expect } from 'vitest';
import { POST } from '../routes/api/queue/+server';
describe('POST /api/queue', () => {
it('should reject invalid URLs', async () => {
const request = new Request('http://localhost/api/queue', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url: 'invalid-url' })
});
const response = await POST({ request } as any);
// ✅ CORRECT - Check status first
expect(response.status).toBe(400);
// ✅ CORRECT - Properly await error response
const data = await response.json();
expect(data.message).toContain('Invalid');
});
it('should reject invalid URLs', async () => {
const request = new Request('http://localhost/api/queue', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ url: 'invalid-url' })
});
const response = await POST({ request } as any);
// ✅ CORRECT - Check status first
expect(response.status).toBe(400);
// ✅ CORRECT - Properly await error response
const data = await response.json();
expect(data.message).toContain('Invalid');
});
});
```
@@ -136,17 +136,17 @@ describe('POST /api/queue', () => {
```typescript
// ❌ WRONG - This will fail
it('should reject invalid input', async () => {
const response = await endpoint({ request } as any);
const data = response.json(); // Missing await!
expect(data.message).toBe('Error'); // data is a Promise
const response = await endpoint({ request } as any);
const data = response.json(); // Missing await!
expect(data.message).toBe('Error'); // data is a Promise
});
// ✅ CORRECT
it('should reject invalid input', async () => {
const response = await endpoint({ request } as any);
expect(response.status).toBe(400);
const data = await response.json(); // Properly awaited
expect(data.message).toBe('Error');
const response = await endpoint({ request } as any);
expect(response.status).toBe(400);
const data = await response.json(); // Properly awaited
expect(data.message).toBe('Error');
});
```
@@ -173,11 +173,11 @@ import { queueProcessor } from './QueueProcessor';
import { beforeEach, afterEach } from 'vitest';
beforeEach(() => {
vi.clearAllMocks(); // Clear call history
vi.clearAllMocks(); // Clear call history
});
afterEach(() => {
vi.restoreAllMocks(); // Restore original implementations
vi.restoreAllMocks(); // Restore original implementations
});
```
@@ -203,16 +203,16 @@ const mockFn = vi.fn() as Mock<() => Promise<string>>;
```typescript
it('should process item', async () => {
const item = queueManager.enqueue('https://instagram.com/p/test');
// Wait for processing with timeout
await vi.waitFor(
() => {
const updated = queueManager.get(item.id);
expect(updated?.status).toBe('success');
},
{ timeout: 5000, interval: 100 }
);
const item = queueManager.enqueue('https://instagram.com/p/test');
// Wait for processing with timeout
await vi.waitFor(
() => {
const updated = queueManager.get(item.id);
expect(updated?.status).toBe('success');
},
{ timeout: 5000, interval: 100 }
);
});
```
@@ -222,20 +222,20 @@ it('should process item', async () => {
import { vi } from 'vitest';
beforeEach(() => {
vi.useFakeTimers();
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
vi.useRealTimers();
});
it('should process after delay', async () => {
queueManager.enqueue('https://test.com');
// Fast-forward time
await vi.advanceTimersByTimeAsync(1000);
// Now check results
queueManager.enqueue('https://test.com');
// Fast-forward time
await vi.advanceTimersByTimeAsync(1000);
// Now check results
});
```
@@ -263,7 +263,7 @@ vi.mock('./module', () => ({ export: vi.fn() }));
// Mock with factory
vi.mock('./module', () => {
return { dynamicExport: () => 'value' };
return { dynamicExport: () => 'value' };
});
// Spy on existing export
@@ -285,9 +285,9 @@ expect(mockFn).toHaveBeenCalledWith('arg1', 'arg2');
expect(mockFn).toHaveBeenLastCalledWith('arg');
// Reset/restore
vi.clearAllMocks(); // Clear call history
vi.resetAllMocks(); // + Reset implementations
vi.restoreAllMocks(); // + Restore original implementations
vi.clearAllMocks(); // Clear call history
vi.resetAllMocks(); // + Reset implementations
vi.restoreAllMocks(); // + Restore original implementations
// Environment variables
vi.stubEnv('VAR_NAME', 'value');

8788
docs/Tandoor (2.3.6).yaml
View File

File diff suppressed because it is too large Load Diff