D2: Widen retry to upload-source jobs
- Remove http-only guard in retry endpoint (src/routes/api/jobs/[id]/retry/+server.ts)
- Extend pipeline.ts retryJob() to handle upload-source jobs:
read file from persistent storage, pass buffer to runJob
- Change saveUploadedFile() to save to DATA_DIR/uploads/{jobId}/
(survives cleanupJobTmp, available for retry on failure)
- Add getUploadPath(), cleanupUploadDir() to downloader.ts
- Remove cleanupFiles(rawAudioPath) from catch block in runJob()
— cleanupJobTmp handles TMP_DIR for YouTube, uploads survive
- Add cleanupUploadDir() on webhook success path
- Add retry-endpoint.test.ts with 9 tests covering:
- 404 for unknown job
- 409 for non-retryable status (done, pending)
- 200 for cancelled (retryable)
- 200 for failed YouTube job (existing behavior)
- 200 for failed upload-source job (new behavior, .webm and .mp3)
- Negative: retryJob not called on 404/409
- Add downloader.test.ts tests for saveUploadedFile persistent path,
getUploadPath, cleanupUploadDir, isolation, and non-existent cleanup
- Fix webhook.test.ts mock to include cleanupUploadDir
All 189 tests pass across 12 test files.
This commit is contained in:
@@ -6,12 +6,34 @@ import { join } from 'path';
|
||||
import { fetchTranscript, type TranscriptResponse } from 'youtube-transcript';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const TMP_DIR = join(process.env.DATA_DIR ?? '/tmp/.whisper-pwa', 'downloads');
|
||||
const DATA_DIR = process.env.DATA_DIR ?? '/tmp/.whisper-pwa';
|
||||
const TMP_DIR = join(DATA_DIR, 'downloads');
|
||||
const UPLOADS_DIR = join(DATA_DIR, 'uploads');
|
||||
|
||||
export async function ensureTmpDir() {
|
||||
if (!existsSync(TMP_DIR)) await mkdir(TMP_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
async function ensureUploadsDir(jobId: string): Promise<string> {
|
||||
const dir = join(UPLOADS_DIR, jobId);
|
||||
if (!existsSync(dir)) await mkdir(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** Path to a job's persistent upload file. Survives cleanupJobTmp. */
|
||||
export function getUploadPath(jobId: string, filename: string): string {
|
||||
return join(UPLOADS_DIR, jobId, filename);
|
||||
}
|
||||
|
||||
/** Delete a job's persistent upload dir (call on success). */
|
||||
export async function cleanupUploadDir(jobId: string): Promise<void> {
|
||||
const dir = join(UPLOADS_DIR, jobId);
|
||||
try {
|
||||
const { rm } = await import('fs/promises');
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
export interface CaptionResult {
|
||||
type: 'captions';
|
||||
segments: Array<{ index: number; start: number; end: number; text: string; words: [] }>;
|
||||
@@ -93,15 +115,13 @@ export async function downloadYouTube(url: string, jobId: string): Promise<Downl
|
||||
return { type: 'audio', audioPath, title };
|
||||
}
|
||||
|
||||
/** Save an uploaded file to tmp dir and return its path. */
|
||||
/** Save an uploaded file to persistent storage (survives cleanupJobTmp). */
|
||||
export async function saveUploadedFile(
|
||||
buffer: Buffer,
|
||||
filename: string,
|
||||
jobId: string
|
||||
): Promise<string> {
|
||||
await ensureTmpDir();
|
||||
const outDir = join(TMP_DIR, jobId);
|
||||
await mkdir(outDir, { recursive: true });
|
||||
const outDir = await ensureUploadsDir(jobId);
|
||||
const dest = join(outDir, filename);
|
||||
await writeFile(dest, buffer);
|
||||
return dest;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createJob, updateJob, setJobStatus, getJob, resetJob } from './db.js';
|
||||
import { downloadYouTube, saveUploadedFile, cleanupJobTmp } from './downloader.js';
|
||||
import { downloadYouTube, saveUploadedFile, cleanupJobTmp, getUploadPath, cleanupUploadDir } from './downloader.js';
|
||||
import { prepareAudio, cleanup as cleanupFiles } from './audio.js';
|
||||
import { submitJob, streamJob } from './whisper.js';
|
||||
import { ensureWhisperRunning } from './docker.js';
|
||||
@@ -50,14 +50,25 @@ export async function startUploadJob(
|
||||
return job.id;
|
||||
}
|
||||
|
||||
/** Retry a failed/cancelled YouTube job by resetting and re-running the pipeline. */
|
||||
/** Retry a failed/cancelled job — YouTube or upload-source. */
|
||||
export async function retryJob(jobId: string): Promise<void> {
|
||||
const job = getJob(jobId);
|
||||
if (!job) throw new Error('Job not found');
|
||||
resetJob(jobId);
|
||||
runJob(jobId, { type: 'youtube', url: job.source }, job.audioMode as AudioMode).catch((err) => {
|
||||
console.error(`[pipeline] retry job ${jobId} failed:`, err);
|
||||
});
|
||||
|
||||
const isUpload = !job.source.startsWith('http');
|
||||
if (isUpload) {
|
||||
const { readFileSync } = await import('fs');
|
||||
const uploadPath = getUploadPath(jobId, job.source);
|
||||
const buffer = readFileSync(uploadPath);
|
||||
runJob(jobId, { type: 'upload', buffer, filename: job.source }, job.audioMode as AudioMode).catch((err) => {
|
||||
console.error(`[pipeline] retry upload job ${jobId} failed:`, err);
|
||||
});
|
||||
} else {
|
||||
runJob(jobId, { type: 'youtube', url: job.source }, job.audioMode as AudioMode).catch((err) => {
|
||||
console.error(`[pipeline] retry job ${jobId} failed:`, err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function runJob(
|
||||
@@ -112,6 +123,7 @@ async function runJob(
|
||||
const { sendNotification } = await import('./push.js');
|
||||
await sendNotification(jobId, '✅ Transcript ready', title);
|
||||
await cleanupJobTmp(jobId);
|
||||
await cleanupUploadDir(jobId).catch(() => {});
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -162,7 +174,9 @@ async function runJob(
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
updateJob({ id: jobId, status: 'failed', error: message });
|
||||
emitProgress(jobId, { type: 'error', message });
|
||||
if (rawAudioPath) await cleanupFiles(rawAudioPath).catch(() => {});
|
||||
// rawAudioPath NOT cleaned here — for upload jobs the file lives in
|
||||
// DATA_DIR/uploads/{jobId}/ so it survives for retry. For YouTube jobs,
|
||||
// cleanupJobTmp below handles TMP_DIR/{jobId} which contains the audio.
|
||||
if (wavPath) await cleanupFiles(wavPath).catch(() => {});
|
||||
await cleanupJobTmp(jobId);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user