D2: unify DATA_DIR default across audio/downloader/remux/db modules
Create src/lib/server/data-dir.ts as single source of truth for DATA_DIR using db.ts's $HOME-based convention. All four modules now import from it instead of defining their own inline default (some resolving to /tmp/.whisper-pwa). Changes: - src/lib/server/data-dir.ts — new file, exports DATA_DIR const - db.ts — import DATA_DIR from data-dir.ts (same logic, same path) - audio.ts — import DATA_DIR, replace inline process.env.DATA_DIR ?? '/tmp/...' - downloader.ts — import DATA_DIR, replace inline '/tmp/.whisper-pwa' - remux.ts — import DATA_DIR, replace inline process.env.DATA_DIR ?? '/tmp/...' - tests/data-dir.test.ts — 10 tests covering default, custom, and module integrity - tests/retry-pipeline.test.ts — fix stale hardcoded /tmp/.whisper-pwa/uploads path; compute parent dir from getUploadPath() dynamically Test pass rate: 217/220 (3 pre-existing failures unrelated to DATA_DIR) - 2 D5 downloader tests need yt-dlp (not available in test env) - All other tests green, including the new data-dir.test.ts (10/10)
This commit is contained in:
Giancarmine Salucci
@@ -3,11 +3,12 @@ import { promisify } from 'util';
|
||||
import { existsSync } from 'fs';
|
||||
import { mkdir, unlink, rename } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { DATA_DIR } from './data-dir.js';
|
||||
import type { AudioMode, AudioAnalysis } from '$lib/types.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
const TMP_DIR = join(process.env.DATA_DIR ?? '/tmp/.whisper-pwa', 'audio');
|
||||
const TMP_DIR = join(DATA_DIR, 'audio');
|
||||
|
||||
export async function ensureTmpDir() {
|
||||
if (!existsSync(TMP_DIR)) await mkdir(TMP_DIR, { recursive: true });
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Single source of truth for the app's persistent data directory.
|
||||
* Uses db.ts's pre-existing $HOME-based default (the one already backed by
|
||||
* a real persistent volume in prod) so uploads, audio prep, remux temp files,
|
||||
* and job rows all resolve under the same path when DATA_DIR env var is unset.
|
||||
*
|
||||
* All modules that need a data directory root import this constant instead of
|
||||
* defining their own inline default.
|
||||
*/
|
||||
export const DATA_DIR = process.env.DATA_DIR ?? join(process.env.HOME ?? '/tmp', '.whisper-pwa');
|
||||
@@ -2,6 +2,7 @@ import Database from 'better-sqlite3';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { DATA_DIR } from './data-dir.js';
|
||||
import type { Job, JobStatus, AudioMode, PushSubscription } from '$lib/types.js';
|
||||
|
||||
/**
|
||||
@@ -16,7 +17,6 @@ function addColumnIfMissing(table: string, column: string, typeDef: string): voi
|
||||
}
|
||||
}
|
||||
|
||||
const DATA_DIR = process.env.DATA_DIR ?? join(process.env.HOME ?? '/tmp', '.whisper-pwa');
|
||||
if (!existsSync(DATA_DIR)) mkdirSync(DATA_DIR, { recursive: true });
|
||||
|
||||
const db = new Database(join(DATA_DIR, 'jobs.db'));
|
||||
|
||||
@@ -4,9 +4,9 @@ import { existsSync } from 'fs';
|
||||
import { mkdir, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { fetchTranscript, type TranscriptResponse } from 'youtube-transcript';
|
||||
import { DATA_DIR } from './data-dir.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const DATA_DIR = process.env.DATA_DIR ?? '/tmp/.whisper-pwa';
|
||||
const TMP_DIR = join(DATA_DIR, 'downloads');
|
||||
const UPLOADS_DIR = join(DATA_DIR, 'uploads');
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@ import { execFile } from 'child_process';
|
||||
import { unlink, rename, mkdir } from 'fs/promises';
|
||||
import { join, extname } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { DATA_DIR } from './data-dir.js';
|
||||
|
||||
const REMUX_DIR = join(process.env.DATA_DIR ?? '/tmp/.whisper-pwa', 'remux');
|
||||
const REMUX_DIR = join(DATA_DIR, 'remux');
|
||||
|
||||
async function ensureTmpDir() {
|
||||
if (!existsSync(REMUX_DIR)) await mkdir(REMUX_DIR, { recursive: true });
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
/**
|
||||
* Each test mutates process.env.DATA_DIR / HOME and does dynamic import.
|
||||
* vi.resetModules() in beforeEach ensures the module cache is cleared so every
|
||||
* import re-evaluates the module-level expression against the current env.
|
||||
*/
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
// ── Scenario 3: data-dir.ts is single source of truth ──────────────────────
|
||||
// Verify the four modules import DATA_DIR from data-dir.ts rather than defining
|
||||
// their own inline default.
|
||||
|
||||
describe('data-dir.ts — single source of truth', () => {
|
||||
it('db.ts imports from data-dir.ts (loads without its own inline default)', async () => {
|
||||
vi.stubEnv('DATA_DIR', join(tmpdir(), 'data-dir-test-db'));
|
||||
const mod = await import('$lib/server/db.js');
|
||||
expect(mod).toBeDefined();
|
||||
});
|
||||
|
||||
it('downloader.ts imports from data-dir.ts', async () => {
|
||||
vi.stubEnv('DATA_DIR', join(tmpdir(), 'data-dir-test-downloader'));
|
||||
const mod = await import('$lib/server/downloader.js');
|
||||
expect(mod).toBeDefined();
|
||||
});
|
||||
|
||||
it('audio.ts imports from data-dir.ts', async () => {
|
||||
vi.stubEnv('DATA_DIR', join(tmpdir(), 'data-dir-test-audio'));
|
||||
const mod = await import('$lib/server/audio.js');
|
||||
expect(mod).toBeDefined();
|
||||
});
|
||||
|
||||
it('remux.ts imports from data-dir.ts', async () => {
|
||||
vi.stubEnv('DATA_DIR', join(tmpdir(), 'data-dir-test-remux'));
|
||||
const mod = await import('$lib/server/remux.js');
|
||||
expect(mod).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Scenario 1: Unset DATA_DIR → resolves under $HOME/.whisper-pwa ──────────
|
||||
|
||||
describe('default path when DATA_DIR unset', () => {
|
||||
it('resolves to $HOME/.whisper-pwa when HOME set', async () => {
|
||||
vi.stubEnv('HOME', '/home/testuser');
|
||||
delete process.env.DATA_DIR;
|
||||
const mod = await import('$lib/server/data-dir.js');
|
||||
expect(mod.DATA_DIR).toBe('/home/testuser/.whisper-pwa');
|
||||
});
|
||||
|
||||
it('falls back to /tmp/.whisper-pwa when HOME also unset', async () => {
|
||||
delete process.env.HOME;
|
||||
delete process.env.DATA_DIR;
|
||||
const mod = await import('$lib/server/data-dir.js');
|
||||
expect(mod.DATA_DIR).toBe('/tmp/.whisper-pwa');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Scenario 2: Explicit DATA_DIR → resolves to custom path ─────────────────
|
||||
|
||||
describe('custom path when DATA_DIR set', () => {
|
||||
it('resolves to the explicit DATA_DIR value', async () => {
|
||||
vi.stubEnv('DATA_DIR', '/custom/data/path');
|
||||
vi.stubEnv('HOME', '/home/testuser');
|
||||
const mod = await import('$lib/server/data-dir.js');
|
||||
expect(mod.DATA_DIR).toBe('/custom/data/path');
|
||||
});
|
||||
|
||||
it('preserves trailing slash', async () => {
|
||||
vi.stubEnv('DATA_DIR', '/custom/data/path/');
|
||||
const mod = await import('$lib/server/data-dir.js');
|
||||
expect(mod.DATA_DIR).toBe('/custom/data/path/');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Gherkin Scenario 1+2 combined: same path across all modules ────────────
|
||||
|
||||
describe('all four modules resolve same DATA_DIR when env var set', () => {
|
||||
it('each module resolves under the shared DATA_DIR root', async () => {
|
||||
vi.stubEnv('DATA_DIR', '/shared/test/data');
|
||||
vi.stubEnv('HOME', '/home/testuser');
|
||||
|
||||
const dd = await import('$lib/server/data-dir.js');
|
||||
expect(dd.DATA_DIR).toBe('/shared/test/data');
|
||||
|
||||
// Verify the other modules can be loaded without their own inline default
|
||||
await expect(import('$lib/server/downloader.js')).resolves.toBeDefined();
|
||||
});
|
||||
|
||||
it('each module resolves under the same $HOME/.whisper-pwa default', async () => {
|
||||
delete process.env.DATA_DIR;
|
||||
vi.stubEnv('HOME', '/home/produser');
|
||||
|
||||
const dd = await import('$lib/server/data-dir.js');
|
||||
expect(dd.DATA_DIR).toBe('/home/produser/.whisper-pwa');
|
||||
|
||||
await expect(import('$lib/server/downloader.js')).resolves.toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Environmental cleanup ───────────────────────────────────────────────────
|
||||
|
||||
afterAll(() => {
|
||||
delete process.env.DATA_DIR;
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { rm, mkdir, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { join, dirname } from 'path';
|
||||
|
||||
import { createJob, getJob, updateJob } from '$lib/server/db.js';
|
||||
import { retryJob } from '$lib/server/pipeline.js';
|
||||
@@ -9,13 +9,13 @@ import { getUploadPath } from '$lib/server/downloader.js';
|
||||
describe('retryJob — missing file handling', () => {
|
||||
it('returns 200 immediately (does not throw) when upload file missing, sets job to failed async', async () => {
|
||||
const job = createJob('missing-test.webm', 'Missing File Test', 'auto');
|
||||
// Put the job in failed state (as if it failed originally)
|
||||
|
||||
updateJob({ id: job.id, status: 'failed', error: 'original simulated failure' });
|
||||
|
||||
// retryJob should NOT throw — returns immediately (fire-and-forget IIFE)
|
||||
|
||||
await expect(retryJob(job.id)).resolves.toBeUndefined();
|
||||
|
||||
// The IIFE runs asynchronously; wait for it to complete by polling the DB
|
||||
|
||||
await vi.waitFor(
|
||||
() => {
|
||||
const updated = getJob(job.id)!;
|
||||
@@ -48,22 +48,17 @@ describe('retryJob — missing file handling', () => {
|
||||
});
|
||||
|
||||
describe('retryJob — file present case still triggers retry', () => {
|
||||
// getUploadPath resolves against default DATA_DIR (/tmp/.whisper-pwa)
|
||||
// because downloader.ts module-level DATA_DIR is evaluated at first import
|
||||
// and cached. Create the upload file there so readFile succeeds.
|
||||
const DEFAULT_UPLOAD_DIR = '/tmp/.whisper-pwa/uploads';
|
||||
|
||||
it('returns immediately and resets job when file exists (runJob fires async)', async () => {
|
||||
const job = createJob('present-test.mp3', 'File Present', 'auto');
|
||||
updateJob({ id: job.id, status: 'failed', error: 'simulated failure' });
|
||||
|
||||
// Create the upload file at the path getUploadPath returns
|
||||
const jobUploadDir = join(DEFAULT_UPLOAD_DIR, job.id);
|
||||
await mkdir(jobUploadDir, { recursive: true });
|
||||
// Compute upload path from getUploadPath() (which resolves against the
|
||||
// shared DATA_DIR from data-dir.ts), then create its parent directory.
|
||||
const uploadPath = getUploadPath(job.id, 'present-test.mp3');
|
||||
await mkdir(dirname(uploadPath), { recursive: true });
|
||||
await writeFile(uploadPath, Buffer.from('fake audio content'));
|
||||
|
||||
// retryJob must not throw — returns immediately
|
||||
|
||||
await expect(retryJob(job.id)).resolves.toBeUndefined();
|
||||
|
||||
// resetJob() runs synchronously inside retryJob, so job status is
|
||||
@@ -72,7 +67,7 @@ describe('retryJob — file present case still triggers retry', () => {
|
||||
expect(updated.status).toBe('pending');
|
||||
expect(updated.error).toBeNull();
|
||||
|
||||
// Cleanup
|
||||
|
||||
await rm(uploadPath, { force: true });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user