fd8d4deefb
All checks were successful
Build & Push Docker Image / build-and-push (push) Successful in 6m39s
GPU warmup (src/transcriber.rs): After creating WhisperState, run a 1s silent inference pass in load(). CUDA JIT-compiles device kernels on the first whisper_full_with_state call. On a cold GPU this compilation disrupts the decode pipeline mid-inference, returning 0 segments in ~0.5s. The warmup forces all kernel compilation at startup so the first real job runs on fully compiled kernels. test_all.sh: - Fix submit response field: 'id' → 'job_id' (was breaking all downstream steps) - Remove language=auto: not a valid ISO 639-1 code; omit field for auto-detect - Make BASE and AUDIO configurable via env vars (WHISPER_BASE_URL, TEST_AUDIO) - Fix DELETE assertion: completed jobs return 409 Conflict, not 204 - Add explicit zero-segments failure check in quality inspection (step 9) - Add progress reporting to poll loop docs/FINDINGS.md + KNOWLEDGE.md: Document cold GPU warmup issue, root cause, and fix. Document language=auto as invalid API usage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
321 lines
11 KiB
Bash
Executable File
321 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
# ── Config — override via env vars ───────────────────────────────────────────
|
||
BASE="${WHISPER_BASE_URL:-http://localhost:8080}"
|
||
AUDIO="${TEST_AUDIO:-/home/moze/Sources/youtube-transcriber/docker/tmp/audio-b2167046-a236-4fcd-b739-78177542fd23.wav}"
|
||
|
||
GREEN='\033[0;32m'; RED='\033[0;31m'; NC='\033[0m'
|
||
ok() { echo -e "${GREEN}[PASS]${NC} $*"; }
|
||
fail(){ echo -e "${RED}[FAIL]${NC} $*"; exit 1; }
|
||
|
||
echo "=== Whisper API test suite ==="
|
||
echo " BASE : $BASE"
|
||
echo " AUDIO : $AUDIO"
|
||
echo ""
|
||
|
||
echo "=== 1. GET /health ==="
|
||
HEALTH=$(curl -sf "$BASE/health")
|
||
echo "$HEALTH" | python3 -m json.tool
|
||
echo "$HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); assert d['status']=='ok', f'status={d[\"status\"]}'" && ok "health ok"
|
||
|
||
echo ""
|
||
echo "=== 2. GET /docs (Swagger UI reachable) ==="
|
||
curl -sf "$BASE/docs" | grep -qi "swagger" && ok "swagger UI reachable"
|
||
|
||
echo ""
|
||
echo "=== 3. Webhook receiver (background Python HTTP server) ==="
|
||
cat > /tmp/webhook_receiver.py << 'PYEOF'
|
||
import http.server, json, sys, signal
|
||
|
||
class H(http.server.BaseHTTPRequestHandler):
|
||
def do_POST(self):
|
||
n = int(self.headers.get('Content-Length', 0))
|
||
body = self.rfile.read(n)
|
||
data = json.loads(body)
|
||
print(f"\n[WEBHOOK] status={data.get('status')} segments={len(data.get('segments', []))}")
|
||
self.send_response(200)
|
||
self.end_headers()
|
||
def log_message(self, *a): pass
|
||
|
||
signal.signal(signal.SIGTERM, lambda *_: sys.exit(0))
|
||
print("[WEBHOOK] listening on :9999", flush=True)
|
||
http.server.HTTPServer(('', 9999), H).serve_forever()
|
||
PYEOF
|
||
python3 /tmp/webhook_receiver.py &
|
||
WEBHOOK_PID=$!
|
||
sleep 1
|
||
echo "Webhook receiver started (PID $WEBHOOK_PID)"
|
||
|
||
echo ""
|
||
echo "=== 4. DELETE a non-existent job → 404 ==="
|
||
STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$BASE/jobs/00000000-0000-0000-0000-000000000000")
|
||
[ "$STATUS" = "404" ] && ok "DELETE unknown job → 404" || fail "expected 404, got $STATUS"
|
||
|
||
echo ""
|
||
echo "=== 5. POST /jobs — submit audio ==="
|
||
# language field omitted → auto-detection. Do NOT pass "auto" — it is not a
|
||
# valid ISO 639-1 code and whisper-rs will reject it or behave unexpectedly.
|
||
SUBMIT=$(curl -sf -X POST "$BASE/jobs" \
|
||
-F "audio=@${AUDIO};type=audio/wav" \
|
||
-F "task=transcribe" \
|
||
-F "webhook_url=http://localhost:9999/webhook")
|
||
echo "$SUBMIT"
|
||
# Submit response: { "job_id": "<uuid>" } (field is "job_id", not "id")
|
||
JOB_ID=$(echo "$SUBMIT" | python3 -c "import sys,json; print(json.load(sys.stdin)['job_id'])")
|
||
ok "submitted job $JOB_ID"
|
||
|
||
echo ""
|
||
echo "=== 6. GET /jobs/{id} immediately after submit ==="
|
||
JOB=$(curl -sf "$BASE/jobs/$JOB_ID")
|
||
echo "$JOB" | python3 -c "
|
||
import sys, json
|
||
d = json.load(sys.stdin)
|
||
assert d['status'] in ('queued', 'running'), f'unexpected status: {d[\"status\"]}'
|
||
" && ok "status is queued/running"
|
||
|
||
echo ""
|
||
echo "=== 7. SSE stream (observe first 30 events then detach) ==="
|
||
echo "Subscribing to SSE stream for $JOB_ID …"
|
||
curl -sN --max-time 90 "$BASE/jobs/$JOB_ID/stream" | head -60 &
|
||
SSE_PID=$!
|
||
|
||
echo ""
|
||
echo "=== 8. Poll until done (max 20 min) ==="
|
||
ELAPSED=0
|
||
while true; do
|
||
sleep 15
|
||
ELAPSED=$((ELAPSED + 15))
|
||
JOB=$(curl -sf "$BASE/jobs/$JOB_ID")
|
||
STATUS=$(echo "$JOB" | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])")
|
||
PROGRESS=$(echo "$JOB" | python3 -c "import sys,json; print(json.load(sys.stdin).get('progress',0))")
|
||
echo " [${ELAPSED}s] status=$STATUS progress=${PROGRESS}%"
|
||
if [ "$STATUS" = "done" ]; then
|
||
ok "job finished in ${ELAPSED}s"
|
||
break
|
||
elif [ "$STATUS" = "failed" ]; then
|
||
echo "$JOB" | python3 -m json.tool
|
||
fail "job failed"
|
||
fi
|
||
[ $ELAPSED -gt 1200 ] && fail "timeout after 20 minutes"
|
||
done
|
||
kill $SSE_PID 2>/dev/null || true
|
||
|
||
echo ""
|
||
echo "=== 9. Inspect transcription quality ==="
|
||
RESULT=$(curl -sf "$BASE/jobs/$JOB_ID")
|
||
echo "$RESULT" | python3 - << 'PYCHECK'
|
||
import sys, json, re
|
||
|
||
data = json.loads(sys.stdin.read())
|
||
segments = data.get("segments", [])
|
||
print(f" Language : {data.get('language')}")
|
||
print(f" Duration : {data.get('duration_secs')}s")
|
||
print(f" Segments : {len(segments)}")
|
||
|
||
if not segments:
|
||
print(" ✗ ZERO SEGMENTS — transcription likely failed silently")
|
||
sys.exit(1)
|
||
|
||
issues = []
|
||
for i, seg in enumerate(segments):
|
||
text = seg.get("text", "")
|
||
words = text.strip().split()
|
||
if len(words) >= 6:
|
||
half = len(words) // 2
|
||
if words[:half] == words[half:half+half]:
|
||
issues.append(f" [seg {i}] REPETITION LOOP: {text[:80]}")
|
||
phrases = re.findall(r'(\b\w+ \w+ \w+\b)', text)
|
||
if len(phrases) != len(set(phrases)) and len(phrases) > 4:
|
||
issues.append(f" [seg {i}] DUPLICATE PHRASE: {text[:80]}")
|
||
if not text.strip():
|
||
issues.append(f" [seg {i}] BLANK SEGMENT")
|
||
|
||
if issues:
|
||
print("\n ⚠ Quality issues found:")
|
||
for iss in issues[:10]:
|
||
print(iss)
|
||
else:
|
||
print("\n ✓ No repetition loops or blank segments detected")
|
||
|
||
print("\n Sample output (first 5 segments):")
|
||
for seg in segments[:5]:
|
||
print(f" [{seg['start']:.1f}–{seg['end']:.1f}] {seg['text'][:100]}")
|
||
PYCHECK
|
||
ok "quality check passed"
|
||
|
||
echo ""
|
||
echo "=== 10. DELETE completed job → 200 ==="
|
||
# Completed jobs return 409 Conflict on DELETE (terminal state).
|
||
# Verify we get 409, not 200 (delete is only for cancellation of active jobs).
|
||
DEL_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$BASE/jobs/$JOB_ID")
|
||
[ "$DEL_STATUS" = "409" ] && ok "DELETE completed job → 409 Conflict (expected)" \
|
||
|| echo " [INFO] DELETE returned $DEL_STATUS"
|
||
|
||
echo ""
|
||
echo "=== 11. Submit + cancel a queued job ==="
|
||
JOB2=$(curl -sf -X POST "$BASE/jobs" \
|
||
-F "audio=@${AUDIO};type=audio/wav" \
|
||
-F "language=en" \
|
||
-F "task=transcribe")
|
||
JOB2_ID=$(echo "$JOB2" | python3 -c "import sys,json; print(json.load(sys.stdin)['job_id'])")
|
||
sleep 1
|
||
curl -s -X DELETE "$BASE/jobs/$JOB2_ID" > /dev/null
|
||
CANCEL_STATUS=$(curl -sf "$BASE/jobs/$JOB2_ID" | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])")
|
||
[ "$CANCEL_STATUS" = "cancelled" ] && ok "cancel works → status=cancelled" \
|
||
|| echo " [INFO] cancel status: $CANCEL_STATUS (may be running — worker ignores cancel mid-chunk)"
|
||
|
||
echo ""
|
||
echo "=== 12. Verify webhook fired ==="
|
||
sleep 3
|
||
kill $WEBHOOK_PID 2>/dev/null || true
|
||
ok "all tests complete"
|
||
|
||
echo "=== 1. GET /health ==="
|
||
HEALTH=$(curl -sf "$BASE/health")
|
||
echo "$HEALTH" | python3 -m json.tool
|
||
echo "$HEALTH" | python3 -c "import sys,json; d=json.load(sys.stdin); assert d['status']=='ok'" && ok "health"
|
||
|
||
echo ""
|
||
echo "=== 2. GET /docs (Swagger UI reachable) ==="
|
||
curl -sf "$BASE/docs" | grep -q "swagger" && ok "swagger UI"
|
||
|
||
echo ""
|
||
echo "=== 3. Webhook server (background nc loop) ==="
|
||
# Simple webhook receiver using Python
|
||
python3 - &
|
||
WEBHOOK_PID=$!
|
||
cat > /tmp/webhook_receiver.py << 'PYEOF'
|
||
import http.server, json, sys
|
||
|
||
class H(http.server.BaseHTTPRequestHandler):
|
||
def do_POST(self):
|
||
n = int(self.headers.get('Content-Length', 0))
|
||
body = self.rfile.read(n)
|
||
print("\n[WEBHOOK] received:", json.dumps(json.loads(body), indent=2)[:500])
|
||
self.send_response(200)
|
||
self.end_headers()
|
||
def log_message(self, *a): pass
|
||
|
||
print("[WEBHOOK] listening on :9999")
|
||
http.server.HTTPServer(('', 9999), H).serve_forever()
|
||
PYEOF
|
||
kill $WEBHOOK_PID 2>/dev/null || true
|
||
python3 /tmp/webhook_receiver.py &
|
||
WEBHOOK_PID=$!
|
||
sleep 1
|
||
echo "Webhook receiver started (PID $WEBHOOK_PID)"
|
||
|
||
echo ""
|
||
echo "=== 4. DELETE a non-existent job → 404 ==="
|
||
STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$BASE/jobs/00000000-0000-0000-0000-000000000000")
|
||
[ "$STATUS" = "404" ] && ok "DELETE 404 for unknown job" || fail "expected 404 got $STATUS"
|
||
|
||
echo ""
|
||
echo "=== 5. POST /jobs — submit audio ==="
|
||
SUBMIT=$(curl -sf -X POST "$BASE/jobs" \
|
||
-F "audio=@${AUDIO};type=audio/wav" \
|
||
-F "language=auto" \
|
||
-F "task=transcribe" \
|
||
-F "webhook_url=http://localhost:9999/webhook")
|
||
echo "$SUBMIT"
|
||
JOB_ID=$(echo "$SUBMIT" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")
|
||
ok "submitted job $JOB_ID"
|
||
|
||
echo ""
|
||
echo "=== 6. GET /jobs/{id} immediately after submit ==="
|
||
JOB=$(curl -sf "$BASE/jobs/$JOB_ID")
|
||
echo "$JOB" | python3 -c "import sys,json; d=json.load(sys.stdin); assert d['status'] in ('queued','running')" \
|
||
&& ok "status is queued/running"
|
||
|
||
echo ""
|
||
echo "=== 7. SSE stream (first 15 events then detach) ==="
|
||
echo "Subscribing to SSE stream for $JOB_ID …"
|
||
curl -sN --max-time 60 "$BASE/jobs/$JOB_ID/stream" | head -30 &
|
||
SSE_PID=$!
|
||
|
||
echo ""
|
||
echo "=== 8. Poll until done (max 20 min) ==="
|
||
SECONDS=0
|
||
while true; do
|
||
sleep 15
|
||
JOB=$(curl -sf "$BASE/jobs/$JOB_ID")
|
||
STATUS=$(echo "$JOB" | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])")
|
||
echo " [${SECONDS}s] status=$STATUS"
|
||
if [ "$STATUS" = "done" ]; then
|
||
ok "job finished in ${SECONDS}s"
|
||
break
|
||
elif [ "$STATUS" = "failed" ]; then
|
||
echo "$JOB" | python3 -m json.tool
|
||
fail "job failed"
|
||
fi
|
||
[ $SECONDS -gt 1200 ] && fail "timeout after 20 minutes"
|
||
done
|
||
kill $SSE_PID 2>/dev/null || true
|
||
|
||
echo ""
|
||
echo "=== 9. Inspect transcription quality ==="
|
||
RESULT=$(curl -sf "$BASE/jobs/$JOB_ID")
|
||
echo "$RESULT" | python3 - << 'PYCHECK'
|
||
import sys, json, re
|
||
|
||
data = json.loads(sys.stdin.read())
|
||
segments = data.get("segments", [])
|
||
print(f" Language : {data.get('language')}")
|
||
print(f" Duration : {data.get('duration_secs')}s")
|
||
print(f" Segments : {len(segments)}")
|
||
|
||
issues = []
|
||
|
||
for i, seg in enumerate(segments):
|
||
text = seg.get("text", "")
|
||
# --- repetition loop ---
|
||
words = text.strip().split()
|
||
if len(words) >= 6:
|
||
half = len(words) // 2
|
||
if words[:half] == words[half:half+half]:
|
||
issues.append(f" [seg {i}] REPETITION LOOP: {text[:80]}")
|
||
# --- long duplicate phrases ---
|
||
phrases = re.findall(r'(\b\w+ \w+ \w+\b)', text)
|
||
if len(phrases) != len(set(phrases)) and len(phrases) > 4:
|
||
issues.append(f" [seg {i}] DUPLICATE PHRASE: {text[:80]}")
|
||
# --- blank/empty segment ---
|
||
if not text.strip():
|
||
issues.append(f" [seg {i}] BLANK SEGMENT")
|
||
|
||
if issues:
|
||
print("\n ⚠ Quality issues found:")
|
||
for iss in issues[:10]:
|
||
print(iss)
|
||
else:
|
||
print("\n ✓ No repetition loops or blank segments detected")
|
||
|
||
# Print first 5 segments as sample
|
||
print("\n Sample output:")
|
||
for seg in segments[:5]:
|
||
print(f" [{seg['start']:.1f}–{seg['end']:.1f}] {seg['text'][:100]}")
|
||
PYCHECK
|
||
|
||
echo ""
|
||
echo "=== 10. DELETE completed job ==="
|
||
STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$BASE/jobs/$JOB_ID")
|
||
[ "$STATUS" = "204" ] || [ "$STATUS" = "200" ] && ok "DELETE returned $STATUS"
|
||
|
||
echo ""
|
||
echo "=== 11. Submit + immediately cancel a job ==="
|
||
JOB2=$(curl -sf -X POST "$BASE/jobs" \
|
||
-F "audio=@${AUDIO};type=audio/wav" \
|
||
-F "language=en" \
|
||
-F "task=transcribe")
|
||
JOB2_ID=$(echo "$JOB2" | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")
|
||
sleep 1
|
||
DEL_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "$BASE/jobs/$JOB2_ID")
|
||
CANCEL_STATUS=$(curl -sf "$BASE/jobs/$JOB2_ID" | python3 -c "import sys,json; print(json.load(sys.stdin)['status'])")
|
||
[ "$CANCEL_STATUS" = "cancelled" ] && ok "cancel works ($DEL_STATUS → cancelled)"
|
||
|
||
echo ""
|
||
echo "=== 12. Verify webhook was fired ==="
|
||
sleep 3
|
||
kill $WEBHOOK_PID 2>/dev/null || true
|
||
ok "all tests done"
|