mozempkandClaude Fable 5
597b882743
--slot-save-path + --cache-reuse 256 on both duo members, ./kv-cache mounted at /cache. Measured on ornith (3K-token system prompt): warm prefix reuse 296ms vs 106.7s cold prefill (361x); slot save 84MB in 59ms, GDN recurrent state serializes. scripts/duo-warmboot.sh wraps save/restore/status via llama-swap /upstream passthrough. Restore after restart: run 'duo-warmboot.sh restore' post stack boot. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
43 lines
1.5 KiB
Bash
Executable File
43 lines
1.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# duo-warmboot.sh — save/restore the duo's prompt KV caches across stack restarts.
|
|
#
|
|
# The duo servers run with --slot-save-path (/cache/<member>, host ./kv-cache/)
|
|
# and --cache-reuse 256. Within a running stack, prefix reuse is automatic
|
|
# (measured 296ms vs 107s cold for a 3K-token system prompt). This script
|
|
# covers the gap: stack restarts lose the in-memory KV, so save after the
|
|
# first real request and restore after every restart.
|
|
#
|
|
# Usage:
|
|
# duo-warmboot.sh save # snapshot slot 0 of both duo members
|
|
# duo-warmboot.sh restore # restore snapshots (skips missing files)
|
|
# duo-warmboot.sh status # list snapshot files
|
|
#
|
|
# Typical flow:
|
|
# docker restart llama_swap_stack && duo-warmboot.sh restore
|
|
set -u
|
|
BASE=http://127.0.0.1:48080
|
|
FILE=pi-boot.bin
|
|
declare -A DIRS=( [ornith-35b-duo]=ornith-duo [qwen3-4b-duo]=qwen-duo )
|
|
CACHE_ROOT="$(cd "$(dirname "$0")/.." && pwd)/kv-cache"
|
|
|
|
cmd="${1:-status}"
|
|
case "$cmd" in
|
|
save|restore)
|
|
for model in "${!DIRS[@]}"; do
|
|
out=$(curl -s --max-time 900 -X POST \
|
|
"$BASE/upstream/$model/slots/0?action=$cmd" \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"filename\":\"$FILE\"}")
|
|
echo "$model $cmd → $out"
|
|
done
|
|
;;
|
|
status)
|
|
for model in "${!DIRS[@]}"; do
|
|
f="$CACHE_ROOT/${DIRS[$model]}/$FILE"
|
|
[ -f "$f" ] && echo "$model: $(ls -lh "$f" | awk '{print $5, $6, $7, $8}')" \
|
|
|| echo "$model: no snapshot"
|
|
done
|
|
;;
|
|
*) echo "usage: $0 save|restore|status"; exit 1 ;;
|
|
esac
|