feat(slot-proxy): live load/prefill/decode progress on :48090

Add a slot-cache-proxy service (profile stack) in front of llama-swap-stack.
Injects return_progress into streaming chat, taps llama.cpp prompt_progress
SSE, serves GET /progress {loading|prefill|generating|idle} with pct/tok_s/
ETA. The pi-load-progress extension polls it; point pi llmruntime baseUrl at
http://localhost:48090/v1. host net + pid:host (load tracker reads the
containerised llama-server /proc/io). Transparent superset of llama-swap
(no endpoint lost; /progress is net-new). Script mirrors llmruntime-x570.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -32,3 +32,6 @@ Thumbs.db
 # Local overrides (never commit secrets or machine-specific tweaks)
 .env.local
 envs/.env.*.local
+
+# slot-cache-proxy runtime (KV bins + proxy.log)
+slot-cache/

compose.yaml

@@ -306,6 +306,40 @@ services:
       llama-net:
         aliases: [llama-current]
 
+  # ── SLOT-CACHE PROXY — live load/prefill/decode progress on :48090 ────────
+  # Sits in front of llama-swap-stack (:48080). Injects return_progress into
+  # streaming chat requests, taps llama.cpp prompt_progress SSE, and serves
+  # GET /progress {loading | prefill | generating | idle} with pct/tok_s/ETA.
+  # The pi-load-progress extension polls it — point pi's llmruntime baseUrl at
+  # http://localhost:48090/v1. host net reaches the published :48080; pid:host
+  # lets the load tracker read the (containerised) llama-server /proc/<pid>/io.
+  #   docker compose --profile stack up -d slot-proxy
+  # Script source of truth: llmruntime-x570 scripts/slot-cache-proxy.py.
+  slot-proxy:
+    image: python:3.12-slim
+    container_name: slot-proxy
+    profiles: [stack]
+    network_mode: host
+    pid: host
+    depends_on:
+      - llama-swap-stack
+    volumes:
+      - ./swap-stack/slot-cache-proxy.py:/app/slot-cache-proxy.py:ro
+      - ./models:/models:ro
+      - ./slot-cache:/slot-cache
+    command:
+      - python3
+      - /app/slot-cache-proxy.py
+      - --listen
+      - "48090"
+      - --upstream
+      - 127.0.0.1:48080
+      - --cache-dir
+      - /slot-cache
+      - --timeout
+      - "7200"
+    restart: unless-stopped
+
   # ── DUO WARMBOOT (RETIRED 2026-07-10) ───────────────────────────────────
   # The auto-save sidecar caused request timeouts: slot save/restore queues on
   # the same slot as generation, so a 200MB periodic save of a deep cache

swap-stack/slot-cache-proxy.py

@@ -0,0 +1,616 @@
+#!/usr/bin/env python3
+"""
+slot-cache-proxy — transparent KV-cache persistence for agent sessions.
+
+Sits between clients (Pi, curl, IDEs) and llama-swap. For models listed in
+PERSIST_MODELS it makes prompt caches survive server restarts and model swaps:
+
+  request in  -> derive session key from (system prompt + first user message)
+              -> if the server slot is cold for this session and a saved cache
+                 file exists: POST /slots/0?action=restore   (~100ms)
+              -> forward the request (streaming passthrough)
+  response out-> POST /slots/0?action=save to <key>.bin      (~300ms, async)
+
+Requires the model entry to run with:  --slots --slot-save-path /slot-cache
+and the llama-swap container to mount that path read-write.
+
+Constraint inherited from dsv4's append-only compressed KV cache: a resumed
+transcript must be byte-identical to the saved one (every assistant turn, same
+chat_template_kwargs). If it diverges the server silently falls back to full
+prefill — correctness is unaffected; we log timings.cache_n so misses are
+visible in slot-cache/proxy.log.
+
+Stdlib only. Run:  python3 slot-cache-proxy.py [--listen 48090] [--upstream 127.0.0.1:48080]
+"""
+import argparse
+import collections
+import hashlib
+import json
+import os
+import re
+import sys
+import threading
+import time
+from datetime import datetime, timezone
+from http.client import HTTPConnection
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+# model ids (incl. aliases) whose sessions we persist -> canonical upstream name.
+# NOTE: this proxy's cache-persistence is a SINGLE-SLOT design (restore/save target
+# slot 0, guarded by a global lock held through the whole generation). It is only
+# valid for --parallel 1 models. QCN now runs --parallel 6 (multi-slot); routing it
+# through the locked path serialized the 6 slots and cost ~22% aggregate throughput
+# (2026-07-15). QCN removed → it takes the unlocked passthrough (0% overhead, full
+# 43 tok/s). Persistence stays for the single-slot flash-284b. flash-162b retired.
+PERSIST_MODELS = {
+    "flash-284b": "flash-284b",
+    "deepseek-v4-flash-284b": "flash-284b",
+}
+SLOT_ID = 0            # --parallel 1: single slot
+KEEP_FILES = 24        # prune saved caches beyond this many (LRU by mtime)
+CHAT_PATH = "/v1/chat/completions"
+
+HOP_BY_HOP = {
+    "connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
+    "te", "trailers", "transfer-encoding", "upgrade",
+}
+
+
+def now():
+    return datetime.now(timezone.utc).astimezone().strftime("%H:%M:%S")
+
+
+class State:
+    def __init__(self):
+        self.lock = threading.Lock()   # serializes the whole intercept path
+        self.hot_key = None            # session key currently in the live slot
+
+
+STATE = State()
+ARGS = None
+LOGF = None
+
+
+# ─────────────────────────────────────────────────────────────────────────
+# Model load-progress tracker (GET /progress).
+#
+# Big local models page 60-210 GB from NVMe on load (10-20 min) with NO
+# progress output from llama.cpp — the ONLY real-time signal is the
+# llama-server subprocess's /proc/<pid>/io read_bytes vs the model's total
+# GGUF size. This container runs with `pid: host` so it sees that pid, and
+# mounts the models dir read-only to size the shards. A background thread
+# samples every ~2 s; GET /progress returns the latest snapshot (no lock,
+# never queues behind a generation). See docs: parallel-wandering-frog plan.
+# ─────────────────────────────────────────────────────────────────────────
+class LoaderState:
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.data = {"state": "idle"}
+        self.samples = collections.deque(maxlen=30)  # (t, read_bytes) rolling
+        self.pid = None
+
+
+LOADER = LoaderState()
+_GGUF_CACHE = {}
+_CLK = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
+
+
+# ─────────────────────────────────────────────────────────────────────────
+# Generation-progress tracker (prefill + decode), surfaced via GET /progress.
+#
+# The dead gap a client sees as "processing" is PREFILL (prompt eval): the
+# server is filling KV for the whole prompt and emits no tokens yet. llama.cpp
+# CAN report it if the request carries "return_progress": true — it then streams
+#   prompt_progress: {total, cache, processed, time_ms}
+# SSE chunks (delta.content null) before the first token. We inject that flag on
+# streaming chat requests, tap the passthrough stream, and publish a live
+# snapshot. `cache` = tokens already in KV (slot-cache restore / shared prefix),
+# so a restored session shows most of the prompt pre-filled.
+#
+# Single shared snapshot: with multiple parallel slots the footer shows the
+# last writer, which is fine for one interactive user. active-count guards the
+# idle reset so an ended gen doesn't wipe a still-running one.
+# ─────────────────────────────────────────────────────────────────────────
+class GenState:
+    def __init__(self):
+        self.lock = threading.Lock()
+        self.data = {"state": "idle"}
+        self.active = 0
+
+
+GENSTATE = GenState()
+
+
+def _gib(b):
+    return round((b or 0) / (1024 ** 3), 1)
+
+
+def find_llama_server():
+    """First llama-server subprocess with --model/--port (llama-swap runs one
+    at a time). Returns (pid, model_path, port) or (None, None, None)."""
+    for pid in os.listdir("/proc"):
+        if not pid.isdigit():
+            continue
+        try:
+            with open(f"/proc/{pid}/cmdline", "rb") as fh:
+                args = [p.decode("utf-8", "ignore") for p in fh.read().split(b"\x00") if p]
+        except OSError:
+            continue
+        if not any(a.rsplit("/", 1)[-1] == "llama-server" for a in args):
+            continue
+        if "--model" not in args or "--port" not in args:
+            continue
+        try:
+            model = args[args.index("--model") + 1]
+            port = int(args[args.index("--port") + 1])
+        except (ValueError, IndexError):
+            continue
+        return int(pid), model, port
+    return None, None, None
+
+
+def gguf_total(model_path):
+    """Sum all `-NNNNN-of-NNNNN.gguf` shards (or a single file). Cached."""
+    if model_path in _GGUF_CACHE:
+        return _GGUF_CACHE[model_path]
+    total = 0
+    m = re.search(r"-(\d{5})-of-(\d{5})\.gguf$", model_path)
+    try:
+        if m:
+            nn, base = m.group(2), model_path[:m.start()]
+            for i in range(1, int(nn) + 1):
+                total += os.path.getsize(f"{base}-{i:05d}-of-{nn}.gguf")
+        else:
+            total = os.path.getsize(model_path)
+    except OSError:
+        total = 0
+    _GGUF_CACHE[model_path] = total
+    return total
+
+
+def read_io_bytes(pid):
+    try:
+        with open(f"/proc/{pid}/io") as fh:
+            for line in fh:
+                if line.startswith("read_bytes:"):
+                    return int(line.split()[1])
+    except OSError:
+        pass
+    return None
+
+
+def proc_age(pid):
+    """Seconds since the process spawned (robust across sampler restarts)."""
+    try:
+        with open("/proc/stat") as fh:
+            btime = next(int(l.split()[1]) for l in fh if l.startswith("btime "))
+        with open(f"/proc/{pid}/stat") as fh:
+            starttime = int(fh.read().rsplit(")", 1)[1].split()[19])
+        return max(0.0, time.time() - (btime + starttime / _CLK))
+    except Exception:
+        return None
+
+
+def running_map():
+    """port -> (model_id, state) from llama-swap /running (:48080, reachable).
+    The model's own 127.0.0.1:<port>/health is NOT reachable (llama-swap is
+    bridge-networked), so llama-swap's state field is our ready/loading signal."""
+    st, data = upstream_json("GET", "/running")
+    out = {}
+    if st == 200 and isinstance(data, dict):
+        for e in data.get("running", []):
+            proxy = e.get("proxy") or ""
+            port = proxy.rsplit(":", 1)[-1] if ":" in proxy else ""
+            if port.isdigit():
+                out[int(port)] = (e.get("model"), e.get("state"))
+    return out
+
+
+def _sample_once():
+    pid, model_path, port = find_llama_server()
+    if not pid:
+        return {"state": "idle"}, None
+    mid, rstate = running_map().get(port, (None, None))
+    if not mid:
+        mid = os.path.basename(model_path).split("-0000")[0]
+    total = gguf_total(model_path)
+    read = read_io_bytes(pid) or 0
+    age = proc_age(pid)
+    elapsed = round(age) if age is not None else None
+    # A llama-server process exists = the model is loading OR loaded. llama-swap
+    # marks it "ready" once the health check passes; anything else (starting, or
+    # not yet listed) means still paging in. read_bytes is cumulative (it keeps
+    # climbing as experts stream post-load), so cap the reported read at total.
+    read_disp = min(read, total) if total else read
+    if rstate == "ready":
+        return {"state": "ready", "model": mid, "pct": 100.0,
+                "read_gib": _gib(read_disp), "total_gib": _gib(total),
+                "eta_sec": 0, "elapsed_sec": elapsed}, pid
+    pct = round(min(100.0, read / total * 100), 1) if total else None
+    return {"state": "loading", "model": mid, "pct": pct,
+            "read_gib": _gib(read_disp), "total_gib": _gib(total),
+            "elapsed_sec": elapsed, "_read": read, "_total": total}, pid
+
+
+def loader_sampler():
+    while True:
+        try:
+            data, pid = _sample_once()
+            with LOADER.lock:
+                if pid != LOADER.pid:
+                    LOADER.samples.clear()
+                    LOADER.pid = pid
+                if data.get("state") == "loading" and data.get("_total"):
+                    LOADER.samples.append((time.time(), data["_read"]))
+                    if len(LOADER.samples) >= 2:
+                        (t0, b0), (t1, b1) = LOADER.samples[0], LOADER.samples[-1]
+                        dt, db = t1 - t0, b1 - b0
+                        if dt > 0 and db > 0:
+                            rate = db / dt
+                            data["rate_mib_s"] = round(rate / (1024 ** 2))
+                            data["eta_sec"] = max(0, round((data["_total"] - b1) / rate))
+                for k in ("_read", "_total"):
+                    data.pop(k, None)
+                LOADER.data = data
+        except Exception as e:
+            with LOADER.lock:
+                LOADER.data = {"state": "idle", "error": str(e)[:100]}
+        time.sleep(2)
+
+
+def log(msg):
+    line = f"[{now()}] {msg}"
+    print(line, flush=True)
+    if LOGF:
+        LOGF.write(line + "\n")
+        LOGF.flush()
+
+
+def session_key(model, messages):
+    """Stable session identity: canonical hash of the first two messages."""
+    if len(messages) < 2:
+        return None
+    basis = json.dumps([model, messages[0], messages[1]],
+                       sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+    return hashlib.sha256(basis.encode()).hexdigest()[:24]
+
+
+def upstream_conn():
+    host, port = ARGS.upstream.rsplit(":", 1)
+    return HTTPConnection(host, int(port), timeout=ARGS.timeout)
+
+
+def upstream_json(method, path, body=None):
+    """Small JSON request to llama-swap. Returns (status, parsed-or-None)."""
+    conn = upstream_conn()
+    try:
+        payload = json.dumps(body).encode() if body is not None else None
+        headers = {"Content-Type": "application/json"} if payload else {}
+        conn.request(method, path, body=payload, headers=headers)
+        resp = conn.getresponse()
+        raw = resp.read()
+        try:
+            return resp.status, json.loads(raw)
+        except Exception:
+            return resp.status, None
+    finally:
+        conn.close()
+
+
+def slot_action(model, action, filename):
+    t0 = time.time()
+    status, data = upstream_json(
+        "POST", f"/upstream/{model}/slots/{SLOT_ID}?action={action}",
+        {"filename": filename})
+    ms = (time.time() - t0) * 1000
+    return status, data, ms
+
+
+def slot_is_cold(model):
+    """True if the live slot holds no prompt (fresh load / swapped back in).
+    Triggers a model load if it isn't resident — same load the chat request
+    would trigger anyway, just earlier."""
+    status, data = upstream_json("GET", f"/upstream/{model}/slots")
+    if status != 200 or not isinstance(data, list) or not data:
+        return True  # can't tell -> treat as cold; restore is cheap/harmless
+    slot = data[0]
+    # this build exposes no "prompt" text; a never-used slot has id_task -1
+    # and zero prompt tokens
+    return slot.get("id_task", -1) < 0 and not slot.get("n_prompt_tokens")
+
+
+def cache_file(key):
+    return os.path.join(ARGS.cache_dir, f"{key}.bin")
+
+
+def prune_cache_dir():
+    try:
+        files = [os.path.join(ARGS.cache_dir, f)
+                 for f in os.listdir(ARGS.cache_dir) if f.endswith(".bin")]
+        files.sort(key=os.path.getmtime, reverse=True)
+        for f in files[KEEP_FILES:]:
+            os.unlink(f)
+            log(f"prune:   {os.path.basename(f)}")
+    except OSError as e:
+        log(f"prune error: {e}")
+
+
+def parse_timings(tail_bytes):
+    """Extract timings from a response tail: final SSE data chunk or plain JSON."""
+    try:
+        text = tail_bytes.decode("utf-8", "ignore")
+        candidates = re.findall(r"data: (\{.*\})", text) or [text]
+        for c in reversed(candidates):
+            try:
+                obj = json.loads(c)
+            except Exception:
+                continue
+            t = obj.get("timings")
+            if t:
+                return t
+    except Exception:
+        pass
+    return None
+
+
+class ScrapeCtx:
+    """Taps a streaming chat response, line by line, and publishes prefill/decode
+    progress to GENSTATE. Cheap: one json.loads per SSE data line."""
+
+    def __init__(self, model):
+        self.model = model
+        self.buf = b""
+        self.phase = "prefill"
+        self.total = self.processed = self.cache = 0
+        self.gen_tokens = 0
+        self.t_first = None
+
+    def begin(self):
+        with GENSTATE.lock:
+            GENSTATE.active += 1
+            GENSTATE.data = {"state": "prefill", "model": self.model, "pct": 0.0}
+
+    def feed(self, chunk):
+        self.buf += chunk
+        while b"\n" in self.buf:
+            line, self.buf = self.buf.split(b"\n", 1)
+            s = line.strip()
+            if not s.startswith(b"data:"):
+                continue
+            payload = s[5:].strip()
+            if not payload or payload == b"[DONE]":
+                continue
+            try:
+                obj = json.loads(payload)
+            except Exception:
+                continue
+            self._event(obj)
+        if len(self.buf) > 262144:            # runaway guard (no newline seen)
+            self.buf = self.buf[-8192:]
+
+    def _event(self, obj):
+        pp = obj.get("prompt_progress")
+        if pp:
+            self.total = pp.get("total", self.total)
+            self.processed = pp.get("processed", self.processed)
+            self.cache = pp.get("cache", self.cache)
+            tms = pp.get("time_ms") or 0
+            pct = round(self.processed / self.total * 100, 1) if self.total else 0.0
+            toks = round(self.processed / (tms / 1000)) if tms > 0 else None
+            eta = round((self.total - self.processed) / toks) if toks else None
+            with GENSTATE.lock:
+                GENSTATE.data = {
+                    "state": "prefill", "model": self.model, "pct": pct,
+                    "done_tok": self.processed, "total_tok": self.total,
+                    "cached_tok": self.cache, "tok_s": toks, "eta_sec": eta}
+            return
+        delta = (obj.get("choices") or [{}])[0].get("delta") or {}
+        if delta.get("content") or delta.get("reasoning_content"):
+            if self.t_first is None:
+                self.t_first = time.time()
+                self.phase = "generating"
+            self.gen_tokens += 1
+            elapsed = time.time() - self.t_first
+            toks = round(self.gen_tokens / elapsed) if elapsed > 0 else None
+            with GENSTATE.lock:
+                GENSTATE.data = {
+                    "state": "generating", "model": self.model,
+                    "out_tok": self.gen_tokens, "tok_s": toks,
+                    "elapsed_sec": round(elapsed)}
+
+    def end(self):
+        with GENSTATE.lock:
+            GENSTATE.active = max(0, GENSTATE.active - 1)
+            if GENSTATE.active == 0:
+                GENSTATE.data = {"state": "idle"}
+
+
+class Handler(BaseHTTPRequestHandler):
+    protocol_version = "HTTP/1.1"
+
+    def log_message(self, *a):  # silence default access log
+        pass
+
+    # ---- generic passthrough for every method ----
+    def _handle(self):
+        body = b""
+        clen = self.headers.get("Content-Length")
+        if clen:
+            body = self.rfile.read(int(clen))
+        elif (self.headers.get("Transfer-Encoding") or "").lower() == "chunked":
+            body = self._read_chunked()
+
+        if self.command == "GET" and self.path.split("?")[0] == "/progress":
+            self._progress()
+        elif self.command == "POST" and self.path.split("?")[0] == CHAT_PATH:
+            self._chat(body)
+        else:
+            self._forward(body, intercept=False)
+
+    do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_OPTIONS = do_HEAD = _handle
+
+    # ---- progress snapshot (never takes STATE.lock) ----
+    # Priority: a model paging in (loading) blocks any generation, so it wins.
+    # Otherwise an in-flight prefill/decode wins over the loader's ready/idle.
+    def _progress(self):
+        with LOADER.lock:
+            load = dict(LOADER.data)
+        if load.get("state") == "loading":
+            snap = load
+        else:
+            with GENSTATE.lock:
+                gen = dict(GENSTATE.data)
+            snap = gen if gen.get("state") in ("prefill", "generating") else load
+        payload = json.dumps(snap).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def _read_chunked(self):
+        out = b""
+        while True:
+            size = int(self.rfile.readline().strip().split(b";")[0], 16)
+            if size == 0:
+                self.rfile.readline()
+                return out
+            out += self.rfile.read(size)
+            self.rfile.readline()
+
+    # ---- the interesting path ----
+    def _chat(self, body):
+        model, key, req, stream = None, None, None, False
+        try:
+            req = json.loads(body)
+            stream = bool(req.get("stream"))
+            model = PERSIST_MODELS.get(req.get("model", ""))
+            if model:
+                key = session_key(model, req.get("messages") or [])
+        except Exception:
+            pass
+
+        # Native prefill telemetry: ask the server to stream prompt_progress.
+        # Only meaningful for streaming requests; leave an explicit flag alone.
+        if req is not None and stream and "return_progress" not in req:
+            req["return_progress"] = True
+            body = json.dumps(req, ensure_ascii=False).encode()
+
+        scrape = {"model": req.get("model")} if (req is not None and stream) else None
+
+        if not (model and key):
+            self._forward(body, intercept=False, scrape=scrape)
+            return
+
+        with STATE.lock:
+            restored = "skip(hot)"
+            if STATE.hot_key != key or slot_is_cold(model):
+                if os.path.exists(cache_file(key)):
+                    st, data, ms = slot_action(model, "restore", f"{key}.bin")
+                    n = (data or {}).get("n_restored", (data or {}).get("n_read"))
+                    restored = f"restore({st},{ms:.0f}ms,n={n})"
+                    STATE.hot_key = key if st == 200 else None
+                else:
+                    restored = "miss(no-file)"
+                    STATE.hot_key = None
+            log(f"chat:    key={key} {restored}")
+
+            timings = self._forward(body, intercept=True, scrape=scrape)
+
+            cache_n = (timings or {}).get("cache_n")
+            prompt_n = (timings or {}).get("prompt_n")
+            st, _, ms = slot_action(model, "save", f"{key}.bin")
+            STATE.hot_key = key if st == 200 else None
+            log(f"done:    key={key} cache_n={cache_n} prompt_n={prompt_n} "
+                f"save({st},{ms:.0f}ms)")
+            if st == 200:
+                prune_cache_dir()
+
+    # ---- forwarding ----
+    def _forward(self, body, intercept, scrape=None):
+        conn = upstream_conn()
+        # Content-Length is recomputed from `body` (return_progress injection can
+        # change its length), so drop the client's copy from the pass-through set.
+        sctx = ScrapeCtx(scrape["model"]) if scrape else None
+        try:
+            headers = {k: v for k, v in self.headers.items()
+                       if k.lower() not in HOP_BY_HOP
+                       and k.lower() not in ("host", "content-length")}
+            headers["Host"] = ARGS.upstream
+            if intercept:
+                headers["Accept-Encoding"] = "identity"  # keep tail parseable
+            if body:
+                headers["Content-Length"] = str(len(body))
+            conn.request(self.command, self.path, body=body or None,
+                         headers=headers)
+            resp = conn.getresponse()
+
+            self.send_response(resp.status)
+            for k, v in resp.getheaders():
+                if k.lower() in HOP_BY_HOP or k.lower() == "content-length":
+                    continue
+                self.send_header(k, v)
+            self.send_header("Transfer-Encoding", "chunked")
+            self.end_headers()
+
+            if sctx:
+                sctx.begin()
+            tail = b""
+            while True:
+                # read1(): one recv, don't block for a full buffer. Critical for
+                # live streaming — prefill emits only tiny (~150 B) prompt_progress
+                # events, so a blocking read(16384) would hoard them until enough
+                # bytes piled up (i.e. until real tokens flow), collapsing the
+                # whole prefill phase into one late burst. read1 forwards each SSE
+                # event the instant llama-server flushes it.
+                chunk = resp.read1(16384)
+                if not chunk:
+                    break
+                if sctx:
+                    sctx.feed(chunk)
+                self.wfile.write(b"%x\r\n%s\r\n" % (len(chunk), chunk))
+                self.wfile.flush()
+                tail = (tail + chunk)[-65536:]
+            self.wfile.write(b"0\r\n\r\n")
+            self.wfile.flush()
+            return parse_timings(tail) if intercept else None
+        except Exception as e:
+            log(f"forward error {self.command} {self.path}: {e}")
+            try:
+                self.send_error(502, str(e))
+            except Exception:
+                pass
+            return None
+        finally:
+            if sctx:
+                sctx.end()
+            conn.close()
+
+
+def main():
+    global ARGS, LOGF
+    p = argparse.ArgumentParser()
+    p.add_argument("--listen", type=int, default=48090)
+    p.add_argument("--upstream", default="127.0.0.1:48080")
+    p.add_argument("--cache-dir", default=os.path.join(
+        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "slot-cache"))
+    p.add_argument("--timeout", type=float, default=3600.0)
+    ARGS = p.parse_args()
+
+    os.makedirs(ARGS.cache_dir, exist_ok=True)
+    LOGF = open(os.path.join(ARGS.cache_dir, "proxy.log"), "a")
+
+    threading.Thread(target=loader_sampler, daemon=True).start()
+
+    srv = ThreadingHTTPServer(("0.0.0.0", ARGS.listen), Handler)
+    log(f"slot-cache-proxy on :{ARGS.listen} -> {ARGS.upstream} "
+        f"(cache: {ARGS.cache_dir}, models: {sorted(set(PERSIST_MODELS.values()))}, "
+        f"progress: GET /progress [load+prefill+decode])")
+    try:
+        srv.serve_forever()
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    main()