From 169e6724d1e075388a369ec3d66a44f144af5168 Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:06:13 -0400 Subject: [PATCH 1/9] test: isolate DASHBOARD_DATA_PATH so pytest cannot pollute the live throughput store Co-Authored-By: Claude Fable 5 --- tests/conftest.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index e3bb3258..5687eb5d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,3 +22,14 @@ "AUDIT_LOG_PATH", str(Path(tempfile.gettempdir()) / "ordo-test-audit.jsonl"), ) + +# ``dashboard/app.py`` resolves DASHBOARD_DATA_PATH (default ``./data/dashboard``) +# at import time and loads/saves the throughput store there. Without an override, a +# local pytest run reads AND WRITES the production data dir — the live store was +# found carrying test_throughput_record_accepts_sample's literal payload +# ("test-model" @ 25.5 tok/s). Point it at a per-run temp dir before any test +# imports dashboard.app. +os.environ.setdefault( + "DASHBOARD_DATA_PATH", + tempfile.mkdtemp(prefix="ordo-test-dashboard-"), +) From 5c8b6a1b77a526cd7f19a63e34d28d817016f34d Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:10:44 -0400 Subject: [PATCH 2/9] dashboard: throughput store v2 - timestamped samples, 7-day eviction, clean reset of alias-conflated v1 data Also stops /api/throughput/benchmark double-recording under the requested alias (the gateway callback already records every completion). Co-Authored-By: Claude Fable 5 --- services/v1-parity/dashboard/app.py | 92 +++++++++++++++-------- tests/test_services_and_throughput.py | 103 +++++++++++++++++++++++--- 2 files changed, 153 insertions(+), 42 deletions(-) diff --git a/services/v1-parity/dashboard/app.py b/services/v1-parity/dashboard/app.py index b3154908..d254e1a9 100644 --- a/services/v1-parity/dashboard/app.py +++ b/services/v1-parity/dashboard/app.py @@ -1432,11 +1432,25 @@ async def mcp_remove(req: McpRemoveRequest): # --- Token Throughput --- -# In-memory store: model -> list of output_tokens_per_sec (rolling, max 500) -_throughput_samples: dict[str, list[float]] = {} -_ttft_samples: dict[str, list[float]] = {} +_throughput_samples: dict[str, list[dict]] = {} # {"tps": float, "ts": epoch} +_ttft_samples: dict[str, list[dict]] = {} # {"ms": float, "ts": epoch} _MAX_SAMPLES_PER_MODEL = 500 _MAX_TRACKED_MODELS = 50 +# v2: samples are timestamped dicts. v1 stored bare floats keyed largely by routing +# ALIAS (one `local-chat` bucket conflating every model ever active behind it, CPU +# failover included) — unattributable, so version bumps trigger a clean reset. +_THROUGHPUT_STORE_VERSION = 2 +_SAMPLE_MAX_AGE_SEC = 7 * 86400 # models with no sample in 7 days leave the store + + +def _evict_stale_models(now: float) -> None: + """Drop models whose newest sample is older than _SAMPLE_MAX_AGE_SEC. + Call while holding _state_lock.""" + cutoff = now - _SAMPLE_MAX_AGE_SEC + for store in (_throughput_samples, _ttft_samples): + stale = [m for m, s in store.items() if not s or s[-1]["ts"] < cutoff] + for m in stale: + del store[m] # Last benchmark result (persists across page refresh until dashboard restart) _last_benchmark: dict | None = None @@ -1451,15 +1465,29 @@ async def mcp_remove(req: McpRemoveRequest): def _load_throughput_state() -> None: - """Load throughput samples and last benchmark from disk (R4).""" + """Load throughput samples and last benchmark from disk (R4). v1 files (no + version field) get a clean reset — their samples are un-timestamped and + alias-conflated; only last_benchmark carries over.""" global _throughput_samples, _ttft_samples, _last_benchmark, _service_usage if not _THROUGHPUT_FILE.exists(): return try: data = json.loads(_THROUGHPUT_FILE.read_text(encoding="utf-8")) - _throughput_samples = {k: v for k, v in (data.get("samples") or {}).items() if isinstance(v, list)} - _ttft_samples = {k: v for k, v in (data.get("ttft_samples") or {}).items() if isinstance(v, list)} _last_benchmark = data.get("last_benchmark") if isinstance(data.get("last_benchmark"), dict) else None + if data.get("version") != _THROUGHPUT_STORE_VERSION: + logger.warning( + "Throughput store is v%s (want v%s) — resetting samples, keeping last_benchmark", + data.get("version", 1), _THROUGHPUT_STORE_VERSION, + ) + return + _throughput_samples = { + k: [s for s in v if isinstance(s, dict) and "tps" in s and "ts" in s] + for k, v in (data.get("samples") or {}).items() if isinstance(v, list) + } + _ttft_samples = { + k: [s for s in v if isinstance(s, dict) and "ms" in s and "ts" in s] + for k, v in (data.get("ttft_samples") or {}).items() if isinstance(v, list) + } _service_usage = [u for u in (data.get("service_usage") or []) if isinstance(u, dict)][-_MAX_SERVICE_USAGE:] except Exception as e: logger.warning("Throughput state load failed: %s", e) @@ -1471,6 +1499,7 @@ def _save_throughput_state() -> None: _THROUGHPUT_FILE.parent.mkdir(parents=True, exist_ok=True) tmp = _THROUGHPUT_FILE.with_suffix(".json.tmp") tmp.write_text(json.dumps({ + "version": _THROUGHPUT_STORE_VERSION, "samples": _throughput_samples, "ttft_samples": _ttft_samples, "last_benchmark": _last_benchmark, @@ -1516,6 +1545,8 @@ class ThroughputRecordRequest(BaseModel): output_tokens_per_sec: float = Field(default=0.0, ge=0, le=1e6) service: str = Field(default="", max_length=64) ttft_ms: float = Field(default=0.0, ge=0, le=1e6) + alias: str = Field(default="", max_length=256) + backend: str = Field(default="", max_length=64) @app.post("/api/throughput/record") @@ -1524,18 +1555,20 @@ async def throughput_record(req: ThroughputRecordRequest): model = req.model.strip() if not model or req.output_tokens_per_sec <= 0: return {"ok": True} + now = time.time() with _state_lock: + _evict_stale_models(now) if model not in _throughput_samples: if len(_throughput_samples) >= _MAX_TRACKED_MODELS: return {"ok": True} _throughput_samples[model] = [] - _throughput_samples[model].append(req.output_tokens_per_sec) + _throughput_samples[model].append({"tps": req.output_tokens_per_sec, "ts": now}) if len(_throughput_samples[model]) > _MAX_SAMPLES_PER_MODEL: _throughput_samples[model] = _throughput_samples[model][-_MAX_SAMPLES_PER_MODEL:] if req.ttft_ms > 0 and (model in _ttft_samples or len(_ttft_samples) < _MAX_TRACKED_MODELS): if model not in _ttft_samples: _ttft_samples[model] = [] - _ttft_samples[model].append(req.ttft_ms) + _ttft_samples[model].append({"ms": req.ttft_ms, "ts": now}) if len(_ttft_samples[model]) > _MAX_SAMPLES_PER_MODEL: _ttft_samples[model] = _ttft_samples[model][-_MAX_SAMPLES_PER_MODEL:] # Service usage (which service is taxing which model) @@ -1543,9 +1576,11 @@ async def throughput_record(req: ThroughputRecordRequest): _service_usage.append({ "model": model, "service": service, + "alias": req.alias.strip()[:256], + "backend": req.backend.strip()[:64], "tps": round(req.output_tokens_per_sec, 1), "ttft_ms": round(req.ttft_ms, 1) if req.ttft_ms > 0 else 0.0, - "ts": time.time(), + "ts": now, }) if len(_service_usage) > _MAX_SERVICE_USAGE: _service_usage[:] = _service_usage[-_MAX_SERVICE_USAGE:] @@ -1596,27 +1631,33 @@ async def throughput_service_usage(): @app.get("/api/throughput/stats") async def throughput_stats(): - """Return per-model throughput stats: peak, p50, p95, p99, latest, sample_count. Includes last_benchmark if available.""" + """Per-model throughput stats over timestamped samples: peak, p50/p95/p99, latest, + sample_count, first_ts/last_ts. Includes last_benchmark if available.""" result: dict[str, dict] = {} + now = time.time() with _state_lock: + _evict_stale_models(now) snapshot = {m: list(s) for m, s in _throughput_samples.items()} ttft_snapshot = {m: list(s) for m, s in _ttft_samples.items()} benchmark = dict(_last_benchmark) if _last_benchmark else None for model, samples in snapshot.items(): if not samples: continue - sorted_s = sorted(samples) - ttfts = ttft_snapshot.get(model, []) + tps_vals = [s["tps"] for s in samples] + sorted_s = sorted(tps_vals) + ttfts = [s["ms"] for s in ttft_snapshot.get(model, [])] sorted_ttfts = sorted(ttfts) result[model] = { - "latest": round(samples[-1], 1), - "peak": round(max(samples), 1), + "latest": round(tps_vals[-1], 1), + "peak": round(max(tps_vals), 1), "p50": round(_percentile(sorted_s, 50), 1), "p95": round(_percentile(sorted_s, 95), 1), "p99": round(_percentile(sorted_s, 99), 1), "ttft_p50_ms": round(_percentile(sorted_ttfts, 50), 1) if sorted_ttfts else 0.0, "ttft_p95_ms": round(_percentile(sorted_ttfts, 95), 1) if sorted_ttfts else 0.0, "sample_count": len(samples), + "first_ts": samples[0]["ts"], + "last_ts": samples[-1]["ts"], } out: dict = {"models": result, "ok": True} if benchmark: @@ -1638,20 +1679,22 @@ async def performance_summary(): for model, samples in snapshot.items(): if not samples: continue - sorted_s = sorted(samples) - ttfts = ttft_snapshot.get(model, []) + tps_vals = [s["tps"] for s in samples] + sorted_s = sorted(tps_vals) + ttfts = [s["ms"] for s in ttft_snapshot.get(model, [])] sorted_ttfts = sorted(ttfts) top_models.append( { "model": model, - "latest_tps": round(samples[-1], 1), + "latest_tps": round(tps_vals[-1], 1), "p95_tps": round(_percentile(sorted_s, 95), 1), "latest_ttft_ms": round(ttfts[-1], 1) if ttfts else 0.0, "p95_ttft_ms": round(_percentile(sorted_ttfts, 95), 1) if sorted_ttfts else 0.0, "sample_count": len(samples), + "last_ts": samples[-1]["ts"], } ) - top_models.sort(key=lambda item: item["sample_count"], reverse=True) + top_models.sort(key=lambda item: item["last_ts"], reverse=True) try: rag = await asyncio.wait_for(rag_status(), timeout=2.0) except TimeoutError: @@ -1756,19 +1799,6 @@ async def throughput_benchmark(req: ThroughputBenchmarkRequest): output_tokens_per_sec = eval_count / elapsed_sec if eval_count > 0 else 0 input_tokens_per_sec = prompt_eval_count / elapsed_sec if prompt_eval_count > 0 else 0 - # Store sample for stats (peak, percentiles) - with _state_lock: - if model not in _throughput_samples: - if len(_throughput_samples) >= _MAX_TRACKED_MODELS: - pass # cap reached — skip storage but still return payload - else: - _throughput_samples[model] = [] - if model in _throughput_samples: - _throughput_samples[model].append(output_tokens_per_sec) - if len(_throughput_samples[model]) > _MAX_SAMPLES_PER_MODEL: - _throughput_samples[model] = _throughput_samples[model][-_MAX_SAMPLES_PER_MODEL:] - _maybe_save_throughput() - payload = { "ok": True, "model": model, diff --git a/tests/test_services_and_throughput.py b/tests/test_services_and_throughput.py index 3fd568be..bf2cd6ff 100644 --- a/tests/test_services_and_throughput.py +++ b/tests/test_services_and_throughput.py @@ -217,7 +217,6 @@ def test_throughput_record_ignores_empty_model(client): # ── /api/throughput/stats ──────────────────────────────────────────────────── def test_throughput_stats_returns_models(client): - # Seed a sample first client.post("/api/throughput/record", json={ "model": "stats-test-model", "output_tokens_per_sec": 30.0, @@ -227,16 +226,98 @@ def test_throughput_stats_returns_models(client): assert r.status_code == 200 data = r.json() assert data["ok"] is True - assert "models" in data - # The seeded model should appear - if "stats-test-model" in data["models"]: - m = data["models"]["stats-test-model"] - assert "latest" in m - assert "peak" in m - assert "p50" in m - assert "p95" in m - assert "sample_count" in m - assert m["sample_count"] >= 1 + m = data["models"]["stats-test-model"] + for key in ("latest", "peak", "p50", "p95", "sample_count", "last_ts", "first_ts"): + assert key in m + assert m["sample_count"] >= 1 + assert m["last_ts"] > 0 + assert m["first_ts"] <= m["last_ts"] + + +def test_throughput_record_accepts_alias_and_backend(client): + """v2 payload: the gateway callback attributes samples to the REAL served GGUF and + passes the requested alias + backend service alongside. Old senders (no alias/backend) + must keep working — both fields optional.""" + r = client.post("/api/throughput/record", json={ + "model": "Attrib-Test-Q6_K.gguf", + "output_tokens_per_sec": 41.0, + "service": "hermes", + "alias": "local-chat", + "backend": "llamacpp", + }) + assert r.status_code == 200 and r.json()["ok"] is True + stats = client.get("/api/throughput/stats").json() + assert "Attrib-Test-Q6_K.gguf" in stats["models"] + + +def test_throughput_samples_evict_after_max_age(client): + """Models with no sample in _SAMPLE_MAX_AGE_SEC disappear from the store — a retired + model must not keep stats forever (the root of the old 'stale model labeled Active' lie).""" + import dashboard.app as dashboard_app + client.post("/api/throughput/record", json={ + "model": "evict-me.gguf", "output_tokens_per_sec": 20.0, + }) + with dashboard_app._state_lock: + for s in dashboard_app._throughput_samples["evict-me.gguf"]: + s["ts"] -= dashboard_app._SAMPLE_MAX_AGE_SEC + 60 + stats = client.get("/api/throughput/stats").json() + assert "evict-me.gguf" not in stats["models"] + + +def test_throughput_store_v1_file_triggers_clean_reset(tmp_path, monkeypatch): + """A version-less (v1) throughput.json is un-timestamped and alias-conflated — + loading must reset samples (keeping last_benchmark), not present legacy junk + as honest history.""" + import json as _json + + import dashboard.app as dashboard_app + legacy = { + "samples": {"local-chat": [40.1, 39.0], "test-model": [25.5]}, + "ttft_samples": {}, + "last_benchmark": {"ok": True, "model": "local-chat", "output_tokens_per_sec": 40.0}, + "service_usage": [], + } + f = tmp_path / "throughput.json" + f.write_text(_json.dumps(legacy), encoding="utf-8") + monkeypatch.setattr(dashboard_app, "_THROUGHPUT_FILE", f) + monkeypatch.setattr(dashboard_app, "_throughput_samples", {}) + monkeypatch.setattr(dashboard_app, "_ttft_samples", {}) + monkeypatch.setattr(dashboard_app, "_service_usage", []) + monkeypatch.setattr(dashboard_app, "_last_benchmark", None) + dashboard_app._load_throughput_state() + assert dashboard_app._throughput_samples == {} + assert dashboard_app._last_benchmark["model"] == "local-chat" + + +def test_throughput_store_v2_roundtrip(tmp_path, monkeypatch): + """v2 save/load round-trips timestamped samples.""" + import dashboard.app as dashboard_app + f = tmp_path / "throughput.json" + monkeypatch.setattr(dashboard_app, "_THROUGHPUT_FILE", f) + sample = {"tps": 33.3, "ts": 1_700_000_000.0} + monkeypatch.setattr(dashboard_app, "_throughput_samples", {"M.gguf": [sample]}) + dashboard_app._save_throughput_state() + monkeypatch.setattr(dashboard_app, "_throughput_samples", {}) + dashboard_app._load_throughput_state() + assert dashboard_app._throughput_samples == {"M.gguf": [sample]} + + +def test_performance_summary_sorts_by_recency(client): + """top_models orders by last_ts desc — a retired model with a huge lifetime + sample_count must not outrank the model serving right now.""" + import dashboard.app as dashboard_app + for _ in range(5): + client.post("/api/throughput/record", json={ + "model": "old-but-many.gguf", "output_tokens_per_sec": 10.0}) + with dashboard_app._state_lock: + for s in dashboard_app._throughput_samples["old-but-many.gguf"]: + s["ts"] -= 3600 + client.post("/api/throughput/record", json={ + "model": "fresh.gguf", "output_tokens_per_sec": 50.0}) + top = client.get("/api/performance/summary").json()["throughput"]["top_models"] + names = [t["model"] for t in top] + assert names.index("fresh.gguf") < names.index("old-but-many.gguf") + assert all("last_ts" in t for t in top) # ── /api/throughput/service-usage ──────────────────────────────────────────── From c738128c32abcfb432053b46701999dd86c58bb2 Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:16:18 -0400 Subject: [PATCH 3/9] test: assert alias/backend actually captured in service_usage event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_throughput_record_accepts_alias_and_backend previously only checked the POST returned ok and the model appeared in /stats — both true even without alias/backend support (pydantic silently drops unknown fields). Now asserts the recorded _service_usage event carries the real alias and backend values, so a future refactor can't silently drop attribution. Co-Authored-By: Claude Fable 5 --- tests/test_services_and_throughput.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_services_and_throughput.py b/tests/test_services_and_throughput.py index bf2cd6ff..541f3c52 100644 --- a/tests/test_services_and_throughput.py +++ b/tests/test_services_and_throughput.py @@ -248,6 +248,12 @@ def test_throughput_record_accepts_alias_and_backend(client): assert r.status_code == 200 and r.json()["ok"] is True stats = client.get("/api/throughput/stats").json() assert "Attrib-Test-Q6_K.gguf" in stats["models"] + import dashboard.app as dashboard_app + with dashboard_app._state_lock: + evt = next(u for u in reversed(dashboard_app._service_usage) + if u["model"] == "Attrib-Test-Q6_K.gguf") + assert evt["alias"] == "local-chat" + assert evt["backend"] == "llamacpp" def test_throughput_samples_evict_after_max_age(client): From f9a9b7a27181fb87fd19f6e20dc303bd0aa9b73f Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:19:40 -0400 Subject: [PATCH 4/9] dashboard: /api/throughput/stats carries the authoritative active model from ops /model-config Co-Authored-By: Claude Fable 5 --- services/v1-parity/dashboard/app.py | 24 +++++++++++++++++++- tests/test_services_and_throughput.py | 32 +++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/services/v1-parity/dashboard/app.py b/services/v1-parity/dashboard/app.py index d254e1a9..4ddd7239 100644 --- a/services/v1-parity/dashboard/app.py +++ b/services/v1-parity/dashboard/app.py @@ -1629,6 +1629,28 @@ async def throughput_service_usage(): return {"by_model": result, "ok": True} +# Authoritative active model, from the same ops-controller /model-config the Model +# Control tab uses. Cached (positive AND negative) so a 10s-poll dashboard doesn't +# hammer ops; null means "unknown" and the UI says so instead of guessing. +_ACTIVE_MODEL_CACHE_TTL = 30.0 +_active_model_cache: dict = {"checked": 0.0, "value": None} + + +async def _throughput_active_model() -> str | None: + now = time.monotonic() + if now - _active_model_cache["checked"] < _ACTIVE_MODEL_CACHE_TTL: + return _active_model_cache["value"] + code, data = await _ops_request("GET", "/model-config", timeout=10.0) + value = None + if code == 200 and isinstance(data, dict) and data.get("active_model"): + value = str(data["active_model"]) + else: + logger.warning("throughput active-model fetch failed (HTTP %s)", code) + _active_model_cache["checked"] = now + _active_model_cache["value"] = value + return value + + @app.get("/api/throughput/stats") async def throughput_stats(): """Per-model throughput stats over timestamped samples: peak, p50/p95/p99, latest, @@ -1659,7 +1681,7 @@ async def throughput_stats(): "first_ts": samples[0]["ts"], "last_ts": samples[-1]["ts"], } - out: dict = {"models": result, "ok": True} + out: dict = {"models": result, "ok": True, "active_model": await _throughput_active_model()} if benchmark: out["last_benchmark"] = benchmark return out diff --git a/tests/test_services_and_throughput.py b/tests/test_services_and_throughput.py index 541f3c52..b88e9163 100644 --- a/tests/test_services_and_throughput.py +++ b/tests/test_services_and_throughput.py @@ -234,6 +234,38 @@ def test_throughput_stats_returns_models(client): assert m["first_ts"] <= m["last_ts"] +def test_throughput_stats_includes_active_model(client, monkeypatch): + """The tab must never GUESS the active model — /stats carries the ops-controller's + answer (the same authority Model Control uses).""" + import dashboard.app as dashboard_app + + async def _fake_ops(method, path, *a, **k): + assert (method, path) == ("GET", "/model-config") + return 200, {"active_model": "Qwen-Test-Q6_K.gguf", "running": {}} + + monkeypatch.setattr("dashboard.app._ops_request", _fake_ops) + monkeypatch.setattr(dashboard_app, "_active_model_cache", {"checked": 0.0, "value": None}) + r = client.get("/api/throughput/stats") + assert r.json()["active_model"] == "Qwen-Test-Q6_K.gguf" + + +def test_throughput_stats_active_model_null_when_ops_down(client, monkeypatch): + """ops-controller unreachable -> active_model is null (honest unknown), the endpoint + still serves stats, and the failure is negatively cached (one upstream call).""" + import dashboard.app as dashboard_app + calls = {"n": 0} + + async def _fake_ops(method, path, *a, **k): + calls["n"] += 1 + return 503, {"detail": "down"} + + monkeypatch.setattr("dashboard.app._ops_request", _fake_ops) + monkeypatch.setattr(dashboard_app, "_active_model_cache", {"checked": 0.0, "value": None}) + assert client.get("/api/throughput/stats").json()["active_model"] is None + assert client.get("/api/throughput/stats").json()["active_model"] is None + assert calls["n"] == 1, "second read within TTL must hit the cache, not ops" + + def test_throughput_record_accepts_alias_and_backend(client): """v2 payload: the gateway callback attributes samples to the REAL served GGUF and passes the requested alias + backend service alongside. Old senders (no alias/backend) From ccf65e4f01e5930dcf1d5715af063973b3f3c2ec Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:22:18 -0400 Subject: [PATCH 5/9] dashboard: guard active-model fetch with asyncio.Lock to prevent concurrent ops-controller calls Fixes a check-then-act race in _throughput_active_model: two concurrent /stats requests landing after TTL expiry could both pass the staleness check and both call ops-controller. Guards the fetch path with _active_model_fetch_lock and re-checks staleness after acquiring so the second waiter reuses the first's result, restoring "one upstream call per TTL window". Co-Authored-By: Claude Fable 5 --- services/v1-parity/dashboard/app.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/services/v1-parity/dashboard/app.py b/services/v1-parity/dashboard/app.py index 4ddd7239..403d1f13 100644 --- a/services/v1-parity/dashboard/app.py +++ b/services/v1-parity/dashboard/app.py @@ -1634,21 +1634,27 @@ async def throughput_service_usage(): # hammer ops; null means "unknown" and the UI says so instead of guessing. _ACTIVE_MODEL_CACHE_TTL = 30.0 _active_model_cache: dict = {"checked": 0.0, "value": None} +_active_model_fetch_lock = asyncio.Lock() async def _throughput_active_model() -> str | None: now = time.monotonic() if now - _active_model_cache["checked"] < _ACTIVE_MODEL_CACHE_TTL: return _active_model_cache["value"] - code, data = await _ops_request("GET", "/model-config", timeout=10.0) - value = None - if code == 200 and isinstance(data, dict) and data.get("active_model"): - value = str(data["active_model"]) - else: - logger.warning("throughput active-model fetch failed (HTTP %s)", code) - _active_model_cache["checked"] = now - _active_model_cache["value"] = value - return value + async with _active_model_fetch_lock: + # Re-check: a concurrent caller may have refreshed while we waited. + now = time.monotonic() + if now - _active_model_cache["checked"] < _ACTIVE_MODEL_CACHE_TTL: + return _active_model_cache["value"] + code, data = await _ops_request("GET", "/model-config", timeout=10.0) + value = None + if code == 200 and isinstance(data, dict) and data.get("active_model"): + value = str(data["active_model"]) + else: + logger.warning("throughput active-model fetch failed (HTTP %s)", code) + _active_model_cache["checked"] = now + _active_model_cache["value"] = value + return value @app.get("/api/throughput/stats") From 5945256fd330fc0c943a415e36232cf101ff8bc8 Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:25:25 -0400 Subject: [PATCH 6/9] fix(ops-api): recreate model-gateway on any gateway-templated key change, not only ctx A model swap without a ctx change left the gateway advertising the OLD model (stale GGUF-derived pin-aliases, weights_file, vision flag) - violating its "cannot drift from what's running" contract. Co-Authored-By: Claude Fable 5 --- services/ops-api/llamacpp_flags.py | 23 ++++++++ services/ops-api/main.py | 15 ++--- tests/test_model_config_gateway_recreate.py | 61 +++++++++++++++++++++ 3 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 tests/test_model_config_gateway_recreate.py diff --git a/services/ops-api/llamacpp_flags.py b/services/ops-api/llamacpp_flags.py index cde9d376..e90d747d 100644 --- a/services/ops-api/llamacpp_flags.py +++ b/services/ops-api/llamacpp_flags.py @@ -318,3 +318,26 @@ def flag_view(effective): view["MTP_ENABLED"] = "1" if enabled else "0" view["MTP_N_MAX"] = str(n_max if n_max else 2) return view + + +# ── model-gateway coupling ──────────────────────────────────────────────────── +# .env keys the model-gateway entrypoint templates into its LiteLLM config +# (services/model-gateway/entrypoint.sh): the GGUF-derived pin-alias names + +# weights_file (LLAMACPP_MODEL), context/generation limits (CTX_SIZE, N_PREDICT), +# and the vision flag (MMPROJ). When any of these change, the gateway's advertised +# metadata is stale until it is recreated — the apply path must recreate it too. +GATEWAY_CONSUMED_KEYS = ( + "LLAMACPP_MODEL", + "LLAMACPP_CTX_SIZE", + "LLAMACPP_N_PREDICT", + "LLAMACPP_MMPROJ", +) + + +def gateway_recreate_needed(prev_effective, new_effective): + """True when any gateway-templated key's EFFECTIVE value changes (a merely + 'touched' key whose value is identical does not force a restart).""" + return any( + (prev_effective.get(k) or "") != (new_effective.get(k) or "") + for k in GATEWAY_CONSUMED_KEYS + ) diff --git a/services/ops-api/main.py b/services/ops-api/main.py index 742f29d9..348d7e84 100644 --- a/services/ops-api/main.py +++ b/services/ops-api/main.py @@ -175,8 +175,9 @@ # GGUF directory (chat models + mmproj) shown in the model-config UI — the models-gguf # named volume, mounted RO at /gguf-models (compose sets LLAMACPP_MODELS_DIR to match). MODELS_DIR = Path(os.environ.get("LLAMACPP_MODELS_DIR", "/gguf-models")) -# Services that template LLAMACPP_CTX_SIZE and must also recreate when ctx changes. -MODEL_CONFIG_CTX_CONSUMERS = ["model-gateway"] +# Services that template model-config .env keys (see llamacpp_flags.GATEWAY_CONSUMED_KEYS) +# and must recreate when any of those keys' effective value changes. +MODEL_CONFIG_GATEWAY_CONSUMERS = ["model-gateway"] # Services whose GPU pin the dashboard may change. GPU_ASSIGNABLE_SERVICES = {"llamacpp", "llamacpp-embed", "comfyui", "stt", "tts"} @@ -1513,7 +1514,7 @@ async def model_config_get(_: None = Depends(verify_token)): async def model_config_post(body: ModelConfigBody, request: Request, _: None = Depends(verify_token)): """Validate + apply model-config overrides via the ONE write path: persist to - the registry, render into .env, recreate llamacpp (+ ctx consumers).""" + the registry, render into .env, recreate llamacpp (+ gateway consumers).""" errs = lf.validate_all({k: v for k, v in body.overrides.items() if v is not None}) if errs: raise HTTPException(status_code=400, detail={"validation": errs}) @@ -1528,15 +1529,13 @@ async def model_config_post(body: ModelConfigBody, request: Request, config = dict(rec.config) source_file = rec.source.get("file", "") - ctx_touched = False + prev_effective = lf.compute_effective(lf.defaults(), _read_env_values(lf.ENV_KEYS)) for k, v in body.overrides.items(): if k == "LLAMACPP_MODEL": if v: source_file = str(v) config.pop("LLAMACPP_MODEL", None) continue - if k == "LLAMACPP_CTX_SIZE": - ctx_touched = True if v is None: config.pop(k, None) else: @@ -1559,7 +1558,9 @@ async def model_config_post(body: ModelConfigBody, request: Request, rec.updated_by = "model-config" REGISTRY.upsert(rec) - services = ["llamacpp"] + (MODEL_CONFIG_CTX_CONSUMERS if ctx_touched else []) + services = ["llamacpp"] + if lf.gateway_recreate_needed(prev_effective, effective): + services += MODEL_CONFIG_GATEWAY_CONSUMERS for svc in services: _recreate_service(svc, request) diff --git a/tests/test_model_config_gateway_recreate.py b/tests/test_model_config_gateway_recreate.py new file mode 100644 index 00000000..875fd1d0 --- /dev/null +++ b/tests/test_model_config_gateway_recreate.py @@ -0,0 +1,61 @@ +"""The model-config apply path must recreate model-gateway whenever a key the +gateway's entrypoint templates into its LiteLLM config changes — not only ctx. + +Regression: a model swap without a ctx change recreated llamacpp only, leaving the +gateway advertising the OLD model (stale pin-alias names, weights_file, vision flag) +— violating its 'cannot drift from what's running' contract and mis-attributing +throughput samples. llamacpp_flags is a pure module (stdlib re only), so it loads +by file path like the other ops-api unit targets.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +_spec = importlib.util.spec_from_file_location( + "llamacpp_flags_under_test", REPO / "services" / "ops-api" / "llamacpp_flags.py" +) +lf = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(lf) + + +def test_gateway_keys_cover_everything_the_entrypoint_templates(): + assert set(lf.GATEWAY_CONSUMED_KEYS) == { + "LLAMACPP_MODEL", "LLAMACPP_CTX_SIZE", "LLAMACPP_N_PREDICT", "LLAMACPP_MMPROJ", + } + + +def test_model_swap_requires_gateway_recreate(): + prev = {"LLAMACPP_MODEL": "old.gguf", "LLAMACPP_CTX_SIZE": "131072"} + new = {"LLAMACPP_MODEL": "new.gguf", "LLAMACPP_CTX_SIZE": "131072"} + assert lf.gateway_recreate_needed(prev, new) is True + + +def test_ctx_change_requires_gateway_recreate(): + prev = {"LLAMACPP_MODEL": "m.gguf", "LLAMACPP_CTX_SIZE": "131072"} + new = {"LLAMACPP_MODEL": "m.gguf", "LLAMACPP_CTX_SIZE": "65536"} + assert lf.gateway_recreate_needed(prev, new) is True + + +def test_mmproj_toggle_requires_gateway_recreate(): + prev = {"LLAMACPP_MODEL": "m.gguf"} + new = {"LLAMACPP_MODEL": "m.gguf", "LLAMACPP_MMPROJ": "/models/mm.gguf"} + assert lf.gateway_recreate_needed(prev, new) is True + + +def test_unrelated_flag_change_leaves_gateway_alone(): + prev = {"LLAMACPP_MODEL": "m.gguf", "LLAMACPP_FLASH_ATTN": "auto"} + new = {"LLAMACPP_MODEL": "m.gguf", "LLAMACPP_FLASH_ATTN": "on"} + assert lf.gateway_recreate_needed(prev, new) is False + + +def test_touched_but_unchanged_value_is_not_a_change(): + prev = {"LLAMACPP_MODEL": "m.gguf", "LLAMACPP_CTX_SIZE": "131072"} + assert lf.gateway_recreate_needed(prev, dict(prev)) is False + + +def test_missing_vs_empty_are_equivalent(): + assert lf.gateway_recreate_needed( + {"LLAMACPP_MODEL": "m.gguf"}, + {"LLAMACPP_MODEL": "m.gguf", "LLAMACPP_MMPROJ": ""}, + ) is False From dc07c7194d0c09774733660c6d4db9fc36d23da1 Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:28:48 -0400 Subject: [PATCH 7/9] model-gateway: throughput samples attribute to the GGUF that actually served, not the alias The local-chat bucket conflated every model ever active behind it and mixed CPU-failover tok/s into GPU percentiles. Resolution uses the served deployment model_info.weights_file the router attaches to kwargs (verified against the pinned LiteLLM v1.82.3 source), with the legacy naming as fallback. Co-Authored-By: Claude Fable 5 --- services/model-gateway/throughput_callback.py | 74 +++++++++++++-- tests/test_throughput_callback.py | 95 +++++++++++++++++++ 2 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 tests/test_throughput_callback.py diff --git a/services/model-gateway/throughput_callback.py b/services/model-gateway/throughput_callback.py index 6706fe0c..6bb4d2db 100644 --- a/services/model-gateway/throughput_callback.py +++ b/services/model-gateway/throughput_callback.py @@ -6,7 +6,7 @@ callbacks: ["throughput_callback.throughput_recorder_instance"] The dashboard's `/api/throughput/record` endpoint accepts: - {model, output_tokens_per_sec, ttft_ms, service} + {model, output_tokens_per_sec, ttft_ms, service, alias?, backend?} This callback fires after every successful completion and POSTs a fire-and- forget sample. Never raises into the inference path — telemetry failures are @@ -19,6 +19,7 @@ import os from datetime import datetime from typing import Any +from urllib.parse import urlparse import httpx from litellm.integrations.custom_logger import CustomLogger @@ -90,13 +91,65 @@ def _detect_service(kwargs: dict[str, Any]) -> str: return "unknown" +def _deployment_model_info(kwargs: dict[str, Any]) -> dict: + """The served deployment's model_info (including our custom `weights_file`). + + LiteLLM v1.82.3's Router._update_kwargs_with_deployment attaches it to the + request kwargs — top-level, and inside the router metadata (which the logger + later moves under litellm_params). Check every location; first hit wins.""" + for candidate in ( + kwargs.get("model_info"), + ((kwargs.get("litellm_params") or {}).get("metadata") or {}).get("model_info"), + (kwargs.get("metadata") or {}).get("model_info"), + ): + if isinstance(candidate, dict) and candidate.get("weights_file"): + return candidate + return {} + + +def _served_api_base(kwargs: dict[str, Any]) -> str: + slo = kwargs.get("standard_logging_object") or {} + return str( + slo.get("api_base") + or ((kwargs.get("litellm_params") or {}).get("metadata") or {}).get("api_base") + or (kwargs.get("metadata") or {}).get("api_base") + or "" + ) + + +def _detect_backend(kwargs: dict[str, Any]) -> str: + """Backend service that served the request, from the deployment api_base host + (e.g. 'llamacpp' vs 'llamacpp-cpu' — the failover distinction). '' if unknown.""" + try: + return (urlparse(_served_api_base(kwargs)).hostname or "")[:64] + except ValueError: + return "" + + +def _detect_alias(kwargs: dict[str, Any]) -> str: + """The model id the CALLER requested (model_group), e.g. 'local-chat'.""" + slo = kwargs.get("standard_logging_object") or {} + group = str(slo.get("model_group") or "").strip() + if group: + return group[:256] + kwarg_model = str(kwargs.get("model") or "").strip() + if "/" in kwarg_model: + kwarg_model = kwarg_model.split("/", 1)[1] + return kwarg_model[:256] + + def _resolve_model(kwargs: dict[str, Any], response_obj: Any) -> str: - """Pick the most useful model identifier for telemetry. + """Identify the model that ACTUALLY served the completion. - Prefers the underlying provider model (e.g. the actual GGUF filename - llama.cpp reports) over the LiteLLM alias, falling back to the alias if - the response doesn't carry one. Strips any `:tag` suffix. - """ + Truth source: the served deployment's model_info.weights_file — the exact GGUF, + entrypoint-substituted from the same .env the llama-server reads — so samples + key by real model, never by a routing alias like `local-chat` (which conflates + every model ever active behind it, CPU failover included). Falls back to the + legacy response/kwargs naming when the router info is missing; telemetry is + never dropped over attribution.""" + weights = str(_deployment_model_info(kwargs).get("weights_file") or "").strip() + if weights: + return weights.replace("\\", "/").rsplit("/", 1)[-1].split(":")[0][:256] resp_model = "" if isinstance(response_obj, dict): resp_model = str(response_obj.get("model") or "").strip() @@ -154,12 +207,19 @@ def _build_payload(kwargs, response_obj, start_time, end_time) -> dict[str, Any] model = _resolve_model(kwargs, response_obj) if not model: return None - return { + payload = { "model": model, "output_tokens_per_sec": round(completion_tokens / duration, 2), "service": _detect_service(kwargs), "ttft_ms": round(_ttft_ms_from_kwargs(kwargs, start_time), 1), } + alias = _detect_alias(kwargs) + backend = _detect_backend(kwargs) + if alias: + payload["alias"] = alias + if backend: + payload["backend"] = backend + return payload def _headers() -> dict[str, str]: diff --git a/tests/test_throughput_callback.py b/tests/test_throughput_callback.py new file mode 100644 index 00000000..d32b7109 --- /dev/null +++ b/tests/test_throughput_callback.py @@ -0,0 +1,95 @@ +"""Unit tests for the model-gateway throughput callback's deployment resolution. + +Samples must key by the GGUF that ACTUALLY served the completion (the deployment's +model_info.weights_file, entrypoint-substituted from the same .env the llama-server +reads), never by the requested routing alias: `local-chat` conflates every model +ever active behind it, and CPU-failover completions (~10x slower) must attribute to +the CPU GGUF instead of polluting GPU percentiles. + +litellm is not a test dependency (huge tree); the callback only needs its +CustomLogger base class, so the module chain is stubbed before the file loads.""" +from __future__ import annotations + +import importlib.util +import sys +from datetime import datetime, timedelta +from pathlib import Path +from types import ModuleType + +REPO = Path(__file__).resolve().parents[1] + +_litellm = ModuleType("litellm") +_integrations = ModuleType("litellm.integrations") +_custom_logger = ModuleType("litellm.integrations.custom_logger") +_custom_logger.CustomLogger = object +_litellm.integrations = _integrations +_integrations.custom_logger = _custom_logger +sys.modules.setdefault("litellm", _litellm) +sys.modules.setdefault("litellm.integrations", _integrations) +sys.modules.setdefault("litellm.integrations.custom_logger", _custom_logger) + +_spec = importlib.util.spec_from_file_location( + "throughput_callback_under_test", + REPO / "services" / "model-gateway" / "throughput_callback.py", +) +tc = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(tc) + +GPU_GGUF = "/models/Qwen-GPU-Q6_K.gguf" +CPU_GGUF = "/models/Qwen-CPU-A3B.gguf" + + +def _kwargs(weights: str | None, api_base: str, alias: str = "local-chat") -> dict: + """Callback kwargs as the LiteLLM v1.82.3 router+logger assemble them: + deployment model_info at top level AND under litellm_params.metadata.""" + mi = {"weights_file": weights} if weights else {} + return { + "model": f"openai/{alias}", + "model_info": dict(mi), + "litellm_params": {"metadata": {"model_info": dict(mi), "api_base": api_base}}, + "standard_logging_object": { + "model_group": alias, + "api_base": api_base, + "user_agent": "python-httpx/0.27", + }, + } + + +def _response(model: str = "local-chat", completion_tokens: int = 64): + return {"model": model, "usage": {"completion_tokens": completion_tokens, "prompt_tokens": 10}} + + +def test_resolves_model_to_served_gguf_not_alias(): + kw = _kwargs(GPU_GGUF, "http://llamacpp:8080/v1") + assert tc._resolve_model(kw, _response()) == "Qwen-GPU-Q6_K.gguf" + + +def test_cpu_failover_attributes_to_cpu_gguf_and_backend(): + kw = _kwargs(CPU_GGUF, "http://llamacpp-cpu:8080/v1") + assert tc._resolve_model(kw, _response()) == "Qwen-CPU-A3B.gguf" + assert tc._detect_backend(kw) == "llamacpp-cpu" + assert tc._detect_alias(kw) == "local-chat" + + +def test_missing_router_info_falls_back_to_legacy_naming(): + """Telemetry must never be dropped when the router info is absent.""" + kw = {"model": "openai/local-chat", "standard_logging_object": {}} + assert tc._resolve_model(kw, _response(model="local-chat")) == "local-chat" + + +def test_build_payload_includes_alias_and_backend(): + start = datetime(2026, 8, 17, 12, 0, 0) + end = start + timedelta(seconds=2) + kw = _kwargs(GPU_GGUF, "http://llamacpp:8080/v1") + payload = tc._build_payload(kw, _response(completion_tokens=64), start, end) + assert payload["model"] == "Qwen-GPU-Q6_K.gguf" + assert payload["alias"] == "local-chat" + assert payload["backend"] == "llamacpp" + assert payload["output_tokens_per_sec"] == 32.0 + + +def test_build_payload_still_none_without_completion_tokens(): + start = datetime(2026, 8, 17, 12, 0, 0) + end = start + timedelta(seconds=2) + kw = _kwargs(GPU_GGUF, "http://llamacpp:8080/v1") + assert tc._build_payload(kw, _response(completion_tokens=0), start, end) is None From cd7465b9cdc8fc049bf2e0886db0737062d4d124 Mon Sep 17 00:00:00 2001 From: Hermes Bot Date: Wed, 19 Aug 2026 15:34:02 -0400 Subject: [PATCH 8/9] dashboard: throughput hero is the registry's active model - guessing heuristics deleted Honest states replace inference: "no traffic yet" for an idle active model, "control plane unreachable" when ops is down, and a recent-models history list instead of mislabeling stale models Active. Benchmark targets the active model's pin-alias. Co-Authored-By: Claude Fable 5 --- .../frontend/src/components/ThroughputTab.jsx | 145 ++++++++++-------- 1 file changed, 79 insertions(+), 66 deletions(-) diff --git a/services/v1-parity/dashboard/frontend/src/components/ThroughputTab.jsx b/services/v1-parity/dashboard/frontend/src/components/ThroughputTab.jsx index c333ddee..9bee37ea 100644 --- a/services/v1-parity/dashboard/frontend/src/components/ThroughputTab.jsx +++ b/services/v1-parity/dashboard/frontend/src/components/ThroughputTab.jsx @@ -1,19 +1,17 @@ -// Throughput tab — inference telemetry + an on-demand benchmark. Port of the legacy -// loadPerfHero + loadThroughputServiceUsage + runThroughputBenchmark. Consumes: -// - GET /api/throughput/stats — per-model tok/s percentiles + last_benchmark +// Throughput tab — inference telemetry + an on-demand benchmark. Consumes: +// - GET /api/throughput/stats — per-model tok/s percentiles (timestamped, +// 7-day retention) + the AUTHORITATIVE active_model (ops /model-config) + last_benchmark // - GET /api/throughput/service-usage — which service drove which model, recent tok/s // - GET /api/performance/summary — context size + fleet summary -// - GET /api/llm/models — to pick the benchmark target model // - POST /api/throughput/benchmark — run a quick tok/s benchmark (confirm + pending) -// Telemetry polls every 10s (paused when hidden); the benchmark is a manual action that -// disables the button while running and refreshes the telemetry when it lands. +// The hero is the registry's active model — never inferred from sample recency or +// counts. Sample keys are real GGUF basenames (the gateway callback attributes each +// completion to the deployment that served it). Telemetry polls every 10s (paused when +// hidden); the benchmark targets the active model's gateway pin-alias. import { useCallback, useState } from 'react' import { api, usePolling } from '../api.js' import { useToast } from './Toast.jsx' -const EMBED_RE = /embed|bge|mxbai|arctic-embed|granite-embedding|paraphrase-multilingual/ -const isEmbeddingModel = (name) => EMBED_RE.test((name || '').toLowerCase()) - // Callers that reach the gateway with the stock OpenAI SDK send its class name as the "service". // Render those as an honest "unidentified caller" instead of dressing a raw SDK class up as a // curated service identity (the real fix is server-side attribution; this stops the UI lying). @@ -66,24 +64,29 @@ export default function ThroughputTab() { const toast = useToast() const { data, error, refresh } = usePolling(async () => { - const [stats, usage, summary, llm] = await Promise.all([ + const [stats, usage, summary] = await Promise.all([ api.get('/api/throughput/stats').catch(() => ({ ok: false, models: {} })), api.get('/api/throughput/service-usage').catch(() => ({ ok: false, by_model: {} })), api.get('/api/performance/summary').catch(() => null), - api.get('/api/llm/models').catch(() => ({ models: [] })), ]) - return { stats, usage, summary, llm } + return { stats, usage, summary } }, 10000) const stats = data?.stats const summary = data?.summary - const llms = (data?.llm?.models || []).filter((m) => !isEmbeddingModel(m.name)) const models = stats?.ok && stats.models ? stats.models : {} - const entries = Object.entries(models) - // Service-usage rows carry REAL per-service timestamps (unlike /stats, which has no age - // eviction). Compute them first so the hero can be picked by recency, not raw sample count. + // Authoritative model state — ops-controller /model-config via /stats. Never guessed: + // null means the control plane is unreachable and the UI says so. + const activeModel = stats?.active_model ?? null + const activeStats = activeModel ? models[activeModel] : null + + // Other models with recent samples (the store evicts after 7 days) — history, not "Active". + const historyRows = Object.entries(models) + .filter(([name]) => name !== activeModel) + .sort((a, b) => (b[1].last_ts || 0) - (a[1].last_ts || 0)) + const byModel = data?.usage?.ok ? (data.usage.by_model || {}) : {} const usageRows = [] Object.entries(byModel).forEach(([model, info]) => { @@ -92,23 +95,6 @@ export default function ThroughputTab() { usageRows.sort((a, b) => (b.svc.last_ts || 0) - (a.svc.last_ts || 0)) const maxTps = Math.max(1, ...usageRows.map((r) => r.svc.last_tps || 0)) - // Hero = the model that served most RECENTLY (real timestamps), NOT the most-sampled one: - // /stats never evicts, so a retired model keeps the top sample_count forever and would be - // mislabeled "Active". Fall back to sample_count only when there's no timestamped usage. - const recentTsByModel = {} - for (const { model, svc } of usageRows) { - if (svc.last_ts) recentTsByModel[model] = Math.max(recentTsByModel[model] || 0, svc.last_ts) - } - const mostRecent = Object.entries(recentTsByModel).sort((a, b) => b[1] - a[1])[0] - let hero = null - if (mostRecent && models[mostRecent[0]]) { - hero = [mostRecent[0], models[mostRecent[0]], mostRecent[1]] - } else { - const s = [...entries].sort((a, b) => (b[1].sample_count || 0) - (a[1].sample_count || 0))[0] - hero = s ? [s[0], s[1], 0] : null - } - const heroTs = hero ? hero[2] : 0 - const ctx = summary?.llamacpp_ctx_size || 0 const ctxLabel = ctx ? (ctx >= 1000 ? `${Math.round(ctx / 1000)}K ctx` : `${ctx} ctx`) : 'no ctx' @@ -118,14 +104,19 @@ export default function ThroughputTab() { const lastBench = stats?.last_benchmark || null const shownBench = result || lastBench + // Gateway pin-alias for the active GGUF — same derivation as the gateway entrypoint + // (basename, .gguf stripped, lowercased). Pin (not local-chat) so the benchmark + // measures the GPU deployment and honestly errors if it's evicted, instead of + // silently measuring the CPU fallback. + const benchTarget = activeModel + ? activeModel.replace(/\.gguf$/i, '').toLowerCase() + : 'local-chat' + const runBenchmark = useCallback(async () => { - // First non-embedding LLM, else the always-valid `local-chat` alias — never the hero, which - // can be a stale/retired model (see the recency note above); benchmarking the wrong model silently. - const model = llms[0]?.name || 'local-chat' - if (!confirm(`Run a throughput benchmark on "${model}"?\n\nThis sends a short generation through the Model Gateway and reports tok/s.`)) return + if (!confirm(`Run a throughput benchmark on "${benchTarget}"?\n\nThis sends a short generation through the Model Gateway and reports tok/s.`)) return setRunning(true) try { - const d = await api.post('/api/throughput/benchmark', { model }) + const d = await api.post('/api/throughput/benchmark', { model: benchTarget }) setResult(d) toast(`Benchmark: ${d.output_tokens_per_sec} tok/s (${d.model})`, 'success') refresh() @@ -134,7 +125,7 @@ export default function ThroughputTab() { } finally { setRunning(false) } - }, [llms, toast, refresh]) + }, [benchTarget, toast, refresh]) return (
@@ -154,35 +145,57 @@ export default function ThroughputTab() {
{[0, 1].map((i) =>
)}
) : ( <> - {/* Hero + percentile rail */} + {/* Active model (authoritative) + percentile rail */}
-
-
- Active model -
- {hero ? hero[0] : 'No samples yet'} -
+ {activeModel === null ? ( +
+ Model control plane unreachable — cannot determine the active model.
-
-
{hero ? fmt(hero[1].latest) : '—'}
-
tok/s latest
+ ) : ( + <> +
+
+ Active model +
+ {activeModel} +
+
+
+
+ {activeStats ? fmt(activeStats.latest) : '—'} +
+
tok/s latest
+
+
+
+ {activeStats ? `${activeStats.sample_count} samples` : 'no traffic yet — run a benchmark'} + · + {ctxLabel} + {activeStats?.last_ts && <>·last sample {fmtAgo(activeStats.last_ts)}} +
+
+ + + + + + +
+ + )} + {historyRows.length > 0 && ( +
+
Recent models (last 7 days)
+ {historyRows.map(([name, m]) => ( +
+ {name} + + p50 {fmt(m.p50)} tok/s · last seen {fmtAgo(m.last_ts)} + +
+ ))}
-
-
- {hero ? `${hero[1].sample_count || 0} samples` : '0 samples'} - · - {ctxLabel} - {hero && <>·{heroTs ? `active ${fmtAgo(heroTs)}` : 'no recent traffic'}} -
- -
- - - - - - -
+ )}
{/* Benchmark runner */} @@ -191,7 +204,7 @@ export default function ThroughputTab() {
Benchmark

- Target: {llms[0]?.name || 'local-chat'} + Target: {benchTarget}