throughput: authoritative model-state tracking (real-GGUF attribution, timestamped store, gateway-recreate drift fix)

services/model-gateway/throughput_callback.py

@@ -6,7 +6,7 @@
       callbacks: ["throughput_callback.throughput_recorder_instance"]
 
 The dashboard's `/api/throughput/record` endpoint accepts:
-    {model, output_tokens_per_sec, ttft_ms, service}
+    {model, output_tokens_per_sec, ttft_ms, service, alias?, backend?}
 
 This callback fires after every successful completion and POSTs a fire-and-
 forget sample. Never raises into the inference path — telemetry failures are
@@ -19,6 +19,7 @@
 import os
 from datetime import datetime
 from typing import Any
+from urllib.parse import urlparse
 
 import httpx
 from litellm.integrations.custom_logger import CustomLogger
@@ -90,13 +91,65 @@ def _detect_service(kwargs: dict[str, Any]) -> str:
     return "unknown"
 
 
+def _deployment_model_info(kwargs: dict[str, Any]) -> dict:
+    """The served deployment's model_info (including our custom `weights_file`).
+
+    LiteLLM v1.82.3's Router._update_kwargs_with_deployment attaches it to the
+    request kwargs — top-level, and inside the router metadata (which the logger
+    later moves under litellm_params). Check every location; first hit wins."""
+    for candidate in (
+        kwargs.get("model_info"),
+        ((kwargs.get("litellm_params") or {}).get("metadata") or {}).get("model_info"),
+        (kwargs.get("metadata") or {}).get("model_info"),
+    ):
+        if isinstance(candidate, dict) and candidate.get("weights_file"):
+            return candidate
+    return {}
+
+
+def _served_api_base(kwargs: dict[str, Any]) -> str:
+    slo = kwargs.get("standard_logging_object") or {}
+    return str(
+        slo.get("api_base")
+        or ((kwargs.get("litellm_params") or {}).get("metadata") or {}).get("api_base")
+        or (kwargs.get("metadata") or {}).get("api_base")
+        or ""
+    )
+
+
+def _detect_backend(kwargs: dict[str, Any]) -> str:
+    """Backend service that served the request, from the deployment api_base host
+    (e.g. 'llamacpp' vs 'llamacpp-cpu' — the failover distinction). '' if unknown."""
+    try:
+        return (urlparse(_served_api_base(kwargs)).hostname or "")[:64]
+    except ValueError:
+        return ""
+
+
+def _detect_alias(kwargs: dict[str, Any]) -> str:
+    """The model id the CALLER requested (model_group), e.g. 'local-chat'."""
+    slo = kwargs.get("standard_logging_object") or {}
+    group = str(slo.get("model_group") or "").strip()
+    if group:
+        return group[:256]
+    kwarg_model = str(kwargs.get("model") or "").strip()
+    if "/" in kwarg_model:
+        kwarg_model = kwarg_model.split("/", 1)[1]
+    return kwarg_model[:256]
+
+
 def _resolve_model(kwargs: dict[str, Any], response_obj: Any) -> str:
-    """Pick the most useful model identifier for telemetry.
+    """Identify the model that ACTUALLY served the completion.
 
-    Prefers the underlying provider model (e.g. the actual GGUF filename
-    llama.cpp reports) over the LiteLLM alias, falling back to the alias if
-    the response doesn't carry one. Strips any `:tag` suffix.
-    """
+    Truth source: the served deployment's model_info.weights_file — the exact GGUF,
+    entrypoint-substituted from the same .env the llama-server reads — so samples
+    key by real model, never by a routing alias like `local-chat` (which conflates
+    every model ever active behind it, CPU failover included). Falls back to the
+    legacy response/kwargs naming when the router info is missing; telemetry is
+    never dropped over attribution."""
+    weights = str(_deployment_model_info(kwargs).get("weights_file") or "").strip()
+    if weights:
+        return weights.replace("\\", "/").rsplit("/", 1)[-1].split(":")[0][:256]
     resp_model = ""
     if isinstance(response_obj, dict):
         resp_model = str(response_obj.get("model") or "").strip()
@@ -154,12 +207,19 @@ def _build_payload(kwargs, response_obj, start_time, end_time) -> dict[str, Any]
     model = _resolve_model(kwargs, response_obj)
     if not model:
         return None
-    return {
+    payload = {
         "model": model,
         "output_tokens_per_sec": round(completion_tokens / duration, 2),
         "service": _detect_service(kwargs),
         "ttft_ms": round(_ttft_ms_from_kwargs(kwargs, start_time), 1),
     }
+    alias = _detect_alias(kwargs)
+    backend = _detect_backend(kwargs)
+    if alias:
+        payload["alias"] = alias
+    if backend:
+        payload["backend"] = backend
+    return payload
 
 
 def _headers() -> dict[str, str]:

services/ops-api/llamacpp_flags.py

@@ -318,3 +318,26 @@ def flag_view(effective):
     view["MTP_ENABLED"] = "1" if enabled else "0"
     view["MTP_N_MAX"] = str(n_max if n_max else 2)
     return view
+
+
+# ── model-gateway coupling ────────────────────────────────────────────────────
+# .env keys the model-gateway entrypoint templates into its LiteLLM config
+# (services/model-gateway/entrypoint.sh): the GGUF-derived pin-alias names +
+# weights_file (LLAMACPP_MODEL), context/generation limits (CTX_SIZE, N_PREDICT),
+# and the vision flag (MMPROJ). When any of these change, the gateway's advertised
+# metadata is stale until it is recreated — the apply path must recreate it too.
+GATEWAY_CONSUMED_KEYS = (
+    "LLAMACPP_MODEL",
+    "LLAMACPP_CTX_SIZE",
+    "LLAMACPP_N_PREDICT",
+    "LLAMACPP_MMPROJ",
+)
+
+
+def gateway_recreate_needed(prev_effective, new_effective):
+    """True when any gateway-templated key's EFFECTIVE value changes (a merely
+    'touched' key whose value is identical does not force a restart)."""
+    return any(
+        (prev_effective.get(k) or "") != (new_effective.get(k) or "")
+        for k in GATEWAY_CONSUMED_KEYS
+    )

services/ops-api/main.py

@@ -175,8 +175,9 @@
 # GGUF directory (chat models + mmproj) shown in the model-config UI — the models-gguf
 # named volume, mounted RO at /gguf-models (compose sets LLAMACPP_MODELS_DIR to match).
 MODELS_DIR = Path(os.environ.get("LLAMACPP_MODELS_DIR", "/gguf-models"))
-# Services that template LLAMACPP_CTX_SIZE and must also recreate when ctx changes.
-MODEL_CONFIG_CTX_CONSUMERS = ["model-gateway"]
+# Services that template model-config .env keys (see llamacpp_flags.GATEWAY_CONSUMED_KEYS)
+# and must recreate when any of those keys' effective value changes.
+MODEL_CONFIG_GATEWAY_CONSUMERS = ["model-gateway"]
 
 # Services whose GPU pin the dashboard may change.
 GPU_ASSIGNABLE_SERVICES = {"llamacpp", "llamacpp-embed", "comfyui", "stt", "tts"}
@@ -1513,7 +1514,7 @@ async def model_config_get(_: None = Depends(verify_token)):
 async def model_config_post(body: ModelConfigBody, request: Request,
                             _: None = Depends(verify_token)):
     """Validate + apply model-config overrides via the ONE write path: persist to
-    the registry, render into .env, recreate llamacpp (+ ctx consumers)."""
+    the registry, render into .env, recreate llamacpp (+ gateway consumers)."""
     errs = lf.validate_all({k: v for k, v in body.overrides.items() if v is not None})
     if errs:
         raise HTTPException(status_code=400, detail={"validation": errs})
@@ -1528,15 +1529,13 @@ async def model_config_post(body: ModelConfigBody, request: Request,
 
     config = dict(rec.config)
     source_file = rec.source.get("file", "")
-    ctx_touched = False
+    prev_effective = lf.compute_effective(lf.defaults(), _read_env_values(lf.ENV_KEYS))
     for k, v in body.overrides.items():
         if k == "LLAMACPP_MODEL":
             if v:
                 source_file = str(v)
             config.pop("LLAMACPP_MODEL", None)
             continue
-        if k == "LLAMACPP_CTX_SIZE":
-            ctx_touched = True
         if v is None:
             config.pop(k, None)
         else:
@@ -1559,7 +1558,9 @@ async def model_config_post(body: ModelConfigBody, request: Request,
     rec.updated_by = "model-config"
     REGISTRY.upsert(rec)
 
-    services = ["llamacpp"] + (MODEL_CONFIG_CTX_CONSUMERS if ctx_touched else [])
+    services = ["llamacpp"]
+    if lf.gateway_recreate_needed(prev_effective, effective):
+        services += MODEL_CONFIG_GATEWAY_CONSUMERS
     for svc in services:
         _recreate_service(svc, request)