feat: show restarts and resource limits on the app detail page without metrics-server

api/src/nebari_apps_api/k8s.py

@@ -7,8 +7,9 @@
 from __future__ import annotations
 
 import base64
+import re
 from collections.abc import Callable
-from datetime import UTC
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING, Any, Protocol, cast
 
 if TYPE_CHECKING:
@@ -29,6 +30,10 @@
 REQUEST_TIMEOUT = (5, 30)
 # Page size for list calls; results are followed through `continue` tokens.
 PAGE_SIZE = 500
+# A termination reason is the kubelet's or runtime's CamelCase identifier
+# (OOMKilled, Error, Completed); anything else is dropped, so no free text
+# from the container reaches API (and agent) callers through it.
+_REASON = re.compile(r"[A-Za-z][A-Za-z0-9]{0,63}")
 
 
 class NotFoundError(Exception):
@@ -68,6 +73,8 @@ def write_secret(
 
     def pod_metrics(self, namespace: str, app_name: str) -> list[dict[str, Any]]: ...
 
+    def pod_health(self, namespace: str, app_name: str) -> list[dict[str, Any]]: ...
+
     def cluster_pod_metrics(self) -> list[dict[str, Any]]: ...
 
     def app_restarts(self) -> list[dict[str, Any]]: ...
@@ -229,8 +236,6 @@ def app_events(self, namespace: str, app_name: str) -> list[dict[str, Any]]:
 
     def restart_app(self, namespace: str, app_name: str) -> None:
         """Roll the app's pods, like `kubectl rollout restart`."""
-        from datetime import datetime
-
         stamp = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
         patch = {"spec": {"template": {"metadata": {"annotations": {"kubectl.kubernetes.io/restartedAt": stamp}}}}}
         try:
@@ -308,6 +313,24 @@ def pod_metrics(self, namespace: str, app_name: str) -> list[dict[str, Any]]:
             )
         return out
 
+    def pod_health(self, namespace: str, app_name: str) -> list[dict[str, Any]]:
+        """Health and resources of the App's live pods, from pod status and spec.
+
+        Needs no metrics server: one pod list call, by the App's label. Each
+        entry carries the `app` container's restart count (the authz
+        sidecar's are left out), how its previous run ended
+        (`lastTermination`: reason, exit code and finish time; never the
+        termination message, which the container writes), when it started
+        running, and its CPU (millicores) and memory (Mi) requests and limits.
+        Pods that have finished (phase Succeeded or Failed, e.g. evicted) are
+        skipped.
+        """
+        try:
+            pods = _list_typed(self._core.list_namespaced_pod, namespace, label_selector=f"{APP_LABEL}={app_name}")
+        except Exception as exc:
+            raise self._wrap(exc) from exc
+        return [_pod_health(pod) for pod in pods if pod.metadata and pod.metadata.name and _live(pod)]
+
     def cluster_pod_metrics(self) -> list[dict[str, Any]]:
         """Per-pod usage for every app pod in the cluster, keyed by app + namespace.
 
@@ -354,6 +377,57 @@ def app_restarts(self) -> list[dict[str, Any]]:
         return out
 
 
+def _live(pod: V1Pod) -> bool:
+    return (pod.status.phase if pod.status else None) not in ("Succeeded", "Failed")
+
+
+def _pod_health(pod: V1Pod) -> dict[str, Any]:
+    containers = pod.spec.containers if pod.spec else []
+    spec = next((c for c in containers if c.name == APP_CONTAINER), None)
+    statuses = (pod.status.container_statuses if pod.status else None) or []
+    status = next((cs for cs in statuses if cs.name == APP_CONTAINER), None)
+    resources = spec.resources if spec else None
+    terminated = status.last_state.terminated if status and status.last_state else None
+    running = status.state.running if status and status.state else None
+    last = None
+    if terminated:
+        reason = terminated.reason or ""
+        last = {
+            "reason": reason if _REASON.fullmatch(reason) else "",
+            "exitCode": terminated.exit_code or 0,
+            "finishedAt": _iso(terminated.finished_at),
+        }
+    return {
+        "name": pod.metadata.name if pod.metadata else "",
+        "restarts": (status.restart_count or 0) if status else 0,
+        "lastTermination": last,
+        "startedAt": (_iso(running.started_at) or None) if running else None,
+        "limits": _resources(resources.limits if resources else None),
+        "requests": _resources(resources.requests if resources else None),
+    }
+
+
+def _resources(values: dict[str, str] | None) -> dict[str, str]:
+    """CPU and memory quantities normalized the way usage is ("500m", "512Mi")."""
+    values = values or {}
+    cpu, memory = values.get("cpu"), values.get("memory")
+    return {
+        "cpu": f"{_cpu_to_milli(str(cpu))}m" if cpu else "",
+        "memory": f"{round(_mem_to_mi(str(memory)))}Mi" if memory else "",
+    }
+
+
+def _iso(value: datetime | str | None) -> str:
+    """A timestamp as RFC 3339 in UTC ("" when unset)."""
+    if not value:
+        return ""
+    if isinstance(value, str):
+        return value
+    if value.tzinfo is None:
+        value = value.replace(tzinfo=UTC)
+    return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
 def _b64(value: str) -> str:
     return base64.b64encode(value.encode()).decode()
 

api/src/nebari_apps_api/main.py

@@ -466,12 +466,25 @@ def app_events(namespace: str, name: str, store: Store, user: Me) -> list[dict[s
 
     @app.get(PREFIX + "/apps/{namespace}/{name}/metrics", response_model=AppMetrics)
     def app_metrics(namespace: str, name: str, store: Store, user: Me) -> AppMetrics:
+        """Per-pod usage (metrics.k8s.io) joined with health from the pods.
+
+        available=false means no metrics server: cpu/memory are then "", but
+        restarts, lastTermination, startedAt, limits and requests still come
+        back, from pod status and spec.
+        """
         authorized_app(store, user, namespace, name)
+        pods = {p["name"]: p for p in store.pod_health(namespace, name)}
         try:
-            pods = store.pod_metrics(namespace, name)
+            usage = store.pod_metrics(namespace, name)
+            available = True
         except NotFoundError:
-            return AppMetrics(available=False, pods=[])
-        return AppMetrics(available=True, pods=[PodMetric.model_validate(p) for p in pods])
+            usage, available = [], False
+        for u in usage:
+            # Usage for a pod the list no longer has (it just finished or
+            # went away) is dropped: the pod list is the set of live pods.
+            if u["name"] in pods:
+                pods[u["name"]].update(cpu=u["cpu"], memory=u["memory"])
+        return AppMetrics(available=available, pods=[PodMetric.model_validate(p) for p in pods.values()])
 
     # -------------------------------------------------------------- analytics
     @app.get(PREFIX + "/analytics/summary", response_model=AnalyticsSummary)

api/src/nebari_apps_api/models.py

@@ -268,14 +268,46 @@ class AppOut(BaseModel):
     role: Role = ""
 
 
+class PodResources(BaseModel):
+    """The `app` container's CPU and memory, as requests or limits ("" when unset)."""
+
+    cpu: str = ""  # e.g. "500m" (millicores)
+    memory: str = ""  # e.g. "512Mi"
+
+
+class PodTermination(BaseModel):
+    """How the `app` container's previous run ended (its lastState.terminated)."""
+
+    # The kubelet's or runtime's CamelCase reason, e.g. OOMKilled, Error,
+    # Completed; "" when absent or not a plain identifier (it never carries
+    # the container's own termination message).
+    reason: str = ""
+    exitCode: int = 0
+    finishedAt: str = ""  # RFC 3339, UTC
+
+
 class PodMetric(BaseModel):
+    """One app pod: usage from metrics.k8s.io, health and resources from the pod."""
+
     name: str
-    cpu: str = ""  # e.g. "12m" (millicores)
-    memory: str = ""  # e.g. "34Mi"
+    cpu: str = ""  # e.g. "12m" (millicores); "" without a metrics server
+    memory: str = ""  # e.g. "34Mi"; "" without a metrics server
+    # Restarts of the `app` container (the authz sidecar's are not counted).
+    restarts: int = 0
+    lastTermination: PodTermination | None = None
+    # When the `app` container last started running; None while it is not running.
+    startedAt: str | None = None
+    limits: PodResources = Field(default_factory=PodResources)
+    requests: PodResources = Field(default_factory=PodResources)
 
 
 class AppMetrics(BaseModel):
-    """Instantaneous resource usage, summed per pod, from metrics.k8s.io."""
+    """Per-pod usage and health of an App.
+
+    available is true when metrics.k8s.io answered, so cpu/memory usage is
+    present. The pod-status fields (restarts, lastTermination, startedAt,
+    limits, requests) come from the pods themselves and are filled either way.
+    """
 
     available: bool = False
     pods: list[PodMetric] = Field(default_factory=list)

api/tests/conftest.py

@@ -23,6 +23,8 @@ def __init__(self) -> None:
         self.logs = "hello from pod\n"
         self.restarted: list[tuple[str, str]] = []
         self.metrics_available = True
+        # pod_health's answer; None: one healthy pod per App.
+        self.pods: list[dict[str, Any]] | None = None
         # (namespace, secret name) -> {"data", "owner", "labels", "version"}.
         self.secrets: dict[tuple[str, str], dict[str, Any]] = {}
         self.secret_writes = 0
@@ -111,6 +113,20 @@ def pod_metrics(self, namespace: str, app_name: str) -> list[dict[str, Any]]:
             raise NotFoundError("metrics.k8s.io not available")
         return [{"name": f"app-{app_name}-abc123", "cpu": "12m", "memory": "34Mi"}]
 
+    def pod_health(self, namespace: str, app_name: str) -> list[dict[str, Any]]:
+        if self.pods is not None:
+            return copy.deepcopy(self.pods)
+        return [
+            {
+                "name": f"app-{app_name}-abc123",
+                "restarts": 0,
+                "lastTermination": None,
+                "startedAt": "2026-07-15T00:00:00Z",
+                "limits": {"cpu": "500m", "memory": "512Mi"},
+                "requests": {"cpu": "100m", "memory": "128Mi"},
+            }
+        ]
+
     def cluster_pod_metrics(self) -> list[dict[str, Any]]:
         if not self.metrics_available:
             raise NotFoundError("metrics.k8s.io not available")

api/tests/test_apps.py

@@ -190,7 +190,40 @@ def test_metrics_unavailable(client, store):
     client.post("/api/v1/apps", json=make_app_body())
     resp = client.get("/api/v1/apps/apps/docs-site/metrics")
     assert resp.status_code == 200
-    assert resp.json() == {"available": False, "pods": []}
+    body = resp.json()
+    assert body["available"] is False
+    # Without a metrics server the pod-status data still comes back.
+    [pod] = body["pods"]
+    assert pod["cpu"] == "" and pod["memory"] == ""
+    assert pod["limits"] == {"cpu": "500m", "memory": "512Mi"}
+    assert pod["requests"] == {"cpu": "100m", "memory": "128Mi"}
+    assert pod["startedAt"] == "2026-07-15T00:00:00Z"
+    assert pod["restarts"] == 0 and pod["lastTermination"] is None
+
+
+def test_metrics_join_usage_with_pod_health(client, store):
+    oom = {"reason": "OOMKilled", "exitCode": 137, "finishedAt": "2026-07-15T00:05:00Z"}
+    store.pods = [
+        {"name": "app-docs-site-abc123", "restarts": 4, "lastTermination": oom, "startedAt": "2026-07-15T00:06:00Z"},
+        {"name": "app-docs-site-new", "restarts": 0, "startedAt": None},
+    ]
+    client.post("/api/v1/apps", json=make_app_body())
+    body = client.get("/api/v1/apps/apps/docs-site/metrics").json()
+    assert body["available"] is True
+    first, second = body["pods"]
+    assert first["cpu"] == "12m" and first["memory"] == "34Mi"
+    assert first["restarts"] == 4
+    assert first["lastTermination"] == oom
+    # A pod metrics.k8s.io has not sampled yet: no usage, health still there.
+    assert second["name"] == "app-docs-site-new"
+    assert second["cpu"] == "" and second["startedAt"] is None
+    assert second["limits"] == {"cpu": "", "memory": ""}
+
+
+def test_metrics_drop_usage_of_pods_that_are_gone(client, store):
+    store.pods = []
+    client.post("/api/v1/apps", json=make_app_body())
+    assert client.get("/api/v1/apps/apps/docs-site/metrics").json() == {"available": True, "pods": []}
 
 
 def test_logs_and_events(client):

api/tests/test_k8s_resilience.py

@@ -30,7 +30,11 @@ def pod(name, rs):
             labels={"apps.nebari.dev/app": "demo"},
             owner_references=[SimpleNamespace(kind="ReplicaSet", name=rs)],
         ),
-        status=SimpleNamespace(container_statuses=[SimpleNamespace(restart_count=0)]),
+        spec=SimpleNamespace(containers=[]),
+        status=SimpleNamespace(
+            phase="Running",
+            container_statuses=[SimpleNamespace(name="app", restart_count=0, state=None, last_state=None)],
+        ),
     )
 
 
@@ -78,6 +82,7 @@ def test_every_call_has_a_timeout(kube):
     kube.app_events("apps", "demo")
     kube.restart_app("apps", "demo")
     kube.pod_metrics("apps", "demo")
+    kube.pod_health("apps", "demo")
     kube.cluster_pod_metrics()
     kube.app_restarts()
     calls = kube._custom.mock_calls + kube._core.mock_calls + kube._apps.mock_calls