feat(capacity): improve FlavorGroupCapacity CRD observability

helm/bundles/cortex-nova/templates/alerts.yaml

@@ -588,6 +588,28 @@ spec:
           This may mean hypervisors in that AZ are fully utilized for the corresponding
           flavor group and no further committed resources can be placed there.
 
+    - alert: CortexNovaCommittedResourceCapacityNotReady
+      expr: |
+        cortex_committed_resource_capacity_ready{service="cortex-nova-metrics"} == 0
+      for: 10m
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "FlavorGroupCapacity for {{ "{{" }} $labels.flavor_group {{ "}}" }} in {{ "{{" }} $labels.az {{ "}}" }} has been not-ready for >10 minutes"
+        description: >
+          The FlavorGroupCapacity CRD for flavor group {{ "{{" }} $labels.flavor_group {{ "}}" }}
+          in availability zone {{ "{{" }} $labels.az {{ "}}" }} has had Ready=False for more than
+          10 minutes. The capacity controller failed to complete all scheduler probes for this
+          (flavor group x AZ) pair. The capacity API (report-capacity) is serving stale total
+          capacity values for this group without usage data — Limes receives capacity but no
+          usage, causing silent staleness. Investigate the capacity controller logs for probe
+          errors and check scheduler availability.
+
     # Committed Resource Usage API
     - alert: CortexNovaCommittedResourceUsageErrors
       expr: |

internal/scheduling/reservations/capacity/controller.go

@@ -552,9 +552,14 @@ func (c *Reconciler) reconcileAZ(
 		}
 	}
 
-	// Write one CRD per group. Skip groups with failed probes — their CRDs retain last good state.
+	// Write one CRD per group. For groups with failed probes, mark Ready=False so the
+	// capacity API can detect staleness and return 5xx rather than serving stale data silently.
 	for _, r := range results {
 		if !r.allFresh {
+			if err := c.markCRDNotReady(ctx, r.groupName, az); err != nil {
+				logger.Error(err, "failed to mark FlavorGroupCapacity CRD not-ready",
+					"flavorGroup", r.groupName, "az", az)
+			}
 			continue
 		}
 		if err := c.writeCRD(ctx, r.groupName, r.groupData, az,
@@ -667,6 +672,32 @@ func (c *Reconciler) writeCRD(
 	return nil
 }
 
+// markCRDNotReady sets Ready=False on an existing FlavorGroupCapacity CRD without touching
+// any other status fields, preserving the last-known capacity values for operator inspection.
+// If the CRD does not exist yet it is a no-op.
+func (c *Reconciler) markCRDNotReady(ctx context.Context, groupName, az string) error {
+	crdName := crdNameFor(groupName, az)
+	var existing v1alpha1.FlavorGroupCapacity
+	if err := c.client.Get(ctx, types.NamespacedName{Name: crdName}, &existing); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return fmt.Errorf("failed to get FlavorGroupCapacity %s: %w", crdName, err)
+	}
+	patch := client.MergeFrom(existing.DeepCopy())
+	meta.SetStatusCondition(&existing.Status.Conditions, metav1.Condition{
+		Type:               v1alpha1.FlavorGroupCapacityConditionReady,
+		ObservedGeneration: existing.Generation,
+		Status:             metav1.ConditionFalse,
+		Reason:             "ReconcileFailed",
+		Message:            "one or more scheduler probes failed; capacity data may be stale",
+	})
+	if err := c.client.Status().Patch(ctx, &existing, patch); err != nil {
+		return fmt.Errorf("failed to patch FlavorGroupCapacity %s status: %w", crdName, err)
+	}
+	return nil
+}
+
 // probeScheduler calls the scheduler and returns slot count, host count, and candidate host names.
 // ignoreAllocations=true (total probe) uses raw effective capacity; false (placeable probe) subtracts allocations.
 func (c *Reconciler) probeScheduler(

internal/scheduling/reservations/capacity/controller_test.go

@@ -81,7 +81,7 @@ func newFlavorGroupKnowledge(t *testing.T, groupName string, smallestMemoryMB ui
 }
 
 // newHypervisor creates a Hypervisor CRD with a topology AZ label, memory and CPU effective capacity.
-func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *hv1.Hypervisor {
+func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor {
 	hv := &hv1.Hypervisor{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   name,
@@ -96,9 +96,6 @@ func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *h
 			hv1.ResourceCPU:    *cpuQty,
 		}
 	}
-	for _, id := range instanceIDs {
-		hv.Status.Instances = append(hv.Status.Instances, hv1.Instance{ID: id})
-	}
 	return hv
 }
 
@@ -200,7 +197,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) {
 	)
 
 	scheme := newTestScheme(t)
-	hv := newHypervisor("host-1", az, memBytes, "vm1")
+	hv := newHypervisor("host-1", az, memBytes)
 	knowledge := newFlavorGroupKnowledge(t, groupName, memMB)
 
 	fakeClient := fake.NewClientBuilder().
@@ -312,6 +309,85 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) {
 	}
 }
 
+func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) {
+	const (
+		groupName = "hana-v2"
+		az        = "qa-de-1a"
+		memMB     = 2048
+		memBytes  = int64(memMB) * 1024 * 1024
+	)
+
+	scheme := newTestScheme(t)
+	knowledge := newFlavorGroupKnowledge(t, groupName, memMB)
+	crdName := crdNameFor(groupName, az)
+
+	// Pre-create a CRD that was previously Ready=True.
+	existing := &v1alpha1.FlavorGroupCapacity{
+		ObjectMeta: metav1.ObjectMeta{Name: crdName},
+		Spec: v1alpha1.FlavorGroupCapacitySpec{
+			FlavorGroup:      groupName,
+			AvailabilityZone: az,
+		},
+	}
+
+	fakeClient := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(knowledge, existing).
+		WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}).
+		Build()
+
+	// Set Ready=True on the pre-existing CRD.
+	patch := client.MergeFrom(existing.DeepCopy())
+	existing.Status.Conditions = []metav1.Condition{{
+		Type:   v1alpha1.FlavorGroupCapacityConditionReady,
+		Status: metav1.ConditionTrue,
+		Reason: "ReconcileSucceeded",
+	}}
+	if err := fakeClient.Status().Patch(context.Background(), existing, patch); err != nil {
+		t.Fatalf("failed to set Ready=True: %v", err)
+	}
+
+	// Scheduler returns 500 to simulate error.
+	failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer failServer.Close()
+
+	ctrl := newController(t, fakeClient, Config{
+		SchedulerURL:      failServer.URL,
+		TotalPipeline:     "kvm-report-capacity",
+		PlaceablePipeline: "kvm-general-purpose",
+	})
+
+	smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2}
+	groupData := compute.FlavorGroupFeature{
+		SmallestFlavor: smallFlavor,
+		Flavors:        []compute.FlavorInGroup{smallFlavor},
+	}
+	hv := newHypervisor("host-1", az, memBytes)
+	hvByName := map[string]hv1.Hypervisor{"host-1": *hv}
+
+	ctrl.reconcileAZ(context.Background(), az,
+		map[string]compute.FlavorGroupFeature{groupName: groupData},
+		hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{})
+
+	var crd v1alpha1.FlavorGroupCapacity
+	if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil {
+		t.Fatalf("failed to get CRD: %v", err)
+	}
+
+	// CRD must now have Ready=False.
+	var readyStatus metav1.ConditionStatus
+	for _, c := range crd.Status.Conditions {
+		if c.Type == v1alpha1.FlavorGroupCapacityConditionReady {
+			readyStatus = c.Status
+		}
+	}
+	if readyStatus != metav1.ConditionFalse {
+		t.Errorf("Ready condition = %q, want False after scheduler error", readyStatus)
+	}
+}
+
 func TestReconcileAZ_IdempotentUpdate(t *testing.T) {
 	const (
 		groupName = "hana-v2"
@@ -842,7 +918,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) {
 	)
 
 	scheme := newTestScheme(t)
-	hv := newHypervisor("host-1", az, memBytes, "vm1")
+	hv := newHypervisor("host-1", az, memBytes)
 	knowledge := newFlavorGroupKnowledge(t, groupName, memMB)
 
 	// Pre-create CRD with non-zero RunningInstances to simulate prior state.

internal/scheduling/reservations/capacity/metrics.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
 	"github.com/prometheus/client_golang/prometheus"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -31,6 +32,7 @@ type Monitor struct {
 	freeCapacityGiB            *prometheus.GaugeVec
 	exclusivelyFreeCapacityGiB *prometheus.GaugeVec
 	exclusivelyFreeSlots       *prometheus.GaugeVec
+	readyGauge                 *prometheus.GaugeVec
 }
 
 // NewMonitor creates a new Monitor that reads FlavorGroupCapacity CRDs.
@@ -77,6 +79,10 @@ func NewMonitor(c client.Client) Monitor {
 			Name: "cortex_committed_resource_exclusively_free_slots",
 			Help: "Number of smallest-flavor VM slots available after the cross-group capacity split.",
 		}, capacityFlavorLabels),
+		readyGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "cortex_committed_resource_capacity_ready",
+			Help: "1 if the FlavorGroupCapacity CRD is Ready (all scheduler probes succeeded), 0 otherwise.",
+		}, capacityLabels),
 	}
 }
 
@@ -92,6 +98,7 @@ func (m *Monitor) Describe(ch chan<- *prometheus.Desc) {
 	m.freeCapacityGiB.Describe(ch)
 	m.exclusivelyFreeCapacityGiB.Describe(ch)
 	m.exclusivelyFreeSlots.Describe(ch)
+	m.readyGauge.Describe(ch)
 }
 
 // Collect implements prometheus.Collector — lists all FlavorGroupCapacity CRDs and exports gauges.
@@ -115,6 +122,7 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) {
 	m.freeCapacityGiB.Reset()
 	m.exclusivelyFreeCapacityGiB.Reset()
 	m.exclusivelyFreeSlots.Reset()
+	m.readyGauge.Reset()
 
 	for _, crd := range list.Items {
 		groupAZLabels := prometheus.Labels{
@@ -138,6 +146,12 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) {
 		}
 		m.exclusivelyFreeSlots.With(groupAZFlavorLabels).Set(float64(crd.Status.ExclusivelyFreeSlots))
 
+		readyVal := 0.0
+		if apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) {
+			readyVal = 1.0
+		}
+		m.readyGauge.With(groupAZLabels).Set(readyVal)
+
 		for _, f := range crd.Status.Flavors {
 			flavorLabels := prometheus.Labels{
 				"flavor_group": crd.Spec.FlavorGroup,
@@ -161,4 +175,5 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) {
 	m.freeCapacityGiB.Collect(ch)
 	m.exclusivelyFreeCapacityGiB.Collect(ch)
 	m.exclusivelyFreeSlots.Collect(ch)
+	m.readyGauge.Collect(ch)
 }

internal/scheduling/reservations/commitments/api/report_capacity.go

@@ -5,6 +5,7 @@ package api
 
 import (
 	"encoding/json"
+	"errors"
 	"net/http"
 	"strconv"
 	"time"
@@ -76,8 +77,13 @@ func (api *HTTPAPI) HandleReportCapacity(w http.ResponseWriter, r *http.Request)
 	calculator := commitments.NewCapacityCalculator(api.client, api.config)
 	report, err := calculator.CalculateCapacity(ctx, req)
 	if err != nil {
-		logger.Error(err, "failed to calculate capacity")
-		statusCode = http.StatusInternalServerError
+		if errors.Is(err, commitments.ErrCapacityNotReady) {
+			logger.Info("capacity data not ready, returning 503", "reason", err.Error())
+			statusCode = http.StatusServiceUnavailable
+		} else {
+			logger.Error(err, "failed to calculate capacity")
+			statusCode = http.StatusInternalServerError
+		}
 		http.Error(w, "Failed to calculate capacity: "+err.Error(), statusCode)
 		api.recordCapacityMetrics(statusCode, startTime)
 		return

internal/scheduling/reservations/commitments/api/report_capacity_test.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"slices"
@@ -184,7 +185,8 @@ func TestCapacityCalculator(t *testing.T) {
 		wantCapacity       uint64
 		wantUsage          *uint64 // nil = expect absent
 		cfg                *commitments.APIConfig
-		wantResourceCount  int // 0 = don't check
+		wantResourceCount  int  // 0 = don't check
+		wantNotReady       bool // expect ErrCapacityNotReady
 	}
 	u := func(v uint64) *uint64 { return &v }
 
@@ -196,10 +198,10 @@ func TestCapacityCalculator(t *testing.T) {
 			checkAZ: "az-one", wantCapacity: 1000, wantUsage: u(200),
 		},
 		{
-			// stale CRD: last-known capacity still reported, usage omitted
-			name:             "stale CRD: capacity reported, usage absent",
+			// stale CRD: CalculateCapacity returns ErrCapacityNotReady → caller returns 503
+			name:             "stale CRD: returns ErrCapacityNotReady",
 			runningInstances: 200, exclusiveFreeBytes: 800 * flavorMemBytes, ready: false,
-			checkAZ: "az-one", wantCapacity: 1000, wantUsage: nil,
+			checkAZ: "az-one", wantNotReady: true,
 		},
 		{
 			// CRD only covers az-one; az-two has no CRD → capacity=0
@@ -233,6 +235,12 @@ func TestCapacityCalculator(t *testing.T) {
 				}
 				report, err := calc.CalculateCapacity(context.Background(),
 					liquid.ServiceCapacityRequest{AllAZs: allAZs})
+				if tc.wantNotReady {
+					if !errors.Is(err, commitments.ErrCapacityNotReady) {
+						t.Fatalf("expected ErrCapacityNotReady, got %v", err)
+					}
+					return
+				}
 				if err != nil {
 					t.Fatalf("unexpected error: %v", err)
 				}

internal/scheduling/reservations/commitments/capacity.go

@@ -5,6 +5,7 @@ package commitments
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/sapcc/go-api-declarations/liquid"
@@ -16,6 +17,11 @@ import (
 	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations"
 )
 
+// ErrCapacityNotReady is returned by CalculateCapacity when one or more FlavorGroupCapacity
+// CRDs have Ready=False, indicating the controller's last probe cycle failed. Callers should
+// return 503 Service Unavailable rather than serving potentially stale data.
+var ErrCapacityNotReady = errors.New("one or more FlavorGroupCapacity CRDs are not ready")
+
 // CapacityCalculator computes capacity reports for Limes LIQUID API.
 type CapacityCalculator struct {
 	client client.Client
@@ -59,7 +65,6 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S
 		Resources:   make(map[liquid.ResourceName]*liquid.ResourceCapacityReport),
 	}
 
-	logger := LoggerFromContext(ctx)
 	for groupName, groupData := range flavorGroups {
 		resCfg := c.conf.ResourceConfigForGroup(groupName)
 		// Skip groups not configured for capacity reporting.
@@ -84,8 +89,7 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S
 			}
 
 			if !apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) {
-				logger.Info("FlavorGroupCapacity CRD is stale, reporting capacity without usage",
-					"flavorGroup", groupName, "az", az)
+				return liquid.ServiceCapacityReport{}, fmt.Errorf("%w: flavorGroup=%s az=%s", ErrCapacityNotReady, groupName, string(az))
 			}
 
 			// ExclusivelyFreeSlots is pre-computed by the controller using min(memSlots, cpuSlots).