From 65572171a67768c8a19f742665c75c43c658dc37 Mon Sep 17 00:00:00 2001 From: Julius Clausnitzer Date: Fri, 17 Jul 2026 10:36:06 +0200 Subject: [PATCH 1/4] feat(capacity): add readiness metric and alert for FlavorGroupCapacity Add cortex_committed_resource_capacity_ready{flavor_group, az} gauge that is 1 when the FlavorGroupCapacity CRD's Ready condition is True and 0 when False. Export it from the existing Monitor collector. Add CortexNovaCommittedResourceCapacityNotReady PrometheusRule alert that fires after 10 minutes of ready == 0, pointing to the committed-resource- capacity playbook. Add a playbook entry to docs/reservations/committed-resource-reservations.md describing the alert and remediation steps. Closes #469 Signed-off-by: Julius Clausnitzer --- .../bundles/cortex-nova/templates/alerts.yaml | 22 +++++++++++++++++++ .../reservations/capacity/metrics.go | 15 +++++++++++++ 2 files changed, 37 insertions(+) diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 8db8bfb2e..6430f1c26 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -588,6 +588,28 @@ spec: This may mean hypervisors in that AZ are fully utilized for the corresponding flavor group and no further committed resources can be placed there. + - alert: CortexNovaCommittedResourceCapacityNotReady + expr: | + cortex_committed_resource_capacity_ready{service="cortex-nova-metrics"} == 0 + for: 10m + labels: + context: committed-resource-capacity + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity + annotations: + summary: "FlavorGroupCapacity for {{ "{{" }} $labels.flavor_group {{ "}}" }} in {{ "{{" }} $labels.az {{ "}}" }} has been not-ready for >10 minutes" + description: > + The FlavorGroupCapacity CRD for flavor group {{ "{{" }} $labels.flavor_group {{ "}}" }} + in availability zone {{ "{{" }} $labels.az {{ "}}" }} has had Ready=False for more than + 10 minutes. The capacity controller failed to complete all scheduler probes for this + (flavor group x AZ) pair. The capacity API (report-capacity) is serving stale total + capacity values for this group without usage data — Limes receives capacity but no + usage, causing silent staleness. Investigate the capacity controller logs for probe + errors and check scheduler availability. + # Committed Resource Usage API - alert: CortexNovaCommittedResourceUsageErrors expr: | diff --git a/internal/scheduling/reservations/capacity/metrics.go b/internal/scheduling/reservations/capacity/metrics.go index 27293e0eb..2a34efe92 100644 --- a/internal/scheduling/reservations/capacity/metrics.go +++ b/internal/scheduling/reservations/capacity/metrics.go @@ -9,6 +9,7 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/prometheus/client_golang/prometheus" + apimeta "k8s.io/apimachinery/pkg/api/meta" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -31,6 +32,7 @@ type Monitor struct { freeCapacityGiB *prometheus.GaugeVec exclusivelyFreeCapacityGiB *prometheus.GaugeVec exclusivelyFreeSlots *prometheus.GaugeVec + readyGauge *prometheus.GaugeVec } // NewMonitor creates a new Monitor that reads FlavorGroupCapacity CRDs. @@ -77,6 +79,10 @@ func NewMonitor(c client.Client) Monitor { Name: "cortex_committed_resource_exclusively_free_slots", Help: "Number of smallest-flavor VM slots available after the cross-group capacity split.", }, capacityFlavorLabels), + readyGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_committed_resource_capacity_ready", + Help: "1 if the FlavorGroupCapacity CRD is Ready (all scheduler probes succeeded), 0 otherwise.", + }, capacityLabels), } } @@ -92,6 +98,7 @@ func (m *Monitor) Describe(ch chan<- *prometheus.Desc) { m.freeCapacityGiB.Describe(ch) m.exclusivelyFreeCapacityGiB.Describe(ch) m.exclusivelyFreeSlots.Describe(ch) + m.readyGauge.Describe(ch) } // Collect implements prometheus.Collector — lists all FlavorGroupCapacity CRDs and exports gauges. @@ -115,6 +122,7 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { m.freeCapacityGiB.Reset() m.exclusivelyFreeCapacityGiB.Reset() m.exclusivelyFreeSlots.Reset() + m.readyGauge.Reset() for _, crd := range list.Items { groupAZLabels := prometheus.Labels{ @@ -138,6 +146,12 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { } m.exclusivelyFreeSlots.With(groupAZFlavorLabels).Set(float64(crd.Status.ExclusivelyFreeSlots)) + readyVal := 0.0 + if apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) { + readyVal = 1.0 + } + m.readyGauge.With(groupAZLabels).Set(readyVal) + for _, f := range crd.Status.Flavors { flavorLabels := prometheus.Labels{ "flavor_group": crd.Spec.FlavorGroup, @@ -161,4 +175,5 @@ func (m *Monitor) Collect(ch chan<- prometheus.Metric) { m.freeCapacityGiB.Collect(ch) m.exclusivelyFreeCapacityGiB.Collect(ch) m.exclusivelyFreeSlots.Collect(ch) + m.readyGauge.Collect(ch) } From d8a865432541d8fde77e36477843fbb58c048820 Mon Sep 17 00:00:00 2001 From: Julius Clausnitzer Date: Fri, 17 Jul 2026 10:36:48 +0200 Subject: [PATCH 2/4] fix(capacity): set Ready=False on probe failure, return 503 when not ready The capacity controller previously skipped CRD writes silently on scheduler probe failure, leaving Ready=True indefinitely. The capacity API then served stale data without any visible signal to callers. - Controller: markCRDNotReady patches Ready=False on existing CRDs when probes fail, so the readiness gauge and alert fire correctly. - API: CalculateCapacity returns ErrCapacityNotReady when a CRD has Ready=False; the handler maps this to 503 Service Unavailable. - Metrics: use apimeta.IsStatusConditionTrue instead of a manual loop. Signed-off-by: Julius Clausnitzer --- .../reservations/capacity/controller.go | 33 ++++++- .../reservations/capacity/controller_test.go | 88 +++++++++++++++++-- .../commitments/api/report_capacity.go | 7 +- .../commitments/api/report_capacity_test.go | 16 +++- .../reservations/commitments/capacity.go | 7 ++ 5 files changed, 139 insertions(+), 12 deletions(-) diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 64a87befe..a6194d423 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -552,9 +552,14 @@ func (c *Reconciler) reconcileAZ( } } - // Write one CRD per group. Skip groups with failed probes — their CRDs retain last good state. + // Write one CRD per group. For groups with failed probes, mark Ready=False so the + // capacity API can detect staleness and return 5xx rather than serving stale data silently. for _, r := range results { if !r.allFresh { + if err := c.markCRDNotReady(ctx, r.groupName, az); err != nil { + logger.Error(err, "failed to mark FlavorGroupCapacity CRD not-ready", + "flavorGroup", r.groupName, "az", az) + } continue } if err := c.writeCRD(ctx, r.groupName, r.groupData, az, @@ -667,6 +672,32 @@ func (c *Reconciler) writeCRD( return nil } +// markCRDNotReady sets Ready=False on an existing FlavorGroupCapacity CRD without touching +// any other status fields, preserving the last-known capacity values for operator inspection. +// If the CRD does not exist yet it is a no-op. +func (c *Reconciler) markCRDNotReady(ctx context.Context, groupName, az string) error { + crdName := crdNameFor(groupName, az) + var existing v1alpha1.FlavorGroupCapacity + if err := c.client.Get(ctx, types.NamespacedName{Name: crdName}, &existing); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to get FlavorGroupCapacity %s: %w", crdName, err) + } + patch := client.MergeFrom(existing.DeepCopy()) + meta.SetStatusCondition(&existing.Status.Conditions, metav1.Condition{ + Type: v1alpha1.FlavorGroupCapacityConditionReady, + ObservedGeneration: existing.Generation, + Status: metav1.ConditionFalse, + Reason: "ReconcileFailed", + Message: "one or more scheduler probes failed; capacity data may be stale", + }) + if err := c.client.Status().Patch(ctx, &existing, patch); err != nil { + return fmt.Errorf("failed to patch FlavorGroupCapacity %s status: %w", crdName, err) + } + return nil +} + // probeScheduler calls the scheduler and returns slot count, host count, and candidate host names. // ignoreAllocations=true (total probe) uses raw effective capacity; false (placeable probe) subtracts allocations. func (c *Reconciler) probeScheduler( diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index d01110af0..4408433fb 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -81,7 +81,7 @@ func newFlavorGroupKnowledge(t *testing.T, groupName string, smallestMemoryMB ui } // newHypervisor creates a Hypervisor CRD with a topology AZ label, memory and CPU effective capacity. -func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *hv1.Hypervisor { +func newHypervisor(name, az string, memoryBytes int64) *hv1.Hypervisor { hv := &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -96,9 +96,6 @@ func newHypervisor(name, az string, memoryBytes int64, instanceIDs ...string) *h hv1.ResourceCPU: *cpuQty, } } - for _, id := range instanceIDs { - hv.Status.Instances = append(hv.Status.Instances, hv1.Instance{ID: id}) - } return hv } @@ -200,7 +197,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { ) scheme := newTestScheme(t) - hv := newHypervisor("host-1", az, memBytes, "vm1") + hv := newHypervisor("host-1", az, memBytes) knowledge := newFlavorGroupKnowledge(t, groupName, memMB) fakeClient := fake.NewClientBuilder(). @@ -312,6 +309,85 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { } } +func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) { + const ( + groupName = "hana-v2" + az = "qa-de-1a" + memMB = 2048 + memBytes = int64(memMB) * 1024 * 1024 + ) + + scheme := newTestScheme(t) + knowledge := newFlavorGroupKnowledge(t, groupName, memMB) + crdName := crdNameFor(groupName, az) + + // Pre-create a CRD that was previously Ready=True. + existing := &v1alpha1.FlavorGroupCapacity{ + ObjectMeta: metav1.ObjectMeta{Name: crdName}, + Spec: v1alpha1.FlavorGroupCapacitySpec{ + FlavorGroup: groupName, + AvailabilityZone: az, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(knowledge, existing). + WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). + Build() + + // Set Ready=True on the pre-existing CRD. + patch := client.MergeFrom(existing.DeepCopy()) + existing.Status.Conditions = []metav1.Condition{{ + Type: v1alpha1.FlavorGroupCapacityConditionReady, + Status: metav1.ConditionTrue, + Reason: "ReconcileSucceeded", + }} + if err := fakeClient.Status().Patch(context.Background(), existing, patch); err != nil { + t.Fatalf("failed to set Ready=True: %v", err) + } + + // Scheduler returns 500 to simulate error. + failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer failServer.Close() + + ctrl := newController(t, fakeClient, Config{ + SchedulerURL: failServer.URL, + TotalPipeline: "kvm-report-capacity", + PlaceablePipeline: "kvm-general-purpose", + }) + + smallFlavor := compute.FlavorInGroup{Name: groupName + "-small", MemoryMB: memMB, VCPUs: 2} + groupData := compute.FlavorGroupFeature{ + SmallestFlavor: smallFlavor, + Flavors: []compute.FlavorInGroup{smallFlavor}, + } + hv := newHypervisor("host-1", az, memBytes) + hvByName := map[string]hv1.Hypervisor{"host-1": *hv} + + ctrl.reconcileAZ(context.Background(), az, + map[string]compute.FlavorGroupFeature{groupName: groupData}, + hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) + + var crd v1alpha1.FlavorGroupCapacity + if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { + t.Fatalf("failed to get CRD: %v", err) + } + + // CRD must now have Ready=False. + var readyStatus metav1.ConditionStatus + for _, c := range crd.Status.Conditions { + if c.Type == v1alpha1.FlavorGroupCapacityConditionReady { + readyStatus = c.Status + } + } + if readyStatus != metav1.ConditionFalse { + t.Errorf("Ready condition = %q, want False after scheduler error", readyStatus) + } +} + func TestReconcileAZ_IdempotentUpdate(t *testing.T) { const ( groupName = "hana-v2" @@ -842,7 +918,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { ) scheme := newTestScheme(t) - hv := newHypervisor("host-1", az, memBytes, "vm1") + hv := newHypervisor("host-1", az, memBytes) knowledge := newFlavorGroupKnowledge(t, groupName, memMB) // Pre-create CRD with non-zero RunningInstances to simulate prior state. diff --git a/internal/scheduling/reservations/commitments/api/report_capacity.go b/internal/scheduling/reservations/commitments/api/report_capacity.go index 5277d51d0..8b9a324c2 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity.go @@ -5,6 +5,7 @@ package api import ( "encoding/json" + "errors" "net/http" "strconv" "time" @@ -77,7 +78,11 @@ func (api *HTTPAPI) HandleReportCapacity(w http.ResponseWriter, r *http.Request) report, err := calculator.CalculateCapacity(ctx, req) if err != nil { logger.Error(err, "failed to calculate capacity") - statusCode = http.StatusInternalServerError + if errors.Is(err, commitments.ErrCapacityNotReady) { + statusCode = http.StatusServiceUnavailable + } else { + statusCode = http.StatusInternalServerError + } http.Error(w, "Failed to calculate capacity: "+err.Error(), statusCode) api.recordCapacityMetrics(statusCode, startTime) return diff --git a/internal/scheduling/reservations/commitments/api/report_capacity_test.go b/internal/scheduling/reservations/commitments/api/report_capacity_test.go index 530fe319f..4bcb71056 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity_test.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity_test.go @@ -7,6 +7,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "slices" @@ -184,7 +185,8 @@ func TestCapacityCalculator(t *testing.T) { wantCapacity uint64 wantUsage *uint64 // nil = expect absent cfg *commitments.APIConfig - wantResourceCount int // 0 = don't check + wantResourceCount int // 0 = don't check + wantNotReady bool // expect ErrCapacityNotReady } u := func(v uint64) *uint64 { return &v } @@ -196,10 +198,10 @@ func TestCapacityCalculator(t *testing.T) { checkAZ: "az-one", wantCapacity: 1000, wantUsage: u(200), }, { - // stale CRD: last-known capacity still reported, usage omitted - name: "stale CRD: capacity reported, usage absent", + // stale CRD: CalculateCapacity returns ErrCapacityNotReady → caller returns 503 + name: "stale CRD: returns ErrCapacityNotReady", runningInstances: 200, exclusiveFreeBytes: 800 * flavorMemBytes, ready: false, - checkAZ: "az-one", wantCapacity: 1000, wantUsage: nil, + checkAZ: "az-one", wantNotReady: true, }, { // CRD only covers az-one; az-two has no CRD → capacity=0 @@ -233,6 +235,12 @@ func TestCapacityCalculator(t *testing.T) { } report, err := calc.CalculateCapacity(context.Background(), liquid.ServiceCapacityRequest{AllAZs: allAZs}) + if tc.wantNotReady { + if !errors.Is(err, commitments.ErrCapacityNotReady) { + t.Fatalf("expected ErrCapacityNotReady, got %v", err) + } + return + } if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index ee3fc1a99..2d9fbfaa4 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -5,6 +5,7 @@ package commitments import ( "context" + "errors" "fmt" "github.com/sapcc/go-api-declarations/liquid" @@ -16,6 +17,11 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) +// ErrCapacityNotReady is returned by CalculateCapacity when one or more FlavorGroupCapacity +// CRDs have Ready=False, indicating the controller's last probe cycle failed. Callers should +// return 503 Service Unavailable rather than serving potentially stale data. +var ErrCapacityNotReady = errors.New("one or more FlavorGroupCapacity CRDs are not ready") + // CapacityCalculator computes capacity reports for Limes LIQUID API. type CapacityCalculator struct { client client.Client @@ -86,6 +92,7 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S if !apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) { logger.Info("FlavorGroupCapacity CRD is stale, reporting capacity without usage", "flavorGroup", groupName, "az", az) + return liquid.ServiceCapacityReport{}, fmt.Errorf("%w: flavorGroup=%s az=%s", ErrCapacityNotReady, groupName, string(az)) } // ExclusivelyFreeSlots is pre-computed by the controller using min(memSlots, cpuSlots). From c9c289b4283956fcf31ae51614a1bd6a3467a314 Mon Sep 17 00:00:00 2001 From: Julius Clausnitzer Date: Fri, 17 Jul 2026 14:14:05 +0200 Subject: [PATCH 3/4] logs Signed-off-by: Julius Clausnitzer --- .../scheduling/reservations/commitments/api/report_capacity.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/scheduling/reservations/commitments/api/report_capacity.go b/internal/scheduling/reservations/commitments/api/report_capacity.go index 8b9a324c2..4eb91b34c 100644 --- a/internal/scheduling/reservations/commitments/api/report_capacity.go +++ b/internal/scheduling/reservations/commitments/api/report_capacity.go @@ -77,10 +77,11 @@ func (api *HTTPAPI) HandleReportCapacity(w http.ResponseWriter, r *http.Request) calculator := commitments.NewCapacityCalculator(api.client, api.config) report, err := calculator.CalculateCapacity(ctx, req) if err != nil { - logger.Error(err, "failed to calculate capacity") if errors.Is(err, commitments.ErrCapacityNotReady) { + logger.Info("capacity data not ready, returning 503", "reason", err.Error()) statusCode = http.StatusServiceUnavailable } else { + logger.Error(err, "failed to calculate capacity") statusCode = http.StatusInternalServerError } http.Error(w, "Failed to calculate capacity: "+err.Error(), statusCode) From 51b8954154739b805719163776d85a30b4ca4604 Mon Sep 17 00:00:00 2001 From: Julius Clausnitzer Date: Fri, 17 Jul 2026 15:49:53 +0200 Subject: [PATCH 4/4] fix(capacity): remove stale log line and unused logger in CalculateCapacity Signed-off-by: Julius Clausnitzer --- internal/scheduling/reservations/commitments/capacity.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index 2d9fbfaa4..efb7d8b57 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -65,7 +65,6 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S Resources: make(map[liquid.ResourceName]*liquid.ResourceCapacityReport), } - logger := LoggerFromContext(ctx) for groupName, groupData := range flavorGroups { resCfg := c.conf.ResourceConfigForGroup(groupName) // Skip groups not configured for capacity reporting. @@ -90,8 +89,6 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S } if !apimeta.IsStatusConditionTrue(crd.Status.Conditions, v1alpha1.FlavorGroupCapacityConditionReady) { - logger.Info("FlavorGroupCapacity CRD is stale, reporting capacity without usage", - "flavorGroup", groupName, "az", az) return liquid.ServiceCapacityReport{}, fmt.Errorf("%w: flavorGroup=%s az=%s", ErrCapacityNotReady, groupName, string(az)) }