feat: CR controller checks for host overload scenario and resolves via reservation re-placements

cmd/manager/main.go

@@ -626,10 +626,14 @@ func main() {
 		monitor := reservations.NewMonitor(multiclusterClient)
 		metrics.Registry.MustRegister(&monitor)
 
+		reservationControllerMonitor := commitments.NewReservationControllerMonitor()
+		metrics.Registry.MustRegister(&reservationControllerMonitor)
+
 		if err := (&commitments.CommitmentReservationController{
-			Client: multiclusterClient,
-			Scheme: mgr.GetScheme(),
-			Conf:   commitmentsConfig.ReservationController,
+			Client:  multiclusterClient,
+			Scheme:  mgr.GetScheme(),
+			Conf:    commitmentsConfig.ReservationController,
+			Monitor: &reservationControllerMonitor,
 		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "CommitmentReservation")
 			os.Exit(1)

helm/bundles/cortex-nova/templates/alerts.yaml

@@ -749,4 +749,41 @@ spec:
           resource router is mapping the same object to multiple clusters, or an
           object was created out-of-band on the wrong cluster. Investigate the
           affected resources and the routing configuration.
+
+    {{- if .Values.kvm.enabled }}
+    - alert: CortexNovaHostReservationsOversubscribed
+      # Fires when the sum of running VM allocations + reservation blocks (committed +
+      # failover) exceeds the host's effective capacity for CPU or memory.
+      # This can happen due to: concurrent slot creation with stale informer cache,
+      # operator-driven VM migrations where the slot stays on the old host, or
+      # capacity changes (e.g. hardware replacement changing EffectiveCapacity).
+      # The 10m hold-off tolerates the known migration window: after a VM departs,
+      # the slot remains on the old host until the usage reconciler cleans it up.
+      # Note: `reserved` only counts Ready reservations — violations during the
+      # initial unready window (slot just created) are not captured by this alert.
+      expr: |
+        (
+          cortex_kvm_host_capacity_usage{type="utilized"}
+          + on(compute_host, availability_zone, resource) cortex_kvm_host_capacity_usage{type="reserved"}
+          + on(compute_host, availability_zone, resource) cortex_kvm_host_capacity_usage{type="failover"}
+          - on(compute_host, availability_zone, resource) cortex_kvm_host_capacity_total
+        ) > 0
+      for: 10m
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "Host {{ "{{" }} $labels.compute_host {{ "}}" }} reservation blocks exceed capacity for {{ "{{" }} $labels.resource {{ "}}" }}"
+        description: >
+          The total of running VM allocations and reservation blocks (committed resource +
+          failover) on host {{ "{{" }} $labels.compute_host {{ "}}" }} exceeds its effective
+          capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize1024 {{ "}}" }}.
+          This means the host is over-subscribed and committed resource guarantees may not be
+          honourable. Common causes: operator-driven VM migration with slot not yet reclaimed, a hardware
+          capacity change, or out of sync issues. If problem remains, inspect the reservations on this host and check the CR controller logs.
+    {{- end }}
 {{- end }}

helm/bundles/cortex-nova/values.yaml

@@ -177,14 +177,17 @@ cortex-scheduling-controllers:
         "*": "kvm-general-purpose-load-balancing"
       pipelineDefault: "kvm-general-purpose-load-balancing"
       # How often to re-verify active Reservation CRDs (healthy state)
-      requeueIntervalActive: "5m"
+      requeueIntervalActive: "30m"
       # Back-off interval when knowledge is unavailable
       requeueIntervalRetry: "1m"
       # Back-off interval while a VM allocation is still within allocationGracePeriod
       requeueIntervalGracePeriod: "1m"
       # How long after a VM is allocated to a reservation before it is expected to appear
       # on the target host; allocations not confirmed within this window are removed
       allocationGracePeriod: "15m"
+      # How long to wait after first detecting host over-subscription before evicting
+      # reservation slots. Gives other controllers (e.g. failover) time to self-heal.
+      oversubscriptionGracePeriod: "2m"
       # URL of the nova external scheduler API for placement decisions
       schedulerURL: "http://localhost:8080/scheduler/nova/external"
       # Keystone credentials used to resolve domain IDs to domain names for the

internal/scheduling/reservations/capacity_accounting.go

@@ -75,6 +75,45 @@ func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv
 	return true
 }
 
+// HostFreeCapacity computes the remaining free capacity on hv after subtracting
+// hv.Status.Allocation and UnusedReservationCapacity for all reservations on this host.
+// Negative values indicate over-subscription for that resource.
+// Returns nil when the hypervisor has no capacity data.
+// Reservations not targeting this host (via Spec.TargetHost or Status.Host) are ignored.
+func HostFreeCapacity(hostReservations []v1alpha1.Reservation, hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity {
+	effCap := hv.Status.EffectiveCapacity
+	if effCap == nil {
+		effCap = hv.Status.Capacity
+	}
+	if effCap == nil {
+		return nil
+	}
+
+	free := make(map[hv1.ResourceName]resource.Quantity, len(effCap))
+	for rn, qty := range effCap {
+		free[rn] = qty.DeepCopy()
+	}
+	for rn, allocated := range hv.Status.Allocation {
+		if f, ok := free[rn]; ok {
+			f.Sub(allocated)
+			free[rn] = f
+		}
+	}
+	for i := range hostReservations {
+		res := &hostReservations[i]
+		if res.Spec.TargetHost != hv.Name && res.Status.Host != hv.Name {
+			continue
+		}
+		for rn, block := range UnusedReservationCapacity(res, false) {
+			if f, ok := free[rn]; ok {
+				f.Sub(block)
+				free[rn] = f
+			}
+		}
+	}
+	return free
+}
+
 // UnusedReservationCapacity returns the resources a Reservation should block on its host(s).
 // This is the single source of truth used by both the capacity controller and
 // filter_has_enough_capacity to ensure consistent accounting.

internal/scheduling/reservations/capacity_accounting_test.go

@@ -172,6 +172,190 @@ func TestUnusedReservationCapacity(t *testing.T) {
 	}
 }
 
+func TestHostFreeCapacity(t *testing.T) {
+	gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) }
+	cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) }
+
+	hvWithCap := func(name string, memGiB, cpuCores int64) hv1.Hypervisor {
+		return hv1.Hypervisor{
+			ObjectMeta: metav1.ObjectMeta{Name: name},
+			Status: hv1.HypervisorStatus{
+				EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
+					hv1.ResourceMemory: gib(memGiB),
+					hv1.ResourceCPU:    cpu(cpuCores),
+				},
+			},
+		}
+	}
+	crSlot := func(name, host string, memGiB, cpuCores int64) v1alpha1.Reservation {
+		return v1alpha1.Reservation{
+			ObjectMeta: metav1.ObjectMeta{Name: name},
+			Spec: v1alpha1.ReservationSpec{
+				Type:       v1alpha1.ReservationTypeCommittedResource,
+				TargetHost: host,
+				Resources: map[hv1.ResourceName]resource.Quantity{
+					hv1.ResourceMemory: gib(memGiB),
+					hv1.ResourceCPU:    cpu(cpuCores),
+				},
+			},
+			Status: v1alpha1.ReservationStatus{Host: host},
+		}
+	}
+	freeMemGiB := func(free map[hv1.ResourceName]resource.Quantity) int64 {
+		q := free[hv1.ResourceMemory]
+		return q.Value() / (1024 * 1024 * 1024)
+	}
+	freeCPU := func(free map[hv1.ResourceName]resource.Quantity) int64 {
+		q := free[hv1.ResourceCPU]
+		return q.Value()
+	}
+
+	t.Run("no capacity data returns nil", func(t *testing.T) {
+		hv := hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host"}}
+		if got := HostFreeCapacity(nil, hv); got != nil {
+			t.Errorf("expected nil, got %v", got)
+		}
+	})
+
+	t.Run("no reservations and no allocation: free = effective capacity", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		free := HostFreeCapacity(nil, hv)
+		if freeMemGiB(free) != 1024 {
+			t.Errorf("expected 1024 GiB free, got %d", freeMemGiB(free))
+		}
+		if freeCPU(free) != 256 {
+			t.Errorf("expected 256 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("allocation subtracted from capacity", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{
+			hv1.ResourceMemory: gib(512),
+			hv1.ResourceCPU:    cpu(128),
+		}
+		free := HostFreeCapacity(nil, hv)
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free))
+		}
+		if freeCPU(free) != 128 {
+			t.Errorf("expected 128 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("reservation blocks subtracted", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-1", "host", 512, 128),
+		}
+		free := HostFreeCapacity(slots, hv)
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free))
+		}
+		if freeCPU(free) != 128 {
+			t.Errorf("expected 128 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("over-subscribed: negative free values", func(t *testing.T) {
+		// 5 x 1TiB slots on a 4TiB host — the production scenario
+		hv := hvWithCap("host", 4096, 256)
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-0", "host", 1024, 128),
+			crSlot("slot-1", "host", 1024, 128),
+			crSlot("slot-2", "host", 1024, 128),
+			crSlot("slot-3", "host", 1024, 128),
+			crSlot("slot-4", "host", 1024, 128),
+		}
+		free := HostFreeCapacity(slots, hv)
+		if freeMemGiB(free) != -1024 {
+			t.Errorf("expected -1024 GiB (over-subscribed), got %d GiB", freeMemGiB(free))
+		}
+		if freeCPU(free) != -384 {
+			t.Errorf("expected -384 CPU (over-subscribed), got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("allocation + reservations combined", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{
+			hv1.ResourceMemory: gib(256),
+			hv1.ResourceCPU:    cpu(64),
+		}
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-1", "host", 512, 128),
+		}
+		free := HostFreeCapacity(slots, hv)
+		// 1024 - 256 (alloc) - 512 (slot) = 256 GiB free
+		if freeMemGiB(free) != 256 {
+			t.Errorf("expected 256 GiB free, got %d", freeMemGiB(free))
+		}
+		// 256 - 64 (alloc) - 128 (slot) = 64 free
+		if freeCPU(free) != 64 {
+			t.Errorf("expected 64 CPU free, got %d", freeCPU(free))
+		}
+	})
+
+	t.Run("confirmed VM reduces slot block (not double counted)", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		// 256 GiB confirmed VM already counted in Allocation
+		hv.Status.Allocation = map[hv1.ResourceName]resource.Quantity{
+			hv1.ResourceMemory: gib(256),
+		}
+		// 512 GiB slot with 256 GiB confirmed VM → block = 512-256 = 256 GiB
+		slot := v1alpha1.Reservation{
+			ObjectMeta: metav1.ObjectMeta{Name: "slot-1"},
+			Spec: v1alpha1.ReservationSpec{
+				Type:       v1alpha1.ReservationTypeCommittedResource,
+				TargetHost: "host",
+				Resources:  map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(512)},
+				CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{
+					Allocations: map[string]v1alpha1.CommittedResourceAllocation{
+						"vm-1": {Resources: map[hv1.ResourceName]resource.Quantity{hv1.ResourceMemory: gib(256)}},
+					},
+				},
+			},
+			Status: v1alpha1.ReservationStatus{
+				Host: "host",
+				CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{
+					Allocations: map[string]string{"vm-1": "host"},
+				},
+			},
+		}
+		free := HostFreeCapacity([]v1alpha1.Reservation{slot}, hv)
+		// 1024 - 256 (alloc/vm) - 256 (remaining slot block) = 512
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB free, got %d", freeMemGiB(free))
+		}
+	})
+
+	t.Run("reservations on other hosts are ignored even if passed in", func(t *testing.T) {
+		hv := hvWithCap("host", 1024, 256)
+		slots := []v1alpha1.Reservation{
+			crSlot("slot-other", "other-host", 1024, 256), // different host — must not block
+		}
+		free := HostFreeCapacity(slots, hv)
+		if freeMemGiB(free) != 1024 {
+			t.Errorf("expected 1024 GiB free (other host ignored), got %d", freeMemGiB(free))
+		}
+	})
+
+	t.Run("falls back to Capacity when EffectiveCapacity nil", func(t *testing.T) {
+		hv := hv1.Hypervisor{
+			ObjectMeta: metav1.ObjectMeta{Name: "host"},
+			Status: hv1.HypervisorStatus{
+				Capacity: map[hv1.ResourceName]resource.Quantity{
+					hv1.ResourceMemory: gib(512),
+				},
+			},
+		}
+		free := HostFreeCapacity(nil, hv)
+		if freeMemGiB(free) != 512 {
+			t.Errorf("expected 512 GiB, got %d", freeMemGiB(free))
+		}
+	})
+}
+
 func TestHostHasCapacityForReservation(t *testing.T) {
 	gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) }
 	cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) }

internal/scheduling/reservations/commitments/committed_resource_controller_test.go

@@ -22,6 +22,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
+	"github.com/cobaltcore-dev/cortex/internal/scheduling/reservations"
 )
 
 // ============================================================================
@@ -137,6 +138,24 @@ func newCRTestClient(scheme *runtime.Scheme, objects ...client.Object) client.Cl
 			}
 			return uuids
 		}).
+		WithIndex(&v1alpha1.Reservation{}, reservations.IdxReservationByHost, func(obj client.Object) []string {
+			res, ok := obj.(*v1alpha1.Reservation)
+			if !ok {
+				return nil
+			}
+			hosts := make(map[string]struct{})
+			if res.Spec.TargetHost != "" {
+				hosts[res.Spec.TargetHost] = struct{}{}
+			}
+			if res.Status.Host != "" {
+				hosts[res.Status.Host] = struct{}{}
+			}
+			result := make([]string, 0, len(hosts))
+			for h := range hosts {
+				result = append(result, h)
+			}
+			return result
+		}).
 		Build()
 }
 

internal/scheduling/reservations/commitments/config.go

@@ -62,6 +62,9 @@ type ReservationControllerConfig struct {
 	// reservation during which it's expected to appear on the target host.
 	// VMs not confirmed within this period are considered stale and removed.
 	AllocationGracePeriod metav1.Duration `json:"allocationGracePeriod"`
+	// OversubscriptionGracePeriod is how long to wait after detecting host over-subscription
+	// before evicting slots. Gives other controllers (e.g. failover) time to self-heal.
+	OversubscriptionGracePeriod metav1.Duration `json:"oversubscriptionGracePeriod,omitempty"`
 	// SchedulerURL is the endpoint of the nova external scheduler.
 	SchedulerURL string `json:"schedulerURL"`
 	// PipelineDefault is the fallback pipeline when no FlavorGroupPipelines entry matches.

internal/scheduling/reservations/commitments/field_index.go

@@ -131,7 +131,6 @@ func indexProjectQuotaByProjectID(ctx context.Context, mcl *multicluster.Client)
 	return err
 }
 
-// indexReservationByAllocationVMUUID registers an index over all VM UUIDs present in
 // Spec.CommittedResourceReservation.Allocations. This allows the reservation controller
 // to efficiently find all other Reservation CRDs carrying a specific VM UUID without
 // scanning every reservation in the cluster.