From 04a7617d34fe161bcb3d79f16a9fbc703e5d2a91 Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 3 Aug 2026 15:38:16 +0200 Subject: [PATCH 01/16] Implement in-flight reservations controller (#957) Introduce a controller that owns the lifecycle of in-flight reservations, which are placement decisions that have been committed but whose VM has not yet appeared on a hypervisor. Assisted-by: Claude Code:claude-opus-4-8 [Bash] [Read] --------- Signed-off-by: Philipp Matthes Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Marcel <156897072+mblos@users.noreply.github.com> --- cmd/manager/main.go | 13 + .../bundles/cortex-nova/templates/alerts.yaml | 37 + helm/bundles/cortex-nova/templates/kpis.yaml | 15 + helm/bundles/cortex-nova/values.yaml | 1 + .../plugins/deployment/reservation_state.go | 117 +++ .../deployment/reservation_state_test.go | 377 +++++++++ internal/knowledge/kpis/supported_kpis.go | 11 +- .../reservations/inflight/controller.go | 426 ++++++++++ .../reservations/inflight/controller_test.go | 727 ++++++++++++++++++ .../reservations/inflight/vm_client.go | 120 +++ .../reservations/inflight/vm_client_test.go | 167 ++++ 11 files changed, 2006 insertions(+), 5 deletions(-) create mode 100644 internal/knowledge/kpis/plugins/deployment/reservation_state.go create mode 100644 internal/knowledge/kpis/plugins/deployment/reservation_state_test.go create mode 100644 internal/scheduling/reservations/inflight/controller.go create mode 100644 internal/scheduling/reservations/inflight/controller_test.go create mode 100644 internal/scheduling/reservations/inflight/vm_client.go create mode 100644 internal/scheduling/reservations/inflight/vm_client_test.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index fa18b7a9f..fca0c0550 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -63,6 +63,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments" commitmentsapi "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/commitments/api" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/failover" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/inflight" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/quota" "github.com/cobaltcore-dev/cortex/pkg/conf" "github.com/cobaltcore-dev/cortex/pkg/monitoring" @@ -499,6 +500,18 @@ func main() { os.Exit(1) } } + if slices.Contains(mainConfig.EnabledControllers, "inflight-reservation-controller") { + setupLog.Info("enabling controller", + "controller", "inflight-reservation-controller") + config := conf.GetConfigOrDie[inflight.NovaVMClientConfig]() + vmClient := inflight.NewNovaVMClient(config) + controller := &inflight.Controller{Client: multiclusterClient, VMClient: vmClient} + if err := controller.SetupWithManager(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create controller", + "controller", "inflight-reservation-controller") + os.Exit(1) + } + } if slices.Contains(mainConfig.EnabledControllers, "nova-deschedulings-executor") { setupLog.Info("enabling controller", "controller", "nova-deschedulings-executor") executorConfig := conf.GetConfigOrDie[nova.DeschedulingsExecutorConfig]() diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index e6e08e8e3..09e9bae98 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -303,6 +303,43 @@ spec: configuration. It is recommended to investigate the pipeline status and logs for more details. + - alert: CortexNovaInFlightReservationsBacklogGrowing + # Alert when the number of in-flight reservations stuck with + # reason=InstanceNotFound is trending upwards over a long window, + # rather than staying roughly the same or draining down. A healthy + # scheduler-hypervisor loop should quickly resolve in-flight + # reservations once the VM shows up on any hypervisor; a sustained + # positive derivative indicates the backlog is not being worked off. + # + # deriv() over 1h returns the per-second slope of the gauge; multiplied + # by 3600 it becomes "expected growth per hour". We require this to be + # meaningfully positive AND the current count to be non-trivial, so we + # don't page on a single lingering reservation. + expr: | + deriv(cortex_reservation_state{domain="nova",type="InFlightReservation",state="unknown",reason="InstanceNotFound"}[1h]) * 3600 > 5 + and + cortex_reservation_state{domain="nova",type="InFlightReservation",state="unknown",reason="InstanceNotFound"} > 10 + for: 2h + labels: + context: reservations + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/unready + annotations: + summary: "In-flight reservation backlog with reason=InstanceNotFound keeps growing" + description: > + The number of in-flight reservations that have not yet observed + their VM on any hypervisor (reason=InstanceNotFound) has been + increasing steadily for at least two hours. Normally these + reservations resolve once the hypervisor operator reports the + instance; a sustained positive trend suggests either VMs are not + landing on hypervisors, or the hypervisor status feed is stalled, + or the in-flight reservation controller is not pruning cleanly. + Investigate the hypervisor operator, the nova build pipeline, and + the in-flight reservation controller logs. + - alert: CortexNovaImagePropertiesHypervisorTypeUndetermined expr: | sum by (pipeline, step) (rate(cortex_filter_weigher_pipeline_step_events_total{service="cortex-nova-metrics", event="image_properties_hv_type_undetermined"}[5m])) > 0.1 diff --git a/helm/bundles/cortex-nova/templates/kpis.yaml b/helm/bundles/cortex-nova/templates/kpis.yaml index 5717cd62e..418389e95 100644 --- a/helm/bundles/cortex-nova/templates/kpis.yaml +++ b/helm/bundles/cortex-nova/templates/kpis.yaml @@ -146,6 +146,21 @@ spec: --- apiVersion: cortex.cloud/v1alpha1 kind: KPI +metadata: + name: cortex-nova-reservation-state +spec: + schedulingDomain: nova + impl: reservation_state_kpi + opts: + reservationSchedulingDomain: nova + description: | + This KPI tracks the state of reservation resources managed by cortex, + labelled by reservation type (e.g. InFlightReservation) and the reason + on the Ready condition. It is used to alert on sustained increases of + unready in-flight reservations (e.g. reason=InstanceNotFound). +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI metadata: name: vmware-project-utilization spec: diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 27acd10e1..6524c095e 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -138,6 +138,7 @@ cortex-scheduling-controllers: component: nova-scheduling enabledControllers: - nova-pipeline-controllers + - inflight-reservation-controller - nova-deschedulings-executor - hypervisor-overcommit-controller - committed-resource-reservations-controller diff --git a/internal/knowledge/kpis/plugins/deployment/reservation_state.go b/internal/knowledge/kpis/plugins/deployment/reservation_state.go new file mode 100644 index 000000000..ba6963d53 --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/reservation_state.go @@ -0,0 +1,117 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "time" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var reservationStateKPILogger = ctrl.Log.WithName("reservation-state-kpi") + +type ReservationStateKPIOpts struct { + // The scheduling domain to filter reservations by. + ReservationSchedulingDomain v1alpha1.SchedulingDomain `json:"reservationSchedulingDomain"` +} + +// KPI observing the state of reservation resources managed by cortex. +// Metrics are labeled by the reservation type (e.g. InFlightReservation, +// CommittedResourceReservation, FailoverReservation) so that operators can +// alert on specific reservation kinds. The state label mirrors the +// v1alpha1.ReservationConditionReady condition status and the reason label +// carries the condition's Reason so that alerts can target specific failure +// modes (e.g. reason="InstanceNotFound" for in-flight reservations whose VM +// has not spawned on any hypervisor yet). +type ReservationStateKPI struct { + // Common base for all KPIs that provides standard functionality. + plugins.BaseKPI[ReservationStateKPIOpts] + + // Prometheus descriptor for the reservation state metric. + counter *prometheus.Desc +} + +func (ReservationStateKPI) GetName() string { return "reservation_state_kpi" } + +// Initialize the KPI. +func (k *ReservationStateKPI) Init(db *db.DB, client client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(db, client, opts); err != nil { + return err + } + k.counter = prometheus.NewDesc( + "cortex_reservation_state", + "State of cortex managed reservations", + []string{"domain", "type", "state", "reason"}, + nil, + ) + return nil +} + +// Conform to the prometheus collector interface by providing the descriptor. +func (k *ReservationStateKPI) Describe(ch chan<- *prometheus.Desc) { ch <- k.counter } + +// Collect the reservation state metrics. +func (k *ReservationStateKPI) Collect(ch chan<- prometheus.Metric) { + // Bound the list call so a slow API server can't hang the Prometheus + // scrape indefinitely; if it fails we log so the disappearance of the + // reservation metric is not silent. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // Get all reservations. The scheduling domain filter is applied per item + // since a Reservation is cluster-scoped and may cover multiple domains. + reservationList := &v1alpha1.ReservationList{} + if err := k.Client.List(ctx, reservationList); err != nil { + reservationStateKPILogger.Error(err, "Failed to list reservations") + return + } + // Aggregate counts by (type, state, reason) so that we emit one time + // series per bucket rather than one per reservation. This keeps metric + // cardinality bounded regardless of how many reservations exist. + type bucket struct { + reservationType string + state string + reason string + } + counts := map[bucket]float64{} + for _, r := range reservationList.Items { + if r.Spec.SchedulingDomain != k.Options.ReservationSchedulingDomain { + continue + } + state := "unknown" + reason := "" + cond := meta.FindStatusCondition(r.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond != nil { + reason = cond.Reason + switch cond.Status { + case metav1.ConditionTrue: + state = "ready" + case metav1.ConditionFalse: + state = "error" + default: + state = "unknown" + } + } + counts[bucket{ + reservationType: string(r.Spec.Type), + state: state, + reason: reason, + }]++ + } + for b, v := range counts { + ch <- prometheus.MustNewConstMetric( + k.counter, prometheus.GaugeValue, v, + string(k.Options.ReservationSchedulingDomain), + b.reservationType, b.state, b.reason, + ) + } +} diff --git a/internal/knowledge/kpis/plugins/deployment/reservation_state_test.go b/internal/knowledge/kpis/plugins/deployment/reservation_state_test.go new file mode 100644 index 000000000..19f543ad9 --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/reservation_state_test.go @@ -0,0 +1,377 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestReservationStateKPI_Init(t *testing.T) { + kpi := &ReservationStateKPI{} + if err := kpi.Init(nil, nil, conf.NewRawOpts(`{"reservationSchedulingDomain": "nova"}`)); err != nil { + t.Fatalf("expected no error, got %v", err) + } +} + +func TestReservationStateKPI_GetName(t *testing.T) { + kpi := &ReservationStateKPI{} + expectedName := "reservation_state_kpi" + if name := kpi.GetName(); name != expectedName { + t.Errorf("expected name %q, got %q", expectedName, name) + } +} + +func TestReservationStateKPI_Describe(t *testing.T) { + kpi := &ReservationStateKPI{} + if err := kpi.Init(nil, nil, conf.NewRawOpts(`{"reservationSchedulingDomain": "nova"}`)); err != nil { + t.Fatalf("expected no error, got %v", err) + } + ch := make(chan *prometheus.Desc, 1) + kpi.Describe(ch) + close(ch) + descCount := 0 + for range ch { + descCount++ + } + if descCount != 1 { + t.Errorf("expected 1 descriptor, got %d", descCount) + } +} + +func TestReservationStateKPI_Collect(t *testing.T) { + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + tests := []struct { + name string + reservations []v1alpha1.Reservation + operator v1alpha1.SchedulingDomain + expectedCount int + description string + }{ + { + name: "no reservations", + reservations: []v1alpha1.Reservation{}, + operator: "nova", + expectedCount: 0, + description: "should not collect metrics when no reservations exist", + }, + { + name: "single ready in-flight reservation", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: v1.ObjectMeta{Name: "r1"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionTrue, + Reason: "ReservationReady", + }, + }, + }, + }, + }, + operator: "nova", + expectedCount: 1, + description: "should collect a single ready metric", + }, + { + name: "unknown in-flight reservation with InstanceNotFound reason", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: v1.ObjectMeta{Name: "r2"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionUnknown, + Reason: "InstanceNotFound", + }, + }, + }, + }, + }, + operator: "nova", + expectedCount: 1, + description: "should collect a metric labelled with reason=InstanceNotFound", + }, + { + name: "reservation without any conditions falls back to unknown", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: v1.ObjectMeta{Name: "r3"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + }, + }, + operator: "nova", + expectedCount: 1, + description: "reservations without conditions should still emit a metric", + }, + { + name: "multiple in-flight reservations with the same reason are aggregated", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: v1.ObjectMeta{Name: "r-a"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionUnknown, + Reason: "InstanceNotFound", + }, + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{Name: "r-b"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionUnknown, + Reason: "InstanceNotFound", + }, + }, + }, + }, + }, + operator: "nova", + expectedCount: 1, + description: "two reservations with the same (type,state,reason) should share one time series", + }, + { + name: "different reservation types emit separate metrics", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: v1.ObjectMeta{Name: "r-if"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionTrue, + Reason: "ReservationReady", + }, + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{Name: "r-cr"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionTrue, + Reason: "ReservationReady", + }, + }, + }, + }, + }, + operator: "nova", + expectedCount: 2, + description: "in-flight and committed-resource reservations should be labelled separately", + }, + { + name: "filter by scheduling domain", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: v1.ObjectMeta{Name: "r-nova"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionTrue, + }, + }, + }, + }, + { + ObjectMeta: v1.ObjectMeta{Name: "r-other"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "ironcore", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionTrue, + }, + }, + }, + }, + }, + operator: "nova", + expectedCount: 1, + description: "only reservations matching the configured domain should be counted", + }, + { + name: "error reservation is labelled state=error", + reservations: []v1alpha1.Reservation{ + { + ObjectMeta: v1.ObjectMeta{Name: "r-err"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionFalse, + Reason: "UnexpectedType", + }, + }, + }, + }, + }, + operator: "nova", + expectedCount: 1, + description: "false Ready condition should surface as state=error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := make([]v1alpha1.Reservation, len(tt.reservations)) + copy(objects, tt.reservations) + + clientBuilder := fake.NewClientBuilder().WithScheme(scheme) + for i := range objects { + clientBuilder = clientBuilder.WithObjects(&objects[i]) + } + client := clientBuilder.Build() + + kpi := &ReservationStateKPI{} + if err := kpi.Init(nil, client, conf.NewRawOpts(`{"reservationSchedulingDomain": "`+string(tt.operator)+`"}`)); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + ch := make(chan prometheus.Metric, 16) + kpi.Collect(ch) + close(ch) + + metricsCount := 0 + for range ch { + metricsCount++ + } + if metricsCount != tt.expectedCount { + t.Errorf("%s: expected %d metrics, got %d", tt.description, tt.expectedCount, metricsCount) + } + }) + } +} + +// TestReservationStateKPI_CollectLabels verifies that the metric emitted for +// an in-flight reservation with reason InstanceNotFound carries the labels +// the InFlightReservationUnready alert relies on. +func TestReservationStateKPI_CollectLabels(t *testing.T) { + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + res := &v1alpha1.Reservation{ + ObjectMeta: v1.ObjectMeta{Name: "r-nf"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: "nova", + }, + Status: v1alpha1.ReservationStatus{ + Conditions: []v1.Condition{ + { + Type: v1alpha1.ReservationConditionReady, + Status: v1.ConditionUnknown, + Reason: "InstanceNotFound", + }, + }, + }, + } + // Also register a second identical reservation so that we can assert the + // counter carries the aggregate count (2). + res2 := res.DeepCopy() + res2.Name = "r-nf-2" + + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(res, res2).Build() + + kpi := &ReservationStateKPI{} + if err := kpi.Init(nil, client, conf.NewRawOpts(`{"reservationSchedulingDomain": "nova"}`)); err != nil { + t.Fatalf("expected no error, got %v", err) + } + ch := make(chan prometheus.Metric, 4) + kpi.Collect(ch) + close(ch) + + found := false + for m := range ch { + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + labels := map[string]string{} + for _, l := range metric.Label { + labels[l.GetName()] = l.GetValue() + } + if labels["type"] != string(v1alpha1.ReservationTypeInFlight) { + continue + } + found = true + if labels["domain"] != "nova" { + t.Errorf("expected domain=nova, got %q", labels["domain"]) + } + if labels["state"] != "unknown" { + t.Errorf("expected state=unknown, got %q", labels["state"]) + } + if labels["reason"] != "InstanceNotFound" { + t.Errorf("expected reason=InstanceNotFound, got %q", labels["reason"]) + } + if got := metric.Gauge.GetValue(); got != 2 { + t.Errorf("expected aggregate count 2, got %f", got) + } + } + if !found { + t.Fatal("expected an in-flight reservation metric to be collected") + } +} diff --git a/internal/knowledge/kpis/supported_kpis.go b/internal/knowledge/kpis/supported_kpis.go index 155e3aab9..d1222e27b 100644 --- a/internal/knowledge/kpis/supported_kpis.go +++ b/internal/knowledge/kpis/supported_kpis.go @@ -29,9 +29,10 @@ var supportedKPIs = map[string]plugins.KPI{ "netapp_storage_pool_cpu_usage_kpi": &storage.NetAppStoragePoolCPUUsageKPI{}, - "datasource_state_kpi": &deployment.DatasourceStateKPI{}, - "knowledge_state_kpi": &deployment.KnowledgeStateKPI{}, - "decision_state_kpi": &deployment.DecisionStateKPI{}, - "kpi_state_kpi": &deployment.KPIStateKPI{}, - "pipeline_state_kpi": &deployment.PipelineStateKPI{}, + "datasource_state_kpi": &deployment.DatasourceStateKPI{}, + "knowledge_state_kpi": &deployment.KnowledgeStateKPI{}, + "decision_state_kpi": &deployment.DecisionStateKPI{}, + "kpi_state_kpi": &deployment.KPIStateKPI{}, + "pipeline_state_kpi": &deployment.PipelineStateKPI{}, + "reservation_state_kpi": &deployment.ReservationStateKPI{}, } diff --git a/internal/scheduling/reservations/inflight/controller.go b/internal/scheduling/reservations/inflight/controller.go new file mode 100644 index 000000000..4292fa334 --- /dev/null +++ b/internal/scheduling/reservations/inflight/controller.go @@ -0,0 +1,426 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package inflight + +import ( + "context" + "errors" + "reflect" + "slices" + "time" + + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var ( + idxReservationByTargetHost = "spec.targetHost" + idxReservationByTargetHostFn = func(obj client.Object) []string { + res, ok := obj.(*v1alpha1.Reservation) + if !ok { + return nil + } + if res.Spec.TargetHost == "" { + return nil + } + return []string{res.Spec.TargetHost} + } +) + +// Controller owns the lifecycle of in-flight reservations. +type Controller struct { + client.Client + + // VMClient is a client to call the source of truth for VMs. + VMClient VMClient +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.20.2/pkg/reconcile +// +// For more details about the method shape, read up here: +// - https://ahmet.im/blog/controller-pitfalls/#reconcile-method-shape +func (c *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + log.V(1).Info("Reconciling resource") + + obj := new(v1alpha1.Reservation) + if err := c.Get(ctx, req.NamespacedName, obj); err != nil { + if apierrors.IsNotFound(err) { + // If the custom resource is not found then it usually means + // that it was deleted or not created. + log.Info("Resource not found. Ignoring since object must be deleted") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + log.Error(err, "Failed to get resource") + return ctrl.Result{}, err + } + + // Sanity checks which should always succeed due to the predicate. + if obj.Spec.Type != v1alpha1.ReservationTypeInFlight { + log.Error(errors.New("unexpected reservation type"), + "Received a reservation with an unexpected type", + "reservationType", obj.Spec.Type) + orig := obj.DeepCopy() + meta.SetStatusCondition(&obj.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionFalse, + Reason: "UnexpectedType", + Message: "Expected reservation type to be InFlightReservation", + }) + return ctrl.Result{}, c.Status().Patch(ctx, obj, client.MergeFrom(orig)) + } + if obj.Spec.InFlightReservation == nil { + log.Error(errors.New("missing in-flight reservation spec"), + "Received a reservation with missing in-flight reservation spec") + orig := obj.DeepCopy() + meta.SetStatusCondition(&obj.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionFalse, + Reason: "MissingSpec", + Message: "In-flight reservation spec is required when type is InFlightReservation", + }) + return ctrl.Result{}, c.Status().Patch(ctx, obj, client.MergeFrom(orig)) + } + + // Get a list of all hypervisors and check if the instance + // has spawned on any of them. + hvs := new(hv1.HypervisorList) + if err := c.List(ctx, hvs); err != nil { + log.Error(err, "Failed to list hypervisors") + return ctrl.Result{}, err + } + found := false + hypervisorName := "" + for _, hv := range hvs.Items { + for _, instance := range hv.Status.Instances { + if instance.ID == obj.Spec.InFlightReservation.VMID { + found = true + hypervisorName = hv.Name + break + } + } + if found { + break + } + } + + if !found { + // The instance has not spawned on any hypervisor (yet). + // Requeue and check again later. We'll alert on this if there are + // too many requeues without the instance spawning. + + // TODO: delete reservation if spec.endTime is exceeded + log.V(1).Info("Instance has not spawned on any hypervisor yet, requeuing", + "vmID", obj.Spec.InFlightReservation.VMID) + orig := obj.DeepCopy() + meta.SetStatusCondition(&obj.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionUnknown, + Reason: "InstanceNotFound", + Message: "The instance has not spawned on any hypervisor yet", + }) + if err := c.Status().Patch(ctx, obj, client.MergeFrom(orig)); err != nil { + log.Error(err, "Failed to update reservation status") + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // We cannot free this reservation if the instance is currently being + // resized (=migrated to the same hypervisor), i.e. the reservation + // doesn't match the actual size of the vm yet. To check the vm size, + // we need to query the source of truth for vms. Only relevant when the + // instance actually landed on the reservation's target host — for a + // stale reservation (instance on a different host) this branch would + // otherwise fire an unnecessary vmClient call and mark the reservation + // VMSizeMismatch when it should just fall through to the + // awaiting-deletion no-op below. + if hypervisorName == obj.Spec.TargetHost && slices.Contains([]v1alpha1.SchedulingIntent{ + novaapi.RebuildIntent, + novaapi.ResizeIntent, + }, obj.Spec.InFlightReservation.Intent) { + vmSize, err := c.VMClient.GetCurrentVMSize(ctx, obj.Spec.InFlightReservation.VMID) + if err != nil { + log.Error(err, "Failed to get current VM size from vmClient", + "vmID", obj.Spec.InFlightReservation.VMID) + return ctrl.Result{}, err + } + // Compare the resource maps semantically: reflect.DeepEqual on + // resource.Quantity can return false for numerically equal values + // because Quantity's unexported cached string/format state may + // differ between values coming from API decoding (reservation + // spec) and values freshly constructed by the vm client. Using + // Quantity.Cmp avoids getting stuck in VMSizeMismatch forever. + sizeMatches := len(vmSize) == len(obj.Spec.Resources) + if sizeMatches { + for k, want := range obj.Spec.Resources { + got, ok := vmSize[k] + if !ok || got.Cmp(want) != 0 { + sizeMatches = false + break + } + } + } + if !sizeMatches { + log.V(1).Info("VM size does not match reservation size yet, requeuing", + "vmID", obj.Spec.InFlightReservation.VMID, + "reservationSize", obj.Spec.Resources, + "currentVMSize", vmSize) + orig := obj.DeepCopy() + meta.SetStatusCondition(&obj.Status.Conditions, metav1.Condition{ + Type: v1alpha1.ReservationConditionReady, + Status: metav1.ConditionUnknown, + Reason: "VMSizeMismatch", + Message: "The current VM size does not match the reservation size yet", + }) + if err := c.Status().Patch(ctx, obj, client.MergeFrom(orig)); err != nil { + log.Error(err, "Failed to update reservation status") + return ctrl.Result{}, err + } + // TODO: EndTime check needed to catch canceled scenarios. Waiting for VM crd to consider this scenario + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + } + + // If the instance spawned on the hypervisor expected by the reservation, + // we batch-delete all reservations for that instance, including the + // reconciled one. + if hypervisorName == obj.Spec.TargetHost { + log.V(1).Info("Instance has spawned on the expected hypervisor, deleting reservations", + "vmID", obj.Spec.InFlightReservation.VMID, + "hypervisor", hypervisorName) + reservations := new(v1alpha1.ReservationList) + if err := c.List(ctx, reservations, client.MatchingFields{ + idxReservationByTargetHost: hypervisorName, + }); err != nil { + log.Error(err, "Failed to list reservations for hypervisor", + "hypervisor", hypervisorName) + return ctrl.Result{}, err + } + for _, res := range reservations.Items { + if res.Spec.InFlightReservation == nil { + continue // Not an in-flight reservation, skip. + } + if res.Spec.InFlightReservation.VMID != obj.Spec.InFlightReservation.VMID { + continue // Not the same instance, skip. + } + if err := c.Delete(ctx, &res); err != nil { + log.Error(err, "Failed to delete reservation", + "reservation", res.Name, + "vmID", res.Spec.InFlightReservation.VMID) + return ctrl.Result{}, err + } + log.V(1).Info("Deleted reservation", + "reservation", res.Name, + "vmID", res.Spec.InFlightReservation.VMID) + } + return ctrl.Result{}, nil + } + + // This reservation will be deleted by the controller when the + // instance spawns on the expected hypervisor, so we don't need to do + // anything else here. + log.V(1).Info("Reservation stale -- awaiting deletion", + "vmID", obj.Spec.InFlightReservation.VMID, + "expectedHypervisor", obj.Spec.TargetHost, + "actualHypervisor", hypervisorName) + return ctrl.Result{}, nil +} + +// handleReservations generates a new event handler for in flight reservations. +func (c *Controller) handleReservations() handler.EventHandler { + handler := handler.Funcs{} + handler.CreateFunc = func(ctx context.Context, evt event.CreateEvent, + queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + + queue.Add(ctrl.Request{NamespacedName: client.ObjectKey{ + Name: evt.Object.(*v1alpha1.Reservation).Name, // cluster-scoped crd + }}) + } + handler.UpdateFunc = func(ctx context.Context, evt event.UpdateEvent, + queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + + queue.Add(ctrl.Request{NamespacedName: client.ObjectKey{ + Name: evt.ObjectOld.(*v1alpha1.Reservation).Name, // cluster-scoped crd + }}) + } + handler.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, + queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + + queue.Add(ctrl.Request{NamespacedName: client.ObjectKey{ + Name: evt.Object.(*v1alpha1.Reservation).Name, // cluster-scoped crd + }}) + } + return handler +} + +// predicateReservations generates a new predicate for in flight reservations. +func (c *Controller) predicateReservations() predicate.Predicate { + return predicate.NewPredicateFuncs(func(object client.Object) bool { + reservation, ok := object.(*v1alpha1.Reservation) + if !ok { + return false // Not a Reservation object. + } + if reservation.Spec.Type != v1alpha1.ReservationTypeInFlight { + return false // Not an in-flight reservation. + } + if reservation.Spec.SchedulingDomain != v1alpha1.SchedulingDomainNova { + return false // Not a Nova reservation. + } + return true // Reconcile. + }) +} + +// handleHypervisors generates a new event handler for hypervisors. +func (c *Controller) handleHypervisors() handler.EventHandler { + handler := handler.Funcs{} + enqueueCorrespondingReservations := func(ctx context.Context, hvName string, + queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + + log := ctrl.LoggerFrom(ctx) + log.V(1).Info("Enqueuing reservations corresponding to hypervisor", + "hypervisor", hvName) + // Requeue all reservations targeting this hypervisor, since the + // instance list has changed and we might find the instance for + // some in-flight reservation now. + reservations := &v1alpha1.ReservationList{} + if err := c.List(ctx, reservations, client.MatchingFields{ + idxReservationByTargetHost: hvName, + }); err != nil { + log.Error(err, "Failed to list reservations for hypervisor", + "hypervisor", hvName) + return + } + for _, res := range reservations.Items { + log.V(1).Info("Enqueuing reservation for reconciliation", + "reservation", res.Name, + "targetHost", res.Spec.TargetHost, + "hypervisor", hvName) + queue.Add(ctrl.Request{NamespacedName: client.ObjectKey{ + Name: res.Name, // cluster-scoped crd + }}) + } + } + handler.CreateFunc = func(ctx context.Context, evt event.CreateEvent, + queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + + hv := evt.Object.(*hv1.Hypervisor) + enqueueCorrespondingReservations(ctx, hv.Name, queue) + } + handler.UpdateFunc = func(ctx context.Context, evt event.UpdateEvent, + queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + + hv := evt.ObjectNew.(*hv1.Hypervisor) + enqueueCorrespondingReservations(ctx, hv.Name, queue) + } + handler.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, + queue workqueue.TypedRateLimitingInterface[reconcile.Request]) { + + hv := evt.Object.(*hv1.Hypervisor) + enqueueCorrespondingReservations(ctx, hv.Name, queue) + } + return handler +} + +// predicateHypervisors generates a new predicate for hypervisors. Update +// events are filtered to only trigger when the Status.Instances list actually +// changes, since that is the only field this controller consumes; without +// this filter, unrelated status updates from the hypervisor operator would +// cause a list + enqueue of every reservation targeting the host. +func (c *Controller) predicateHypervisors() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(evt event.CreateEvent) bool { + _, ok := evt.Object.(*hv1.Hypervisor) + return ok + }, + DeleteFunc: func(evt event.DeleteEvent) bool { + _, ok := evt.Object.(*hv1.Hypervisor) + return ok + }, + GenericFunc: func(evt event.GenericEvent) bool { + _, ok := evt.Object.(*hv1.Hypervisor) + return ok + }, + UpdateFunc: func(evt event.UpdateEvent) bool { + oldHV, ok := evt.ObjectOld.(*hv1.Hypervisor) + if !ok { + return false + } + newHV, ok := evt.ObjectNew.(*hv1.Hypervisor) + if !ok { + return false + } + return !reflect.DeepEqual(oldHV.Status.Instances, newHV.Status.Instances) + }, + } +} + +// SetupWithManager sets up the controller with the Manager and a multicluster +// client. The multicluster client is used to watch for changes in the +// Reservation CRD across all clusters and trigger reconciliations accordingly. +func (c *Controller) SetupWithManager(ctx context.Context, mgr ctrl.Manager) (err error) { + // Check that the provided client is a multicluster client, since we need + // that to watch for hypervisors across clusters. Do this before adding + // any runnables so a misconfigured setup fails fast. + mcl, ok := c.Client.(*multicluster.Client) + if !ok { + return errors.New("provided client must be a multicluster client") + } + // Add the vm client as runnable to the manager. + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + return c.VMClient.StartWithKubernetesSecrets(ctx, c.Client) + })); err != nil { + return err + } + bldr := multicluster.BuildController(mcl, mgr) + // The reservation crd & hypervisor crd may be distributed across multiple + // remote clusters. + bldr, err = bldr.WatchesMulticluster(&v1alpha1.Reservation{}, + c.handleReservations(), + c.predicateReservations(), + ) + if err != nil { + return err + } + // Index reservations by their target host so we can requeue reservations + // for which the list of instances on a hypervisor has changed. + if err := mcl.IndexField(ctx, + &v1alpha1.Reservation{}, + &v1alpha1.ReservationList{}, + idxReservationByTargetHost, + idxReservationByTargetHostFn, + ); err != nil { + return err + } + // Watch hypervisor changes and requeue reservations targeting + // the changed hypervisor. + bldr, err = bldr.WatchesMulticluster(&hv1.Hypervisor{}, + c.handleHypervisors(), + c.predicateHypervisors(), + ) + if err != nil { + return err + } + return bldr.Named("inflight-reservation-controller"). + Complete(c) +} diff --git a/internal/scheduling/reservations/inflight/controller_test.go b/internal/scheduling/reservations/inflight/controller_test.go new file mode 100644 index 000000000..6f61d3a76 --- /dev/null +++ b/internal/scheduling/reservations/inflight/controller_test.go @@ -0,0 +1,727 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package inflight + +import ( + "context" + "errors" + "testing" + "time" + + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// stubVMClient is a test double for vmClient. If err is non-nil, GetCurrentVMSize +// returns it; otherwise it returns size. +type stubVMClient struct { + size map[hv1.ResourceName]resource.Quantity + err error +} + +func (s *stubVMClient) StartWithKubernetesSecrets(ctx context.Context, client client.Client) error { + return nil +} + +func (s *stubVMClient) GetCurrentVMSize(ctx context.Context, vmID string) (map[hv1.ResourceName]resource.Quantity, error) { + return s.size, s.err +} + +// newTestScheme returns a runtime.Scheme with all required types registered. +func newTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + if err := v1alpha1.AddToScheme(s); err != nil { + t.Fatalf("failed to add v1alpha1 scheme: %v", err) + } + if err := hv1.AddToScheme(s); err != nil { + t.Fatalf("failed to add hypervisor scheme: %v", err) + } + return s +} + +// newTestClient builds a fake client with the indices the controller relies on. +func newTestClient(scheme *runtime.Scheme, objects ...client.Object) client.Client { + return fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithStatusSubresource(&v1alpha1.Reservation{}). + WithIndex(&v1alpha1.Reservation{}, idxReservationByTargetHost, idxReservationByTargetHostFn). + Build() +} + +// newInFlightReservation builds an in-flight reservation with the given name and VM ID. +func newInFlightReservation(name, vmID, targetHost string) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + TargetHost: targetHost, + InFlightReservation: &v1alpha1.InFlightReservationSpec{ + VMID: vmID, + }, + }, + } +} + +// newHypervisor builds a Hypervisor with the given name and instance IDs. +func newHypervisor(name string, instanceIDs ...string) *hv1.Hypervisor { + hv := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: name}} + for _, id := range instanceIDs { + hv.Status.Instances = append(hv.Status.Instances, hv1.Instance{ID: id}) + } + return hv +} + +// assertReadyCondition fetches the named reservation and asserts the Ready condition's +// status and reason. Fails fast if the condition is missing. +func assertReadyCondition(t *testing.T, k8sClient client.Client, name string, wantStatus metav1.ConditionStatus, wantReason string) { + t.Helper() + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, &got); err != nil { + t.Fatalf("failed to get reservation %q: %v", name, err) + } + cond := meta.FindStatusCondition(got.Status.Conditions, v1alpha1.ReservationConditionReady) + if cond == nil { + t.Fatalf("Ready condition was not set on %q", name) + return + } + if cond.Status != wantStatus { + t.Errorf("%s: Ready status = %q, want %q", name, cond.Status, wantStatus) + } + if cond.Reason != wantReason { + t.Errorf("%s: Ready reason = %q, want %q", name, cond.Reason, wantReason) + } +} + +func TestReconcile_NotFoundIsIgnored(t *testing.T) { + scheme := newTestScheme(t) + k8sClient := newTestClient(scheme) + c := &Controller{Client: k8sClient} + + res, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "missing"}, + }) + if err != nil { + t.Fatalf("Reconcile returned error for missing object: %v", err) + } + if res.RequeueAfter != 0 { + t.Errorf("expected empty result, got %+v", res) + } +} + +func TestReconcile_UnexpectedTypeSetsConditionFalse(t *testing.T) { + scheme := newTestScheme(t) + wrong := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "wrong-type"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + }, + } + k8sClient := newTestClient(scheme, wrong) + c := &Controller{Client: k8sClient} + + if _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "wrong-type"}, + }); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + assertReadyCondition(t, k8sClient, "wrong-type", metav1.ConditionFalse, "UnexpectedType") +} + +func TestReconcile_MissingSpecSetsConditionFalse(t *testing.T) { + scheme := newTestScheme(t) + noSpec := &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "no-spec"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + // InFlightReservation deliberately nil. + }, + } + k8sClient := newTestClient(scheme, noSpec) + c := &Controller{Client: k8sClient} + + if _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "no-spec"}, + }); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + assertReadyCondition(t, k8sClient, "no-spec", metav1.ConditionFalse, "MissingSpec") +} + +func TestReconcile_InstanceNotSpawnedRequeues(t *testing.T) { + scheme := newTestScheme(t) + res := newInFlightReservation("res-1", "vm-uuid-1", "host-1") + hv := newHypervisor("host-1", "other-vm-uuid") + k8sClient := newTestClient(scheme, res, hv) + c := &Controller{Client: k8sClient} + + result, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + if result.RequeueAfter != 10*time.Second { + t.Errorf("RequeueAfter = %v, want 10s", result.RequeueAfter) + } + + // Reservation still exists. + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("reservation was unexpectedly deleted: %v", err) + } + assertReadyCondition(t, k8sClient, "res-1", metav1.ConditionUnknown, "InstanceNotFound") +} + +func TestReconcile_InstanceOnDifferentHostAwaitsDeletion(t *testing.T) { + // Instance landed on a *different* host than the target. The reservation + // is now stale but the hypervisor operator (not this controller) removes + // it, so Reconcile should be a no-op that leaves the reservation intact. + scheme := newTestScheme(t) + res := newInFlightReservation("res-1", "vm-uuid-1", "host-1") + hv1Obj := newHypervisor("host-1") + hv2Obj := newHypervisor("host-2", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv1Obj, hv2Obj) + c := &Controller{Client: k8sClient} + + result, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + if result.RequeueAfter != 0 { + t.Errorf("expected empty result, got %+v", result) + } + + // Reservation still exists — it's waiting for the hypervisor operator. + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("expected reservation to still exist, got error: %v", err) + } +} + +func TestReconcile_InstanceOnTargetHostDeletesReservation(t *testing.T) { + scheme := newTestScheme(t) + res := newInFlightReservation("res-1", "vm-uuid-1", "host-1") + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv) + c := &Controller{Client: k8sClient} + + if _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err == nil { + t.Fatal("expected reservation to be deleted, but Get succeeded") + } +} + +func TestReconcile_InstanceOnTargetHostBatchDeletesReservationsForSameVM(t *testing.T) { + // Multiple in-flight reservations pointing at the same target host for the + // same VM (e.g. left over from earlier scheduling attempts). Once the + // instance is confirmed on that host, all of them must be cleaned up in a + // single reconcile. + scheme := newTestScheme(t) + res1 := newInFlightReservation("res-1", "vm-uuid-1", "host-1") + res2 := newInFlightReservation("res-2", "vm-uuid-1", "host-1") + // A reservation for a different VM on the same host must be left alone. + other := newInFlightReservation("res-other", "vm-uuid-2", "host-1") + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res1, res2, other, hv) + c := &Controller{Client: k8sClient} + + if _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err == nil { + t.Fatal("expected res-1 to be deleted, but Get succeeded") + } + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-2"}, &got); err == nil { + t.Fatal("expected res-2 to be deleted, but Get succeeded") + } + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-other"}, &got); err != nil { + t.Fatalf("expected res-other to survive, got error: %v", err) + } +} + +// newResizeReservation builds an in-flight reservation with the given intent +// and resource requirements. Used to exercise the resize/rebuild size-check +// branch of Reconcile. +// +//nolint:unparam +func newResizeReservation(name, vmID, targetHost string, intent v1alpha1.SchedulingIntent, resources map[hv1.ResourceName]resource.Quantity) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + TargetHost: targetHost, + Resources: resources, + InFlightReservation: &v1alpha1.InFlightReservationSpec{ + VMID: vmID, + Intent: intent, + }, + }, + } +} + +func TestReconcile_ResizeSizeMismatchRequeues(t *testing.T) { + // For a resize/rebuild reservation, the instance being present on the + // target host isn't sufficient — the VM must have grown/shrunk to the + // reserved size. Until it has, the controller must requeue and set the + // Ready condition to VMSizeMismatch. + scheme := newTestScheme(t) + reserved := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("4"), + "memory": resource.MustParse("8Gi"), + } + current := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("2"), + "memory": resource.MustParse("4Gi"), + } + res := newResizeReservation("res-1", "vm-uuid-1", "host-1", novaapi.ResizeIntent, reserved) + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv) + c := &Controller{Client: k8sClient, VMClient: &stubVMClient{size: current}} + + result, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + if result.RequeueAfter != 10*time.Second { + t.Errorf("RequeueAfter = %v, want 10s", result.RequeueAfter) + } + + // Reservation must still exist and carry the VMSizeMismatch condition. + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("reservation was unexpectedly deleted: %v", err) + } + assertReadyCondition(t, k8sClient, "res-1", metav1.ConditionUnknown, "VMSizeMismatch") +} + +func TestReconcile_ResizeSizeMatchesDeletesReservation(t *testing.T) { + // Once the VM has been resized to the reserved dimensions, the resize + // reservation can be freed like a normal in-flight reservation. + scheme := newTestScheme(t) + reserved := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("4"), + "memory": resource.MustParse("8Gi"), + } + res := newResizeReservation("res-1", "vm-uuid-1", "host-1", novaapi.ResizeIntent, reserved) + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv) + c := &Controller{Client: k8sClient, VMClient: &stubVMClient{size: reserved}} + + if _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err == nil { + t.Fatal("expected reservation to be deleted, but Get succeeded") + } +} + +// TestReconcile_ResizeSizeMatchesSemanticallyDeletesReservation guards the +// semantic quantity comparison in Reconcile: the reservation spec and the vm +// client return numerically-identical quantities that were constructed +// differently (MustParse vs NewQuantity), so reflect.DeepEqual returns false +// even though the resources match. If the controller ever regresses to +// reflect.DeepEqual, this test starts failing. +func TestReconcile_ResizeSizeMatchesSemanticallyDeletesReservation(t *testing.T) { + scheme := newTestScheme(t) + // Reservation spec side: built like the API server would (MustParse). + reserved := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("4"), + "memory": resource.MustParse("8Gi"), + } + // VM client side: built like the nova vm client does (NewQuantity). + current := map[hv1.ResourceName]resource.Quantity{ + "cpu": *resource.NewQuantity(4, resource.DecimalSI), + "memory": *resource.NewQuantity(8*1024*1024*1024, resource.BinarySI), + } + res := newResizeReservation("res-1", "vm-uuid-1", "host-1", novaapi.ResizeIntent, reserved) + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv) + c := &Controller{Client: k8sClient, VMClient: &stubVMClient{size: current}} + + if _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err == nil { + t.Fatal("expected reservation to be deleted (semantic size match), but Get succeeded") + } +} + +// TestReconcile_ResizeStaleReservationSkipsVMClient covers the guard that +// stops the size-check branch from running when the instance landed on a host +// other than the reservation's target. A stale resize reservation should fall +// through to the awaiting-deletion no-op — not fire a vmClient call and get +// marked VMSizeMismatch. VMClient is left nil so an accidental call panics. +func TestReconcile_ResizeStaleReservationSkipsVMClient(t *testing.T) { + scheme := newTestScheme(t) + reserved := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("4"), + } + res := newResizeReservation("res-1", "vm-uuid-1", "host-1", novaapi.ResizeIntent, reserved) + // Instance ended up on host-2, not the target host-1. + hv1Obj := newHypervisor("host-1") + hv2Obj := newHypervisor("host-2", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv1Obj, hv2Obj) + c := &Controller{Client: k8sClient, VMClient: nil} + + result, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + if result.RequeueAfter != 0 { + t.Errorf("expected empty result for stale reservation, got %+v", result) + } + + // Reservation must still exist and must NOT have been marked with + // VMSizeMismatch — it's simply awaiting the hypervisor operator to + // prune it. + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("expected reservation to still exist, got error: %v", err) + } + if cond := meta.FindStatusCondition(got.Status.Conditions, v1alpha1.ReservationConditionReady); cond != nil && cond.Reason == "VMSizeMismatch" { + t.Errorf("stale reservation was incorrectly marked VMSizeMismatch") + } +} + +func TestReconcile_RebuildSizeMismatchRequeues(t *testing.T) { + // Same branch as resize but exercised via the rebuild intent to make sure + // both intents actually trip the vmClient check. + scheme := newTestScheme(t) + reserved := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("4"), + } + current := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("2"), + } + res := newResizeReservation("res-1", "vm-uuid-1", "host-1", novaapi.RebuildIntent, reserved) + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv) + c := &Controller{Client: k8sClient, VMClient: &stubVMClient{size: current}} + + result, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + if result.RequeueAfter != 10*time.Second { + t.Errorf("RequeueAfter = %v, want 10s", result.RequeueAfter) + } + assertReadyCondition(t, k8sClient, "res-1", metav1.ConditionUnknown, "VMSizeMismatch") +} + +func TestReconcile_ResizeVMClientErrorReturnsError(t *testing.T) { + // If the source of truth for VMs can't be reached we must surface the + // error so controller-runtime backs off — silently deleting the + // reservation here would be a resource-accounting bug. + scheme := newTestScheme(t) + reserved := map[hv1.ResourceName]resource.Quantity{ + "cpu": resource.MustParse("4"), + } + res := newResizeReservation("res-1", "vm-uuid-1", "host-1", novaapi.ResizeIntent, reserved) + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv) + vmErr := errors.New("vm client boom") + c := &Controller{Client: k8sClient, VMClient: &stubVMClient{err: vmErr}} + + _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }) + if !errors.Is(err, vmErr) { + t.Fatalf("Reconcile err = %v, want %v", err, vmErr) + } + + // Reservation must survive the error so it can be retried. + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err != nil { + t.Fatalf("reservation was unexpectedly deleted: %v", err) + } +} + +func TestReconcile_NonResizeIntentSkipsVMClient(t *testing.T) { + // For non-resize/rebuild intents the vmClient must not be consulted at + // all — we leave it nil to make an accidental call panic loudly. + scheme := newTestScheme(t) + res := newResizeReservation("res-1", "vm-uuid-1", "host-1", novaapi.LiveMigrationIntent, + map[hv1.ResourceName]resource.Quantity{"cpu": resource.MustParse("4")}) + hv := newHypervisor("host-1", "vm-uuid-1") + k8sClient := newTestClient(scheme, res, hv) + c := &Controller{Client: k8sClient, VMClient: nil} + + if _, err := c.Reconcile(context.Background(), ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "res-1"}, + }); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + + var got v1alpha1.Reservation + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "res-1"}, &got); err == nil { + t.Fatal("expected reservation to be deleted, but Get succeeded") + } +} + +func TestIdxReservationByTargetHostFn(t *testing.T) { + tests := []struct { + name string + obj client.Object + want []string + }{ + { + name: "wrong type", + obj: &hv1.Hypervisor{}, + want: nil, + }, + { + name: "empty target host", + obj: &v1alpha1.Reservation{ + Spec: v1alpha1.ReservationSpec{TargetHost: ""}, + }, + want: nil, + }, + { + name: "target host set", + obj: &v1alpha1.Reservation{ + Spec: v1alpha1.ReservationSpec{TargetHost: "host-1"}, + }, + want: []string{"host-1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := idxReservationByTargetHostFn(tt.obj) + if len(got) != len(tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("got[%d] = %q, want %q", i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestPredicateReservations(t *testing.T) { + c := &Controller{} + pred := c.predicateReservations() + + tests := []struct { + name string + obj client.Object + want bool + }{ + { + name: "wrong type", + obj: &hv1.Hypervisor{}, + want: false, + }, + { + name: "wrong reservation type", + obj: &v1alpha1.Reservation{ + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeFailover, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + }, + }, + want: false, + }, + { + name: "wrong scheduling domain", + obj: &v1alpha1.Reservation{ + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: v1alpha1.SchedulingDomainPods, + }, + }, + want: false, + }, + { + name: "in-flight nova reservation", + obj: &v1alpha1.Reservation{ + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeInFlight, + SchedulingDomain: v1alpha1.SchedulingDomainNova, + }, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := pred.Create(event.CreateEvent{Object: tt.obj}); got != tt.want { + t.Errorf("Create = %v, want %v", got, tt.want) + } + if got := pred.Update(event.UpdateEvent{ObjectNew: tt.obj, ObjectOld: tt.obj}); got != tt.want { + t.Errorf("Update = %v, want %v", got, tt.want) + } + if got := pred.Delete(event.DeleteEvent{Object: tt.obj}); got != tt.want { + t.Errorf("Delete = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPredicateHypervisors(t *testing.T) { + c := &Controller{} + pred := c.predicateHypervisors() + + if got := pred.Create(event.CreateEvent{Object: &hv1.Hypervisor{}}); !got { + t.Errorf("Create(Hypervisor) = false, want true") + } + if got := pred.Create(event.CreateEvent{Object: &v1alpha1.Reservation{}}); got { + t.Errorf("Create(Reservation) = true, want false") + } + + // Update events must only pass when Status.Instances actually changes, + // so unrelated status churn on the Hypervisor doesn't trigger a list + + // enqueue of all reservations targeting the host. + same := newHypervisor("host-1", "vm-1") + if got := pred.Update(event.UpdateEvent{ObjectOld: same, ObjectNew: same.DeepCopy()}); got { + t.Errorf("Update(no instance change) = true, want false") + } + changed := same.DeepCopy() + changed.Status.Instances = append(changed.Status.Instances, hv1.Instance{ID: "vm-2"}) + if got := pred.Update(event.UpdateEvent{ObjectOld: same, ObjectNew: changed}); !got { + t.Errorf("Update(instance added) = false, want true") + } +} + +// mockWorkQueue captures items added during handler invocations. +type mockWorkQueue struct { + workqueue.TypedRateLimitingInterface[reconcile.Request] + items []reconcile.Request +} + +func (m *mockWorkQueue) Add(item reconcile.Request) { + m.items = append(m.items, item) +} + +func TestHandleReservations(t *testing.T) { + c := &Controller{} + h := c.handleReservations() + res := &v1alpha1.Reservation{ObjectMeta: metav1.ObjectMeta{Name: "res-1"}} + ctx := context.Background() + + t.Run("Create", func(t *testing.T) { + q := &mockWorkQueue{} + h.Create(ctx, event.CreateEvent{Object: res}, q) + if len(q.items) != 1 || q.items[0].Name != "res-1" { + t.Errorf("queue = %+v, want one entry for res-1", q.items) + } + }) + t.Run("Update", func(t *testing.T) { + q := &mockWorkQueue{} + h.Update(ctx, event.UpdateEvent{ObjectOld: res, ObjectNew: res}, q) + if len(q.items) != 1 || q.items[0].Name != "res-1" { + t.Errorf("queue = %+v, want one entry for res-1", q.items) + } + }) + t.Run("Delete", func(t *testing.T) { + q := &mockWorkQueue{} + h.Delete(ctx, event.DeleteEvent{Object: res}, q) + if len(q.items) != 1 || q.items[0].Name != "res-1" { + t.Errorf("queue = %+v, want one entry for res-1", q.items) + } + }) +} + +func TestHandleHypervisors_EnqueuesMatchingReservations(t *testing.T) { + scheme := newTestScheme(t) + matching := newInFlightReservation("res-1", "vm-1", "host-1") + other := newInFlightReservation("res-2", "vm-2", "host-2") + k8sClient := newTestClient(scheme, matching, other) + c := &Controller{Client: k8sClient} + h := c.handleHypervisors() + + hv := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host-1"}} + ctx := context.Background() + + t.Run("Create", func(t *testing.T) { + q := &mockWorkQueue{} + h.Create(ctx, event.CreateEvent{Object: hv}, q) + if len(q.items) != 1 || q.items[0].Name != "res-1" { + t.Errorf("queue = %+v, want only res-1", q.items) + } + }) + t.Run("Update", func(t *testing.T) { + q := &mockWorkQueue{} + h.Update(ctx, event.UpdateEvent{ObjectOld: hv, ObjectNew: hv}, q) + if len(q.items) != 1 || q.items[0].Name != "res-1" { + t.Errorf("queue = %+v, want only res-1", q.items) + } + }) + t.Run("Delete", func(t *testing.T) { + q := &mockWorkQueue{} + h.Delete(ctx, event.DeleteEvent{Object: hv}, q) + if len(q.items) != 1 || q.items[0].Name != "res-1" { + t.Errorf("queue = %+v, want only res-1", q.items) + } + }) +} + +func TestHandleHypervisors_NoMatchingReservations(t *testing.T) { + scheme := newTestScheme(t) + other := newInFlightReservation("res-2", "vm-2", "host-2") + k8sClient := newTestClient(scheme, other) + c := &Controller{Client: k8sClient} + h := c.handleHypervisors() + + hv := &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host-1"}} + q := &mockWorkQueue{} + h.Create(context.Background(), event.CreateEvent{Object: hv}, q) + if len(q.items) != 0 { + t.Errorf("queue = %+v, want empty", q.items) + } +} + +func TestSetupWithManager_RejectsNonMulticlusterClient(t *testing.T) { + scheme := newTestScheme(t) + c := &Controller{Client: newTestClient(scheme), VMClient: &stubVMClient{}} + err := c.SetupWithManager(context.Background(), nil) + if err == nil { + t.Fatal("expected error for non-multicluster client, got nil") + } +} diff --git a/internal/scheduling/reservations/inflight/vm_client.go b/internal/scheduling/reservations/inflight/vm_client.go new file mode 100644 index 000000000..bcdcdebe7 --- /dev/null +++ b/internal/scheduling/reservations/inflight/vm_client.go @@ -0,0 +1,120 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package inflight + +import ( + "context" + "errors" + "net/http" + + "github.com/cobaltcore-dev/cortex/pkg/keystone" + "github.com/cobaltcore-dev/cortex/pkg/sso" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/gophercloud/gophercloud/v2" + "github.com/gophercloud/gophercloud/v2/openstack/compute/v2/servers" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type VMClient interface { + StartWithKubernetesSecrets(ctx context.Context, client client.Client) error + // GetCurrentVMSize returns the size of the VM with the given ID, + // or an error if the VM cannot be found. + GetCurrentVMSize(ctx context.Context, vmID string) (map[hv1.ResourceName]resource.Quantity, error) +} + +type novaVMClient struct { + // config is the configuration for the novaVMClient, including keystone and SSO credentials. + config NovaVMClientConfig + // sc is the service client for the OpenStack Nova API. + sc *gophercloud.ServiceClient +} + +type NovaVMClientConfig struct { + // Secret ref to keystone credentials stored in a k8s secret. + KeystoneSecretRef corev1.SecretReference `json:"keystoneSecretRef"` + // Secret ref to SSO credentials stored in a k8s secret, if applicable. + SSOSecretRef *corev1.SecretReference `json:"ssoSecretRef"` +} + +func NewNovaVMClient(config NovaVMClientConfig) VMClient { + return &novaVMClient{ + config: config, + } +} + +func (c *novaVMClient) StartWithKubernetesSecrets(ctx context.Context, client client.Client) error { + log := ctrl.LoggerFrom(ctx) + log.Info("starting novaVMClient with Kubernetes secrets") + var authenticatedHTTP = http.DefaultClient + if c.config.SSOSecretRef != nil { + var err error + authenticatedHTTP, err = sso.Connector{Client: client}. + FromSecretRef(ctx, *c.config.SSOSecretRef) + if err != nil { + log.Error(err, "failed to create SSO authenticated HTTP client") + return err + } + log.Info("successfully created SSO authenticated HTTP client") + } + authenticatedKeystone, err := keystone. + Connector{Client: client, HTTPClient: authenticatedHTTP}. + FromSecretRef(ctx, c.config.KeystoneSecretRef) + if err != nil { + log.Error(err, "failed to create authenticated keystone client") + return err + } + log.Info("successfully created authenticated keystone client") + // Automatically fetch the nova endpoint from the keystone service catalog. + provider := authenticatedKeystone.Client() + serviceType := "compute" + url, err := authenticatedKeystone.FindEndpoint( + authenticatedKeystone.Availability(), serviceType, + ) + if err != nil { + log.Error(err, "failed to find nova endpoint in keystone service catalog") + return err + } + log.Info("successfully found nova endpoint in keystone service catalog", "url", url) + c.sc = &gophercloud.ServiceClient{ + ProviderClient: provider, + Endpoint: url, + Type: serviceType, + // Since microversion 2.53, the hypervisor id and service id is a UUID. + // We need that to find placement resource providers for hypervisors. + Microversion: "2.53", + } + return nil +} + +// GetCurrentVMSize returns the size of the VM with the given ID, or an +// error if the VM cannot be found. +func (c *novaVMClient) GetCurrentVMSize(ctx context.Context, vmID string) (map[hv1.ResourceName]resource.Quantity, error) { + log := ctrl.LoggerFrom(ctx) + if c.sc == nil { + log.Error(nil, "nova service client not initialized yet") + return nil, errors.New("nova service client not initialized yet") + } + var server struct { + Flavor struct { + RAM int64 `json:"ram"` + VCPUs int64 `json:"vcpus"` + } `json:"flavor"` + } + err := servers.Get(ctx, c.sc, vmID).ExtractInto(&server) + if err != nil { + log.Error(err, "failed to get server details from nova", "vmID", vmID) + return nil, err + } + size := map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceCPU: *resource. + NewQuantity(server.Flavor.VCPUs, resource.DecimalSI), + hv1.ResourceMemory: *resource. + NewQuantity(server.Flavor.RAM*1024*1024, resource.BinarySI), + } + log.Info("successfully retrieved VM size from nova", "vmID", vmID, "size", size) + return size, nil +} diff --git a/internal/scheduling/reservations/inflight/vm_client_test.go b/internal/scheduling/reservations/inflight/vm_client_test.go new file mode 100644 index 000000000..d8488fe05 --- /dev/null +++ b/internal/scheduling/reservations/inflight/vm_client_test.go @@ -0,0 +1,167 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package inflight + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/cobaltcore-dev/cortex/pkg/keystone" + testlibKeystone "github.com/cobaltcore-dev/cortex/pkg/keystone/testing" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + "github.com/gophercloud/gophercloud/v2" + "k8s.io/apimachinery/pkg/api/resource" +) + +// setupNovaMockServer starts an httptest server backed by handler and returns +// a mock keystone client whose provider client can drive the gophercloud +// service client at that server's URL. +func setupNovaMockServer(handler http.HandlerFunc) (*httptest.Server, keystone.KeystoneClient) { + server := httptest.NewServer(handler) + return server, &testlibKeystone.MockKeystoneClient{Url: server.URL + "/"} +} + +// newTestNovaVMClient wires a novaVMClient against the given test server and +// keystone mock. +func newTestNovaVMClient(server *httptest.Server, k keystone.KeystoneClient) *novaVMClient { + return &novaVMClient{ + sc: &gophercloud.ServiceClient{ + ProviderClient: k.Client(), + Endpoint: server.URL + "/", + Type: "compute", + Microversion: "2.53", + }, + } +} + +func TestNovaVMClient_GetCurrentVMSize(t *testing.T) { + const vmID = "vm-abc" + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + t.Fatalf("expected GET method, got %s", r.Method) + } + if !strings.HasSuffix(r.URL.Path, "/servers/"+vmID) { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"server": {"id": "vm-abc", "flavor": {"ram": 2048, "vcpus": 4}}}`)) + if err != nil { + t.Fatalf("failed to write response: %v", err) + } + } + server, k := setupNovaMockServer(handler) + defer server.Close() + c := newTestNovaVMClient(server, k) + + size, err := c.GetCurrentVMSize(t.Context(), vmID) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + // CPU is DecimalSI-encoded quantity of vcpus. + cpu, ok := size[hv1.ResourceCPU] + if !ok { + t.Fatalf("expected CPU resource in size map: %+v", size) + } + if got := cpu.Value(); got != 4 { + t.Errorf("expected 4 vCPUs, got %d", got) + } + // Memory is reported in bytes: ram (MiB) * 1024 * 1024. + mem, ok := size[hv1.ResourceMemory] + if !ok { + t.Fatalf("expected Memory resource in size map: %+v", size) + } + wantMem := int64(2048) * 1024 * 1024 + if got := mem.Value(); got != wantMem { + t.Errorf("expected %d bytes memory, got %d", wantMem, got) + } + if mem.Format != resource.BinarySI { + t.Errorf("expected BinarySI format for memory, got %v", mem.Format) + } + if cpu.Format != resource.DecimalSI { + t.Errorf("expected DecimalSI format for cpu, got %v", cpu.Format) + } +} + +func TestNovaVMClient_GetCurrentVMSize_ZeroValues(t *testing.T) { + // Server that returns a flavor with zero ram/vcpus — we still expect + // a valid, non-nil size map (values are zero) and no error. + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`{"server": {"id": "vm-zero", "flavor": {"ram": 0, "vcpus": 0}}}`)) + if err != nil { + t.Fatalf("failed to write response: %v", err) + } + } + server, k := setupNovaMockServer(handler) + defer server.Close() + c := newTestNovaVMClient(server, k) + + size, err := c.GetCurrentVMSize(t.Context(), "vm-zero") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + cpu := size[hv1.ResourceCPU] + if got := cpu.Value(); got != 0 { + t.Errorf("expected 0 vCPUs, got %d", got) + } + mem := size[hv1.ResourceMemory] + if got := mem.Value(); got != 0 { + t.Errorf("expected 0 bytes memory, got %d", got) + } +} + +func TestNovaVMClient_GetCurrentVMSize_NotFound(t *testing.T) { + handler := func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, err := w.Write([]byte(`{"itemNotFound": {"message": "Instance not found", "code": 404}}`)) + if err != nil { + t.Fatalf("failed to write response: %v", err) + } + } + server, k := setupNovaMockServer(handler) + defer server.Close() + c := newTestNovaVMClient(server, k) + + _, err := c.GetCurrentVMSize(t.Context(), "missing-vm") + if err == nil { + t.Fatal("expected error for 404, got nil") + } +} + +func TestNovaVMClient_GetCurrentVMSize_ServerError(t *testing.T) { + handler := func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + } + server, k := setupNovaMockServer(handler) + defer server.Close() + c := newTestNovaVMClient(server, k) + + _, err := c.GetCurrentVMSize(t.Context(), "any-vm") + if err == nil { + t.Fatal("expected error for 500, got nil") + } +} + +func TestNovaVMClient_GetCurrentVMSize_MalformedJSON(t *testing.T) { + handler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte(`not a json`)) + if err != nil { + t.Fatalf("failed to write response: %v", err) + } + } + server, k := setupNovaMockServer(handler) + defer server.Close() + c := newTestNovaVMClient(server, k) + + _, err := c.GetCurrentVMSize(t.Context(), "any-vm") + if err == nil { + t.Fatal("expected error for malformed JSON, got nil") + } +} From 9115327e3c8a0671aea9d65e9f6baafbb83f752f Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Mon, 3 Aug 2026 22:34:11 +0200 Subject: [PATCH 02/16] Add dynamic labels to pipeline step event metrics (#1108) --- .../bundles/cortex-nova/templates/alerts.yaml | 13 +- .../lib/filter_weigher_pipeline_monitor.go | 22 ++- .../filter_weigher_pipeline_step_monitor.go | 118 ++++++++++++++- ...lter_weigher_pipeline_step_monitor_test.go | 138 ++++++++++++++++-- .../filter_weigher_pipeline_step_result.go | 18 ++- .../filters/filter_image_properties.go | 18 ++- .../filters/filter_image_properties_test.go | 37 ++++- 7 files changed, 322 insertions(+), 42 deletions(-) diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 09e9bae98..6c1a10250 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -342,7 +342,7 @@ spec: - alert: CortexNovaImagePropertiesHypervisorTypeUndetermined expr: | - sum by (pipeline, step) (rate(cortex_filter_weigher_pipeline_step_events_total{service="cortex-nova-metrics", event="image_properties_hv_type_undetermined"}[5m])) > 0.1 + sum by (pipeline, step, intent) (rate(cortex_filter_weigher_pipeline_step_events_total{service="cortex-nova-metrics", event="image_properties_hv_type_undetermined"}[5m])) > 0.1 for: 15m labels: context: scheduling @@ -356,11 +356,12 @@ spec: description: > The `filter_image_properties` step in pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` is frequently unable to determine the hypervisor type from the image properties - of incoming scheduling requests. In this case the filter is skipped and all hosts - are returned, so kvm-only images may be placed on non-kvm hypervisors (or vice - versa). This may indicate that images are missing the expected hypervisor - type property, or that the property format has changed. Investigate the - image metadata of the affected requests. + of incoming scheduling requests for intent `{{ "{{" }} $labels.intent {{ "}}" }}`. + In this case the filter is skipped and all hosts are returned, so kvm-only + images may be placed on non-kvm hypervisors (or vice versa). This may indicate + that images are missing the expected hypervisor type property, or that the + property format has changed. Investigate the image metadata of the affected + requests. {{- if .Values.kvm.enabled }} - alert: CortexNovaDoesntFindValidKVMHosts diff --git a/internal/scheduling/lib/filter_weigher_pipeline_monitor.go b/internal/scheduling/lib/filter_weigher_pipeline_monitor.go index 341152322..785e18953 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_monitor.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_monitor.go @@ -22,8 +22,6 @@ type FilterWeigherPipelineMonitor struct { stepReorderingsObserver *prometheus.HistogramVec // A histogram to observe the impact of the step on the hosts. stepImpactObserver *prometheus.HistogramVec - // A counter for named events reported by a step during its run. - stepEventCounter *prometheus.CounterVec // A histogram to measure how long the pipeline takes to run in total. pipelineRunTimer *prometheus.HistogramVec // A histogram to observe the number of hosts going into the scheduler pipeline. @@ -32,6 +30,10 @@ type FilterWeigherPipelineMonitor struct { hostNumberOutObserver *prometheus.HistogramVec // Counter for the number of requests processed by the scheduler. requestCounter *prometheus.CounterVec + + // stepEventCollector is a collector for named events reported by steps + // during their run. The collector supports a dynamic set of labels per event. + stepEventCollector *pipelineStepEventCollector } // Create a new scheduler monitor and register the necessary Prometheus metrics. @@ -66,10 +68,6 @@ func NewPipelineMonitor() FilterWeigherPipelineMonitor { Help: "Impact of the step on the hosts", Buckets: prometheus.ExponentialBucketsRange(0.01, 1000, 20), }, []string{"pipeline", "step", "stat", "unit"}), - stepEventCounter: prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_filter_weigher_pipeline_step_events_total", - Help: "Number of named events reported by a scheduler pipeline step", - }, []string{"pipeline", "step", "event"}), pipelineRunTimer: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "cortex_filter_weigher_pipeline_run_duration_seconds", Help: "Duration of scheduler pipeline run", @@ -89,6 +87,8 @@ func NewPipelineMonitor() FilterWeigherPipelineMonitor { Name: "cortex_filter_weigher_pipeline_requests_total", Help: "Total number of requests processed by the scheduler.", }, []string{"pipeline"}), + // Additional collectors. + stepEventCollector: newPipelineStepEventCollector(), } } @@ -122,12 +122,17 @@ func (m *FilterWeigherPipelineMonitor) observePipelineResult(request FilterWeigh } func (m *FilterWeigherPipelineMonitor) Describe(ch chan<- *prometheus.Desc) { + // stepEventCollector is intentionally not described here: its label names + // are dynamic and only known when events are recorded, so a fixed + // descriptor cannot be provided upfront. The remaining metrics are + // described normally so the registry can validate them at registration + // time. In practice the current prometheus client allows Collect to emit + // event metrics with label sets that differ from the described metrics. m.stepRunTimer.Describe(ch) m.stepHostWeight.Describe(ch) m.stepRemovedHostsObserver.Describe(ch) m.stepReorderingsObserver.Describe(ch) m.stepImpactObserver.Describe(ch) - m.stepEventCounter.Describe(ch) m.pipelineRunTimer.Describe(ch) m.hostNumberInObserver.Describe(ch) m.hostNumberOutObserver.Describe(ch) @@ -140,9 +145,10 @@ func (m *FilterWeigherPipelineMonitor) Collect(ch chan<- prometheus.Metric) { m.stepRemovedHostsObserver.Collect(ch) m.stepReorderingsObserver.Collect(ch) m.stepImpactObserver.Collect(ch) - m.stepEventCounter.Collect(ch) m.pipelineRunTimer.Collect(ch) m.hostNumberInObserver.Collect(ch) m.hostNumberOutObserver.Collect(ch) m.requestCounter.Collect(ch) + + m.stepEventCollector.Collect(ch) } diff --git a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go index 2ad84e9fd..3fcdeebcc 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor.go @@ -12,6 +12,7 @@ import ( "sort" "strconv" "strings" + "sync" "github.com/prometheus/client_golang/prometheus" ) @@ -36,8 +37,10 @@ type FilterWeigherPipelineStepMonitor[RequestType FilterWeigherPipelineRequest] stepReorderingsObserver *prometheus.HistogramVec // A metric measuring the impact of the step on the hosts. stepImpactObserver *prometheus.HistogramVec - // A counter for named events reported by the step during its run. - stepEventCounter *prometheus.CounterVec + + // stepEventCollector is a collector for named events reported by the + // step during its run. + stepEventCollector *pipelineStepEventCollector } // Schedule using the wrapped step and measure the time it takes. @@ -60,7 +63,7 @@ func monitorStep[RequestType FilterWeigherPipelineRequest](stepName string, m Fi removedHostsObserver: removedHostsObserver, stepReorderingsObserver: m.stepReorderingsObserver, stepImpactObserver: m.stepImpactObserver, - stepEventCounter: m.stepEventCounter, + stepEventCollector: m.stepEventCollector, } } @@ -87,11 +90,9 @@ func (s *FilterWeigherPipelineStepMonitor[RequestType]) RunWrapped( ) // Count named events reported by the step during its run. - if s.stepEventCounter != nil { + if s.stepEventCollector != nil { for _, event := range stepResult.Events { - s.stepEventCounter. - WithLabelValues(s.pipelineName, s.stepName, event). - Inc() + s.stepEventCollector.Record(s.pipelineName, s.stepName, event.Name, event.Labels) } } @@ -272,3 +273,106 @@ func impact(before, after []string, stats map[string]float64, topK int) (float64 return impact, nil } + +// pipelineStepEventCollector is a prometheus.Collector that counts named events +// reported by scheduler pipeline steps. It supports a dynamic set of labels per +// event: the label names are not fixed upfront but derived from the labels +// attached to each event. +type pipelineStepEventCollector struct { + sync.Mutex + // counts holds the event counter values keyed by a deterministic signature. + counts map[string]float64 + // events maps the signature to the full event context (pipeline, step, + // event name and dynamic labels). + events map[string]recordedStepEvent +} + +type recordedStepEvent struct { + pipeline string + step string + eventName string + labels map[string]string +} + +func newPipelineStepEventCollector() *pipelineStepEventCollector { + return &pipelineStepEventCollector{ + counts: make(map[string]float64), + events: make(map[string]recordedStepEvent), + } +} + +// reservedStepEventLabels are the fixed label names used by the event metric. +// Dynamic labels with these names are dropped because prometheus.Desc does not +// allow duplicate label names. +var reservedStepEventLabels = map[string]struct{}{ + "pipeline": {}, + "step": {}, + "event": {}, +} + +// Record increments the counter for the given event. The same event can be +// recorded multiple times; each call increments the matching counter. +// Dynamic labels that collide with the fixed label names (pipeline, step, +// event) are silently dropped to avoid panicking prometheus.MustNewConstMetric. +func (c *pipelineStepEventCollector) Record(pipeline, step, eventName string, labels map[string]string) { + c.Lock() + defer c.Unlock() + cleaned := make(map[string]string, len(labels)) + for k, v := range labels { + if _, reserved := reservedStepEventLabels[k]; reserved { + continue + } + cleaned[k] = v + } + key := stepEventKey(pipeline, step, eventName, cleaned) + if _, ok := c.counts[key]; !ok { + c.counts[key] = 0 + c.events[key] = recordedStepEvent{ + pipeline: pipeline, + step: step, + eventName: eventName, + labels: cleaned, + } + } + c.counts[key]++ +} + +// Describe is intentionally a no-op. Because the label names are dynamic and +// only known at collect time, we cannot describe the metric upfront. +func (c *pipelineStepEventCollector) Describe(ch chan<- *prometheus.Desc) { +} + +// Collect emits one metric per recorded event. Each metric shares the same +// fully-qualified name but may have a different set of labels depending on the +// labels provided by the step that reported the event. +func (c *pipelineStepEventCollector) Collect(ch chan<- prometheus.Metric) { + c.Lock() + defer c.Unlock() + for key, count := range c.counts { + ev := c.events[key] + labelNames := slices.Sorted(maps.Keys(ev.labels)) + labelValues := make([]string, len(labelNames)) + for i, name := range labelNames { + labelValues[i] = ev.labels[name] + } + desc := prometheus.NewDesc( + "cortex_filter_weigher_pipeline_step_events_total", + "Number of named events reported by a scheduler pipeline step", + append([]string{"pipeline", "step", "event"}, labelNames...), + nil, + ) + values := append([]string{ev.pipeline, ev.step, ev.eventName}, labelValues...) + ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, count, values...) + } +} + +// stepEventKey returns a deterministic signature for an event record. +func stepEventKey(pipeline, step, eventName string, labels map[string]string) string { + keys := slices.Sorted(maps.Keys(labels)) + parts := make([]string, 0, 3+2*len(keys)) + parts = append(parts, pipeline, step, eventName) + for _, k := range keys { + parts = append(parts, k, labels[k]) + } + return strings.Join(parts, "\x00") +} diff --git a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go index 5ae402111..f17252d42 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_step_monitor_test.go @@ -6,6 +6,7 @@ package lib import ( "log/slog" "os" + "strings" "testing" "github.com/prometheus/client_golang/prometheus" @@ -59,20 +60,19 @@ func TestStepMonitorRun(t *testing.T) { } func TestStepMonitorRunEvents(t *testing.T) { - stepEventCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_filter_weigher_pipeline_step_events_total", - Help: "Number of named events reported by a scheduler pipeline step", - }, []string{"pipeline", "step", "event"}) + stepEventCollector := newPipelineStepEventCollector() monitor := &FilterWeigherPipelineStepMonitor[mockFilterWeigherPipelineRequest]{ - stepName: "mock_step", - pipelineName: "mock_pipeline", - stepEventCounter: stepEventCounter, + stepName: "mock_step", + pipelineName: "mock_pipeline", + stepEventCollector: stepEventCollector, } step := &mockWeigher[mockFilterWeigherPipelineRequest]{ RunFunc: func(traceLog *slog.Logger, request mockFilterWeigherPipelineRequest) (*FilterWeigherPipelineStepResult, error) { return &FilterWeigherPipelineStepResult{ Activations: map[string]float64{"host1": 0.0}, - Events: []string{"hypervisor_type_undetermined"}, + Events: []FilterWeigherPipelineStepEvent{ + {Name: "hypervisor_type_undetermined"}, + }, }, nil }, } @@ -83,9 +83,52 @@ func TestStepMonitorRunEvents(t *testing.T) { if _, err := monitor.RunWrapped(slog.Default(), request, step); err != nil { t.Fatalf("Run() error = %v, want nil", err) } - got := testutil.ToFloat64(stepEventCounter.WithLabelValues("mock_pipeline", "mock_step", "hypervisor_type_undetermined")) - if got != 1 { - t.Errorf("stepEventCounter = %v, want 1", got) + if got := len(stepEventCollector.counts); got != 1 { + t.Fatalf("stepEventCollector.counts = %v, want 1", got) + } + for _, count := range stepEventCollector.counts { + if count != 1 { + t.Errorf("stepEventCollector count = %v, want 1", count) + } + } +} + +func TestStepMonitorRunEventsWithDynamicLabels(t *testing.T) { + stepEventCollector := newPipelineStepEventCollector() + monitor := &FilterWeigherPipelineStepMonitor[mockFilterWeigherPipelineRequest]{ + stepName: "mock_step", + pipelineName: "mock_pipeline", + stepEventCollector: stepEventCollector, + } + step := &mockWeigher[mockFilterWeigherPipelineRequest]{ + RunFunc: func(traceLog *slog.Logger, request mockFilterWeigherPipelineRequest) (*FilterWeigherPipelineStepResult, error) { + return &FilterWeigherPipelineStepResult{ + Activations: map[string]float64{"host1": 0.0}, + Events: []FilterWeigherPipelineStepEvent{ + { + Name: "hypervisor_type_undetermined", + Labels: map[string]string{"intent": "create"}, + }, + }, + }, nil + }, + } + request := mockFilterWeigherPipelineRequest{ + Hosts: []string{"host1"}, + Weights: map[string]float64{"host1": 0.0}, + } + if _, err := monitor.RunWrapped(slog.Default(), request, step); err != nil { + t.Fatalf("Run() error = %v, want nil", err) + } + registry := prometheus.NewRegistry() + registry.MustRegister(stepEventCollector) + expected := ` + # HELP cortex_filter_weigher_pipeline_step_events_total Number of named events reported by a scheduler pipeline step + # TYPE cortex_filter_weigher_pipeline_step_events_total counter + cortex_filter_weigher_pipeline_step_events_total{event="hypervisor_type_undetermined",intent="create",pipeline="mock_pipeline",step="mock_step"} 1 + ` + if err := testutil.GatherAndCompare(registry, strings.NewReader(expected)); err != nil { + t.Fatalf("GatherAndCompare() error = %v", err) } } @@ -180,3 +223,76 @@ func TestImpact(t *testing.T) { }) } } + +func TestPipelineStepEventCollector_DynamicLabels(t *testing.T) { + collector := newPipelineStepEventCollector() + collector.Record("pipeline", "step", "event_a", map[string]string{"intent": "create"}) + collector.Record("pipeline", "step", "event_a", map[string]string{"intent": "create"}) + collector.Record("pipeline", "step", "event_a", map[string]string{"intent": "resize"}) + collector.Record("pipeline", "step", "event_b", map[string]string{"foo": "bar"}) + + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + + expected := ` + # HELP cortex_filter_weigher_pipeline_step_events_total Number of named events reported by a scheduler pipeline step + # TYPE cortex_filter_weigher_pipeline_step_events_total counter + cortex_filter_weigher_pipeline_step_events_total{event="event_a",intent="create",pipeline="pipeline",step="step"} 2 + cortex_filter_weigher_pipeline_step_events_total{event="event_a",intent="resize",pipeline="pipeline",step="step"} 1 + cortex_filter_weigher_pipeline_step_events_total{event="event_b",foo="bar",pipeline="pipeline",step="step"} 1 + ` + if err := testutil.GatherAndCompare(registry, strings.NewReader(expected)); err != nil { + t.Fatalf("GatherAndCompare() error = %v", err) + } +} + +func TestPipelineStepEventCollector_NoLabels(t *testing.T) { + collector := newPipelineStepEventCollector() + collector.Record("pipeline", "step", "event_c", nil) + + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + + expected := ` + # HELP cortex_filter_weigher_pipeline_step_events_total Number of named events reported by a scheduler pipeline step + # TYPE cortex_filter_weigher_pipeline_step_events_total counter + cortex_filter_weigher_pipeline_step_events_total{event="event_c",pipeline="pipeline",step="step"} 1 + ` + if err := testutil.GatherAndCompare(registry, strings.NewReader(expected)); err != nil { + t.Fatalf("GatherAndCompare() error = %v", err) + } +} + +func TestPipelineStepEventCollector_DescribeIsNoOp(t *testing.T) { + collector := newPipelineStepEventCollector() + ch := make(chan *prometheus.Desc, 1) + collector.Describe(ch) + close(ch) + if len(ch) != 0 { + t.Errorf("expected Describe to send no descriptors, got %d", len(ch)) + } +} + +func TestPipelineStepEventCollector_DropsReservedLabels(t *testing.T) { + collector := newPipelineStepEventCollector() + // Reserved label names must be dropped so the emitted metric does not + // contain duplicate label names. + collector.Record("pipeline", "step", "event_d", map[string]string{ + "intent": "create", + "pipeline": "ignored", + "step": "ignored", + "event": "ignored", + }) + + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + + expected := ` + # HELP cortex_filter_weigher_pipeline_step_events_total Number of named events reported by a scheduler pipeline step + # TYPE cortex_filter_weigher_pipeline_step_events_total counter + cortex_filter_weigher_pipeline_step_events_total{event="event_d",intent="create",pipeline="pipeline",step="step"} 1 + ` + if err := testutil.GatherAndCompare(registry, strings.NewReader(expected)); err != nil { + t.Fatalf("GatherAndCompare() error = %v", err) + } +} diff --git a/internal/scheduling/lib/filter_weigher_pipeline_step_result.go b/internal/scheduling/lib/filter_weigher_pipeline_step_result.go index 8ccee7cf3..f7595cadd 100644 --- a/internal/scheduling/lib/filter_weigher_pipeline_step_result.go +++ b/internal/scheduling/lib/filter_weigher_pipeline_step_result.go @@ -27,9 +27,21 @@ type FilterWeigherPipelineStepResult struct { // Named events reported by the step during its run, e.g. // "hypervisor_type_undetermined". Each event is counted by the pipeline // monitor as cortex_filter_weigher_pipeline_step_events_total, labeled by - // pipeline, step and event. Use this to expose noteworthy step conditions - // (skipped filtering, missing data, ...) as Prometheus metrics. - Events []string + // pipeline, step, event and any additional labels provided by the step. + // Use this to expose noteworthy step conditions (skipped filtering, missing + // data, ...) as Prometheus metrics. + Events []FilterWeigherPipelineStepEvent +} + +// FilterWeigherPipelineStepEvent is a named event reported by a scheduling +// pipeline step. Additional labels can be attached to the event to enrich the +// exported Prometheus metric. +type FilterWeigherPipelineStepEvent struct { + // Name of the event, e.g. "image_properties_hv_type_undetermined". + Name string + // Additional labels to attach to the exported metric. Label names must be + // valid Prometheus label names. + Labels map[string]string } type FilterWeigherPipelineStepStatistics struct { diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties.go b/internal/scheduling/nova/plugins/filters/filter_image_properties.go index 2fe1ada44..dfc0d6446 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties.go @@ -22,9 +22,17 @@ type FilterImagePropertiesStep struct { func (s *FilterImagePropertiesStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Resolve the scheduling intent once and fall back to a readable default + // when it cannot be determined. + intent, intentErr := request.GetIntent() + intentLabel := "unknown" + if intentErr == nil { + intentLabel = string(intent) + } + // Apply this filter to all requests, unless we know from the request's // intent that image metadata is expected to not be set. - if intent, err := request.GetIntent(); err == nil { + if intentErr == nil { intentsExpectedToNotHaveImageMeta := []v1alpha1.SchedulingIntent{ // Cortex-internal intents in which scheduling requests are sent // mainly based on flavor-related metadata, independent of the @@ -50,8 +58,12 @@ func (s *FilterImagePropertiesStep) Run(traceLog *slog.Logger, request api.Exter traceLog.Warn("could not determine hypervisor type from image properties", "error", err) // Expose this event through the step monitor to alert on high-frequency - // occurrences of this situation. - result.Events = append(result.Events, "image_properties_hv_type_undetermined") + // occurrences of this situation. Include the request intent as a dynamic + // label so the alert can pinpoint which scheduling intents are affected. + result.Events = append(result.Events, lib.FilterWeigherPipelineStepEvent{ + Name: "image_properties_hv_type_undetermined", + Labels: map[string]string{"intent": intentLabel}, + }) return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go index 64dd4bb26..8490b2b76 100644 --- a/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_image_properties_test.go @@ -8,6 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -49,10 +50,11 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { } tests := []struct { - name string - request api.ExternalSchedulerRequest - expectedHosts []string - filteredHosts []string + name string + request api.ExternalSchedulerRequest + expectedHosts []string + filteredHosts []string + expectedEvents []lib.FilterWeigherPipelineStepEvent }{ { name: "kvm image keeps all hosts", @@ -77,12 +79,18 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { request: requestWith([]string{"host1", "host2", "host3"}, nil), expectedHosts: []string{"host1", "host2", "host3"}, filteredHosts: []string{}, + expectedEvents: []lib.FilterWeigherPipelineStepEvent{ + {Name: "image_properties_hv_type_undetermined", Labels: map[string]string{"intent": "unknown"}}, + }, }, { name: "unsupported hypervisor type keeps all hosts", request: requestWith([]string{"host1", "host2"}, map[string]any{"img_hv_type": "xen"}), expectedHosts: []string{"host1", "host2"}, filteredHosts: []string{}, + expectedEvents: []lib.FilterWeigherPipelineStepEvent{ + {Name: "image_properties_hv_type_undetermined", Labels: map[string]string{"intent": "unknown"}}, + }, }, { name: "empty host list", @@ -149,6 +157,27 @@ func TestFilterImagePropertiesStep_Run(t *testing.T) { if len(result.Activations) != len(tt.expectedHosts) { t.Errorf("expected %d hosts, got %d", len(tt.expectedHosts), len(result.Activations)) } + + if len(result.Events) != len(tt.expectedEvents) { + t.Errorf("expected %d events, got %d", len(tt.expectedEvents), len(result.Events)) + } + for i, expectedEvent := range tt.expectedEvents { + if i >= len(result.Events) { + break + } + gotEvent := result.Events[i] + if gotEvent.Name != expectedEvent.Name { + t.Errorf("expected event name %q, got %q", expectedEvent.Name, gotEvent.Name) + } + if len(gotEvent.Labels) != len(expectedEvent.Labels) { + t.Errorf("expected event labels %v, got %v", expectedEvent.Labels, gotEvent.Labels) + } + for k, v := range expectedEvent.Labels { + if gotEvent.Labels[k] != v { + t.Errorf("expected event label %q=%q, got %q", k, v, gotEvent.Labels[k]) + } + } + } }) } } From 330032806a617a883ead0a4b50ba806d7170198d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:34:33 +0200 Subject: [PATCH 03/16] Renovate: Update module go.xyrillian.de/gg to v1.12.0 (#1111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Adoption](https://docs.renovatebot.com/merge-confidence/) | [Passing](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---|---|---| | [go.xyrillian.de/gg](https://git.xyrillian.de/go-gg) | `v1.11.1` → `v1.13.0` | ![age](https://developer.mend.io/api/mc/badges/age/go/go.xyrillian.de%2fgg/v1.13.0?slim=true) | ![adoption](https://developer.mend.io/api/mc/badges/adoption/go/go.xyrillian.de%2fgg/v1.13.0?slim=true) | ![passing](https://developer.mend.io/api/mc/badges/compatibility/go/go.xyrillian.de%2fgg/v1.11.1/v1.13.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/go/go.xyrillian.de%2fgg/v1.11.1/v1.13.0?slim=true) | --- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 20ecab4bd..bb9f2a62c 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 github.com/sapcc/go-bits v0.0.0-20260730170321-f6f727520601 - go.xyrillian.de/gg v1.11.1 + go.xyrillian.de/gg v1.13.0 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 k8s.io/client-go v0.36.3 diff --git a/go.sum b/go.sum index 480a3f594..ec13033bb 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= -go.xyrillian.de/gg v1.11.1 h1:7P3kDFkTmR7jx2riYi0GwX5uhgrsL37QSrN16xH/n8E= -go.xyrillian.de/gg v1.11.1/go.mod h1:DoO4fQSWIrBRlNlCjVyrYM0kAEBt/Jg2GkMH+cGRZ0k= +go.xyrillian.de/gg v1.13.0 h1:K1RlyRxe2+7oaXALZBtuJqqwSEPneTjH1gPD18WuSaU= +go.xyrillian.de/gg v1.13.0/go.mod h1:DoO4fQSWIrBRlNlCjVyrYM0kAEBt/Jg2GkMH+cGRZ0k= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= From f1be13f9e52446c17bf7a479e8b9943479b37c2d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:36:22 +0200 Subject: [PATCH 04/16] Renovate: Update kube-prometheus-stack Docker tag to v88 (#1112) --- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 35e4f1a77..d69cdd95f 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 87.21.0 + version: 88.1.3 From 3b0acef7063212774a1ac46016fd50bd24c6247b Mon Sep 17 00:00:00 2001 From: Philipp Matthes Date: Tue, 4 Aug 2026 14:56:46 +0200 Subject: [PATCH 05/16] Skip non-candidate hypervisors in nova filters and weighers (#1117) During onboarding or offboarding, hypervisor resources may have incomplete status fields like empty hypervisor type, missing capacity, or missing zone labels. With filter_status_conditions running first in every pipeline, subsequent filters and weighers should only inspect hosts that are still candidates. This change adds an early candidate guard to all nova filters and weighers that list Hypervisor CRs, so they skip hosts not present in result.Activations. It also emits a pipeline event when filter_capabilities encounters an unknown hypervisor type on a candidate host, and adds a matching Prometheus alert. Assisted-by: Claude Code:thalamus/moonshotai/Kimi-K2.7-Code [Bash] [Read] Signed-off-by: Philipp Matthes --- .../bundles/cortex-nova/templates/alerts.yaml | 20 +++ .../cortex-nova/templates/pipelines_kvm.yaml | 120 ++++++++-------- .../filters/filter_aggregate_metadata.go | 3 + .../filters/filter_allowed_projects.go | 3 + .../plugins/filters/filter_capabilities.go | 7 + .../filters/filter_capabilities_test.go | 128 +++++++++++++++++- .../nova/plugins/filters/filter_correct_az.go | 3 + .../filters/filter_external_customer.go | 3 + .../filters/filter_has_accelerators.go | 3 + .../filters/filter_has_enough_capacity.go | 3 + .../filters/filter_has_requested_traits.go | 3 + .../filter_instance_group_anti_affinity.go | 3 + .../filters/filter_requested_destination.go | 3 + .../filters/filter_status_conditions.go | 3 + .../nova/plugins/weighers/kvm_binpack.go | 3 + .../kvm_instance_group_soft_affinity.go | 3 + 16 files changed, 247 insertions(+), 64 deletions(-) diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 6c1a10250..133d8468a 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -363,6 +363,26 @@ spec: property format has changed. Investigate the image metadata of the affected requests. + - alert: CortexNovaCapabilitiesUnknownHypervisorType + expr: | + sum by (pipeline, step, hypervisor_type) (rate(cortex_filter_weigher_pipeline_step_events_total{service="cortex-nova-metrics", event="filter_capabilities_unknown_hypervisor_type"}[5m])) > 0.1 + for: 15m + labels: + context: scheduling + dashboard: cortex-status-dashboard/cortex-status-dashboard + service: cortex + severity: warning + support_group: workload-management + playbook: docs/support/playbook/cortex/alerts/scheduling + annotations: + summary: "Nova capabilities filter frequently encounters unknown hypervisor types" + description: > + The `filter_capabilities` step in pipeline `{{ "{{" }} $labels.pipeline {{ "}}" }}` + is frequently unable to determine the hypervisor type + (`{{ "{{" }} $labels.hypervisor_type {{ "}}" }}`) for candidate hosts. + This may indicate that hypervisors are reporting incomplete domain capabilities. + Investigate the affected hosts and hypervisor operator status. + {{- if .Values.kvm.enabled }} - alert: CortexNovaDoesntFindValidKVMHosts expr: sum by (az, hvtype) (increase(cortex_vm_faults{hvtype=~"CH|QEMU",faultmsg=~".*No valid host was found.*",faultmsg!~".*No such host.*"}[5m])) > 0 diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index fa160c508..e72905e27 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -17,6 +17,11 @@ spec: # Fetch all placement candidates, ignoring nova's preselection. ignorePreselection: true filters: + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. - name: filter_correct_az description: | This step will filter out hosts whose aggregate information indicates they @@ -31,11 +36,6 @@ spec: This step will filter out hosts for which the hypervisor type does not match the one specified in the image properties, for example, filtering out all known KVM hypervisors if the image requires a different hypervisor type. - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - name: filter_capabilities description: | This step will filter out hosts that do not meet the compute capabilities @@ -131,8 +131,8 @@ spec: instance group are already running on that host. The more instances of the same group on a host, the lower (for soft-anti-affinity) or higher (for soft-affinity) the weight, which makes it less likely or more likely, - respectively, for the scheduler to choose that host for new instances of - the same group. + respectively, for the scheduler to choose that host for new instances of the + same group. - name: kvm_binpack multiplier: -1.0 # inverted = balancing params: @@ -174,6 +174,11 @@ spec: # Fetch all placement candidates, ignoring nova's preselection. ignorePreselection: true filters: + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. - name: filter_correct_az description: | This step will filter out hosts whose aggregate information indicates they @@ -188,11 +193,6 @@ spec: This step will filter out hosts for which the hypervisor type does not match the one specified in the image properties, for example, filtering out all known KVM hypervisors if the image requires a different hypervisor type. - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - name: filter_capabilities description: | This step will filter out hosts that do not meet the compute capabilities @@ -287,8 +287,8 @@ spec: instance group are already running on that host. The more instances of the same group on a host, the lower (for soft-anti-affinity) or higher (for soft-affinity) the weight, which makes it less likely or more likely, - respectively, for the scheduler to choose that host for new instances of - the same group. + respectively, for the scheduler to choose that host for new instances of the + same group. - name: kvm_binpack params: - {key: resourceWeights, floatMapValue: {"memory": 1.0}} @@ -323,11 +323,20 @@ spec: type: filter-weigher ignorePreselection: true filters: + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. - name: filter_host_instructions description: | This step will consider the `ignore_hosts` and `force_hosts` instructions from the nova scheduler request spec to filter out or exclusively allow certain hosts. + - name: filter_correct_az + description: | + This step will filter out hosts whose aggregate information indicates they + are not placed in the requested availability zone. - name: filter_has_enough_capacity description: | This step will filter out hosts that do not have enough available capacity @@ -345,15 +354,6 @@ spec: description: | This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if the nova flavor extra specs request accelerators via "accel:device_profile". - - name: filter_correct_az - description: | - This step will filter out hosts whose aggregate information indicates they - are not placed in the requested availability zone. - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - name: filter_allowed_projects description: | This step filters hosts based on allowed projects defined in the @@ -454,11 +454,21 @@ spec: on a host, this pipeline validates the host is still suitable for the VM. type: filter-weigher filters: + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. - name: filter_host_instructions description: | This step will consider the `ignore_hosts` and `force_hosts` instructions from the nova scheduler request spec to filter out or exclusively allow certain hosts. + - name: filter_correct_az + description: | + This step will filter out hosts whose aggregate information indicates they + are not placed in the requested availability zone. This ensures VMs can + only reuse reservations in their own AZ. - name: filter_has_requested_traits description: | This step filters hosts that do not have the requested traits given by the @@ -469,16 +479,6 @@ spec: description: | This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if the nova flavor extra specs request accelerators via "accel:device_profile". - - name: filter_correct_az - description: | - This step will filter out hosts whose aggregate information indicates they - are not placed in the requested availability zone. This ensures VMs can - only reuse reservations in their own AZ. - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - name: filter_external_customer description: | This step prefix-matches the domain name for external customer domains and @@ -517,11 +517,21 @@ spec: fails for any VM, the reservation is deleted (nack). type: filter-weigher filters: + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. - name: filter_host_instructions description: | This step will consider the `ignore_hosts` and `force_hosts` instructions from the nova scheduler request spec to filter out or exclusively allow certain hosts. + - name: filter_correct_az + description: | + This step will filter out hosts whose aggregate information indicates they + are not placed in the requested availability zone. This ensures reservation + validation respects AZ boundaries. - name: filter_has_enough_capacity description: | This step will filter out hosts that do not have enough available capacity @@ -538,16 +548,6 @@ spec: description: | This step will filter out hosts without the trait `COMPUTE_ACCELERATORS` if the nova flavor extra specs request accelerators via "accel:device_profile". - - name: filter_correct_az - description: | - This step will filter out hosts whose aggregate information indicates they - are not placed in the requested availability zone. This ensures reservation - validation respects AZ boundaries. - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - name: filter_external_customer description: | This step prefix-matches the domain name for external customer domains and @@ -592,6 +592,9 @@ spec: # Fetch all placement candidates, ignoring nova's preselection. ignorePreselection: true filters: + - name: filter_status_conditions + description: | + Excludes hosts that are not ready or are disabled. - name: filter_correct_az description: | Restricts host candidates to the requested availability zone. @@ -610,9 +613,6 @@ spec: description: | Ensures hosts meet the compute capabilities required by the flavor extra specs (e.g., architecture, maxphysaddr bits). - - name: filter_status_conditions - description: | - Excludes hosts that are not ready or are disabled. weighers: [] --- apiVersion: cortex.cloud/v1alpha1 @@ -630,6 +630,11 @@ spec: ignorePreselection: true createHistory: false filters: + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. - name: filter_correct_az description: | This step will filter out hosts whose aggregate information indicates they @@ -644,11 +649,6 @@ spec: This step will filter out hosts for which the hypervisor type does not match the one specified in the image properties, for example, filtering out all known KVM hypervisors if the image requires a different hypervisor type. - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - name: filter_capabilities description: | This step will filter out hosts that do not meet the compute capabilities @@ -746,8 +746,8 @@ spec: instance group are already running on that host. The more instances of the same group on a host, the lower (for soft-anti-affinity) or higher (for soft-affinity) the weight, which makes it less likely or more likely, - respectively, for the scheduler to choose that host for new instances of - the same group. + respectively, for the scheduler to choose that host for new instances of the + same group. - name: kvm_binpack multiplier: -1.0 # inverted = balancing params: @@ -787,6 +787,11 @@ spec: ignorePreselection: true createHistory: false filters: + - name: filter_status_conditions + description: | + This step will filter out hosts for which the hypervisor status conditions + do not meet the expected values, for example, that the hypervisor is ready + and not disabled. - name: filter_correct_az description: | This step will filter out hosts whose aggregate information indicates they @@ -801,11 +806,6 @@ spec: This step will filter out hosts for which the hypervisor type does not match the one specified in the image properties, for example, filtering out all known KVM hypervisors if the image requires a different hypervisor type. - - name: filter_status_conditions - description: | - This step will filter out hosts for which the hypervisor status conditions - do not meet the expected values, for example, that the hypervisor is ready - and not disabled. - name: filter_capabilities description: | This step will filter out hosts that do not meet the compute capabilities @@ -902,8 +902,8 @@ spec: instance group are already running on that host. The more instances of the same group on a host, the lower (for soft-anti-affinity) or higher (for soft-affinity) the weight, which makes it less likely or more likely, - respectively, for the scheduler to choose that host for new instances of - the same group. + respectively, for the scheduler to choose that host for new instances of the + same group. - name: kvm_binpack params: - {key: resourceWeights, floatMapValue: {"memory": 1.0}} diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go index 08a7e55c3..f65c37379 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go @@ -40,6 +40,9 @@ func (s *FilterAggregateMetadata) Run(traceLog *slog.Logger, request api.Externa restrictedProjectsByHost := make(map[string][]string) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } for _, aggregate := range hv.Status.Aggregates { // Any metadata key prefixed with "filter_tenant_id" restricts the // aggregate to the referenced projects. Multiple numbered keys (e.g. diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go index bd0849cf1..536129b71 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go @@ -42,6 +42,9 @@ func (s *FilterAllowedProjectsStep) Run(traceLog *slog.Logger, request api.Exter } for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } if len(hv.Spec.AllowedProjects) == 0 { // Hypervisor is available for all projects. traceLog.Info("host allows all projects, keeping", "host", hv.Name) diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities.go b/internal/scheduling/nova/plugins/filters/filter_capabilities.go index cda9a9a20..7facd24ac 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities.go @@ -93,9 +93,16 @@ func (s *FilterCapabilitiesStep) Run(traceLog *slog.Logger, request api.External hvCaps := make(map[string]map[string]string) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } caps, err := hvToNovaCapabilities(hv) if err != nil { traceLog.Warn("hypervisor has unknown capabilities, using empty defaults", "host", hv.Name, "error", err) + result.Events = append(result.Events, lib.FilterWeigherPipelineStepEvent{ + Name: "filter_capabilities_unknown_hypervisor_type", + Labels: map[string]string{"hypervisor_type": hv.Status.DomainCapabilities.HypervisorType}, + }) caps = make(map[string]string) } hvCaps[hv.Name] = caps diff --git a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go index 76c2131f0..e63d68885 100644 --- a/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_capabilities_test.go @@ -8,6 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -194,13 +195,40 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { }, }, }, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{ + Name: "host-empty-type", + }, + Status: hv1.HypervisorStatus{ + DomainCapabilities: hv1.DomainCapabilities{ + HypervisorType: "", + }, + Capabilities: hv1.Capabilities{ + HostCpuArch: "x86_64", + }, + }, + }, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{ + Name: "host-unknown-type", + }, + Status: hv1.HypervisorStatus{ + DomainCapabilities: hv1.DomainCapabilities{ + HypervisorType: "xen", + }, + Capabilities: hv1.Capabilities{ + HostCpuArch: "x86_64", + }, + }, + }, } tests := []struct { - name string - request api.ExternalSchedulerRequest - expectedHosts []string - filteredHosts []string + name string + request api.ExternalSchedulerRequest + expectedHosts []string + filteredHosts []string + expectedEvents []lib.FilterWeigherPipelineStepEvent }{ { name: "No extra specs in request - all hosts pass", @@ -391,6 +419,76 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { expectedHosts: []string{"host4"}, filteredHosts: []string{"host1", "host2", "host3"}, }, + { + name: "Ignore hypervisors not in request even with unknown type", + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + Flavor: api.NovaObject[api.NovaFlavor]{ + Data: api.NovaFlavor{ + ExtraSpecs: map[string]string{ + "capabilities:hypervisor_type": "CH", + }, + }, + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{ + {ComputeHost: "host1"}, + }, + }, + expectedHosts: []string{"host1"}, + filteredHosts: []string{}, + expectedEvents: []lib.FilterWeigherPipelineStepEvent{}, + }, + { + name: "Candidate hypervisor with empty hypervisor type emits event and is filtered out", + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + Flavor: api.NovaObject[api.NovaFlavor]{ + Data: api.NovaFlavor{ + ExtraSpecs: map[string]string{ + "capabilities:hypervisor_type": "CH", + }, + }, + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{ + {ComputeHost: "host-empty-type"}, + }, + }, + expectedHosts: []string{}, + filteredHosts: []string{"host-empty-type"}, + expectedEvents: []lib.FilterWeigherPipelineStepEvent{ + {Name: "filter_capabilities_unknown_hypervisor_type", Labels: map[string]string{"hypervisor_type": ""}}, + }, + }, + { + name: "Candidate hypervisor with unknown hypervisor type emits event and is filtered out", + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + Flavor: api.NovaObject[api.NovaFlavor]{ + Data: api.NovaFlavor{ + ExtraSpecs: map[string]string{ + "capabilities:hypervisor_type": "CH", + }, + }, + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{ + {ComputeHost: "host-unknown-type"}, + }, + }, + expectedHosts: []string{}, + filteredHosts: []string{"host-unknown-type"}, + expectedEvents: []lib.FilterWeigherPipelineStepEvent{ + {Name: "filter_capabilities_unknown_hypervisor_type", Labels: map[string]string{"hypervisor_type": "xen"}}, + }, + }, { name: "No matching hosts", request: api.ExternalSchedulerRequest{ @@ -578,6 +676,28 @@ func TestFilterCapabilitiesStep_Run(t *testing.T) { if len(result.Activations) != len(tt.expectedHosts) { t.Errorf("expected %d hosts, got %d", len(tt.expectedHosts), len(result.Activations)) } + + // Check events + if len(result.Events) != len(tt.expectedEvents) { + t.Errorf("expected %d events, got %d", len(tt.expectedEvents), len(result.Events)) + } + for i, expectedEvent := range tt.expectedEvents { + if i >= len(result.Events) { + break + } + gotEvent := result.Events[i] + if gotEvent.Name != expectedEvent.Name { + t.Errorf("expected event name %q, got %q", expectedEvent.Name, gotEvent.Name) + } + if len(gotEvent.Labels) != len(expectedEvent.Labels) { + t.Errorf("expected event labels %v, got %v", expectedEvent.Labels, gotEvent.Labels) + } + for k, v := range expectedEvent.Labels { + if gotEvent.Labels[k] != v { + t.Errorf("expected event label %q=%q, got %q", k, v, gotEvent.Labels[k]) + } + } + } }) } } diff --git a/internal/scheduling/nova/plugins/filters/filter_correct_az.go b/internal/scheduling/nova/plugins/filters/filter_correct_az.go index ed7f68188..465e01546 100644 --- a/internal/scheduling/nova/plugins/filters/filter_correct_az.go +++ b/internal/scheduling/nova/plugins/filters/filter_correct_az.go @@ -34,6 +34,9 @@ func (s *FilterCorrectAZStep) Run(traceLog *slog.Logger, request api.ExternalSch // "topology.kubernetes.io/zone" on the hv crd. var computeHostsInAZ = make(map[string]struct{}) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } az, ok := hv.Labels[corev1.LabelTopologyZone] if !ok { traceLog.Warn("host missing zone label, keeping", "host", hv.Name) diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer.go b/internal/scheduling/nova/plugins/filters/filter_external_customer.go index b574e4ad1..80bd7cf9c 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer.go @@ -77,6 +77,9 @@ func (s *FilterExternalCustomerStep) Run(traceLog *slog.Logger, request api.Exte } hvsWithTrait := make(map[string]struct{}) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } traits := hv.Status.Traits traits = append(traits, hv.Spec.CustomTraits...) if !slices.Contains(traits, "CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE") { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go index dcccdc010..2a3c8e4ff 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_accelerators.go @@ -33,6 +33,9 @@ func (s *FilterHasAcceleratorsStep) Run(traceLog *slog.Logger, request api.Exter } hvsWithTrait := make(map[string]struct{}) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } traits := hv.Status.Traits traits = append(traits, hv.Spec.CustomTraits...) if !slices.Contains(traits, "COMPUTE_ACCELERATORS") { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index 117b032ae..a66166fa4 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -82,6 +82,9 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa return nil, err } for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } var sourceMap map[hv1.ResourceName]resource.Quantity if hv.Status.EffectiveCapacity == nil { traceLog.Warn("hypervisor with nil effective capacity, use capacity instead (overprovisioning not considered)", "host", hv.Name) diff --git a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go index aa35d2fc9..ac7ed8748 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_requested_traits.go @@ -62,6 +62,9 @@ func (s *FilterHasRequestedTraits) Run(traceLog *slog.Logger, request api.Extern hostsMatchingAllTraits := map[string]struct{}{} for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } allRequiredPresent := true traits := hv.Status.Traits traits = append(traits, hv.Spec.CustomTraits...) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go index 137ddf04f..7cf3ebbfd 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go @@ -68,6 +68,9 @@ func (s *FilterInstanceGroupAntiAffinityStep) Run( } hvsByName := make(map[string]hv1.Hypervisor) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } hvsByName[hv.Name] = hv } diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go index 971848dcd..71cd48844 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go @@ -128,6 +128,9 @@ func (s *FilterRequestedDestinationStep) Run(traceLog *slog.Logger, request api. } hvsByName := make(map[string]hv1.Hypervisor) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } hvsByName[hv.Name] = hv } s.processRequestedAggregates(traceLog, rd.Data.Aggregates, hvsByName, result.Activations) diff --git a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go index 3d7f2aae6..60b616d71 100644 --- a/internal/scheduling/nova/plugins/filters/filter_status_conditions.go +++ b/internal/scheduling/nova/plugins/filters/filter_status_conditions.go @@ -43,6 +43,9 @@ func (s *FilterStatusConditionsStep) Run(traceLog *slog.Logger, request api.Exte var hostsReady = make(map[string]struct{}) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } allMet := true for conditionType, expectedStatus := range expected { cd := meta.FindStatusCondition(hv.Status.Conditions, conditionType) diff --git a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go index e1509a4cc..39772f94e 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_binpack.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_binpack.go @@ -80,6 +80,9 @@ func (s *KVMBinpackStep) Run(traceLog *slog.Logger, request api.ExternalSchedule } hvsByName := make(map[string]hv1.Hypervisor, len(hvs.Items)) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } hvsByName[hv.Name] = hv } vmResources := s.calcVMResources(request) diff --git a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go index 5f13897f0..6085edb70 100644 --- a/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go +++ b/internal/scheduling/nova/plugins/weighers/kvm_instance_group_soft_affinity.go @@ -58,6 +58,9 @@ func (s *KVMInstanceGroupSoftAffinityStep) Run(traceLog *slog.Logger, request ap } hvsByName := make(map[string]hv1.Hypervisor, len(hvs.Items)) for _, hv := range hvs.Items { + if _, ok := result.Activations[hv.Name]; !ok { + continue + } hvsByName[hv.Name] = hv } From 6cad951d67801db5cfda252d13010c3bbfeaa0c5 Mon Sep 17 00:00:00 2001 From: juliusclausnitzer Date: Tue, 4 Aug 2026 15:24:00 +0200 Subject: [PATCH 06/16] feat(reservations): follow VM live migration in CR reservation reconciler (#1048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Implements issue #373. When the reservation reconciler detects that a confirmed VM has disappeared from its expected host, it now searches all hypervisor CRDs to determine whether the VM live-migrated rather than treating it as terminated. - **New host has capacity**: `Spec.TargetHost` is updated to the new host. The existing TargetHost→Status.Host sync path (Branch B in `Reconcile`) advances `Status.Host` on the next cycle, keeping capacity accounting consistent during the transition. - **New host has no capacity**: `TargetHost` is left unchanged; the VM's actual location is recorded in `Status.Allocations`; the new `VMMisplaced` condition is set with reason `MigratedToFullHost`. - **VM not found anywhere**: existing stale-removal behaviour, unchanged. Only the migrating VM's host is considered per-reconcile. Other allocated VMs in the same reservation are not moved. The HV list is fetched lazily — only when a confirmed VM is actually missing from its expected host — so the common path (no migration) pays no extra API cost. --------- Signed-off-by: juliusclausnitzer Signed-off-by: Julius Clausnitzer --- .../reservations/capacity_accounting.go | 65 +++++ .../reservations/capacity_accounting_test.go | 160 ++++++++++++ .../commitments/reservation_controller.go | 130 +++++++++- .../reservation_controller_test.go | 236 +++++++++++++++++- 4 files changed, 583 insertions(+), 8 deletions(-) diff --git a/internal/scheduling/reservations/capacity_accounting.go b/internal/scheduling/reservations/capacity_accounting.go index 2ccca9685..ab305d5d5 100644 --- a/internal/scheduling/reservations/capacity_accounting.go +++ b/internal/scheduling/reservations/capacity_accounting.go @@ -10,6 +10,71 @@ import ( "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) +// HostHasCapacityForReservation reports whether hv has sufficient remaining capacity to +// absorb res moving to it — i.e. whether the unfilled portion of res's slot fits alongside +// everything already committed on the host. +// +// 1. Start from EffectiveCapacity (or Capacity when EffectiveCapacity is nil). +// 2. Subtract hv.Status.Allocation (VMs physically running on this host). +// 3. For each other reservation assigned to this host (via Spec.TargetHost or Status.Host), +// subtract its UnusedReservationCapacity. +// 4. Check that the remainder is ≥ UnusedReservationCapacity(res): the unfilled portion of +// res's slot. Confirmed VMs in res already appear in hv.Status.Allocation (step 2), so +// comparing against the full slot would count them twice. +// +// res itself is excluded from step 3 to avoid subtracting its own block from free capacity. +// Returns false when the hypervisor has no capacity data. +func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv1.Hypervisor, res *v1alpha1.Reservation) bool { + effCap := hv.Status.EffectiveCapacity + if effCap == nil { + effCap = hv.Status.Capacity + } + if effCap == nil { + return false + } + + free := make(map[hv1.ResourceName]resource.Quantity, len(effCap)) + for rn, qty := range effCap { + free[rn] = qty.DeepCopy() + } + + for rn, allocated := range hv.Status.Allocation { + if f, ok := free[rn]; ok { + f.Sub(allocated) + free[rn] = f + } + } + + for i := range allReservations { + other := &allReservations[i] + if other.Name == res.Name && other.Namespace == res.Namespace { + continue + } + // Only block resources from reservations that target or are confirmed on this host. + targetsThisHost := other.Spec.TargetHost == hv.Name || other.Status.Host == hv.Name + if !targetsThisHost { + continue + } + for resourceName, block := range UnusedReservationCapacity(other, false) { + if f, ok := free[resourceName]; ok { + f.Sub(block) + free[resourceName] = f + } + } + } + + for resourceName, required := range UnusedReservationCapacity(res, false) { + remaining, ok := free[resourceName] + if !ok { + return false + } + if remaining.Cmp(required) < 0 { + return false + } + } + return true +} + // UnusedReservationCapacity returns the resources a Reservation should block on its host(s). // This is the single source of truth used by both the capacity controller and // filter_has_enough_capacity to ensure consistent accounting. diff --git a/internal/scheduling/reservations/capacity_accounting_test.go b/internal/scheduling/reservations/capacity_accounting_test.go index 815a0a07f..d13a1c400 100644 --- a/internal/scheduling/reservations/capacity_accounting_test.go +++ b/internal/scheduling/reservations/capacity_accounting_test.go @@ -8,6 +8,7 @@ import ( hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/cobaltcore-dev/cortex/api/v1alpha1" ) @@ -170,3 +171,162 @@ func TestUnusedReservationCapacity(t *testing.T) { }) } } + +func TestHostHasCapacityForReservation(t *testing.T) { + gib := func(n int64) resource.Quantity { return *resource.NewQuantity(n*1024*1024*1024, resource.BinarySI) } + cpu := func(n int64) resource.Quantity { return *resource.NewQuantity(n, resource.DecimalSI) } + + hvWithCapacity := func(name string, memGiB, cpuCores int64) hv1.Hypervisor { + return hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(memGiB), + hv1.ResourceCPU: cpu(cpuCores), + }, + }, + } + } + + resWithSlot := func(name, targetHost string, memGiB, cpuCores int64) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: targetHost, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(memGiB), + hv1.ResourceCPU: cpu(cpuCores), + }, + }, + Status: v1alpha1.ReservationStatus{Host: targetHost}, + } + } + deref := func(r *v1alpha1.Reservation) v1alpha1.Reservation { return *r } + + tests := []struct { + name string + hv hv1.Hypervisor + res *v1alpha1.Reservation + others []v1alpha1.Reservation + wantFits bool + }{ + { + name: "empty host: slot fits easily", + hv: hvWithCapacity("host-new", 960, 80), + res: resWithSlot("res-1", "host-old", 480, 40), + wantFits: true, + }, + { + name: "host fully consumed by another reservation: no capacity", + hv: hvWithCapacity("host-new", 480, 40), + res: resWithSlot("res-target", "host-old", 480, 40), + others: []v1alpha1.Reservation{ + deref(resWithSlot("res-blocker", "host-new", 480, 40)), + }, + wantFits: false, + }, + { + name: "host partially consumed, enough room left", + hv: hvWithCapacity("host-new", 960, 80), + res: resWithSlot("res-target", "host-old", 480, 40), + others: []v1alpha1.Reservation{ + deref(resWithSlot("res-blocker", "host-new", 480, 40)), + }, + wantFits: true, + }, + { + name: "host partially consumed, exactly at boundary: fits", + hv: hvWithCapacity("host-new", 960, 80), + res: resWithSlot("res-target", "host-old", 480, 40), + others: []v1alpha1.Reservation{ + deref(resWithSlot("res-blocker-a", "host-new", 240, 20)), + deref(resWithSlot("res-blocker-b", "host-new", 240, 20)), + }, + wantFits: true, + }, + { + name: "host partially consumed, one resource short (CPU)", + hv: hvWithCapacity("host-new", 960, 60), + res: resWithSlot("res-target", "host-old", 480, 40), + others: []v1alpha1.Reservation{ + deref(resWithSlot("res-blocker", "host-new", 480, 40)), + }, + // 960-480=480 memory OK, but 60-40=20 CPU < 40 required + wantFits: false, + }, + { + name: "target reservation itself excluded from blocking calculation", + hv: hvWithCapacity("host-new", 480, 40), + res: resWithSlot("res-target", "host-new", 480, 40), + others: []v1alpha1.Reservation{ + // Same name as res — should be ignored + deref(resWithSlot("res-target", "host-new", 480, 40)), + }, + wantFits: true, + }, + { + name: "reservations on other hosts do not count", + hv: hvWithCapacity("host-new", 480, 40), + res: resWithSlot("res-target", "host-old", 480, 40), + others: []v1alpha1.Reservation{ + deref(resWithSlot("res-on-other-host", "host-unrelated", 480, 40)), + }, + wantFits: true, + }, + { + name: "hv with no capacity data: always false", + hv: hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-nocap"}, + }, + res: resWithSlot("res-target", "host-old", 480, 40), + wantFits: false, + }, + { + name: "hv allocation already consumed memory: no room", + hv: hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host-new"}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(480), + hv1.ResourceCPU: cpu(40), + }, + Allocation: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(100), + }, + }, + }, + res: resWithSlot("res-target", "host-old", 480, 40), + wantFits: false, // 480-100 = 380 GiB remaining < 480 GiB slot (no confirmed VMs, so full slot is required) + }, + { + name: "reservation targeting via Status.Host (not TargetHost) still blocks", + hv: hvWithCapacity("host-new", 480, 40), + res: resWithSlot("res-target", "host-old", 480, 40), + others: []v1alpha1.Reservation{ + // TargetHost empty but Status.Host = host-new + { + ObjectMeta: metav1.ObjectMeta{Name: "res-status-host"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: gib(480), + hv1.ResourceCPU: cpu(40), + }, + }, + Status: v1alpha1.ReservationStatus{Host: "host-new"}, + }, + }, + wantFits: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := HostHasCapacityForReservation(tt.others, tt.hv, tt.res) + if got != tt.wantFits { + t.Errorf("HostHasCapacityForReservation() = %v, want %v", got, tt.wantFits) + } + }) + } +} diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index 598f3a667..f6bf5a385 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -382,10 +382,16 @@ type reconcileAllocationsResult struct { // reconcileAllocations verifies all allocations in Spec against actual VM state using the // Hypervisor CRD as the sole source of truth. // -// For new allocations (within grace period): the VM may not yet appear in the HV CRD -// (still spawning), so we skip verification and requeue with a short interval. -// For older allocations: we check the HV CRD; VMs not found are considered leaving and -// removed from the reservation. +// New allocations within the grace period are skipped — the VM may not yet appear in the +// HV CRD while it is still spawning. Older allocations are verified; VMs no longer present +// on their expected host are handled as follows: +// +// Live migration: when a confirmed VM is found on a different host, the reservation follows +// it only when the reservation has exactly one allocated VM and the new host has capacity. +// In all other cases (multiple VMs, or new host at capacity), the migrated VM is removed +// from the reservation so the slot remains available for re-use on the original host. +// Moving TargetHost when other VMs are present would cause those remaining VMs to appear +// misplaced on the next reconcile cycle. func (r *CommitmentReservationController) reconcileAllocations(ctx context.Context, res *v1alpha1.Reservation) (*reconcileAllocationsResult, error) { logger := LoggerFromContext(ctx) result := &reconcileAllocationsResult{} @@ -436,11 +442,37 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte existingStatusAllocations[k] = v } + // allHVs and allReservations are fetched lazily — only needed when a confirmed VM is + // missing from its expected host and we need to scan for a live migration. + var allHVs *hv1.HypervisorList + var allReservations *v1alpha1.ReservationList + + ensureHVsAndReservations := func() error { + if allHVs != nil && allReservations != nil { + return nil + } + hvs := &hv1.HypervisorList{} + if err := r.List(ctx, hvs); err != nil { + return fmt.Errorf("failed to list hypervisors: %w", err) + } + res := &v1alpha1.ReservationList{} + if err := r.List(ctx, res); err != nil { + return fmt.Errorf("failed to list reservations: %w", err) + } + allHVs = hvs + allReservations = res + return nil + } + // Build new Status.Allocations map based on HV CRD state. newStatusAllocations := make(map[string]string) // Track allocations to remove from Spec (stale/leaving VMs). var allocationsToRemove []string + // migrationTargetHost is set when the reservation has exactly one VM, that VM + // live-migrated to a new host, and the new host has capacity. + migrationTargetHost := "" + for vmUUID, allocation := range res.Spec.CommittedResourceReservation.Allocations { allocationAge := now.Sub(allocation.CreationTimestamp.Time) isInGracePeriod := allocationAge < r.Conf.AllocationGracePeriod.Duration @@ -464,7 +496,12 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte logger.V(1).Info("verified VM allocation via Hypervisor CRD", "vm", vmUUID, "host", expectedHost) - } else { + continue + } + + // VM not on the expected host. For unconfirmed post-grace VMs this is a clean + // stale allocation — remove it without further searching. + if !isConfirmed { allocationsToRemove = append(allocationsToRemove, vmUUID) logger.Info("removing stale allocation (VM not found on hypervisor)", "vm", vmUUID, @@ -472,6 +509,73 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte "expectedHost", expectedHost, "allocationAge", allocationAge, "gracePeriod", r.Conf.AllocationGracePeriod.Duration) + continue + } + + // Confirmed VM missing from expected host — could be a live migration. + // Scan all HVs lazily; the list is shared across any further misses this cycle. + if err := ensureHVsAndReservations(); err != nil { + return nil, err + } + + var foundHost string + var foundHV hv1.Hypervisor + for _, hv := range allHVs.Items { + if hv.Name == expectedHost { + continue // already checked via hvInstanceSet above + } + for _, inst := range hv.Status.Instances { + if inst.ID == vmUUID { + foundHost = hv.Name + foundHV = hv + break + } + } + if foundHost != "" { + break + } + } + + if foundHost == "" { + // VM is not on any known hypervisor. This covers two cases: + // 1. The VM was terminated or evacuated — correct to remove. + // 2. The VM is mid-live-migration: it has left host-old's HV CRD but + // host-new's CRD has not been updated yet. In this window the VM + // is incorrectly treated as gone and removed from the reservation. + // A VM CRD with lifecycle state (migrating/active) would close this + // gap; without one we accept this narrow race as a known limitation. + allocationsToRemove = append(allocationsToRemove, vmUUID) + logger.Info("removing confirmed allocation (VM not found on any hypervisor)", + "vm", vmUUID, + "reservation", res.Name, + "expectedHost", expectedHost) + continue + } + + // VM found on a different host — live migration detected. + // + // Follow the VM only when this is the sole VM in the reservation and the new + // host has capacity. Moving TargetHost with multiple VMs present would cause + // the remaining VMs to appear misplaced on the next reconcile. When there are + // multiple VMs, or the new host is at capacity, remove this VM so the slot + // on the original host remains available for re-use. + isSingleVM := len(res.Spec.CommittedResourceReservation.Allocations) == 1 + if isSingleVM && reservations.HostHasCapacityForReservation(allReservations.Items, foundHV, res) { + logger.Info("VM live-migrated to host with capacity, updating TargetHost", + "vm", vmUUID, + "reservation", res.Name, + "oldHost", expectedHost, + "newHost", foundHost) + migrationTargetHost = foundHost + newStatusAllocations[vmUUID] = foundHost + } else { + logger.Info("removing VM from reservation after live migration: either multiple VMs present or new host lacks capacity", + "vm", vmUUID, + "reservation", res.Name, + "expectedHost", expectedHost, + "actualHost", foundHost, + "singleVM", isSingleVM) + allocationsToRemove = append(allocationsToRemove, vmUUID) } } @@ -487,10 +591,19 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte specChanged = true } + // Advance both TargetHost and Status.Host in the same patch cycle to avoid a + // transient state where Status.Host lags behind TargetHost and blocks capacity + // accounting on the old host during the next reconcile. + if migrationTargetHost != "" { + res.Spec.TargetHost = migrationTargetHost + res.Status.Host = migrationTargetHost + specChanged = true + } + // Update Status.Allocations res.Status.CommittedResourceReservation.Allocations = newStatusAllocations - // Patch Spec if changed (stale allocations removed) + // Patch Spec if changed (stale allocations removed and/or TargetHost updated) if specChanged { if err := r.Patch(ctx, res, client.MergeFrom(old)); err != nil { if client.IgnoreNotFound(err) == nil { @@ -509,8 +622,11 @@ func (r *CommitmentReservationController) reconcileAllocations(ctx context.Conte // the status update. Otherwise MergeFrom(old) would see no diff // and the status patch would be a no-op. old = res.DeepCopy() - // Re-apply the status update that was overwritten by the re-fetch. + // Re-apply status updates that were overwritten by the re-fetch. res.Status.CommittedResourceReservation.Allocations = newStatusAllocations + if migrationTargetHost != "" { + res.Status.Host = migrationTargetHost + } } // Proactively remove this VM UUID from all other candidate reservations that still diff --git a/internal/scheduling/reservations/commitments/reservation_controller_test.go b/internal/scheduling/reservations/commitments/reservation_controller_test.go index 651852c2c..a776ac0cf 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller_test.go +++ b/internal/scheduling/reservations/commitments/reservation_controller_test.go @@ -7,8 +7,10 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" + "strconv" "testing" "time" @@ -446,7 +448,7 @@ func newTestCRReservation(allocations map[string]metav1.Time) *v1alpha1.Reservat // newTestHypervisorCRD creates a test Hypervisor CRD with instances. // -//nolint:unparam // name parameter allows future test flexibility + func newTestHypervisorCRD(name string, instances []hv1.Instance) *hv1.Hypervisor { return &hv1.Hypervisor{ ObjectMeta: metav1.ObjectMeta{ @@ -982,3 +984,235 @@ func TestCommitmentReservationController_DomainNameHint(t *testing.T) { }) } } + +// ============================================================================ +// Tests: live migration detection in reconcileAllocations +// ============================================================================ + +// newHVWithCapacity creates a Hypervisor CRD with the given instances and effective capacity. +func newHVWithCapacity(name string, memGiB, cpuCores int64, instances []hv1.Instance) *hv1.Hypervisor { + return &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Status: hv1.HypervisorStatus{ + EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", memGiB)), + hv1.ResourceCPU: resource.MustParse(strconv.FormatInt(cpuCores, 10)), + }, + Instances: instances, + }, + } +} + +// newConfirmedCRReservation creates a ready CR reservation with one confirmed VM on host. +// slotMemGiB/slotCPU define the full reservation slot; vmMemGiB/vmCPU define what the VM +// actually consumes — these may be smaller, leaving an unfilled remainder in the slot. +func newConfirmedCRReservation(name, host, vmUUID string, slotMemGiB, slotCPU, vmMemGiB, vmCPU int64) *v1alpha1.Reservation { + return &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: host, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", slotMemGiB)), + hv1.ResourceCPU: resource.MustParse(strconv.FormatInt(slotCPU, 10)), + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationSpec{ + ProjectID: "test-project", + ResourceName: "test-flavor", + Allocations: map[string]v1alpha1.CommittedResourceAllocation{ + vmUUID: { + CreationTimestamp: metav1.NewTime(time.Now().Add(-1 * time.Hour)), + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", vmMemGiB)), + hv1.ResourceCPU: resource.MustParse(strconv.FormatInt(vmCPU, 10)), + }, + }, + }, + }, + }, + Status: v1alpha1.ReservationStatus{ + Host: host, + Conditions: []metav1.Condition{ + {Type: v1alpha1.ReservationConditionReady, Status: metav1.ConditionTrue, Reason: "ReservationActive"}, + }, + CommittedResourceReservation: &v1alpha1.CommittedResourceReservationStatus{ + Allocations: map[string]string{vmUUID: host}, + }, + }, + } +} + +func TestReconcileAllocations_LiveMigration(t *testing.T) { + const ( + vmUUID = "vm-uuid" + vm2UUID = "vm-uuid-2" + oldHost = "host-old" + newHost = "host-new" + ) + + config := ReservationControllerConfig{AllocationGracePeriod: metav1.Duration{Duration: 15 * time.Minute}} + + tests := []struct { + name string + // reservation to use; nil uses the default single-VM reservation + reservation *v1alpha1.Reservation + // extra objects beyond the base reservation and old host HV + extraObjects []client.Object + // expected outcomes after first reconcile pass + wantTargetHost string + wantStatusHost string // expected in Status.Allocations[vmUUID]; "" means absent + wantSpecHasVM bool + // if true, run a second reconcile pass and assert state is stable + assertSecondPass bool + }{ + { + name: "single VM, new host has capacity: follow the VM", + extraObjects: []client.Object{ + newHVWithCapacity(newHost, 960, 80, []hv1.Instance{{ID: vmUUID, Active: true}}), + }, + wantTargetHost: newHost, + wantStatusHost: newHost, + wantSpecHasVM: true, + assertSecondPass: true, + }, + { + name: "single VM, new host at capacity: remove VM, slot stays on old host", + extraObjects: []client.Object{ + // VM (240Gi/20) running on newHost; a full-slot blocker leaves no room for + // the 240Gi/20 slot remainder. + newHVWithCapacity(newHost, 480, 40, []hv1.Instance{{ID: vmUUID, Active: true}}), + &v1alpha1.Reservation{ + ObjectMeta: metav1.ObjectMeta{Name: "res-blocker"}, + Spec: v1alpha1.ReservationSpec{ + Type: v1alpha1.ReservationTypeCommittedResource, + TargetHost: newHost, + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("480Gi"), + hv1.ResourceCPU: resource.MustParse("40"), + }, + }, + Status: v1alpha1.ReservationStatus{Host: newHost}, + }, + }, + wantTargetHost: oldHost, + wantStatusHost: "", + wantSpecHasVM: false, + }, + { + name: "multiple VMs, one migrated: remove migrated VM, never update TargetHost", + reservation: func() *v1alpha1.Reservation { + res := newConfirmedCRReservation("res-1", oldHost, vmUUID, 480, 40, 240, 20) + // Add a second VM confirmed on oldHost. + res.Spec.CommittedResourceReservation.Allocations[vm2UUID] = v1alpha1.CommittedResourceAllocation{ + CreationTimestamp: metav1.NewTime(time.Now().Add(-1 * time.Hour)), + Resources: map[hv1.ResourceName]resource.Quantity{ + hv1.ResourceMemory: resource.MustParse("240Gi"), + hv1.ResourceCPU: resource.MustParse("20"), + }, + } + res.Status.CommittedResourceReservation.Allocations[vm2UUID] = oldHost + return res + }(), + extraObjects: []client.Object{ + // vmUUID has migrated to newHost with plenty of capacity; vm2UUID stays on oldHost. + // Supply oldHost HV explicitly so vm2UUID is present on it. + newTestHypervisorCRD(oldHost, []hv1.Instance{{ID: vm2UUID, Active: true}}), + newHVWithCapacity(newHost, 960, 80, []hv1.Instance{{ID: vmUUID, Active: true}}), + }, + wantTargetHost: oldHost, + wantStatusHost: "", + wantSpecHasVM: false, + }, + { + name: "VM gone from all hosts: remove allocation", + extraObjects: []client.Object{newTestHypervisorCRD("host-other", []hv1.Instance{{ID: "other-vm"}})}, + wantTargetHost: oldHost, + wantStatusHost: "", + wantSpecHasVM: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := newCRTestScheme(t) + + res := tt.reservation + if res == nil { + res = newConfirmedCRReservation("res-1", oldHost, vmUUID, 480, 40, 240, 20) + } + + var objects []client.Object + objects = append(objects, res) + + // Add an oldHost HV unless the test provides its own via extraObjects. + addsOldHostHV := false + for _, obj := range tt.extraObjects { + if hv, ok := obj.(*hv1.Hypervisor); ok && hv.Name == oldHost { + addsOldHostHV = true + break + } + } + if !addsOldHostHV { + objects = append(objects, newTestHypervisorCRD(oldHost, []hv1.Instance{})) + } + objects = append(objects, tt.extraObjects...) + + k8sClient := newCRTestClient(scheme, objects...) + controller := &CommitmentReservationController{Client: k8sClient, Scheme: scheme, Conf: config} + ctx := WithNewGlobalRequestID(context.Background()) + + if _, err := controller.reconcileAllocations(ctx, res); err != nil { + t.Fatalf("reconcileAllocations() error = %v", err) + } + + var updated v1alpha1.Reservation + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(res), &updated); err != nil { + t.Fatalf("failed to get updated reservation: %v", err) + } + + if updated.Spec.TargetHost != tt.wantTargetHost { + t.Errorf("Spec.TargetHost = %q, want %q", updated.Spec.TargetHost, tt.wantTargetHost) + } + _, specHasVM := updated.Spec.CommittedResourceReservation.Allocations[vmUUID] + if specHasVM != tt.wantSpecHasVM { + t.Errorf("VM in Spec.Allocations = %v, want %v", specHasVM, tt.wantSpecHasVM) + } + var statusHost string + if updated.Status.CommittedResourceReservation != nil { + statusHost = updated.Status.CommittedResourceReservation.Allocations[vmUUID] + } + if statusHost != tt.wantStatusHost { + t.Errorf("Status.Allocations[%s] = %q, want %q", vmUUID, statusHost, tt.wantStatusHost) + } + + // For the migration-follow case: run a second reconcile to confirm state is + // stable and Status.Host was advanced to the new host in the first pass. + if tt.assertSecondPass { + if _, err := controller.reconcileAllocations(ctx, &updated); err != nil { + t.Fatalf("second reconcileAllocations() error = %v", err) + } + var updated2 v1alpha1.Reservation + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(res), &updated2); err != nil { + t.Fatalf("failed to get reservation after second pass: %v", err) + } + if updated2.Spec.TargetHost != tt.wantTargetHost { + t.Errorf("second pass: Spec.TargetHost = %q, want %q", updated2.Spec.TargetHost, tt.wantTargetHost) + } + if updated2.Status.Host != tt.wantTargetHost { + t.Errorf("second pass: Status.Host = %q, want %q", updated2.Status.Host, tt.wantTargetHost) + } + _, specHasVM2 := updated2.Spec.CommittedResourceReservation.Allocations[vmUUID] + if !specHasVM2 { + t.Errorf("second pass: VM unexpectedly removed from Spec.Allocations") + } + var statusHost2 string + if updated2.Status.CommittedResourceReservation != nil { + statusHost2 = updated2.Status.CommittedResourceReservation.Allocations[vmUUID] + } + if statusHost2 != tt.wantStatusHost { + t.Errorf("second pass: Status.Allocations[%s] = %q, want %q", vmUUID, statusHost2, tt.wantStatusHost) + } + } + }) + } +} From 7d1cdc08956ebd2aea6ac070b278921c2c024e61 Mon Sep 17 00:00:00 2001 From: Marcel <156897072+mblos@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:25:24 +0200 Subject: [PATCH 07/16] fix(CR): subtract reservation CPU blocks when counting placeable slots (#1118) Fixes capacity slot counting to use both memory and CPU as binding constraints, not memory alone. Signed-off-by: Marcel <156897072+mblos@users.noreply.github.com> --- .../reservations/capacity/controller.go | 44 +++++++++++-------- .../reservations/capacity/controller_test.go | 16 +++---- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index a6194d423..de47fc825 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -238,10 +238,10 @@ func (c *Reconciler) reconcileAll(ctx context.Context) error { azs := availabilityZones(hvList.Items) - blockedByReservations, err := c.blockedMemoryByHost(ctx) + blockedByReservations, err := c.blockedResourcesByHost(ctx) if err != nil { - logger.Error(err, "failed to compute blocked memory by host, placeable slot counts may be overstated") - blockedByReservations = map[string]int64{} + logger.Error(err, "failed to compute blocked resources by host, placeable slot counts may be overstated") + blockedByReservations = map[string]map[string]int64{} } usageByKey := c.computeVMUsage(ctx, flavorGroups, hvList.Items) @@ -320,9 +320,10 @@ func (c *Reconciler) computeVMUsage( } // hvRemainingResources returns remaining schedulable resources after subtracting -// current allocations and (for memory) active reservation blocks. +// current allocations and active reservation blocks. +// blockedResources uses ResourceMemory/ResourceCores keys. // Returns nil if the hypervisor has no capacity data. -func hvRemainingResources(hv hv1.Hypervisor, blockedMemBytes int64) map[string]int64 { +func hvRemainingResources(hv hv1.Hypervisor, blockedResources map[string]int64) map[string]int64 { effCap := hv.Status.EffectiveCapacity if effCap == nil { effCap = hv.Status.Capacity @@ -338,7 +339,7 @@ func hvRemainingResources(hv hv1.Hypervisor, blockedMemBytes int64) map[string]i if alloc, ok := hv.Status.Allocation[hv1.ResourceMemory]; ok { mem -= alloc.Value() } - mem -= blockedMemBytes + mem -= blockedResources[ResourceMemory] if mem < 0 { mem = 0 } @@ -350,6 +351,7 @@ func hvRemainingResources(hv hv1.Hypervisor, blockedMemBytes int64) map[string]i if alloc, ok := hv.Status.Allocation[hv1.ResourceCPU]; ok { cpu -= alloc.Value() } + cpu -= blockedResources[ResourceCores] if cpu < 0 { cpu = 0 } @@ -367,7 +369,7 @@ func (c *Reconciler) probeGroup( groupData compute.FlavorGroupFeature, az string, hvByName map[string]hv1.Hypervisor, - blockedByReservations map[string]int64, + blockedByReservations map[string]map[string]int64, ) (probeGroupResult, error) { logger := LoggerFromContext(ctx) @@ -445,7 +447,7 @@ func (c *Reconciler) probeGroup( func buildSplitInputs( results []probeGroupResult, hvByName map[string]hv1.Hypervisor, - blockedByReservations map[string]int64, + blockedByReservations map[string]map[string]int64, az string, logger logr.Logger, ) (groupInputs []GroupInput, hosts map[string]HostState) { @@ -502,7 +504,7 @@ func (c *Reconciler) reconcileAZ( az string, flavorGroups map[string]compute.FlavorGroupFeature, hvByName map[string]hv1.Hypervisor, - blockedByReservations map[string]int64, + blockedByReservations map[string]map[string]int64, usageByKey map[vmUsageKey]vmUsage, ) { @@ -706,7 +708,7 @@ func (c *Reconciler) probeScheduler( az, pipeline string, hvByName map[string]hv1.Hypervisor, ignoreAllocations bool, - blockedByReservations map[string]int64, + blockedByReservations map[string]map[string]int64, ) (capacity, hosts int64, candidateHosts []string, err error) { flavorBytes := int64(flavor.MemoryMB) * 1024 * 1024 //nolint:gosec @@ -793,15 +795,16 @@ func (c *Reconciler) probeScheduler( return capacity, hosts, candidateHosts, nil } -// blockedMemoryByHost returns total reservation-blocked bytes per host. +// blockedResourcesByHost returns total reservation-blocked resources per host. // Both TargetHost and Status.Host are blocked; migration blocks both simultaneously. -func (c *Reconciler) blockedMemoryByHost(ctx context.Context) (map[string]int64, error) { +// The inner map uses ResourceMemory and ResourceCores keys. +func (c *Reconciler) blockedResourcesByHost(ctx context.Context) (map[string]map[string]int64, error) { var list v1alpha1.ReservationList if err := c.client.List(ctx, &list); err != nil { return nil, fmt.Errorf("failed to list reservations: %w", err) } - blocked := make(map[string]int64) + blocked := make(map[string]map[string]int64) for i := range list.Items { res := &list.Items[i] @@ -817,13 +820,16 @@ func (c *Reconciler) blockedMemoryByHost(ctx context.Context) (map[string]int64, } resourcesToBlock := reservations.UnusedReservationCapacity(res, false) - memQty, ok := resourcesToBlock[hv1.ResourceMemory] - if !ok { - continue - } - memBytes := memQty.Value() for host := range hostsToBlock { - blocked[host] += memBytes + if blocked[host] == nil { + blocked[host] = make(map[string]int64) + } + if qty, ok := resourcesToBlock[hv1.ResourceMemory]; ok { + blocked[host][ResourceMemory] += qty.Value() + } + if qty, ok := resourcesToBlock[hv1.ResourceCPU]; ok { + blocked[host][ResourceCores] += qty.Value() + } } } return blocked, nil diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 4408433fb..4b4d49f8a 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -225,7 +225,7 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) + hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdNameFor(groupName, az)}, &crd); err != nil { @@ -297,7 +297,7 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - map[string]hv1.Hypervisor{}, map[string]int64{}, map[vmUsageKey]vmUsage{}) + map[string]hv1.Hypervisor{}, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) // Stale probes → CRD must NOT be written; last good state is preserved. var list v1alpha1.FlavorGroupCapacityList @@ -369,7 +369,7 @@ func TestReconcileAZ_MarksExistingCRDNotReadyOnSchedulerError(t *testing.T) { ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) + hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -434,9 +434,9 @@ func TestReconcileAZ_IdempotentUpdate(t *testing.T) { groups := map[string]compute.FlavorGroupFeature{groupName: groupData} // First call - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) // Second call — should not error on the already-existing CRD. - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -984,7 +984,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { } // Now run reconcileAZ to verify the CRD gets zeroed out. - ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, usageByKey) + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]map[string]int64{}, usageByKey) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -1032,8 +1032,8 @@ func TestProbeScheduler_SubtractsReservationBlocksWhenNotIgnored(t *testing.T) { } // Placeable probe with 1 reservation block: 3 - 1 (alloc) - 1 (reservation) = 1 slot. - blockedByReservations := map[string]int64{ - "host-1": memBytes, // 1 reservation blocking 1 slot's worth of memory + blockedByReservations := map[string]map[string]int64{ + "host-1": {ResourceMemory: memBytes}, // 1 reservation blocking 1 slot's worth of memory } placeableCap, _, _, err := c.probeScheduler(context.Background(), flavor, "az-a", "placeable-pipeline", hvByName, false, blockedByReservations) if err != nil { From 41af7155298a3ad38b181e5857db6860fedb1f1e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 07:03:06 +0200 Subject: [PATCH 08/16] Renovate: Update kube-prometheus-stack Docker tag to v88.1.4 (#1119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | patch | `88.1.3` → `88.1.4` | --- ### Release Notes
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v88.1.4`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-88.1.4) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. #### What's Changed - \[kube-prometheus-stack] Update kube-prometheus-stack dependency non-major updates by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7164](https://redirect.github.com/prometheus-community/helm-charts/pull/7164) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index d69cdd95f..d21ac19a6 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 88.1.3 + version: 88.1.4 From 7343f7cd2b23188a83fe62f00531029d9c75edb9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:25:19 +0200 Subject: [PATCH 09/16] Renovate: Update debian:trixie-slim Docker digest to 3a39a05 (#1120) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | debian | | digest | `020c0d2` → `3a39a05` | | debian | final | digest | `020c0d2` → `3a39a05` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/18) for more information. --- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - At any time (no schedule defined) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- postgres/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 706109e9b..36a12142d 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:trixie-slim@sha256:020c0d20b9880058cbe785a9db107156c3c75c2ac944a6aa7ab59f2add76a7bd +FROM debian:trixie-slim@sha256:3a39a0592364683e6bab97937b72cad5a8fa6dcbbee90edb3bb48c7f8e94f258 # explicitly set user/group IDs RUN set -eux; \ From a04b82554d9418b88df5b3e99606d4335ef2eb5e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:25:35 +0200 Subject: [PATCH 10/16] Renovate: Update kube-prometheus-stack Docker tag to v88.1.5 (#1121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | patch | `88.1.4` → `88.1.5` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/18) for more information. --- ### Release Notes
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v88.1.5`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-88.1.5) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. #### What's Changed - \[kube-prometheus-stack] Update Helm release grafana to v12.10.3 by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7165](https://redirect.github.com/prometheus-community/helm-charts/pull/7165) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index d21ac19a6..19967acf5 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 88.1.4 + version: 88.1.5 From a9532c8eacaf95236d77789fae904ef0b496033c Mon Sep 17 00:00:00 2001 From: Markus Wieland Date: Wed, 5 Aug 2026 13:28:58 +0200 Subject: [PATCH 11/16] feat: add kpi that tracks CR count per configured cluster (#1054) - Add `CountPerGVK` method to multicluster client - This uses the PartialObjectMetadataList so only object metadata is fetched, making it efficient even for large object counts - Counts the number of objects per cluster and exports it as metric with the configured labels of the remote cluster --------- Signed-off-by: Markus Wieland --- .../cortex-nova/templates/kpis_kvm.yaml | 20 ++ .../multicluster_object_count_kpi.go | 193 ++++++++++++ .../multicluster_object_count_kpi_test.go | 284 ++++++++++++++++++ internal/knowledge/kpis/supported_kpis.go | 2 + pkg/multicluster/client.go | 61 +++- pkg/multicluster/client_test.go | 103 +++++++ 6 files changed, 662 insertions(+), 1 deletion(-) create mode 100644 internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go create mode 100644 internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go diff --git a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml index 10ff5c45f..1ec5a2d2d 100644 --- a/helm/bundles/cortex-nova/templates/kpis_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/kpis_kvm.yaml @@ -41,4 +41,24 @@ spec: - name: nova-flavors - name: identity-projects - name: identity-domains +--- +apiVersion: cortex.cloud/v1alpha1 +kind: KPI +metadata: + name: multicluster-object-count +spec: + schedulingDomain: nova + impl: multicluster_object_count_kpi + opts: + gvks: + - cortex.cloud/v1alpha1/HistoryList + - cortex.cloud/v1alpha1/ReservationList + - cortex.cloud/v1alpha1/CommittedResourceList + - kvm.cloud.sap/v1/HypervisorList + description: | + This KPI reports the number of objects of each configured GVK per cluster + (home and remote), labelled by the configured cluster labels. Home-cluster + metrics use is_home=true with empty routing-label values. It uses + PartialObjectMetadataList so only object metadata is fetched, making it + efficient even for large object counts. {{- end }} \ No newline at end of file diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go new file mode 100644 index 000000000..77365a96d --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi.go @@ -0,0 +1,193 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "fmt" + "log/slog" + "strings" + "unicode" + + "github.com/cobaltcore-dev/cortex/internal/knowledge/db" + "github.com/cobaltcore-dev/cortex/internal/knowledge/kpis/plugins" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MulticlusterObjectCountKPIOpts configures which GVKs to count per cluster. +// Each entry is a "group/version/Kind" string, e.g. +// "cortex.cloud/v1alpha1/HypervisorList". +type MulticlusterObjectCountKPIOpts struct { + GVKs []string `json:"gvks"` +} + +type gvkDesc struct { + gvk schema.GroupVersionKind +} + +// multiclusterReader is the subset of *multicluster.Client this KPI needs. +// Keeping it narrow lets the KPI logic be unit-tested without wiring real or +// fake clusters into a multicluster.Client. +type multiclusterReader interface { + ConfiguredRouteLabels(gvk schema.GroupVersionKind) []map[string]string + ListMetadataPerCluster(ctx context.Context, gvk schema.GroupVersionKind, opts ...client.ListOption) ([]multicluster.ClusterObjectMetadata, error) +} + +// MulticlusterObjectCountKPI reports the number of objects of each configured +// GVK per cluster (home and remote), labelled by the cluster's routing labels. +// Home-cluster metrics carry is_home=true with empty routing-label values. +type MulticlusterObjectCountKPI struct { + plugins.BaseKPI[MulticlusterObjectCountKPIOpts] + mcl multiclusterReader + descs []gvkDesc + labelKeys []string // global union of snake_case routing-label keys across all GVKs + sharedDesc *prometheus.Desc // single descriptor shared by all GVKs +} + +func (MulticlusterObjectCountKPI) GetName() string { return "multicluster_object_count_kpi" } + +func (k *MulticlusterObjectCountKPI) Init(_ *db.DB, c client.Client, opts conf.RawOpts) error { + if err := k.BaseKPI.Init(nil, c, opts); err != nil { + return err + } + mcl, ok := c.(*multicluster.Client) + if !ok { + return fmt.Errorf("multicluster_object_count_kpi requires a *multicluster.Client, got %T", c) + } + k.mcl = mcl + + // Parse all GVKs, then build the shared descriptor via buildObjectCountSchema. + var gvks []schema.GroupVersionKind + for _, raw := range k.Options.GVKs { + gvk, err := parseGVK(raw) + if err != nil { + return fmt.Errorf("invalid GVK %q: %w", raw, err) + } + gvks = append(gvks, gvk) + } + + var err error + k.labelKeys, k.sharedDesc, err = buildObjectCountSchema(mcl, gvks) + if err != nil { + return err + } + + for _, gvk := range gvks { + k.descs = append(k.descs, gvkDesc{gvk: gvk}) + } + return nil +} + +func (k *MulticlusterObjectCountKPI) Describe(ch chan<- *prometheus.Desc) { + if k.sharedDesc != nil { + ch <- k.sharedDesc + } +} + +func (k *MulticlusterObjectCountKPI) Collect(ch chan<- prometheus.Metric) { + ctx := context.Background() + for _, d := range k.descs { + perCluster, err := k.mcl.ListMetadataPerCluster(ctx, d.gvk) + if err != nil { + slog.Error("multicluster_object_count_kpi: failed to list object metadata", + "gvk", d.gvk, "err", err) + continue + } + for _, c := range perCluster { + isHome := "false" + if c.IsHome { + isHome = "true" + } + // Label values must match the descriptor order: group, version, kind, + // is_home, then one value per routing label key. For remote clusters the + // routing label value comes from the cluster's registration labels (e.g. + // Labels["availabilityZone"] → label value for "availability_zone"). The + // home cluster has no routing labels, so those positions are empty strings. + labelVals := make([]string, 0, 4+len(k.labelKeys)) + labelVals = append(labelVals, d.gvk.Group, d.gvk.Version, d.gvk.Kind, isHome) + for _, key := range k.labelKeys { + labelVals = append(labelVals, labelValueForSnakeKey(key, c.Labels)) + } + ch <- prometheus.MustNewConstMetric(k.sharedDesc, prometheus.GaugeValue, + float64(len(c.Items)), labelVals...) + } + } +} + +// buildObjectCountSchema computes the union of snake_case routing-label keys +// across all given GVKs and returns the label keys and a single shared +// prometheus.Desc. Fails if any routing-label key collides with a fixed label +// (group, version, kind, is_home). +func buildObjectCountSchema(r multiclusterReader, gvks []schema.GroupVersionKind) ([]string, *prometheus.Desc, error) { + fixedLabels := map[string]bool{"group": true, "version": true, "kind": true, "is_home": true} + keySet := map[string]bool{} + var labelKeys []string + for _, gvk := range gvks { + for _, lm := range r.ConfiguredRouteLabels(gvk) { + for key := range lm { + snake := toSnakeCase(key) + if fixedLabels[snake] { + return nil, nil, fmt.Errorf("routing label key %q collides with fixed label", snake) + } + if !keySet[snake] { + keySet[snake] = true + labelKeys = append(labelKeys, snake) + } + } + } + } + // Fixed labels: group/version/kind identify the resource type; is_home + // distinguishes the home cluster (no routing labels) from remote clusters. + // The routing label keys follow (e.g. availability_zone). + varLabels := append([]string{"group", "version", "kind", "is_home"}, labelKeys...) + desc := prometheus.NewDesc( + "cortex_multicluster_object_count", + "Number of objects of a given GVK per cluster", + varLabels, + nil, + ) + return labelKeys, desc, nil +} + +// parseGVK parses a "group/version/Kind" string. +func parseGVK(s string) (schema.GroupVersionKind, error) { + parts := strings.SplitN(s, "/", 3) + if len(parts) != 3 { + return schema.GroupVersionKind{}, fmt.Errorf("expected group/version/Kind, got: %s", s) + } + if parts[1] == "" || parts[2] == "" { + return schema.GroupVersionKind{}, fmt.Errorf("expected group/version/Kind, got: %s", s) + } + if strings.ContainsRune(parts[2], '/') { + return schema.GroupVersionKind{}, fmt.Errorf("expected group/version/Kind, got: %s", s) + } + return schema.GroupVersionKind{Group: parts[0], Version: parts[1], Kind: parts[2]}, nil +} + +// toSnakeCase converts camelCase to snake_case (e.g. availabilityZone → availability_zone). +func toSnakeCase(s string) string { + var b strings.Builder + for i, r := range s { + if unicode.IsUpper(r) && i > 0 { + b.WriteByte('_') + } + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + +// labelValueForSnakeKey finds the value in labels whose key, when snake_cased, +// matches snakeKey. Returns empty string for nil labels or no match (home cluster). +func labelValueForSnakeKey(snakeKey string, labels map[string]string) string { + for k, v := range labels { + if toSnakeCase(k) == snakeKey { + return v + } + } + return "" +} diff --git a/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go new file mode 100644 index 000000000..df6b89b47 --- /dev/null +++ b/internal/knowledge/kpis/plugins/deployment/multicluster_object_count_kpi_test.go @@ -0,0 +1,284 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "testing" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/pkg/conf" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// fakeReader is a stub multiclusterReader that returns canned per-cluster +// metadata. It lets the KPI be tested without any real or fake clusters. +type fakeReader struct { + routeLabels []map[string]string + perCluster []multicluster.ClusterObjectMetadata + err error +} + +func (f *fakeReader) ConfiguredRouteLabels(schema.GroupVersionKind) []map[string]string { + return f.routeLabels +} + +func (f *fakeReader) ListMetadataPerCluster(context.Context, schema.GroupVersionKind, ...client.ListOption) ([]multicluster.ClusterObjectMetadata, error) { + return f.perCluster, f.err +} + +// items builds n PartialObjectMetadata entries to represent a count of n. +func items(n int) []metav1.PartialObjectMetadata { + out := make([]metav1.PartialObjectMetadata, n) + return out +} + +func TestMulticlusterObjectCountKPI_GetName(t *testing.T) { + kpi := &MulticlusterObjectCountKPI{} + if got := kpi.GetName(); got != "multicluster_object_count_kpi" { + t.Errorf("GetName() = %q, want %q", got, "multicluster_object_count_kpi") + } +} + +func TestMulticlusterObjectCountKPI_Init_RejectsNonMCLClient(t *testing.T) { + scheme, err := v1alpha1.SchemeBuilder.Build() + if err != nil { + t.Fatal(err) + } + plainClient := fake.NewClientBuilder().WithScheme(scheme).Build() + kpi := &MulticlusterObjectCountKPI{} + if err := kpi.Init(nil, plainClient, conf.NewRawOpts(`{"gvks":[]}`)); err == nil { + t.Fatal("expected error when passing a non-*multicluster.Client") + } +} + +// initWithReader builds a KPI whose descriptors are derived from the given +// route labels, bypassing Init's *multicluster.Client type assertion so the +// KPI logic can be tested against a fakeReader. Uses the production +// buildObjectCountSchema helper so the test exercises the same schema path. +func initWithReader(t *testing.T, r *fakeReader, gvkStrs ...string) *MulticlusterObjectCountKPI { + t.Helper() + kpi := &MulticlusterObjectCountKPI{} + kpi.mcl = r + + var gvks []schema.GroupVersionKind + for _, gvkStr := range gvkStrs { + gvk, err := parseGVK(gvkStr) + if err != nil { + t.Fatalf("parseGVK(%q): %v", gvkStr, err) + } + gvks = append(gvks, gvk) + kpi.descs = append(kpi.descs, gvkDesc{gvk: gvk}) + } + + labelKeys, desc, err := buildObjectCountSchema(r, gvks) + if err != nil { + t.Fatalf("buildObjectCountSchema: %v", err) + } + kpi.labelKeys = labelKeys + kpi.sharedDesc = desc + return kpi +} + +func TestMulticlusterObjectCountKPI_Init_RejectsInvalidGVK(t *testing.T) { + kpi := &MulticlusterObjectCountKPI{} + // A *multicluster.Client with no configured GVKs still passes the type + // assertion; the invalid GVK string must be rejected during parsing. + mcl := &multicluster.Client{} + opts := conf.NewRawOpts(`{"gvks":["not-a-valid-gvk"]}`) + if err := kpi.Init(nil, mcl, opts); err == nil { + t.Fatal("expected error for invalid GVK string") + } +} + +func TestMulticlusterObjectCountKPI_Init_RejectsEmptyVersionOrKind(t *testing.T) { + cases := []string{"apps//DeploymentList", "/v1/"} + for _, raw := range cases { + if _, err := parseGVK(raw); err == nil { + t.Errorf("parseGVK(%q): expected error for empty segment, got nil", raw) + } + } +} + +func TestMulticlusterObjectCountKPI_Describe(t *testing.T) { + r := &fakeReader{routeLabels: []map[string]string{{"availabilityZone": "az-1"}}} + kpi := initWithReader(t, r, "/v1/ConfigMapList") + + ch := make(chan *prometheus.Desc, 5) + kpi.Describe(ch) + close(ch) + var count int + for range ch { + count++ + } + if count != 1 { + t.Errorf("Describe: expected 1 descriptor, got %d", count) + } +} + +func TestMulticlusterObjectCountKPI_Collect(t *testing.T) { + r := &fakeReader{ + routeLabels: []map[string]string{ + {"availabilityZone": "az-1"}, + {"availabilityZone": "az-2"}, + }, + perCluster: []multicluster.ClusterObjectMetadata{ + {Labels: map[string]string{"availabilityZone": "az-1"}, Items: items(2)}, + {Labels: map[string]string{"availabilityZone": "az-2"}, Items: items(1)}, + }, + } + kpi := initWithReader(t, r, "/v1/ConfigMapList") + + ch := make(chan prometheus.Metric, 10) + kpi.Collect(ch) + close(ch) + + // Collect into a map: availabilityZone -> count value. + byAZ := map[string]float64{} + for m := range ch { + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + var az, isHome string + for _, lp := range metric.Label { + switch lp.GetName() { + case "availability_zone": + az = lp.GetValue() + case "is_home": + isHome = lp.GetValue() + } + } + if isHome != "false" { + t.Errorf("az %q: expected is_home=false for remote cluster, got %q", az, isHome) + } + byAZ[az] = metric.Gauge.GetValue() + } + + if len(byAZ) != 2 { + t.Fatalf("expected metrics for 2 clusters, got %d: %v", len(byAZ), byAZ) + } + if byAZ["az-1"] != 2 { + t.Errorf("az-1: expected count 2, got %g", byAZ["az-1"]) + } + if byAZ["az-2"] != 1 { + t.Errorf("az-2: expected count 1, got %g", byAZ["az-2"]) + } +} + +func TestMulticlusterObjectCountKPI_Collect_HomeCluster(t *testing.T) { + r := &fakeReader{ + perCluster: []multicluster.ClusterObjectMetadata{ + {Items: items(1), IsHome: true}, + }, + } + kpi := initWithReader(t, r, "/v1/ConfigMapList") + + ch := make(chan prometheus.Metric, 10) + kpi.Collect(ch) + close(ch) + + var collected int + for m := range ch { + collected++ + var metric dto.Metric + if err := m.Write(&metric); err != nil { + t.Fatalf("failed to write metric: %v", err) + } + var isHome string + for _, lp := range metric.Label { + if lp.GetName() == "is_home" { + isHome = lp.GetValue() + } + } + if isHome != "true" { + t.Errorf("expected is_home=true for home cluster, got %q", isHome) + } + if got := metric.Gauge.GetValue(); got != 1 { + t.Errorf("expected count 1, got %g", got) + } + } + if collected != 1 { + t.Fatalf("expected 1 metric, got %d", collected) + } +} + +func TestMulticlusterObjectCountKPI_SnakeCaseLabels(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"availabilityZone", "availability_zone"}, + {"az", "az"}, + {"myLabelKey", "my_label_key"}, + {"alreadysnake", "alreadysnake"}, + } + for _, tt := range tests { + if got := toSnakeCase(tt.input); got != tt.want { + t.Errorf("toSnakeCase(%q) = %q, want %q", tt.input, got, tt.want) + } + } +} + +// TestMulticlusterObjectCountKPI_UnionLabelSchema verifies that two GVKs with +// differing routing-label sets share a single descriptor whose label schema is +// the union of both sets, and that Prometheus collection succeeds without +// duplicate-descriptor errors. +func TestMulticlusterObjectCountKPI_UnionLabelSchema(t *testing.T) { + // GVK A clusters use "availabilityZone"; GVK B clusters use "region". + // The union descriptor must carry both keys. + r := &fakeReader{ + routeLabels: []map[string]string{ + {"availabilityZone": "az-1"}, + {"region": "us-east"}, + }, + perCluster: []multicluster.ClusterObjectMetadata{ + {Labels: map[string]string{"availabilityZone": "az-1"}, Items: items(3)}, + {Labels: map[string]string{"region": "us-east"}, Items: items(5)}, + }, + } + kpi := initWithReader(t, r, "/v1/ConfigMapList", "apps/v1/DeploymentList") + + // Registering with a real prometheus.Registry would panic if two descriptors + // share the same metric name but have different label schemas. + reg := prometheus.NewRegistry() + if err := reg.Register(kpi); err != nil { + t.Fatalf("Register failed (likely duplicate/inconsistent descriptor): %v", err) + } + + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("Gather failed: %v", err) + } + var total int + for _, mf := range mfs { + total += len(mf.Metric) + } + // Two GVKs × two clusters each = 4 metrics. + if total != 4 { + t.Errorf("expected 4 metrics, got %d", total) + } + + // Every metric must have both union keys present (empty string for missing). + for _, mf := range mfs { + for _, m := range mf.Metric { + labelMap := map[string]string{} + for _, lp := range m.Label { + labelMap[lp.GetName()] = lp.GetValue() + } + if _, ok := labelMap["availability_zone"]; !ok { + t.Errorf("metric missing label availability_zone: %v", labelMap) + } + if _, ok := labelMap["region"]; !ok { + t.Errorf("metric missing label region: %v", labelMap) + } + } + } +} diff --git a/internal/knowledge/kpis/supported_kpis.go b/internal/knowledge/kpis/supported_kpis.go index d1222e27b..617d58008 100644 --- a/internal/knowledge/kpis/supported_kpis.go +++ b/internal/knowledge/kpis/supported_kpis.go @@ -35,4 +35,6 @@ var supportedKPIs = map[string]plugins.KPI{ "kpi_state_kpi": &deployment.KPIStateKPI{}, "pipeline_state_kpi": &deployment.PipelineStateKPI{}, "reservation_state_kpi": &deployment.ReservationStateKPI{}, + + "multicluster_object_count_kpi": &deployment.MulticlusterObjectCountKPI{}, } diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index 583d74574..8b3a65a60 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -13,6 +13,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" @@ -211,7 +212,7 @@ func (c *Client) ClustersForGVK(gvk schema.GroupVersionKind) ([]cluster.Cluster, remotes := c.remoteClusters[gvk] isHome := c.homeGVKs[gvk] if len(remotes) == 0 && !isHome { - return nil, fmt.Errorf("GVK %s is not configured in home or any remote cluster", gvk) + return nil, fmt.Errorf("gvk %s is not configured in home or any remote cluster", gvk) } clusters := make([]cluster.Cluster, 0, len(remotes)+1) for _, r := range remotes { @@ -458,6 +459,64 @@ func (c *Client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts return errors.New("apply operation is not supported in multicluster client") } +// ClusterObjectMetadata is one entry in the result of ListMetadataPerCluster. +// Labels holds the routing labels for the cluster. Items holds the object +// metadata returned by the cluster (no spec or status). IsHome is true for the +// home cluster, which has no routing labels. +type ClusterObjectMetadata struct { + Labels map[string]string + Items []metav1.PartialObjectMetadata + IsHome bool +} + +// ListMetadataPerCluster returns the object metadata of the given GVK for each +// configured cluster. It uses PartialObjectMetadataList so only object metadata +// crosses the wire — no spec or status — making it efficient even for large +// object counts. Callers that only need counts can use len(Items). Clusters +// that return an error are logged and skipped (same policy as List). The home +// cluster is included with IsHome set to true. +func (c *Client) ListMetadataPerCluster(ctx context.Context, gvk schema.GroupVersionKind, opts ...client.ListOption) ([]ClusterObjectMetadata, error) { + log := ctrl.LoggerFrom(ctx) + + c.remoteClustersMu.RLock() + remotes := c.remoteClusters[gvk] + isHome := c.homeGVKs[gvk] + if len(remotes) == 0 && !isHome { + c.remoteClustersMu.RUnlock() + return nil, fmt.Errorf("gvk %s is not configured in home or any remote cluster", gvk) + } + type clusterEntry struct { + cl cluster.Cluster + labels map[string]string + isHome bool + } + entries := make([]clusterEntry, 0, len(remotes)+1) + for _, r := range remotes { + entries = append(entries, clusterEntry{cl: r.cluster, labels: maps.Clone(r.labels)}) + } + if isHome && c.HomeCluster != nil { + entries = append(entries, clusterEntry{cl: c.HomeCluster, isHome: true}) + } + c.remoteClustersMu.RUnlock() + + results := make([]ClusterObjectMetadata, 0, len(entries)) + for _, e := range entries { + partialList := &metav1.PartialObjectMetadataList{} + partialList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: gvk.Group, + Version: gvk.Version, + Kind: gvk.Kind, + }) + if err := e.cl.GetClient().List(ctx, partialList, opts...); err != nil { + log.Error(err, "error listing resource metadata from cluster", + "gvk", gvk, "host", e.cl.GetConfig().Host) + continue + } + results = append(results, ClusterObjectMetadata{Labels: e.labels, Items: partialList.Items, IsHome: e.isHome}) + } + return results, nil +} + // Create routes the object to the matching cluster using the ResourceRouter // and performs a Create operation. // diff --git a/pkg/multicluster/client_test.go b/pkg/multicluster/client_test.go index 1ee07bb7c..41fd3a6b9 100644 --- a/pkg/multicluster/client_test.go +++ b/pkg/multicluster/client_test.go @@ -1857,3 +1857,106 @@ func TestClient_ConfiguredRouteLabels(t *testing.T) { } }) } + +func TestClient_ListMetadataPerCluster(t *testing.T) { + scheme := newTestScheme(t) + ctx := context.Background() + + t.Run("lists object metadata per remote cluster", func(t *testing.T) { + az1 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-1", Namespace: "default"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-2", Namespace: "default"}}, + ) + az2 := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-3", Namespace: "default"}}, + ) + c := &Client{ + HomeScheme: scheme, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{ + configMapListGVK: { + {cluster: az1, labels: map[string]string{"availabilityZone": "az-1"}}, + {cluster: az2, labels: map[string]string{"availabilityZone": "az-2"}}, + }, + }, + } + results, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + byAZ := map[string]int{} + for _, r := range results { + if r.IsHome { + t.Errorf("expected IsHome=false for remote cluster %v", r.Labels) + } + byAZ[r.Labels["availabilityZone"]] = len(r.Items) + } + if byAZ["az-1"] != 2 { + t.Errorf("expected 2 objects in az-1, got %d", byAZ["az-1"]) + } + if byAZ["az-2"] != 1 { + t.Errorf("expected 1 object in az-2, got %d", byAZ["az-2"]) + } + }) + + t.Run("includes home cluster with IsHome set", func(t *testing.T) { + home := newFakeCluster(scheme, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm-home", Namespace: "default"}}, + ) + c := &Client{ + HomeCluster: home, + HomeScheme: scheme, + homeGVKs: map[schema.GroupVersionKind]bool{configMapListGVK: true}, + } + results, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if !results[0].IsHome { + t.Errorf("expected IsHome=true for home cluster") + } + if results[0].Labels != nil { + t.Errorf("expected nil labels for home cluster, got %v", results[0].Labels) + } + if len(results[0].Items) != 1 { + t.Errorf("expected 1 item, got %d", len(results[0].Items)) + } + }) + + t.Run("returns error for unconfigured GVK", func(t *testing.T) { + c := &Client{ + HomeScheme: scheme, + } + _, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err == nil { + t.Fatal("expected error for unconfigured GVK") + } + }) + + t.Run("empty clusters return zero items", func(t *testing.T) { + az1 := newFakeCluster(scheme) // no objects + c := &Client{ + HomeScheme: scheme, + remoteClusters: map[schema.GroupVersionKind][]remoteCluster{ + configMapListGVK: { + {cluster: az1, labels: map[string]string{"availabilityZone": "az-1"}}, + }, + }, + } + results, err := c.ListMetadataPerCluster(ctx, configMapListGVK) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if len(results[0].Items) != 0 { + t.Errorf("expected 0 items, got %d", len(results[0].Items)) + } + }) +} From 1daefe16256c333167cc716ab5f7cd8555b40f4c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:49:32 +0200 Subject: [PATCH 12/16] Renovate: Update github.com/sapcc/go-bits digest to 4bbc84d (#1124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [github.com/sapcc/go-bits](https://redirect.github.com/sapcc/go-bits) | require | digest | `f6f7275` → `4bbc84d` | --- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR is behind base branch, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index bb9f2a62c..129333abe 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/ironcore-dev/ironcore v0.5.0 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 - github.com/sapcc/go-bits v0.0.0-20260730170321-f6f727520601 - go.xyrillian.de/gg v1.13.0 + github.com/sapcc/go-bits v0.0.0-20260806170240-4bbc84d224db + go.xyrillian.de/gg v1.13.3 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 k8s.io/client-go v0.36.3 @@ -29,7 +29,7 @@ require ( github.com/go-openapi/swag/stringutils v0.25.1 // indirect github.com/go-openapi/swag/typeutils v0.25.1 // indirect github.com/go-openapi/swag/yamlutils v0.25.1 // indirect - github.com/gofrs/uuid/v5 v5.5.0 // indirect + github.com/gofrs/uuid/v5 v5.5.1 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect k8s.io/streaming v0.36.3 // indirect diff --git a/go.sum b/go.sum index ec13033bb..de07f6930 100644 --- a/go.sum +++ b/go.sum @@ -102,8 +102,8 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gofrs/uuid/v5 v5.5.0 h1:FkPv6jYQRbZtH3bD8yC7106u+CedTCLF8+t7CLHSZNo= -github.com/gofrs/uuid/v5 v5.5.0/go.mod h1:bbAA98EoIlxyRHIVg6ektCSsZ5n8mSbwgEhvhMYlZgg= +github.com/gofrs/uuid/v5 v5.5.1 h1:z1Ce19/JwNidXpy3tOQc3241lnJLKdKyq/xlNvlD4Ng= +github.com/gofrs/uuid/v5 v5.5.1/go.mod h1:bbAA98EoIlxyRHIVg6ektCSsZ5n8mSbwgEhvhMYlZgg= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= @@ -204,8 +204,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.24.0 h1:sGBvOMVSM1olJlyvNoQSk7NX5uatXHKkztGDBPnTWMs= github.com/sapcc/go-api-declarations v1.24.0/go.mod h1:ZWRTijvgF8o8aHg5stgg7u4DF6jFrd0X97le/uGlZsA= -github.com/sapcc/go-bits v0.0.0-20260730170321-f6f727520601 h1:pF4eF41dp0AxM0IfICn3ttpI5IwQoUmC0IkGuXLTmBA= -github.com/sapcc/go-bits v0.0.0-20260730170321-f6f727520601/go.mod h1:4z4Vd1C7d6hDdbJ5q1CgzE9B8ESWuiMTbkZSJA5gSjA= +github.com/sapcc/go-bits v0.0.0-20260806170240-4bbc84d224db h1:DfseqB6CZzdpbSdL4sGlqEBSYu3JpUpwYFiNzoYTkog= +github.com/sapcc/go-bits v0.0.0-20260806170240-4bbc84d224db/go.mod h1:1wh2+fuMXNrJYttuaqiU7sNQgJhTmsZ/0S0SK1ttwQc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -250,8 +250,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= -go.xyrillian.de/gg v1.13.0 h1:K1RlyRxe2+7oaXALZBtuJqqwSEPneTjH1gPD18WuSaU= -go.xyrillian.de/gg v1.13.0/go.mod h1:DoO4fQSWIrBRlNlCjVyrYM0kAEBt/Jg2GkMH+cGRZ0k= +go.xyrillian.de/gg v1.13.3 h1:Ulz3+eZnO2OUl7Bv+SWaA5ufDmjeLwwmICPP4dCurxA= +go.xyrillian.de/gg v1.13.3/go.mod h1:DoO4fQSWIrBRlNlCjVyrYM0kAEBt/Jg2GkMH+cGRZ0k= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= From 85d9d0cf597703bc30dc5a2448f5e84dffbef8d7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:52:00 +0200 Subject: [PATCH 13/16] Renovate: Update module go.xyrillian.de/gg to v1.13.2 (#1122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | [kube-prometheus-stack](https://redirect.github.com/prometheus-operator/kube-prometheus) ([source](https://redirect.github.com/prometheus-community/helm-charts)) | patch | `88.1.5` → `88.1.6` | --- ### Release Notes
prometheus-community/helm-charts (kube-prometheus-stack) ### [`v88.1.6`](https://redirect.github.com/prometheus-community/helm-charts/releases/tag/kube-prometheus-stack-88.1.6) kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. #### What's Changed - \[kube-prometheus-stack] Update kube-prometheus-stack dependency non-major updates by [@​renovate](https://redirect.github.com/renovate)\[bot] in [#​7170](https://redirect.github.com/prometheus-community/helm-charts/pull/7170) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (in timezone Europe/Berlin) - Branch creation - "after 6pm every weekday,every weekend,before 8am every weekday" - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/cobaltcore-dev/cortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- helm/dev/cortex-prometheus-operator/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 19967acf5..51c0ca83c 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 88.1.5 + version: 88.1.6 From 013d3d3186a5879a8106e449c5515ffb067c1b73 Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 09:28:49 +0200 Subject: [PATCH 14/16] docs: document pipeline step event metrics with dynamic labels (#1128) Pipeline steps can now report named events with dynamic Prometheus labels via FilterWeigherPipelineStepEvent (introduced in PR #1108), but the docs did not mention this mechanism. This adds a brief subsection under Pipelines so developers writing new filters or weighers know how to emit step events and what metric they produce. Assisted-by: Claude Code:claude-sonnet-4-20250514 [Bash] [Read] Signed-off-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> --- docs/apis.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/apis.md b/docs/apis.md index a1edf6a14..7fd08b8c3 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -96,6 +96,10 @@ The `scheduling.Options` struct configures a single pipeline invocation. All fie **Validation constraint:** A `ReadOnly` run must also set `SkipHistory=true`, `SkipInflight=true`, and `SkipCommittedResourceTracking=true`. This is enforced by `Options.Validate()` — omitting any of these fields causes validation to fail with an error before the pipeline executes. +#### Step Events + +Pipeline steps can report named events via the `Events []FilterWeigherPipelineStepEvent` field on their result. Each event carries a `Name` and an optional `Labels map[string]string` for dynamic Prometheus labels. The pipeline monitor exports these as the `cortex_filter_weigher_pipeline_step_events_total` counter with fixed labels `pipeline`, `step`, `event` plus any dynamic labels the step provides. Use events to surface noteworthy step conditions (skipped filtering, missing data) as observable metrics. See `filter_image_properties` for a reference implementation that emits an `image_properties_hv_type_undetermined` event with an `intent` label. + ### Decisions ```bash From ae03a3a8d7a712dac3a932d5c66efb880e2aabbd Mon Sep 17 00:00:00 2001 From: "cortex-ai-agents[bot]" <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:59:34 +0200 Subject: [PATCH 15/16] Release cortex v0.3.6 (#1130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Bump all helm chart versions for cortex v0.3.6 release - Add CHANGELOG entry documenting all changes since v0.3.5 - Update library chart appVersions to sha-85d9d0cf ## Chart versions | Chart | Old | New | |-------|-----|-----| | cortex | 0.3.5 | 0.3.6 | | cortex-shim | 0.1.11 | 0.1.12 | | cortex-postgres | 0.6.11 | 0.6.12 | | cortex-nova | 0.0.85 | 0.0.86 | | cortex-cinder | 0.0.85 | 0.0.86 | | cortex-manila | 0.0.85 | 0.0.86 | | cortex-crds | 0.0.85 | 0.0.86 | | cortex-ironcore | 0.0.85 | 0.0.86 | | cortex-pods | 0.0.85 | 0.0.86 | | cortex-placement-shim | 0.1.11 | 0.1.12 | Once merged to main, PR #1129 (main → release) can be merged to complete the release. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: cortex-ai-agents[bot] <279748396+cortex-ai-agents[bot]@users.noreply.github.com> Co-authored-by: Markus Wieland Co-authored-by: Claude Opus 4.7 --- CHANGELOG.md | 56 +++++++++++++++++++ helm/bundles/cortex-cinder/Chart.yaml | 8 +-- helm/bundles/cortex-crds/Chart.yaml | 4 +- helm/bundles/cortex-ironcore/Chart.yaml | 4 +- helm/bundles/cortex-manila/Chart.yaml | 8 +-- helm/bundles/cortex-nova/Chart.yaml | 8 +-- helm/bundles/cortex-placement-shim/Chart.yaml | 4 +- helm/bundles/cortex-pods/Chart.yaml | 4 +- helm/library/cortex-postgres/Chart.yaml | 4 +- helm/library/cortex-shim/Chart.yaml | 4 +- helm/library/cortex/Chart.yaml | 4 +- 11 files changed, 82 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7691e9f89..d58e91ab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,61 @@ # Changelog +## 2026-08-10 — [#1129](https://github.com/cobaltcore-dev/cortex/pull/1129) + +### cortex v0.3.6 (sha-85d9d0cf) + +New features: +- Implement in-flight reservations controller — adds a new controller that tracks reservations currently being fulfilled by monitoring VM creation state via Nova API, transitioning reservations through their lifecycle stages and cleaning up stale in-flight reservations ([#957](https://github.com/cobaltcore-dev/cortex/pull/957)) +- Add KPI that tracks CR count per configured cluster — introduces the `MulticlusterObjectCountKPI` plugin which counts custom resource objects (e.g. Hypervisors, Reservations) per cluster and exposes them as Prometheus metrics for deployment monitoring ([#1054](https://github.com/cobaltcore-dev/cortex/pull/1054)) +- Follow VM live migration in CR reservation reconciler — the committed resource reservation controller now detects when a VM has been live-migrated to a different host and updates the reservation's target host accordingly, preventing stale capacity accounting ([#1048](https://github.com/cobaltcore-dev/cortex/pull/1048)) + +Bug fixes: +- Subtract reservation CPU blocks when counting placeable slots — the capacity accounting now correctly deducts reserved CPU blocks from available capacity before calculating how many new instances can be placed, preventing over-commitment ([#1118](https://github.com/cobaltcore-dev/cortex/pull/1118)) + +Non-breaking changes: +- Skip non-candidate hypervisors in nova filters and weighers — filters and weighers now skip hypervisors that are not in the candidate set, improving performance and avoiding unnecessary processing ([#1117](https://github.com/cobaltcore-dev/cortex/pull/1117)) +- Add dynamic labels to pipeline step event metrics — scheduling pipeline step monitors can now emit events with dynamic label key-value pairs, enabling richer per-step observability ([#1108](https://github.com/cobaltcore-dev/cortex/pull/1108)) +- Update `go.xyrillian.de/gg` to v1.13.2 ([#1111](https://github.com/cobaltcore-dev/cortex/pull/1111), [#1122](https://github.com/cobaltcore-dev/cortex/pull/1122)) +- Update `github.com/sapcc/go-bits` ([#1124](https://github.com/cobaltcore-dev/cortex/pull/1124)) +- Update `kube-prometheus-stack` to v88.1.5 ([#1112](https://github.com/cobaltcore-dev/cortex/pull/1112), [#1119](https://github.com/cobaltcore-dev/cortex/pull/1119), [#1121](https://github.com/cobaltcore-dev/cortex/pull/1121)) +- Update `debian:trixie-slim` Docker digest ([#1120](https://github.com/cobaltcore-dev/cortex/pull/1120)) + +### cortex-shim v0.1.12 (sha-85d9d0cf) + +Includes updated image sha-85d9d0cf with dependency updates. + +### cortex-postgres v0.6.12 (sha-85d9d0cf) + +Includes updated base image (debian:trixie-slim digest update). + +### cortex-nova v0.0.86 + +Includes updated chart cortex v0.3.6. + +### cortex-cinder v0.0.86 + +Includes updated chart cortex v0.3.6. + +### cortex-manila v0.0.86 + +Includes updated chart cortex v0.3.6. + +### cortex-crds v0.0.86 + +Includes updated chart cortex v0.3.6. + +### cortex-ironcore v0.0.86 + +Includes updated chart cortex v0.3.6. + +### cortex-pods v0.0.86 + +Includes updated chart cortex v0.3.6. + +### cortex-placement-shim v0.1.12 + +Includes updated chart cortex-shim v0.1.12. + ## 2026-08-03 — [#1114](https://github.com/cobaltcore-dev/cortex/pull/1114) ### cortex v0.3.5 (sha-684e0b07) diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 8624354f5..ccfea8439 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.85 +version: 0.0.86 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.11 + version: 0.6.12 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index cf3a9251c..a3e17a750 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.85 +version: 0.0.86 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 2cbd38822..8d7fdcff7 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.85 +version: 0.0.86 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 02e6a713f..b6f293369 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.85 +version: 0.0.86 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.11 + version: 0.6.12 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index aa24e732f..bba789bd2 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,23 +5,23 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.85 +version: 0.0.86 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres - name: cortex-postgres repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.6.11 + version: 0.6.12 # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index f7a9f5384..083001bfb 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.11 +version: 0.1.12 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.11 + version: 0.1.12 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 7c1de6bea..1f9365c63 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.85 +version: 0.0.86 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.5 + version: 0.3.6 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 2a765a364..0e2ebdea8 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -5,5 +5,5 @@ apiVersion: v2 name: cortex-postgres description: Postgres setup for Cortex. type: application -version: 0.6.11 -appVersion: "sha-e06153f8" +version: 0.6.12 +appVersion: "sha-85d9d0cf" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index db01cfba6..c9c0bd0da 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.11 -appVersion: "sha-c325b29e" +version: 0.1.12 +appVersion: "sha-85d9d0cf" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 1dade8eba..ea3009850 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.3.5 -appVersion: "sha-684e0b07" +version: 0.3.6 +appVersion: "sha-85d9d0cf" icon: "https://example.com/icon.png" dependencies: [] From 7d9b1b793c925f8a9dbebc0463eee2374f26e2bb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:07:03 +0200 Subject: [PATCH 16/16] bump app version [skip ci] (#1115) bump app version [skip ci] ``` bumped cortex: sha-85d9d0cf -> sha-013d3d31 bumped cortex-shim: sha-85d9d0cf -> sha-1daefe16 bumped cortex-postgres: sha-85d9d0cf -> sha-7343f7cd ``` Signed-off-by: umswmayj <140147670+umswmayj@users.noreply.github.com> Co-authored-by: umswmayj <140147670+umswmayj@users.noreply.github.com> --- helm/library/cortex-postgres/Chart.yaml | 2 +- helm/library/cortex-shim/Chart.yaml | 2 +- helm/library/cortex/Chart.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/library/cortex-postgres/Chart.yaml b/helm/library/cortex-postgres/Chart.yaml index 0e2ebdea8..51d2b675f 100644 --- a/helm/library/cortex-postgres/Chart.yaml +++ b/helm/library/cortex-postgres/Chart.yaml @@ -6,4 +6,4 @@ name: cortex-postgres description: Postgres setup for Cortex. type: application version: 0.6.12 -appVersion: "sha-85d9d0cf" +appVersion: "sha-7343f7cd" diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index c9c0bd0da..889fae5e0 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -3,6 +3,6 @@ name: cortex-shim description: A Helm chart to distribute cortex shims. type: application version: 0.1.12 -appVersion: "sha-85d9d0cf" +appVersion: "sha-1daefe16" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index ea3009850..baa1e4075 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -3,6 +3,6 @@ name: cortex description: A Helm chart to distribute cortex. type: application version: 0.3.6 -appVersion: "sha-85d9d0cf" +appVersion: "sha-013d3d31" icon: "https://example.com/icon.png" dependencies: []