diff --git a/.github/workflows/claude-assistant.yaml b/.github/workflows/claude-assistant.yaml index 68f896ed7..23596cec9 100644 --- a/.github/workflows/claude-assistant.yaml +++ b/.github/workflows/claude-assistant.yaml @@ -27,10 +27,31 @@ jobs: echo "allowed=false" >> $GITHUB_OUTPUT fi + check-trigger-phrase: + runs-on: ubuntu-latest + needs: check-allowlist + outputs: + triggered: ${{ steps.check.outputs.triggered }} + steps: + - name: Check for trigger phrase in comment body + id: check + env: + COMMENT_BODY: ${{ github.event.comment.body }} + ISSUE_BODY: ${{ github.event.issue.body }} + ISSUE_TITLE: ${{ github.event.issue.title }} + run: | + if [[ "$COMMENT_BODY" == *"@claude"* ]] \ + || [[ "$ISSUE_BODY" == *"@claude"* ]] \ + || [[ "$ISSUE_TITLE" == *"@claude"* ]]; then + echo "triggered=true" >> $GITHUB_OUTPUT + else + echo "triggered=false" >> $GITHUB_OUTPUT + fi + claude: environment: claude - needs: check-allowlist - if: needs.check-allowlist.outputs.allowed == 'true' + needs: [check-allowlist, check-trigger-phrase] + if: needs.check-allowlist.outputs.allowed == 'true' && needs.check-trigger-phrase.outputs.triggered == 'true' runs-on: ubuntu-latest permissions: contents: write diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f94c487d..4a491ea09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,79 @@ # Changelog -## 2026-07-03 — [#1009](https://github.com/cobaltcore-dev/cortex/pull/1009) +## 2026-07-13 — [#1036](https://github.com/cobaltcore-dev/cortex/pull/1036) -### cortex v0.2.2 (sha-f4d7df89) +### cortex v0.3.0 (sha-ee9cd485) + +Breaking changes: +- `NumberOfObjects` field in `DatasourceStatus` changed from `int64` to `*int64` so `omitempty` no longer strips zero values ([#1024](https://github.com/cobaltcore-dev/cortex/pull/1024)) + +Non-breaking changes: +- Migrate CR and failover calls to options-based pipeline selection — eliminates dedicated CR and failover pipelines by encoding behavioral differences as call-time scheduling options ([#950](https://github.com/cobaltcore-dev/cortex/pull/950)) +- Convert capacity controller from timer-based to controller-runtime reconciler with reactive watches and coalesced reconcile keys ([#1025](https://github.com/cobaltcore-dev/cortex/pull/1025)) +- Split quota enforcement `accept_skipped` decision into sub-buckets for granular observability ([#1022](https://github.com/cobaltcore-dev/cortex/pull/1022)) +- Fix alert wording: 'LackingBehind' → 'LaggingBehind' ([#1031](https://github.com/cobaltcore-dev/cortex/pull/1031)) +- Increase datasource lagging behind alert time frame to 30 min ([#1035](https://github.com/cobaltcore-dev/cortex/pull/1035)) +- Add logger context for remote cluster communication ([#1023](https://github.com/cobaltcore-dev/cortex/pull/1023)) +- Update `golang.org/x/sync` v0.21.0→v0.22.0, `golang.org/x/term` v0.44.0→v0.45.0 ([#1026](https://github.com/cobaltcore-dev/cortex/pull/1026)) +- Update `ironcore` v0.4.2→v0.4.3 ([#1029](https://github.com/cobaltcore-dev/cortex/pull/1029)) +- Update `github.com/sapcc/go-bits` ([#1030](https://github.com/cobaltcore-dev/cortex/pull/1030)) + +### cortex-shim v0.1.6 (sha-ee9cd485) + +Includes updated image sha-ee9cd485. + +### cortex-nova v0.0.80 + +Includes updated charts cortex v0.3.0, cortex-postgres v0.6.8. + +- Scope `SyncObjectsDroppedToZero` alert to only fire when previously non-zero ([#1024](https://github.com/cobaltcore-dev/cortex/pull/1024)) +- Add `capacityMinReconcileInterval` config key for capacity controller ([#1025](https://github.com/cobaltcore-dev/cortex/pull/1025)) + +### cortex-cinder v0.0.80 + +Includes updated charts cortex v0.3.0, cortex-postgres v0.6.8. + +- Scope `SyncObjectsDroppedToZero` alert to only fire when previously non-zero ([#1024](https://github.com/cobaltcore-dev/cortex/pull/1024)) + +### cortex-manila v0.0.80 + +Includes updated charts cortex v0.3.0, cortex-postgres v0.6.8. + +- Scope `SyncObjectsDroppedToZero` alert to only fire when previously non-zero ([#1024](https://github.com/cobaltcore-dev/cortex/pull/1024)) + +### cortex-crds v0.0.80 + +Includes updated chart cortex v0.3.0. + +### cortex-ironcore v0.0.80 + +Includes updated chart cortex v0.3.0. + +### cortex-pods v0.0.80 + +Includes updated chart cortex v0.3.0. + +### cortex-placement-shim v0.1.6 + +Includes updated chart cortex-shim v0.1.6. + +## 2026-07-06 — [#1009](https://github.com/cobaltcore-dev/cortex/pull/1009) + +### cortex v0.2.2 (sha-8d888397) + +Breaking changes: +- Remove Nova image datasource — the `images` NovaDatasourceType is no longer available ([#1006](https://github.com/cobaltcore-dev/cortex/pull/1006)) Non-breaking changes: +- Fix stale RunningInstances/RunningResources fields that never zero out when all VMs in a flavor group are removed ([#1017](https://github.com/cobaltcore-dev/cortex/pull/1017)) - Include host information in error logs when multicluster connection fails ([#997](https://github.com/cobaltcore-dev/cortex/pull/997)) - Fix Perses config for local development ([#1007](https://github.com/cobaltcore-dev/cortex/pull/1007)) -- Update `github.com/sapcc/go-bits` ([#1005](https://github.com/cobaltcore-dev/cortex/pull/1005)) -- Update kube-prometheus-stack Docker tag to v87.4.0 ([#998](https://github.com/cobaltcore-dev/cortex/pull/998)) +- Update `github.com/sapcc/go-bits` ([#1005](https://github.com/cobaltcore-dev/cortex/pull/1005), [#1013](https://github.com/cobaltcore-dev/cortex/pull/1013)) +- Update kube-prometheus-stack Docker tag to v87.6.0 ([#998](https://github.com/cobaltcore-dev/cortex/pull/998), [#1014](https://github.com/cobaltcore-dev/cortex/pull/1014)) -### cortex-shim v0.1.5 (sha-c98ddb05) +### cortex-shim v0.1.5 (sha-c8295850) -Includes updated image sha-c98ddb05. +Includes updated image sha-c8295850. ### cortex-nova v0.0.79 diff --git a/api/external/nova/messages.go b/api/external/nova/messages.go index e83d37ced..4779eddeb 100644 --- a/api/external/nova/messages.go +++ b/api/external/nova/messages.go @@ -154,10 +154,15 @@ const ( EvacuateIntent v1alpha1.SchedulingIntent = "evacuate" // CreateIntent indicates that the request is intended for creating a new VM. CreateIntent v1alpha1.SchedulingIntent = "create" - // ReserveForFailoverIntent indicates that the request is for failover reservation scheduling. + // ReserveForFailoverIntent indicates that the request is for creating a new failover reservation slot. ReserveForFailoverIntent v1alpha1.SchedulingIntent = "reserve_for_failover" + // ReuseFailoverReservationIntent indicates that the request is checking whether an existing + // failover reservation slot can be reused by a VM (compatibility check, not a new slot). + ReuseFailoverReservationIntent v1alpha1.SchedulingIntent = "reuse_failover_reservation" // ReserveForCommittedResourceIntent indicates that the request is for CR reservation scheduling. ReserveForCommittedResourceIntent v1alpha1.SchedulingIntent = "reserve_for_committed_resource" + // CapacityProbeIntent indicates that the request is a synthetic capacity probe (not a real VM placement). + CapacityProbeIntent v1alpha1.SchedulingIntent = "capacity_probe" // HintKeyResourceGroup is the scheduler hint key used to pass the resource group // (e.g., flavor group name) for failover reservation scheduling. @@ -188,9 +193,15 @@ func (req ExternalSchedulerRequest) GetIntent() (v1alpha1.SchedulingIntent, erro // Used by cortex failover reservation controller case "reserve_for_failover": return ReserveForFailoverIntent, nil + // Used by cortex failover reservation controller (reuse check) + case "reuse_failover_reservation": + return ReuseFailoverReservationIntent, nil // Used by cortex committed resource reservation controller case "reserve_for_committed_resource": return ReserveForCommittedResourceIntent, nil + // Used by cortex capacity probe controller + case "capacity_probe": + return CapacityProbeIntent, nil default: return CreateIntent, nil } diff --git a/api/v1alpha1/datasource_types.go b/api/v1alpha1/datasource_types.go index fff321c48..6ab7d962e 100644 --- a/api/v1alpha1/datasource_types.go +++ b/api/v1alpha1/datasource_types.go @@ -232,7 +232,7 @@ type DatasourceStatus struct { // When the datasource was last successfully synced. LastSynced metav1.Time `json:"lastSynced,omitempty"` // The number of objects currently stored for this datasource. - NumberOfObjects int64 `json:"numberOfObjects,omitempty"` + NumberOfObjects *int64 `json:"numberOfObjects,omitempty"` // Planned time for the next sync. NextSyncTime metav1.Time `json:"nextSyncTime,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9f654186a..0fa21d2bb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -353,6 +353,11 @@ func (in *DatasourceSpec) DeepCopy() *DatasourceSpec { func (in *DatasourceStatus) DeepCopyInto(out *DatasourceStatus) { *out = *in in.LastSynced.DeepCopyInto(&out.LastSynced) + if in.NumberOfObjects != nil { + in, out := &in.NumberOfObjects, &out.NumberOfObjects + *out = new(int64) + **out = **in + } in.NextSyncTime.DeepCopyInto(&out.NextSyncTime) if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions diff --git a/cmd/manager/main.go b/cmd/manager/main.go index af86e3d84..d8508d1c7 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -765,22 +765,25 @@ func main() { setupLog.Info("enabling controller", "controller", "capacity-controller") capacityConfig := conf.GetConfigOrDie[capacity.Config]() capacityConfig.ApplyDefaults() + if err := capacityConfig.Validate(); err != nil { + setupLog.Error(err, "invalid capacity-controller config") + os.Exit(1) + } capacityMonitor := capacity.NewMonitor(multiclusterClient) if err := metrics.Registry.Register(&capacityMonitor); err != nil { setupLog.Error(err, "failed to register capacity monitor metrics, continuing without metrics") } - capacityController := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource) - if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - return capacityController.Start(ctx) - })); err != nil { - setupLog.Error(err, "unable to add capacity controller to manager") + if err := capacity.NewController(multiclusterClient, capacityConfig, commitmentsVMSource). + SetupWithManager(mgr, multiclusterClient); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "capacity") os.Exit(1) } setupLog.Info("capacity-controller registered", "schedulerURL", capacityConfig.SchedulerURL, "reconcileInterval", capacityConfig.ReconcileInterval, + "minReconcileInterval", capacityConfig.MinReconcileInterval, "totalPipeline", capacityConfig.TotalPipeline, "placeablePipeline", capacityConfig.PlaceablePipeline) } diff --git a/docs/apis.md b/docs/apis.md index 6a1c2bdf7..f73753d85 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -90,8 +90,9 @@ The `scheduling.Options` struct configures a single pipeline invocation. All fie | `MaxCandidates` | `int` | Maximum number of candidate hosts returned after weighing. 0 means no limit. | | `SkipHistory` | `bool` | Skips recording the placement decision in placement history. | | `SkipInflight` | `bool` | Skips creating pessimistic blocking reservations for returned candidates. | +| `SkipCommittedResourceTracking` | `bool` | Skips writing the placed VM UUID into the matching committed resource reservation slot. Required for ReadOnly runs. | -**Validation constraint:** A `ReadOnly` run must also set `SkipHistory=true` and `SkipInflight=true`. This is enforced by `Options.Validate()` — omitting either field causes validation to fail with an error before the pipeline executes. +**Validation constraint:** A `ReadOnly` run must also set `SkipHistory=true`, `SkipInflight=true`, and `SkipCommittedResourceTracking=true`. This is enforced by `Options.Validate()` — omitting any of these fields causes validation to fail with an error before the pipeline executes. ### Decisions @@ -142,7 +143,7 @@ For more details on how committed resources interact with reservations, see [com kubectl get flavorgroupcapacities ``` -FlavorGroupCapacity caches pre-computed capacity data for one flavor group in one availability zone. One CRD exists per (flavor group × AZ) pair, maintained by the capacity controller on a fixed interval. The spec identifies the flavor group and AZ; the status holds per-flavor slot counts (`PlaceableVMs`, `PlaceableHosts`, `TotalCapacityVMSlots`, `TotalCapacityHosts`), aggregate fields (`CommittedCapacity`, `TotalCapacity`, `TotalInstances`), and a `LastReconcileAt` timestamp. The capacity API reads these CRDs instead of probing the scheduler on each request. +FlavorGroupCapacity caches pre-computed capacity data for one flavor group in one availability zone. One CRD exists per (flavor group × AZ) pair, maintained by the capacity controller on a fixed interval. The spec identifies the flavor group and AZ; the status holds per-flavor slot counts (`PlaceableVMs`, `PlaceableHosts`, `TotalCapacityVMSlots`, `TotalCapacityHosts`), aggregate fields (`CommittedCapacity`, `CommittedCapacityBytes`, `TotalCapacity`, `FreeCapacity`, `ExclusivelyFreeCapacity`, `ExclusivelyFreeSlots`, `RunningInstances`, `RunningResources`, `SmallestFlavorName`), and a `LastReconcileAt` timestamp. Where flavor groups share hosts, remaining capacity is fairly split across overlapping groups using a round-robin algorithm so that no group's exclusively-free capacity exceeds the actual installed resources. The capacity API reads these CRDs instead of probing the scheduler on each request. ### ProjectQuota diff --git a/docs/reservations/failover-reservations.md b/docs/reservations/failover-reservations.md index cd642d46c..0f2bf425b 100644 --- a/docs/reservations/failover-reservations.md +++ b/docs/reservations/failover-reservations.md @@ -166,7 +166,7 @@ We use three different scheduler pipelines for failover reservations, each servi **Why:** When reusing a reservation, capacity is already reserved on the target host. We only need to verify that the VM is compatible with the host (traits, capabilities, AZ, etc.) without checking if there's enough free capacity. -Options: `ReadOnly: true, SkipHistory: true` — pure compatibility check, no state mutations. +Options: `ReadOnly: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true` — pure compatibility check, no state mutations. ### `kvm-general-purpose-load-balancing` (new reservation) **Used when:** Creating a new failover reservation. @@ -175,14 +175,14 @@ Options: `ReadOnly: true, SkipHistory: true` — pure compatibility check, no st 1. Is compatible with the VM (traits, capabilities, AZ, etc.) 2. Has enough free capacity to accommodate the VM if it needs to evacuate -Options: `LockReservations: true, SkipHistory: true` — capacity check must see true remaining capacity with all reservation slots locked. +Options: `LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true` — capacity check must see true remaining capacity with all reservation slots locked. ### `kvm-acknowledge-failover-reservation` **Used when:** Validating that an existing reservation is still valid (watch-based reconciliation). **Why:** Periodically we need to verify that a VM could still evacuate to its reserved host. This sends an evacuation-style scheduling request with only the reservation's host as the eligible target. If the scheduler rejects it, the reservation is no longer valid and should be deleted so the periodic controller can create a new one on a valid host. -Options: `ReadOnly: true, SkipHistory: true` — validation only, no state mutations. +Options: `ReadOnly: true, LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true` — validation only, no state mutations. ## Data Model diff --git a/go.mod b/go.mod index eff248f05..fcdc8942b 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/cobaltcore-dev/openstack-hypervisor-operator v1.2.3 github.com/go-gorp/gorp v2.2.0+incompatible github.com/gophercloud/gophercloud/v2 v2.13.0 - github.com/ironcore-dev/ironcore v0.4.2 + github.com/ironcore-dev/ironcore v0.4.3 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 - github.com/sapcc/go-bits v0.0.0-20260703101005-27edea94477c + github.com/sapcc/go-bits v0.0.0-20260709170107-f8ef1547bb62 go.xyrillian.de/gg v1.11.1 k8s.io/api v0.36.2 k8s.io/apimachinery v0.36.2 @@ -111,15 +111,15 @@ require ( golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 - golang.org/x/sys v0.46.0 // indirect - golang.org/x/term v0.44.0 + golang.org/x/sync v0.22.0 + golang.org/x/sys v0.47.0 // indirect + golang.org/x/term v0.45.0 golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/grpc v1.81.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 111d07a4f..d1011eae6 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ironcore-dev/ironcore v0.4.2 h1:qYhW0MIc8boHxG5MhgoCECGPJTKbPYnR/sGYmmdqM6U= -github.com/ironcore-dev/ironcore v0.4.2/go.mod h1:3VRaWvT9GZhtlPuFQHQKxYviEZQw9aloAjS3h+c7jcc= +github.com/ironcore-dev/ironcore v0.4.3 h1:G3TmR4r3LtbmakMkxDa2pPlLCmxAfGVBEdy+QVyxl8Q= +github.com/ironcore-dev/ironcore v0.4.3/go.mod h1:vFpdcyC4QD7o3j01lpux12NET6vNPDAq3hzrFOY9QOc= github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= @@ -174,8 +174,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v1.42.0 h1:CJby8u36xb7v34W78F8WKvqTQP7PCMIPB78IVDB73l4= -github.com/onsi/gomega v1.42.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= +github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -204,8 +204,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.24.0 h1:sGBvOMVSM1olJlyvNoQSk7NX5uatXHKkztGDBPnTWMs= github.com/sapcc/go-api-declarations v1.24.0/go.mod h1:ZWRTijvgF8o8aHg5stgg7u4DF6jFrd0X97le/uGlZsA= -github.com/sapcc/go-bits v0.0.0-20260703101005-27edea94477c h1:jqVSQNGIZ+V7mP+bjKoz/OEVy5qWoGvaYCfwXfohEWw= -github.com/sapcc/go-bits v0.0.0-20260703101005-27edea94477c/go.mod h1:uchHF8FNxncTCoQYDjGSO86AuxSzgIbN5mQzM95RYOA= +github.com/sapcc/go-bits v0.0.0-20260709170107-f8ef1547bb62 h1:HGIsxyetnws+88X5ZAY86AxYGR88vFWMVrXzxAMWg6U= +github.com/sapcc/go-bits v0.0.0-20260709170107-f8ef1547bb62/go.mod h1:uchHF8FNxncTCoQYDjGSO86AuxSzgIbN5mQzM95RYOA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -273,14 +273,14 @@ golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -291,12 +291,12 @@ gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0 gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 h1:tu/dtnW1o3wfaxCOjSLn5IRX4YDcJrtlpzYkhHhGaC4= -google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171/go.mod h1:M5krXqk4GhBKvB596udGL3UyjL4I1+cTbK0orROM9ng= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index bec946fca..3f954cd0b 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.79 +version: 0.0.80 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-cinder/templates/alerts.yaml b/helm/bundles/cortex-cinder/templates/alerts.yaml index 4beea8b53..941c33546 100644 --- a/helm/bundles/cortex-cinder/templates/alerts.yaml +++ b/helm/bundles/cortex-cinder/templates/alerts.yaml @@ -156,7 +156,7 @@ spec: time the service will have a less recent view of the datacenter. - alert: CortexCinderSyncObjectsDroppedToZero - expr: cortex_sync_objects{service="cortex-cinder-metrics"} == 0 + expr: cortex_sync_objects{service="cortex-cinder-metrics"} == 0 and (cortex_sync_objects{service="cortex-cinder-metrics"} offset 1h > 0) for: 60m labels: context: syncobjects diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index b0402277a..13eb15db9 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.79 +version: 0.0.80 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 0b91af43e..4c0beecb2 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.79 +version: 0.0.80 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index b1f69a0d2..b7da9c9e0 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.79 +version: 0.0.80 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-manila/templates/alerts.yaml b/helm/bundles/cortex-manila/templates/alerts.yaml index ef36fe983..769730c1e 100644 --- a/helm/bundles/cortex-manila/templates/alerts.yaml +++ b/helm/bundles/cortex-manila/templates/alerts.yaml @@ -162,7 +162,7 @@ spec: time the service will have a less recent view of the datacenter. - alert: CortexManilaSyncObjectsDroppedToZero - expr: cortex_sync_objects{service="cortex-manila-metrics"} == 0 + expr: cortex_sync_objects{service="cortex-manila-metrics"} == 0 and (cortex_sync_objects{service="cortex-manila-metrics"} offset 1h > 0) for: 60m labels: context: syncobjects diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index ebd6c4de4..27e9b455a 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.79 +version: 0.0.80 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/templates/alerts.yaml b/helm/bundles/cortex-nova/templates/alerts.yaml index 840e969f2..8db8bfb2e 100644 --- a/helm/bundles/cortex-nova/templates/alerts.yaml +++ b/helm/bundles/cortex-nova/templates/alerts.yaml @@ -179,7 +179,7 @@ spec: time the service will have a less recent view of the datacenter. - alert: CortexNovaSyncObjectsDroppedToZero - expr: cortex_sync_objects{service="cortex-nova-metrics", datasource!="openstack_migrations"} == 0 + expr: cortex_sync_objects{service="cortex-nova-metrics", datasource!="openstack_migrations"} == 0 and (cortex_sync_objects{service="cortex-nova-metrics", datasource!="openstack_migrations"} offset 1h > 0) for: 60m labels: context: syncobjects @@ -340,11 +340,11 @@ spec: completed its first reconciliation yet. This may indicate issues with the datasource controller's workqueue overprioritizing other datasources. - - alert: CortexNovaExistingDatasourcesLackingBehind + - alert: CortexNovaExistingDatasourcesLaggingBehind expr: | sum by(datasource) (cortex_datasource_seconds_until_reconcile{queued="true",domain="nova"}) < -600 and on(datasource) cortex_datasource_state{state="ready",domain="nova"} == 1 - for: 10m + for: 30m labels: context: datasources dashboard: cortex-status-dashboard/cortex-status-dashboard @@ -353,10 +353,10 @@ spec: support_group: workload-management playbook: docs/support/playbook/cortex/alerts/datasources annotations: - summary: "Existing datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` is lacking behind" + summary: "Existing datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` is lagging behind" description: > An existing datasource `{{ "{{" }} $labels.datasource {{ "}}" }}` has been queued for - reconciliation for more than 10 minutes. This may indicate issues with + reconciliation for more than 30 minutes. This may indicate issues with the datasource controller's workqueue or that this or another datasource is taking an unusually long time to reconcile. diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index 1171f3b01..27acd10e1 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -154,11 +154,14 @@ cortex-scheduling-controllers: # that use committed resources. Requires also enabling of CR controllers and tasks committedResourceTracking: false # Pipeline used for the empty-state capacity probe (ignores allocations and reservations). - capacityTotalPipeline: "kvm-report-capacity" + capacityTotalPipeline: "kvm-general-purpose-load-balancing" # Pipeline used for the current-state capacity probe (considers current VM allocations). - capacityPlaceablePipeline: "kvm-general-purpose-load-balancing-no-history" - # How often the capacity controller re-runs its scheduler probes. + capacityPlaceablePipeline: "kvm-general-purpose-load-balancing" + # How often the capacity reconciler re-runs its scheduler probes (periodic floor). capacityReconcileInterval: 5m + # Minimum time between two consecutive capacity reconcile runs. + # Prevents back-to-back reconciles on rapid watch events. + capacityMinReconcileInterval: 30s # If true, the external scheduler API will limit the list of hosts in its # response to those included in the scheduling request. novaLimitHostsToRequest: true @@ -166,11 +169,12 @@ cortex-scheduling-controllers: # Set to 0 or negative to disable shuffling. evacuationShuffleK: 3 committedResourceReservationController: - # Maps flavor group IDs to pipeline names; "*" acts as catch-all fallback + # Pipeline selection for CR reservation scheduling. The catch-all default covers + # general-purpose flavors. For HANA flavor groups, add an explicit entry, e.g.: + # "my-hana-group": "kvm-hana-bin-packing" flavorGroupPipelines: - "*": "kvm-general-purpose-load-balancing-no-history" # Catch-all fallback - # Fallback pipeline when no flavorGroupPipelines entry matches - pipelineDefault: "kvm-general-purpose-load-balancing-no-history" + "*": "kvm-general-purpose-load-balancing" + pipelineDefault: "kvm-general-purpose-load-balancing" # How often to re-verify active Reservation CRDs (healthy state) requeueIntervalActive: "5m" # Back-off interval when knowledge is unavailable diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index 07e09aab7..dfc035bda 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.5 +version: 0.1.6 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.5 + version: 0.1.6 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index 6bbcf92ce..f58ac7cd3 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.79 +version: 0.0.80 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.2.2 + version: 0.3.0 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 7e2d4dbba..ca8ea75f5 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 87.10.1 + version: 87.15.1 diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 0e6b0ddfb..3ff6405b7 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.5 -appVersion: "sha-c8295850" +version: 0.1.6 +appVersion: "sha-ee9cd485" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index b746c98f3..64433287e 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.2.2 -appVersion: "sha-8d888397" +version: 0.3.0 +appVersion: "sha-ee9cd485" icon: "https://example.com/icon.png" dependencies: [] diff --git a/internal/knowledge/datasources/plugins/openstack/controller.go b/internal/knowledge/datasources/plugins/openstack/controller.go index 96d056a15..07277ce79 100644 --- a/internal/knowledge/datasources/plugins/openstack/controller.go +++ b/internal/knowledge/datasources/plugins/openstack/controller.go @@ -95,7 +95,7 @@ func (r *OpenStackDatasourceReconciler) Reconcile(ctx context.Context, req ctrl. log.Info("skipping datasource, not an openstack datasource", "name", datasource.Name) return ctrl.Result{}, nil } - if datasource.Status.NextSyncTime.After(time.Now()) && datasource.Status.NumberOfObjects != 0 { + if datasource.Status.NextSyncTime.After(time.Now()) && datasource.Status.NumberOfObjects != nil { if _, seen := r.reconciledOnce.Load(req.NamespacedName); seen { log.Info("skipping datasource sync, not yet time", "name", datasource.Name) return ctrl.Result{RequeueAfter: time.Until(datasource.Status.NextSyncTime.Time)}, nil @@ -262,7 +262,7 @@ func (r *OpenStackDatasourceReconciler) Reconcile(ctx context.Context, req ctrl. datasource.Status.LastSynced = metav1.NewTime(time.Now()) nextTime := time.Now().Add(datasource.Spec.OpenStack.SyncInterval.Duration) datasource.Status.NextSyncTime = metav1.NewTime(nextTime) - datasource.Status.NumberOfObjects = nResults + datasource.Status.NumberOfObjects = &nResults patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, datasource, patch); err != nil { log.Error(err, "failed to patch datasource status", "name", datasource.Name) diff --git a/internal/knowledge/datasources/plugins/openstack/controller_test.go b/internal/knowledge/datasources/plugins/openstack/controller_test.go index 899e83237..a809e099b 100644 --- a/internal/knowledge/datasources/plugins/openstack/controller_test.go +++ b/internal/knowledge/datasources/plugins/openstack/controller_test.go @@ -495,11 +495,11 @@ func TestUpdatePredicateIgnoresStatusConditionChanges(t *testing.T) { name: "status NumberOfObjects changes - should trigger reconcile", oldObj: &v1alpha1.Datasource{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, - Status: v1alpha1.DatasourceStatus{NumberOfObjects: 10}, + Status: v1alpha1.DatasourceStatus{NumberOfObjects: new(int64(10))}, }, newObj: &v1alpha1.Datasource{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, - Status: v1alpha1.DatasourceStatus{NumberOfObjects: 20}, + Status: v1alpha1.DatasourceStatus{NumberOfObjects: new(int64(20))}, }, expected: true, }, diff --git a/internal/knowledge/datasources/plugins/prometheus/controller.go b/internal/knowledge/datasources/plugins/prometheus/controller.go index dab9ea3b3..bccc1fcf1 100644 --- a/internal/knowledge/datasources/plugins/prometheus/controller.go +++ b/internal/knowledge/datasources/plugins/prometheus/controller.go @@ -71,7 +71,7 @@ func (r *PrometheusDatasourceReconciler) Reconcile(ctx context.Context, req ctrl log.Info("skipping datasource, not a prometheus datasource", "name", datasource.Name) return ctrl.Result{}, nil } - if datasource.Status.NextSyncTime.After(time.Now()) && datasource.Status.NumberOfObjects != 0 { + if datasource.Status.NextSyncTime.After(time.Now()) && datasource.Status.NumberOfObjects != nil { if _, seen := r.reconciledOnce.Load(req.NamespacedName); seen { log.Info("skipping datasource sync, not yet time", "name", datasource.Name) return ctrl.Result{RequeueAfter: time.Until(datasource.Status.NextSyncTime.Time)}, nil @@ -201,7 +201,7 @@ func (r *PrometheusDatasourceReconciler) Reconcile(ctx context.Context, req ctrl }) datasource.Status.LastSynced = metav1.NewTime(time.Now()) datasource.Status.NextSyncTime = metav1.NewTime(nextSync) - datasource.Status.NumberOfObjects = nResults + datasource.Status.NumberOfObjects = &nResults patch := client.MergeFrom(old) if err := r.Status().Patch(ctx, datasource, patch); err != nil { log.Error(err, "failed to patch datasource status", "name", datasource.Name) diff --git a/internal/knowledge/kpis/controller_test.go b/internal/knowledge/kpis/controller_test.go index 708a19cd0..0aeb96485 100644 --- a/internal/knowledge/kpis/controller_test.go +++ b/internal/knowledge/kpis/controller_test.go @@ -333,7 +333,7 @@ func TestController_Reconcile(t *testing.T) { }, }, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 1, + NumberOfObjects: new(int64(1)), Conditions: []metav1.Condition{ { Type: v1alpha1.DatasourceConditionReady, @@ -393,7 +393,7 @@ func TestController_Reconcile(t *testing.T) { }, }, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 0, + NumberOfObjects: new(int64(0)), Conditions: []metav1.Condition{ { Type: v1alpha1.DatasourceConditionReady, @@ -622,7 +622,7 @@ func TestController_handleKPIChange(t *testing.T) { DatabaseSecretRef: corev1.SecretReference{Name: "db-secret", Namespace: "default"}, }, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 1, + NumberOfObjects: new(int64(1)), Conditions: []metav1.Condition{ { Type: v1alpha1.DatasourceConditionReady, @@ -638,7 +638,7 @@ func TestController_handleKPIChange(t *testing.T) { DatabaseSecretRef: corev1.SecretReference{Name: "db-secret", Namespace: "default"}, }, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 0, + NumberOfObjects: new(int64(0)), Conditions: []metav1.Condition{ { Type: v1alpha1.DatasourceConditionReady, @@ -703,7 +703,7 @@ func TestController_handleKPIChange(t *testing.T) { DatabaseSecretRef: corev1.SecretReference{Name: "db-secret", Namespace: "default"}, }, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 1, + NumberOfObjects: new(int64(1)), Conditions: []metav1.Condition{ { Type: v1alpha1.DatasourceConditionReady, diff --git a/internal/knowledge/kpis/plugins/deployment/datasource_state_test.go b/internal/knowledge/kpis/plugins/deployment/datasource_state_test.go index d6e56bd35..e143bf5f9 100644 --- a/internal/knowledge/kpis/plugins/deployment/datasource_state_test.go +++ b/internal/knowledge/kpis/plugins/deployment/datasource_state_test.go @@ -49,7 +49,7 @@ func TestDatasourceStateKPI_Collect(t *testing.T) { ObjectMeta: v1.ObjectMeta{Name: "ds1"}, Spec: v1alpha1.DatasourceSpec{SchedulingDomain: "test-operator"}, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 10, + NumberOfObjects: new(int64(10)), Conditions: []v1.Condition{}, }, }, @@ -87,7 +87,7 @@ func TestDatasourceStateKPI_Collect(t *testing.T) { ObjectMeta: v1.ObjectMeta{Name: "ds-ready"}, Spec: v1alpha1.DatasourceSpec{SchedulingDomain: "test-operator"}, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 10, + NumberOfObjects: new(int64(10)), Conditions: []v1.Condition{}, }, }, @@ -127,7 +127,7 @@ func TestDatasourceStateKPI_Collect(t *testing.T) { ObjectMeta: v1.ObjectMeta{Name: "ds-correct-operator"}, Spec: v1alpha1.DatasourceSpec{SchedulingDomain: "test-operator"}, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 10, + NumberOfObjects: new(int64(10)), Conditions: []v1.Condition{}, }, }, @@ -135,7 +135,7 @@ func TestDatasourceStateKPI_Collect(t *testing.T) { ObjectMeta: v1.ObjectMeta{Name: "ds-wrong-operator"}, Spec: v1alpha1.DatasourceSpec{SchedulingDomain: "other-operator"}, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 10, + NumberOfObjects: new(int64(10)), Conditions: []v1.Condition{}, }, }, @@ -151,7 +151,7 @@ func TestDatasourceStateKPI_Collect(t *testing.T) { ObjectMeta: v1.ObjectMeta{Name: "ds-unknown"}, Spec: v1alpha1.DatasourceSpec{SchedulingDomain: "test-operator"}, Status: v1alpha1.DatasourceStatus{ - NumberOfObjects: 0, + NumberOfObjects: new(int64(0)), Conditions: []v1.Condition{}, }, }, diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go index 157a80521..f55e8ab7b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go @@ -9,6 +9,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -21,6 +22,14 @@ type FilterAggregateMetadata struct { // the "filter_tenant_id" metadata key set. func (s *FilterAggregateMetadata) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Failover and capacity probe calls are not placed on behalf of a tenant project; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } hvs := &hv1.HypervisorList{} if err := s.Client.List(context.Background(), hvs); err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go index d1ff9cd2d..42c096a2b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go @@ -404,3 +404,46 @@ func TestFilterAggregateMetadata_IndexRegistration(t *testing.T) { t.Errorf("expected factory to return *FilterAggregateMetadata, got %T", filter) } } + +func TestFilterAggregateMetadata_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host1 is in an aggregate restricting to project-x; request is project-y → host1 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host1"}, + Status: hv1.HypervisorStatus{ + Aggregates: []hv1.Aggregate{{ + Name: "restricted", + Metadata: map[string]string{"filter_tenant_id": "project-x"}, + }}, + }, + }, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + } + step := &FilterAggregateMetadata{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + ProjectID: "project-y", + SchedulerHints: map[string]any{"_nova_check_type": intent}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go index a0a486f3d..bd0849cf1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go @@ -9,6 +9,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -21,6 +22,14 @@ type FilterAllowedProjectsStep struct { // Note that hosts without specified projects are still accessible. func (s *FilterAllowedProjectsStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Failover and capacity probe calls are not placed on behalf of a tenant project; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } if request.Spec.Data.ProjectID == "" { traceLog.Info("no project ID in request, skipping filter") return result, nil diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go index 53a4ac958..ccbaf44e5 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go @@ -323,3 +323,41 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }) } } + +func TestFilterAllowedProjectsStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host2 restricts to project-x; request is project-y → host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Spec: hv1.HypervisorSpec{AllowedProjects: []string{"project-x"}}, + }, + } + step := &FilterAllowedProjectsStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + ProjectID: "project-y", + SchedulerHints: map[string]any{"_nova_check_type": intent}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer.go b/internal/scheduling/nova/plugins/filters/filter_external_customer.go index bcbf74716..b574e4ad1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer.go @@ -11,6 +11,7 @@ import ( "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -36,10 +37,15 @@ type FilterExternalCustomerStep struct { func (s *FilterExternalCustomerStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) - // Skip for failover reservation scheduling — domain restrictions don't apply - // since failover reservations are not tied to a specific customer domain. - if intent, err := request.GetIntent(); err == nil && intent == api.ReserveForFailoverIntent { - traceLog.Info("skipping external customer filter for failover reservation intent") + // Failover and capacity probe calls are not associated with a user domain; + // domain-based host restrictions don't apply. CR scheduling does carry a real domain + // and must be checked. + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.CapacityProbeIntent, + }, intent) { + traceLog.Info("skipping external customer filter for non-placement intent", "intent", intent) return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go index 290709f05..cb358e0c1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go @@ -545,3 +545,44 @@ func TestFilterExternalCustomerStepOpts_Validate(t *testing.T) { }) } } + +func TestFilterExternalCustomerStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // Domain matches external prefix; host1 lacks the exclusive trait → host1 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE"}}, + }, + } + step := &FilterExternalCustomerStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterExternalCustomerStepOpts{CustomerDomainNamePrefixes: []string{"iaas-"}} + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + SchedulerHints: map[string]any{ + "domain_name": "iaas-customer", + "_nova_check_type": intent, + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index 8940c0d86..e2fadf981 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -66,6 +66,10 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa opts := request.GetOptions() result := s.IncludeAllHostsFromRequest(request) + // Merge call-time options with static step config. + ignoreAllocations := s.Options.IgnoreAllocations || opts.AssumeEmptyHosts + ignoredReservationTypes := slices.Concat(s.Options.IgnoredReservationTypes, opts.IgnoredReservationTypes) + // This map holds the free resources per host. freeResourcesByHost := make(map[string]map[hv1.ResourceName]resource.Quantity) @@ -87,7 +91,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa } // Subtract allocated resources (skip when ignoring allocations for empty-datacenter capacity queries). - if !s.Options.IgnoreAllocations { + if !ignoreAllocations { for resourceName, allocated := range hv.Status.Allocation { free, ok := freeResourcesByHost[hv.Name][resourceName] if !ok { @@ -110,7 +114,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa } for _, reservation := range reservations.Items { // Check if this reservation type should be ignored — applies regardless of ready state. - if slices.Contains(s.Options.IgnoredReservationTypes, reservation.Spec.Type) { + if slices.Contains(ignoredReservationTypes, reservation.Spec.Type) { traceLog.Debug("ignoring reservation type", "type", reservation.Spec.Type, "reservation", reservation.Name) continue } @@ -169,14 +173,23 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa // 2. During live migrations or other operations, we don't want to use failover capacity. // Note: we cannot use failover reservations from other VMs, as that can invalidate our HA guarantees. intent, err := request.GetIntent() - if err == nil && intent == api.EvacuateIntent { - if reservation.Status.FailoverReservation != nil { - if _, contained := reservation.Status.FailoverReservation.Allocations[request.Spec.Data.InstanceUUID]; contained { - traceLog.Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", - "reservation", reservation.Name, - "instanceUUID", request.Spec.Data.InstanceUUID) - continue + if err == nil { + switch intent { + case api.EvacuateIntent: + if reservation.Status.FailoverReservation != nil { + if _, contained := reservation.Status.FailoverReservation.Allocations[request.Spec.Data.InstanceUUID]; contained { + traceLog.Info("unlocking resources reserved by failover reservation for VM in allocations (evacuation)", + "reservation", reservation.Name, + "instanceUUID", request.Spec.Data.InstanceUUID) + continue + } } + case api.ReuseFailoverReservationIntent: + // Reuse check: the reservation already pre-blocks the right capacity for this VM. + // Don't subtract it from free capacity to avoid double-counting. + traceLog.Debug("skipping failover reservation block for reuse compatibility check", + "reservation", reservation.Name) + continue } } traceLog.Debug("processing failover reservation", "reservation", reservation.Name) @@ -209,7 +222,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa // Oversize spec-only: if a pending VM is larger than the remaining slot, block its full size. // // FailoverReservations: block = Spec.Resources (always fully blocked). - resourcesToBlock := resv.UnusedReservationCapacity(&reservation, s.Options.IgnoreAllocations) + resourcesToBlock := resv.UnusedReservationCapacity(&reservation, ignoreAllocations) // Block the calculated resources on each host for host := range hostsToBlock { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index 8b55556f8..b1b659e59 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -248,7 +248,7 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { //nolint:unparam // vcpus varies in real usage +func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { return newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup, vcpus, memory, "", evacuation, hosts) } @@ -961,6 +961,41 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { } } +func TestFilterHasEnoughCapacity_AssumeEmptyHosts(t *testing.T) { + scheme := buildTestScheme(t) + + // host1: 8 CPU total, 6 CPU allocated to running VMs → 2 free; request needs 4 → fails normally. + // With AssumeEmptyHosts: allocations ignored → 8 free → passes. + hypervisors := []*hv1.Hypervisor{ + newHypervisor("host1", "8", "6", "32Gi", "0"), + } + request := newNovaRequest("vm", "proj", "m1.large", "gp", 4, "1Gi", false, []string{"host1"}) + + objects := make([]client.Object, 0, len(hypervisors)) + for _, h := range hypervisors { + objects = append(objects, h.DeepCopy()) + } + + step := &FilterHasEnoughCapacity{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterHasEnoughCapacityOpts{} + + // Without AssumeEmptyHosts: host1 filtered (only 2 CPU free). + resultWithout, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + assertActivations(t, resultWithout.Activations, []string{}, []string{"host1"}) + + // With AssumeEmptyHosts via call-time options: allocations ignored → host1 passes. + request.Options = scheduling.Options{AssumeEmptyHosts: true} + resultWith, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + assertActivations(t, resultWith.Activations, []string{"host1"}, []string{}) +} + func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) { scheme := buildTestScheme(t) @@ -986,7 +1021,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) step := &FilterHasEnoughCapacity{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - // Ignore CR reservations via pipeline-level opts (call-time opts.IgnoredReservationTypes removed in favour of YAML params). + // Ignore CR reservations via YAML step opts (static, configured per pipeline step). step.Options = FilterHasEnoughCapacityOpts{ LockReserved: true, IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, @@ -999,6 +1034,49 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) assertActivations(t, result.Activations, []string{"host1"}, []string{"host2"}) } +func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTimeMerge(t *testing.T) { + scheme := buildTestScheme(t) + + // Same two-host setup: CR on host1, Failover on host2. + // Each blocks 4 CPU, leaving 4 free; request needs 8 CPU so both hosts fail without ignoring. + // Verify that call-time opts.IgnoredReservationTypes is merged with (not instead of) YAML opts. + hypervisors := []*hv1.Hypervisor{ + newHypervisor("host1", "16", "8", "32Gi", "16Gi"), + newHypervisor("host2", "16", "8", "32Gi", "16Gi"), + } + reservations := []*v1alpha1.Reservation{ + newCommittedReservation("cr-res", "host1", "project-X", "m1.large", "gp-1", "4", "8Gi", nil, nil), + newFailoverReservation("failover-res", "host2", "4", "8Gi", map[string]string{"other-vm": "host3"}), + } + request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) + + objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) + for _, h := range hypervisors { + objects = append(objects, h.DeepCopy()) + } + for _, r := range reservations { + objects = append(objects, r.DeepCopy()) + } + + step := &FilterHasEnoughCapacity{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + // YAML opts ignore CR; call-time opts ignore Failover — both must be respected. + step.Options = FilterHasEnoughCapacityOpts{ + LockReserved: true, + IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, + } + request.Options = scheduling.Options{ + IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, + } + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + // Both CR (host1) and Failover (host2) reservations ignored → both hosts pass. + assertActivations(t, result.Activations, []string{"host1", "host2"}, []string{}) +} + func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) { scheme := buildTestScheme(t) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go index 326864b9d..8c9045e34 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go @@ -8,6 +8,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" ) @@ -22,6 +23,15 @@ func (s *FilterInstanceGroupAffinityStep) Run( ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Instance group hints are only present in user-initiated VM placement requests; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } ig := request.Spec.Data.InstanceGroup if ig == nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go index 7321747e3..e6e6a8803 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go @@ -352,3 +352,25 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }) } } + +func TestFilterInstanceGroupAffinityStep_SkipsForNonPlacementIntent(t *testing.T) { + // Affinity group on host1 only — host2 would normally be filtered. + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := newNovaRequest("vm", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{Policy: "affinity", Hosts: []string{"host1"}}, + } + request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + step := &FilterInstanceGroupAffinityStep{} + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go index 0dee29d9e..137ddf04f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go @@ -9,6 +9,7 @@ import ( "slices" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -25,6 +26,15 @@ func (s *FilterInstanceGroupAntiAffinityStep) Run( ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // Instance group hints are only present in user-initiated VM placement requests; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } ig := request.Spec.Data.InstanceGroup if ig == nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go index 931265b9b..63a9aff4a 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go @@ -547,3 +547,42 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }) } } + +func TestFilterInstanceGroupAntiAffinityStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host2 has the group member vm; max_server_per_host=1 → host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Instances: []hv1.Instance{{ID: "vm-existing"}}}, + }, + } + step := &FilterInstanceGroupAntiAffinityStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := newNovaRequest("vm-new", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{ + Policy: "anti-affinity", + Members: []string{"vm-existing"}, + Rules: map[string]any{"max_server_per_host": 1}, + }, + } + request.Spec.Data.SchedulerHints = map[string]any{"_nova_check_type": intent} + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 4e139f8d8..2278d526f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -81,43 +81,52 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External } // Step 1: Skip intents that don't represent new resource consumption. + // We distinguish two sub-buckets in the decision label so operators can tell + // external non-consuming intents (evacuate / live-migrate — VM already exists, + // being moved) apart from cortex-internal reservation intents (failover / + // committed-resource reservations — new schedules driven by cortex itself). intent, err := request.GetIntent() if err == nil { switch intent { case api.EvacuateIntent, api.LiveMigrationIntent: traceLog.Info("skipping quota enforcement for non-consuming intent", "intent", intent) - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_intent_migration", "", "", "") return result, nil - case api.ReserveForFailoverIntent: + case api.ReserveForFailoverIntent, api.ReuseFailoverReservationIntent: traceLog.Info("skipping quota enforcement for failover reservation intent") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_intent_internal", "", "", "") return result, nil case api.ReserveForCommittedResourceIntent: - // TODO: revisit whether committed resource reservation scheduling should also be quota-checked traceLog.Info("skipping quota enforcement for committed resource reservation intent") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_intent_internal", "", "", "") + return result, nil + case api.CapacityProbeIntent: + traceLog.Info("skipping quota enforcement for capacity probe intent") QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") return result, nil } } // Step 2: Extract project, AZ, and hw_version from the request. + // Missing-data skips share a single decision bucket; the specific missing + // field is in the trace log. projectID := request.Spec.Data.ProjectID az := request.Spec.Data.AvailabilityZone hwVersion := request.Spec.Data.Flavor.Data.ExtraSpecs["hw_version"] if projectID == "" { traceLog.Warn("no project ID in request, skipping quota enforcement") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", az, hwVersion) + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_missing_data", "", az, hwVersion) return result, nil } if az == "" { traceLog.Warn("no availability zone in request, skipping quota enforcement") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", hwVersion) + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_missing_data", "", "", hwVersion) return result, nil } if hwVersion == "" { traceLog.Warn("no hw_version in flavor extra specs, skipping quota enforcement") - QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", az, "") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped_missing_data", "", az, "") return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go index 8d8edc17e..0743b6346 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go @@ -21,7 +21,10 @@ type QuotaEnforcementMetrics struct { // produces. Labels: // - mode: "enforce" | "shadow" // - decision: "accept_cr" | "accept_payg" | "accept_no_quota" | - // "accept_skipped" | "reject" + // "accept_skipped_intent_migration" | + // "accept_skipped_intent_internal" | + // "accept_skipped_missing_data" | + // "reject" // - resource: "ram" | "cores" | "instances" | "" (empty when // the decision is not driven by a single resource) // - availability_zone: AZ string or "" if unknown at decision time diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 32ea16ad4..688b5381d 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -78,7 +78,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { // Metric expectations — every case asserts exactly one increment on the // labeled series and exactly one series in the vector. expectMode string // "enforce" | "shadow" - expectDecision string // accept_cr | accept_payg | accept_no_quota | accept_skipped | reject + expectDecision string // accept_cr | accept_payg | accept_no_quota | accept_skipped_intent_migration | accept_skipped_intent_internal | accept_skipped_missing_data | reject expectResource string // "ram" | "cores" | "instances" | "" expectAZ string expectFG string @@ -263,7 +263,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { expectAccept: true, enforce: true, expectMode: "enforce", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_intent_migration", expectResource: "", expectAZ: "", expectFG: "", @@ -276,7 +276,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { expectAccept: true, enforce: true, expectMode: "enforce", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_intent_migration", expectResource: "", expectAZ: "", expectFG: "", @@ -289,7 +289,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { expectAccept: true, enforce: true, expectMode: "enforce", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_intent_internal", expectResource: "", expectAZ: "", expectFG: "", @@ -302,7 +302,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { expectAccept: true, enforce: true, expectMode: "enforce", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_intent_internal", expectResource: "", expectAZ: "", expectFG: "", @@ -314,7 +314,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { expectAccept: true, enforce: true, expectMode: "enforce", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_missing_data", expectResource: "", expectAZ: "az-1", expectFG: "", @@ -326,7 +326,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { expectAccept: true, enforce: true, expectMode: "enforce", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_missing_data", expectResource: "", expectAZ: "az-1", expectFG: "hana_v2", @@ -338,7 +338,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { expectAccept: true, enforce: true, expectMode: "enforce", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_missing_data", expectResource: "", expectAZ: "", expectFG: "hana_v2", @@ -904,7 +904,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { map[string]any{"_nova_check_type": "live_migrate"}), expectAccept: true, expectMode: "shadow", - expectDecision: "accept_skipped", + expectDecision: "accept_skipped_intent_migration", expectResource: "", expectAZ: "", expectFG: "", @@ -1087,3 +1087,36 @@ func TestQuotaEnforcementMetrics_RecordDecision_NilWarns(t *testing.T) { msg, got, buf.String()) } } + +func TestFilterQuotaEnforcement_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add v1alpha1 to scheme: %v", err) + } + // Enforce mode: quota fully consumed → all hosts would normally be filtered. + objects := []client.Object{ + &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-no-quota-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-no-quota", + AvailabilityZone: "az-1", + Quota: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + Status: v1alpha1.ProjectQuotaStatus{ + PaygUsage: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + }, + } + request := makeQuotaEnforcementRequest("project-no-quota", "az-1", "gp", 1, 1, map[string]any{"_nova_check_type": "capacity_probe"}) + step := &FilterQuotaEnforcement{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterQuotaEnforcementOpts{Enforce: true} + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != len(request.Hosts) { + t.Errorf("expected all %d hosts to pass, got %d", len(request.Hosts), len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go index 8922ab8c4..971848dcd 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go @@ -10,6 +10,7 @@ import ( "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" ) @@ -102,6 +103,15 @@ func (s *FilterRequestedDestinationStep) processRequestedHost( // host filtering. func (s *FilterRequestedDestinationStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + // The requested_destination hint is only set by Nova for user-directed placement; + if intent, err := request.GetIntent(); err == nil && slices.Contains([]v1alpha1.SchedulingIntent{ + api.ReserveForFailoverIntent, + api.ReuseFailoverReservationIntent, + api.ReserveForCommittedResourceIntent, + api.CapacityProbeIntent, + }, intent) { + return result, nil + } rd := request.Spec.Data.RequestedDestination if rd == nil { traceLog.Info("no requested_destination in request, skipping filter") diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go index 5a752160e..fab4c7aec 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go @@ -782,3 +782,40 @@ func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { t.Errorf("expected error when client fails, got none") } } + +func TestFilterRequestedDestinationStep_SkipsForNonPlacementIntent(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // RequestedDestination forces host1 — host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + } + step := &FilterRequestedDestinationStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + for _, intent := range []string{"reserve_for_failover", "reuse_failover_reservation", "reserve_for_committed_resource", "capacity_probe"} { + t.Run(intent, func(t *testing.T) { + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ + Data: api.NovaRequestedDestination{Host: "host1"}, + }, + SchedulerHints: map[string]any{"_nova_check_type": intent}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } + }) + } +} diff --git a/internal/scheduling/reservations/capacity/config.go b/internal/scheduling/reservations/capacity/config.go index 264a0b59d..4b9292042 100644 --- a/internal/scheduling/reservations/capacity/config.go +++ b/internal/scheduling/reservations/capacity/config.go @@ -4,16 +4,24 @@ package capacity import ( + "fmt" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// Config holds configuration for the capacity controller. +// Config holds configuration for the capacity reconciler. type Config struct { - // ReconcileInterval is how often the controller probes the scheduler and updates CRDs. + // ReconcileInterval is the periodic floor: how often the reconciler re-runs even without a + // watch event. Acts as a fallback for changes not covered by watches (e.g. blocked memory drift). ReconcileInterval metav1.Duration `json:"capacityReconcileInterval"` + // MinReconcileInterval is the minimum time between two consecutive reconcile runs. + // If Reconcile() is called sooner than this since the last successful run, it returns early + // with RequeueAfter set to the remaining duration. Prevents back-to-back reconciles on rapid + // watch events (e.g. a batch of CommittedResource updates). + MinReconcileInterval metav1.Duration `json:"capacityMinReconcileInterval"` + // TotalPipeline is the scheduler pipeline used for the empty-state probe. // This pipeline should ignore current VM allocations (e.g. kvm-report-capacity). TotalPipeline string `json:"capacityTotalPipeline"` @@ -32,6 +40,9 @@ func (c *Config) ApplyDefaults() { if c.ReconcileInterval.Duration == 0 { c.ReconcileInterval = defaults.ReconcileInterval } + if c.MinReconcileInterval.Duration == 0 { + c.MinReconcileInterval = defaults.MinReconcileInterval + } if c.TotalPipeline == "" { c.TotalPipeline = defaults.TotalPipeline } @@ -43,11 +54,21 @@ func (c *Config) ApplyDefaults() { } } +// Validate checks that the config is internally consistent after defaults are applied. +func (c *Config) Validate() error { + if c.ReconcileInterval.Duration <= c.MinReconcileInterval.Duration { + return fmt.Errorf("capacityReconcileInterval (%s) must be greater than capacityMinReconcileInterval (%s)", + c.ReconcileInterval.Duration, c.MinReconcileInterval.Duration) + } + return nil +} + func DefaultConfig() Config { return Config{ - ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, - TotalPipeline: "kvm-report-capacity", - PlaceablePipeline: "kvm-general-purpose-load-balancing-no-history", - SchedulerURL: "http://localhost:8080/scheduler/nova/external", + ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, + MinReconcileInterval: metav1.Duration{Duration: 30 * time.Second}, + TotalPipeline: "kvm-general-purpose-load-balancing", + PlaceablePipeline: "kvm-general-purpose-load-balancing", + SchedulerURL: "http://localhost:8080/scheduler/nova/external", } } diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index 1e69b22cb..d3f5c8249 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -19,28 +19,98 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" schedulerapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" + "github.com/cobaltcore-dev/cortex/pkg/multicluster" ) var log = ctrl.Log.WithName("capacity-controller").WithValues("module", "capacity") -// Controller reconciles FlavorGroupCapacity CRDs on a fixed interval. -// For each AZ it probes all flavor groups, runs the round-robin capacity split, then writes -// one FlavorGroupCapacity CRD per (flavor group × AZ) pair. -type Controller struct { +// coalescedKey is the single reconcile key used for all watch events. +// All CRD changes are coalesced into this one key so rapid changes never +// cause rapid-fire scheduler probes. +const coalescedKey = "capacity" + +// flavorGroupsKnowledgePredicate fires only for the "flavor_groups" Knowledge object. +// Other Knowledge objects (different extractors) are irrelevant to capacity. +var flavorGroupsKnowledgePredicate = predicate.NewPredicateFuncs(func(obj client.Object) bool { + k, ok := obj.(*v1alpha1.Knowledge) + if !ok { + return false + } + return k.Spec.Extractor.Name == "flavor_groups" +}) + +// hvCapacityChangePredicate fires only when Allocation, EffectiveCapacity, or Capacity +// changed on a Hypervisor, and only for hypervisors that carry an AZ label. +// Label/annotation-only updates and unrelated status field changes are ignored. +var hvCapacityChangePredicate = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + _, hasAZ := e.Object.GetLabels()["topology.kubernetes.io/zone"] + return hasAZ + }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldAZ := e.ObjectOld.GetLabels()["topology.kubernetes.io/zone"] + newAZ := e.ObjectNew.GetLabels()["topology.kubernetes.io/zone"] + if newAZ == "" { + return false + } + if oldAZ != newAZ { + return true + } + oldHV, ok1 := e.ObjectOld.(*hv1.Hypervisor) + newHV, ok2 := e.ObjectNew.(*hv1.Hypervisor) + if !ok1 || !ok2 { + return false + } + return !capacityMapsEqual(oldHV.Status.Allocation, newHV.Status.Allocation) || + !capacityMapsEqual(oldHV.Status.EffectiveCapacity, newHV.Status.EffectiveCapacity) || + !capacityMapsEqual(oldHV.Status.Capacity, newHV.Status.Capacity) + }, + DeleteFunc: func(e event.DeleteEvent) bool { + _, hasAZ := e.Object.GetLabels()["topology.kubernetes.io/zone"] + return hasAZ + }, + GenericFunc: func(e event.GenericEvent) bool { return false }, +} + +// capacityMapsEqual returns true if two resource maps are equal by value. +func capacityMapsEqual(a, b map[hv1.ResourceName]resource.Quantity) bool { + if len(a) != len(b) { + return false + } + for k, va := range a { + vb, ok := b[k] + if !ok || va.Cmp(vb) != 0 { + return false + } + } + return true +} + +// Reconciler reconciles FlavorGroupCapacity CRDs, driven by both watch events and a +// periodic floor timer. All watch events are coalesced into a single reconcile key so +// rapid changes produce at most one queued reconcile. +type Reconciler struct { client client.Client vmSource reservations.VMSource schedulerClient *reservations.SchedulerClient config Config + + lastReconcileAt time.Time } -func NewController(c client.Client, config Config, vmSource reservations.VMSource) *Controller { - return &Controller{ +func NewController(c client.Client, config Config, vmSource reservations.VMSource) *Reconciler { + return &Reconciler{ client: c, vmSource: vmSource, schedulerClient: reservations.NewSchedulerClient(config.SchedulerURL), @@ -48,23 +118,66 @@ func NewController(c client.Client, config Config, vmSource reservations.VMSourc } } -// Start runs the periodic reconcile loop. Implements manager.Runnable. -func (c *Controller) Start(ctx context.Context) error { - timer := time.NewTimer(0) // fire immediately on start - defer timer.Stop() - - for { - select { - case <-ctx.Done(): - return nil - case <-timer.C: - cycleCtx := WithNewGlobalRequestID(ctx) - if err := c.reconcileAll(cycleCtx); err != nil { - LoggerFromContext(cycleCtx).Error(err, "reconcile cycle failed") - } - timer.Reset(c.config.ReconcileInterval.Duration) - } +// Reconcile implements reconcile.Reconciler. It is called by controller-runtime whenever a +// watched CRD changes, and also on the periodic RequeueAfter floor set by ReconcileInterval. +// If called sooner than MinReconcileInterval since the last successful run, it returns early. +func (c *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result, error) { + elapsed := time.Since(c.lastReconcileAt) + minInterval := c.config.MinReconcileInterval.Duration + + if !c.lastReconcileAt.IsZero() && elapsed < minInterval { + remaining := minInterval - elapsed + LoggerFromContext(ctx).V(1).Info("skipping reconcile: min interval not elapsed", + "elapsed", elapsed.Round(time.Second), + "remaining", remaining.Round(time.Second)) + return ctrl.Result{RequeueAfter: remaining}, nil + } + + cycleCtx := WithNewGlobalRequestID(ctx) + if err := c.reconcileAll(cycleCtx); err != nil { + LoggerFromContext(cycleCtx).Error(err, "reconcile cycle failed") + return ctrl.Result{}, err } + + c.lastReconcileAt = time.Now() + + return ctrl.Result{RequeueAfter: c.config.ReconcileInterval.Duration}, nil +} + +// SetupWithManager registers the reconciler with the controller manager and sets up watches +// on all CRDs that affect capacity output. All events are coalesced to a single key. +func (c *Reconciler) SetupWithManager(mgr ctrl.Manager, mcl *multicluster.Client) error { + log.Info("starting capacity reconciler", + "reconcileInterval", c.config.ReconcileInterval.Duration, + "minReconcileInterval", c.config.MinReconcileInterval.Duration) + + coalesce := func(_ context.Context, _ client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: coalescedKey}}} + } + + bldr := multicluster.BuildController(mcl, mgr) + var err error + + bldr, err = bldr.WatchesMulticluster(&v1alpha1.Knowledge{}, handler.EnqueueRequestsFromMapFunc(coalesce), flavorGroupsKnowledgePredicate) + if err != nil { + return fmt.Errorf("failed to watch Knowledge: %w", err) + } + bldr, err = bldr.WatchesMulticluster(&hv1.Hypervisor{}, handler.EnqueueRequestsFromMapFunc(coalesce), hvCapacityChangePredicate) + if err != nil { + return fmt.Errorf("failed to watch Hypervisor: %w", err) + } + bldr, err = bldr.WatchesMulticluster(&v1alpha1.Reservation{}, handler.EnqueueRequestsFromMapFunc(coalesce)) + if err != nil { + return fmt.Errorf("failed to watch Reservation: %w", err) + } + bldr, err = bldr.WatchesMulticluster(&v1alpha1.Pipeline{}, handler.EnqueueRequestsFromMapFunc(coalesce)) + if err != nil { + return fmt.Errorf("failed to watch Pipeline: %w", err) + } + + return bldr.Named("capacity"). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(c) } type vmUsageKey struct{ group, az string } @@ -79,7 +192,7 @@ type vmUsage struct { } // reconcileAll iterates all AZs, runs the round-robin split per AZ, then writes CRDs. -func (c *Controller) reconcileAll(ctx context.Context) error { +func (c *Reconciler) reconcileAll(ctx context.Context) error { logger := LoggerFromContext(ctx) startTime := time.Now() @@ -131,7 +244,7 @@ func (c *Controller) reconcileAll(ctx context.Context) error { // computeVMUsage fetches running VMs and aggregates usage per (flavorGroup, az). // On error returns an empty map with fresh=false — callers must not overwrite running fields. -func (c *Controller) computeVMUsage( +func (c *Reconciler) computeVMUsage( ctx context.Context, flavorGroups map[string]compute.FlavorGroupFeature, hvs []hv1.Hypervisor, @@ -233,7 +346,7 @@ func hvRemainingResources(hv hv1.Hypervisor, blockedMemBytes int64) map[string]i // reconcileAZ runs the round-robin capacity split for all flavor groups in one AZ, // then writes one FlavorGroupCapacity CRD per group that had all probes succeed. // Groups with failed probes are skipped — their CRDs retain the last good state. -func (c *Controller) reconcileAZ( +func (c *Reconciler) reconcileAZ( ctx context.Context, az string, flavorGroups map[string]compute.FlavorGroupFeature, @@ -414,7 +527,7 @@ func (c *Controller) reconcileAZ( } // writeCRD upserts one FlavorGroupCapacity CRD with fresh computed values. -func (c *Controller) writeCRD( +func (c *Reconciler) writeCRD( ctx context.Context, groupName string, groupData compute.FlavorGroupFeature, @@ -507,7 +620,7 @@ func (c *Controller) writeCRD( // probeScheduler calls the scheduler and returns slot count, host count, and candidate host names. // ignoreAllocations=true (total probe) uses raw effective capacity; false (placeable probe) subtracts allocations. -func (c *Controller) probeScheduler( +func (c *Reconciler) probeScheduler( ctx context.Context, flavor compute.FlavorInGroup, az, pipeline string, @@ -529,6 +642,16 @@ func (c *Controller) probeScheduler( eligibleHosts = append(eligibleHosts, schedulerapi.ExternalSchedulerHost{ComputeHost: name}) } + // Total probe ignores all reservation blocks (raw hardware capacity). + // Placeable probe counts reservations as capacity blocks. + var ignoredReservationTypes []v1alpha1.ReservationType + if ignoreAllocations { + ignoredReservationTypes = []v1alpha1.ReservationType{ + v1alpha1.ReservationTypeCommittedResource, + v1alpha1.ReservationTypeFailover, + } + } + resp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ InstanceUUID: "capacity-" + flavor.Name, ProjectID: "cortex-capacity-probe", @@ -539,8 +662,11 @@ func (c *Controller) probeScheduler( AvailabilityZone: az, Pipeline: pipeline, EligibleHosts: eligibleHosts, + SchedulerHints: map[string]any{"_nova_check_type": string(schedulerapi.CapacityProbeIntent)}, }, scheduling.Options{ ReadOnly: true, + AssumeEmptyHosts: ignoreAllocations, + IgnoredReservationTypes: ignoredReservationTypes, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true, @@ -586,7 +712,7 @@ func (c *Controller) probeScheduler( // blockedMemoryByHost returns total reservation-blocked bytes per host. // Both TargetHost and Status.Host are blocked; migration blocks both simultaneously. -func (c *Controller) blockedMemoryByHost(ctx context.Context) (map[string]int64, error) { +func (c *Reconciler) blockedMemoryByHost(ctx context.Context) (map[string]int64, error) { var list v1alpha1.ReservationList if err := c.client.List(ctx, &list); err != nil { return nil, fmt.Errorf("failed to list reservations: %w", err) @@ -622,7 +748,7 @@ func (c *Controller) blockedMemoryByHost(ctx context.Context) (map[string]int64, // sumCommittedCapacity sums active CommittedResource amounts (memory type, guaranteed/confirmed) // for the given (flavorGroup, az) pair. Returns the total in smallest-flavor slots. -func (c *Controller) sumCommittedCapacity(ctx context.Context, groupName, az string, smallestFlavorBytes int64) (int64, error) { +func (c *Reconciler) sumCommittedCapacity(ctx context.Context, groupName, az string, smallestFlavorBytes int64) (int64, error) { var list v1alpha1.CommittedResourceList if err := c.client.List(ctx, &list); err != nil { return 0, fmt.Errorf("failed to list CommittedResources: %w", err) diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index c33d98be4..0ba4b8f5c 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -11,12 +11,14 @@ import ( "regexp" "sort" "testing" + "time" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -111,8 +113,8 @@ func newMockSchedulerServer(t *testing.T, hosts []string) *httptest.Server { })) } -// newController is a test helper that creates a Controller with a nil VMSource. -func newController(t *testing.T, c client.Client, cfg Config) *Controller { +// newController is a test helper that creates a Reconciler with a nil VMSource. +func newController(t *testing.T, c client.Client, cfg Config) *Reconciler { t.Helper() return NewController(c, cfg, nil) } @@ -848,3 +850,149 @@ func TestProbeScheduler_SubtractsReservationBlocksWhenNotIgnored(t *testing.T) { t.Errorf("placeable capacity = %d, want 1 (3 slots − 1 alloc − 1 reservation)", placeableCap) } } + +func TestProbeScheduler_SetsCapacityProbeIntent(t *testing.T) { + scheme := newTestScheme(t) + hv := newHypervisor("host-1", "az-a", 4096*1024*1024) + + var capturedReq schedulerapi.ExternalSchedulerRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + json.NewEncoder(w).Encode(schedulerapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}}) //nolint:errcheck + })) + defer srv.Close() + + c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{SchedulerURL: srv.URL}, nil) + hvByName := map[string]hv1.Hypervisor{"host-1": *hv} + flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: 4096} + + if _, _, _, err := c.probeScheduler(context.Background(), flavor, "az-a", "test-pipeline", hvByName, true, nil); err != nil { + t.Fatalf("probeScheduler failed: %v", err) + } + hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") + if err != nil { + t.Fatalf("failed to get _nova_check_type hint: %v", err) + } + if hint != string(schedulerapi.CapacityProbeIntent) { + t.Errorf("capacity probe must set _nova_check_type=%q, got %q", schedulerapi.CapacityProbeIntent, hint) + } + if capturedReq.Spec.Data.ProjectID != "cortex-capacity-probe" { + t.Errorf("capacity probe must send ProjectID cortex-capacity-probe, got %q", capturedReq.Spec.Data.ProjectID) + } +} + +// TestReconcile_ReactsToKnowledgeChange verifies that Reconcile() runs reconcileAll() and +// writes FlavorGroupCapacity CRDs when triggered by a watch event (simulated here by calling +// Reconcile directly with the coalesced key). +func TestReconcile_ReactsToKnowledgeChange(t *testing.T) { + const ( + groupName = "hana-v2" + az = "qa-de-1a" + memMB = 4096 + memBytes = int64(memMB) * 1024 * 1024 + ) + + scheme := newTestScheme(t) + hv := newHypervisor("host-1", az, memBytes) + knowledge := newFlavorGroupKnowledge(t, groupName, memMB) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(knowledge, hv). + WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). + Build() + + schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) + defer schedulerServer.Close() + + r := NewController(fakeClient, Config{ + SchedulerURL: schedulerServer.URL, + TotalPipeline: "kvm-report-capacity", + PlaceablePipeline: "kvm-general-purpose", + ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, + MinReconcileInterval: metav1.Duration{Duration: 30 * time.Second}, + }, nil) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: coalescedKey}} + result, err := r.Reconcile(context.Background(), req) + if err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + if result.RequeueAfter != 5*time.Minute { + t.Errorf("RequeueAfter = %v, want 5m (periodic floor)", result.RequeueAfter) + } + + // reconcileAll should have written one CRD for the single (group × AZ) pair. + var list v1alpha1.FlavorGroupCapacityList + if err := fakeClient.List(context.Background(), &list); err != nil { + t.Fatalf("failed to list CRDs: %v", err) + } + if len(list.Items) != 1 { + t.Errorf("expected 1 FlavorGroupCapacity CRD after reactive reconcile, got %d", len(list.Items)) + } +} + +// TestReconcile_MinIntervalEarlyReturn verifies that a second Reconcile() call within +// MinReconcileInterval returns early (no reconcileAll) with RequeueAfter set to the +// remaining cooldown duration. +func TestReconcile_MinIntervalEarlyReturn(t *testing.T) { + const ( + groupName = "hana-v2" + az = "qa-de-1a" + memMB = 4096 + memBytes = int64(memMB) * 1024 * 1024 + ) + + scheme := newTestScheme(t) + hv := newHypervisor("host-1", az, memBytes) + knowledge := newFlavorGroupKnowledge(t, groupName, memMB) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(knowledge, hv). + WithStatusSubresource(&v1alpha1.FlavorGroupCapacity{}, &v1alpha1.Knowledge{}). + Build() + + schedulerServer := newMockSchedulerServer(t, []string{"host-1"}) + defer schedulerServer.Close() + + minInterval := 30 * time.Second + r := NewController(fakeClient, Config{ + SchedulerURL: schedulerServer.URL, + TotalPipeline: "kvm-report-capacity", + PlaceablePipeline: "kvm-general-purpose", + ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, + MinReconcileInterval: metav1.Duration{Duration: minInterval}, + }, nil) + + req := ctrl.Request{NamespacedName: types.NamespacedName{Name: coalescedKey}} + + // First call: should run reconcileAll and succeed. + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("first Reconcile returned error: %v", err) + } + + // Second call immediately after: should return early without running reconcileAll. + result, err := r.Reconcile(context.Background(), req) + if err != nil { + t.Fatalf("second Reconcile returned error: %v", err) + } + if result.RequeueAfter == 0 { + t.Fatal("second Reconcile: expected non-zero RequeueAfter (min interval early return), got 0") + } + if result.RequeueAfter > minInterval { + t.Errorf("second Reconcile: RequeueAfter %v exceeds MinReconcileInterval %v", result.RequeueAfter, minInterval) + } + + // Only one CRD should exist — the second call must not have triggered reconcileAll. + var list v1alpha1.FlavorGroupCapacityList + if err := fakeClient.List(context.Background(), &list); err != nil { + t.Fatalf("failed to list CRDs: %v", err) + } + if len(list.Items) != 1 { + t.Errorf("expected 1 CRD (only first reconcile ran), got %d", len(list.Items)) + } +} diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index 92fcc6246..ce968b4f4 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -1193,3 +1193,41 @@ func TestCRLifecycle(t *testing.T) { } }) } + +func TestCRScheduling_SetsReserveForCommittedResourceIntent(t *testing.T) { + // CR slot scheduling has a real project ID and must run tenant-context filters + // (filter_allowed_projects, filter_aggregate_metadata, etc.). Verify the intent is + // reserve_for_committed_resource so filters can identify the call type correctly. + var capturedReq schedulerdelegationapi.ExternalSchedulerRequest + var schedulerCalled bool + captureScheduler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + schedulerCalled = true + if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} + json.NewEncoder(w).Encode(resp) //nolint:errcheck + }) + + env := newIntgEnv(t, []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")}, captureScheduler, nil) + defer env.close() + + cr := intgCR("test-cr", "commit-uuid-1", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + env.reconcileCR(t, cr.Name) + env.reconcileChildReservations(t, cr.Name) + + if !schedulerCalled { + t.Fatal("scheduler was never called — test did not exercise the scheduling path") + } + hint, err := capturedReq.Spec.Data.GetSchedulerHintStr("_nova_check_type") + if err != nil { + t.Fatalf("failed to get _nova_check_type hint: %v", err) + } + if hint != string(schedulerdelegationapi.ReserveForCommittedResourceIntent) { + t.Errorf("CR slot scheduling must set _nova_check_type=%q, got %q", schedulerdelegationapi.ReserveForCommittedResourceIntent, hint) + } +} diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index a145ec676..b7ad34d5f 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -435,6 +435,22 @@ func TestIntegration(t *testing.T) { ExpectedMinRes: 1, // Both HANA VMs can share reservation on host3 UseTraitsFilter: true, }, + { + Name: "HANA VM uses kvm-hana-bin-packing pipeline (trait:CUSTOM_HANA_EXCLUSIVE_HOST=required)", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-hana-exclusive-1", Name: "vm-hana-exclusive-1", Active: true}}, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}), + newHypervisor("host2", 16, 32, 0, 0, nil, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}), + }, + VMs: []reservations.VM{ + newVMWithExtraSpecs("vm-hana-exclusive-1", "m1.hana", "project-A", "host1", 8192, 4, + map[string]string{"trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required"}), + }, + FlavorRequirements: map[string]int{"m1.hana": 1}, + ExpectedMinRes: 1, + ExpectedMaxRes: 1, + VerifyVMReservation: []string{"vm-hana-exclusive-1"}, + UseTraitsFilter: true, + }, } for _, tc := range testCases { @@ -1153,7 +1169,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h }, { ObjectMeta: metav1.ObjectMeta{ - Name: PipelineReuseFailoverReservation, + Name: PipelineNewFailoverReservation, }, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, @@ -1181,6 +1197,17 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h }, }, } + pipelines = append(pipelines, v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "kvm-hana-bin-packing"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{{Name: "kvm_failover_evacuation"}}, + }, + }) ctx := context.Background() for _, pipeline := range pipelines { @@ -1343,7 +1370,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, }, { ObjectMeta: metav1.ObjectMeta{ - Name: PipelineReuseFailoverReservation, + Name: PipelineNewFailoverReservation, }, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, @@ -1371,6 +1398,18 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, }, }, } + pipelines = append(pipelines, v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "kvm-hana-bin-packing"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{{Name: "kvm_failover_evacuation"}}, + }, + }) ctx := context.Background() for _, pipeline := range pipelines { diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 5f5903de9..efd3876f3 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -8,6 +8,7 @@ import ( "fmt" "slices" "sort" + "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" @@ -17,22 +18,29 @@ import ( // Pipeline names for failover reservation scheduling const ( - // PipelineReuseFailoverReservation is used to check if a VM can reuse an existing reservation. - // It validates host compatibility without checking capacity (since reservation already has capacity). - PipelineReuseFailoverReservation = "kvm-valid-host-reuse-failover-reservation" - // PipelineNewFailoverReservation is used to find a host for creating a new reservation. - // It validates host compatibility AND checks capacity. // Uses the general-purpose pipeline; LockReservations and SkipHistory are set via Options. PipelineNewFailoverReservation = "kvm-general-purpose-load-balancing" - - // PipelineAcknowledgeFailoverReservation is used to validate that a failover reservation - // is still valid for all its allocated VMs. It sends an evacuation-style scheduling request - // for each VM with only the reservation's host as the eligible target. - PipelineAcknowledgeFailoverReservation = "kvm-acknowledge-failover-reservation" ) -func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm reservations.VM, allHypervisors []string, pipeline string, resSpec resolvedReservationSpec, opts scheduling.Options) ([]string, error) { +// DefaultFailoverOptions is the base scheduling.Options for all failover scheduling calls. +// Per-call overrides (ReadOnly, LockReservations) are applied on top. +var DefaultFailoverOptions = scheduling.Options{ + SkipHistory: true, + SkipInflight: true, + SkipCommittedResourceTracking: true, +} + +// inferFailoverPipeline returns the standard pipeline for a failover scheduling call based on +// the VM's flavor extra specs — the same HANA vs general-purpose split used by Nova placement. +func inferFailoverPipeline(extraSpecs map[string]string) string { + if strings.ToLower(extraSpecs["trait:CUSTOM_HANA_EXCLUSIVE_HOST"]) == "required" { + return "kvm-hana-bin-packing" + } + return "kvm-general-purpose-load-balancing" +} + +func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm reservations.VM, allHypervisors []string, pipeline string, resSpec resolvedReservationSpec, intent v1alpha1.SchedulingIntent, opts scheduling.Options) ([]string, error) { logger := LoggerFromContext(ctx) // Build list of eligible hypervisors (excluding VM's current hypervisor) @@ -82,7 +90,7 @@ func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx contex Pipeline: pipeline, AvailabilityZone: vm.AvailabilityZone, SchedulerHints: map[string]any{ - "_nova_check_type": string(api.ReserveForFailoverIntent), + "_nova_check_type": string(intent), api.HintKeyResourceGroup: resSpec.ResourceGroup(vm.FlavorName), }, } @@ -123,7 +131,9 @@ func (c *FailoverReservationController) tryReuseExistingReservation( logger := LoggerFromContext(ctx) - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineReuseFailoverReservation, resSpec, scheduling.Options{ReadOnly: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + reuseOpts := DefaultFailoverOptions + reuseOpts.ReadOnly = true + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, inferFailoverPipeline(vm.FlavorExtraSpecs), resSpec, api.ReuseFailoverReservationIntent, reuseOpts) if err != nil { logger.Error(err, "failed to get potential hypervisors for VM", "vmUUID", vm.UUID) return nil @@ -213,7 +223,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( VCPUs: vcpus, EligibleHosts: []api.ExternalSchedulerHost{{ComputeHost: reservationHost}}, IgnoreHosts: []string{vm.CurrentHypervisor}, - Pipeline: PipelineAcknowledgeFailoverReservation, + Pipeline: inferFailoverPipeline(flavorExtraSpecs), AvailabilityZone: vm.AvailabilityZone, SchedulerHints: map[string]any{"_nova_check_type": string(api.EvacuateIntent)}, } @@ -222,9 +232,12 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( "vmUUID", vm.UUID, "reservationHost", reservationHost, "vmCurrentHost", vm.CurrentHypervisor, - "pipeline", PipelineAcknowledgeFailoverReservation) + "pipeline", scheduleReq.Pipeline) - resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, scheduling.Options{ReadOnly: true, LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + evacuateOpts := DefaultFailoverOptions + evacuateOpts.ReadOnly = true + evacuateOpts.LockReservations = true + resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, evacuateOpts) if err != nil { logger.Error(err, "failed to validate VM for reservation host", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, fmt.Errorf("failed to validate VM for reservation host: %w", err) @@ -266,7 +279,9 @@ func (c *FailoverReservationController) scheduleAndBuildNewFailoverReservation( // Get potential hypervisors from scheduler using the reservation spec resources // (which may be sized to the LargestFlavor from the flavor group) - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation, resSpec, scheduling.Options{LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + newResOpts := DefaultFailoverOptions + newResOpts.LockReservations = true + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation, resSpec, api.ReserveForFailoverIntent, newResOpts) if err != nil { return nil, fmt.Errorf("failed to get potential hypervisors for VM: %w", err) } diff --git a/internal/scheduling/reservations/failover/reservation_scheduling_test.go b/internal/scheduling/reservations/failover/reservation_scheduling_test.go index c80ef4bf9..6ae2ab820 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling_test.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling_test.go @@ -5,14 +5,20 @@ package failover import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "testing" + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) // ============================================================================ @@ -466,3 +472,103 @@ func buildSchedulingTestVMWithResources(uuid, hypervisor string, memoryMB, vcpus }, } } + +// captureSchedulerRequest spins up a test HTTP server that captures one scheduler request +// and returns a single host. The captured request is written into *out. +func captureSchedulerRequest(t *testing.T, host string, out *novaapi.ExternalSchedulerRequest) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(out); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp := novaapi.ExternalSchedulerResponse{Hosts: []string{host}} + _ = json.NewEncoder(w).Encode(resp) //nolint:errcheck + })) +} + +// buildMinimalController returns a FailoverReservationController wired to a scheduler at the given URL. +func buildMinimalController(t *testing.T, schedulerURL string) *FailoverReservationController { + t.Helper() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add v1alpha1 to scheme: %v", err) + } + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("add hv1 to scheme: %v", err) + } + k8sClient := fake.NewClientBuilder().WithScheme(scheme).Build() + schedulerClient := reservations.NewSchedulerClient(schedulerURL + "/scheduler/nova/external") + config := FailoverConfig{} + config.ApplyDefaults() + return NewFailoverReservationController(k8sClient, nil, config, schedulerClient, nil) +} + +// ============================================================================ +// Test: scheduling options sent to the scheduler +// ============================================================================ + +func TestFailoverSchedulerOptions(t *testing.T) { + vm := buildSchedulingTestVM("vm-1", "host-1") + vm.AvailabilityZone = "az1" + resolved := resolveVMSpecForScheduling(context.Background(), vm, false, nil) + reservation := buildSchedulingTestReservation("res-1", "host-2", nil) + reservation.Status.FailoverReservation = &v1alpha1.FailoverReservationStatus{Allocations: map[string]string{}} + + tests := []struct { + name string + call func(c *FailoverReservationController, ctx context.Context) + wantReadOnly bool + wantLockReservations bool + }{ + { + // Read-only compatibility check — must not write scheduling state. + // Tenant-context filters must run so only hosts the VM can actually reach are returned. + name: "tryReuseExistingReservation", + call: func(c *FailoverReservationController, ctx context.Context) { + _ = c.tryReuseExistingReservation(ctx, vm, []v1alpha1.Reservation{reservation}, []string{"host-1", "host-2"}, resolved) + }, + wantReadOnly: true, + wantLockReservations: false, + }, + { + // Validates whether a VM can land on the reservation host during evacuation. + // LockReservations ensures failover capacity is not unlocked for other VMs. + // Tenant-context filters (aggregate metadata, allowed projects, instance group) must run. + name: "validateVMViaSchedulerEvacuation", + call: func(c *FailoverReservationController, ctx context.Context) { + _, _ = c.validateVMViaSchedulerEvacuation(ctx, vm, "host-2") //nolint:errcheck + }, + wantReadOnly: true, + wantLockReservations: true, + }, + { + // Finds a host for a new reservation — writes state, so not read-only. + // LockReservations ensures existing reservations are treated as unavailable. + // Tenant-context filters must run. + name: "scheduleAndBuildNewFailoverReservation", + call: func(c *FailoverReservationController, ctx context.Context) { + _, _ = c.scheduleAndBuildNewFailoverReservation(ctx, vm, []string{"host-1", "host-2"}, nil, nil, resolved) //nolint:errcheck + }, + wantReadOnly: false, + wantLockReservations: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var capturedReq novaapi.ExternalSchedulerRequest + server := captureSchedulerRequest(t, "host-2", &capturedReq) + defer server.Close() + + tt.call(buildMinimalController(t, server.URL), context.Background()) + + if capturedReq.Options.ReadOnly != tt.wantReadOnly { + t.Errorf("ReadOnly = %v, want %v", capturedReq.Options.ReadOnly, tt.wantReadOnly) + } + if capturedReq.Options.LockReservations != tt.wantLockReservations { + t.Errorf("LockReservations = %v, want %v", capturedReq.Options.LockReservations, tt.wantLockReservations) + } + }) + } +} diff --git a/pkg/multicluster/client.go b/pkg/multicluster/client.go index fcaecaf93..a8e597c8d 100644 --- a/pkg/multicluster/client.go +++ b/pkg/multicluster/client.go @@ -163,6 +163,7 @@ func (c *Client) AddRemote(ctx context.Context, host, caCert string, insecureSki } cl, err := cluster.New(&restConfigCopy, func(o *cluster.Options) { o.Scheme = c.HomeScheme + o.Logger = ctrl.LoggerFrom(ctx).WithValues("host", host) }) if err != nil { return nil, err diff --git a/tools/plutono/Dockerfile b/tools/plutono/Dockerfile index 025d8be8a..403de7ec2 100644 --- a/tools/plutono/Dockerfile +++ b/tools/plutono/Dockerfile @@ -2,6 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # Plutono is a fork of Grafana 7.5.17 under the Apache 2.0 License. -FROM ghcr.io/credativ/plutono:v7.5.49 +FROM ghcr.io/credativ/plutono:v7.5.50 COPY provisioning /etc/plutono/provisioning COPY plutono.ini /etc/plutono/plutono.ini \ No newline at end of file