diff --git a/Tiltfile b/Tiltfile index e836cdbc5..bc0c1c407 100644 --- a/Tiltfile +++ b/Tiltfile @@ -47,6 +47,12 @@ if len(env_set_overrides) > 0: else: print("=== No CORTEX_ environment variables found ===") +region = os.getenv('OS_REGION_NAME') +if region: + print("=== Deriving region-scoped URLs from OS_REGION_NAME=" + region + " ===") + env_set_overrides.append('openstack.url=https://identity-3.' + region + '.cloud.sap/v3') + env_set_overrides.append('prometheus.url=https://metrics-internal.scaleout.' + region + '.cloud.sap/') + load('ext://helm_resource', 'helm_resource', 'helm_repo') helm_repo( 'Prometheus Community Helm Repo', diff --git a/api/v1alpha1/flavor_group_capacity_types.go b/api/v1alpha1/flavor_group_capacity_types.go index 7e9ee36c0..dbdb54277 100644 --- a/api/v1alpha1/flavor_group_capacity_types.go +++ b/api/v1alpha1/flavor_group_capacity_types.go @@ -113,11 +113,17 @@ type FlavorGroupCapacityStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Cluster -// +kubebuilder:printcolumn:name="FlavorGroup",type="string",JSONPath=".spec.flavorGroup" +// +kubebuilder:printcolumn:name="Group",type="string",JSONPath=".spec.flavorGroup" // +kubebuilder:printcolumn:name="AZ",type="string",JSONPath=".spec.availabilityZone" // +kubebuilder:printcolumn:name="Running",type="integer",JSONPath=".status.runningInstances" -// +kubebuilder:printcolumn:name="LastReconcile",type="date",JSONPath=".status.lastReconcileAt" +// +kubebuilder:printcolumn:name="Avail",type="integer",JSONPath=".status.exclusivelyFreeSlots" // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type=='Ready')].status" +// +kubebuilder:printcolumn:name="Reconciled",type="date",JSONPath=".status.lastReconcileAt" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",priority=1 +// +kubebuilder:printcolumn:name="Free_Mem",type="string",JSONPath=".status.freeCapacity.memory",priority=1 +// +kubebuilder:printcolumn:name="Excl_Mem",type="string",JSONPath=".status.exclusivelyFreeCapacity.memory",priority=1 +// +kubebuilder:printcolumn:name="Free_CPU",type="string",JSONPath=".status.freeCapacity.cores",priority=1 +// +kubebuilder:printcolumn:name="Excl_CPU",type="string",JSONPath=".status.exclusivelyFreeCapacity.cores",priority=1 // FlavorGroupCapacity caches pre-computed capacity data for one flavor group in one AZ. // One CRD exists per (flavor group × AZ) pair, updated by the capacity controller on a fixed interval. diff --git a/docs/reservations/committed-resource-reservations.md b/docs/reservations/committed-resource-reservations.md index 3775f34b2..4c3f448be 100644 --- a/docs/reservations/committed-resource-reservations.md +++ b/docs/reservations/committed-resource-reservations.md @@ -1,50 +1,34 @@ # Committed Resource Reservation System -Cortex reserves hypervisor capacity for customers who pre-commit resources (committed resources, CRs), and exposes usage and capacity data via APIs. +Cortex reserves hypervisor capacity for customers who pre-commit resources (committed resources, CRs), and exposes usage and capacity data to Limes via the LIQUID API. + +Implementation: `internal/scheduling/reservations/commitments/` - [Committed Resource Reservation System](#committed-resource-reservation-system) + - [Architecture Overview](#architecture-overview) + - [Limes State → Cortex Action](#limes-state--cortex-action) + - [Resource Types](#resource-types) + - [Commitment Lifecycle](#commitment-lifecycle) + - [Reservation Lifecycle](#reservation-lifecycle) + - [Capacity Blocking](#capacity-blocking) + - [InFlightReservation](#inflightreservation) + - [APIs](#apis) + - [Change-Commitments](#change-commitments) + - [Quota](#quota) + - [Report-Usage](#report-usage) + - [Report-Capacity](#report-capacity) + - [Capacity Reporting Reference](#capacity-reporting-reference) + - [FlavorGroupCapacity CRD — per-flavor fields](#flavorgroupcapacity-crd--per-flavor-fields) + - [FlavorGroupCapacity CRD — group-level fields](#flavorgroupcapacity-crd--group-level-fields) + - [Prometheus metrics](#prometheus-metrics) + - [Report-Capacity REST endpoint](#report-capacity-rest-endpoint) + - [Syncer Task](#syncer-task) + - [Placement Observability](#placement-observability) - [Configuration and Observability](#configuration-and-observability) - - [Lifecycle Management](#lifecycle-management) - - [State (CRDs)](#state-crds) - - [CR Commitment Lifecycle](#cr-commitment-lifecycle) - - [Resource types](#resource-types) - - [CommittedResource Controller](#committedresource-controller) - - [Reservation Lifecycle](#reservation-lifecycle) - - [VM Lifecycle](#vm-lifecycle) - - [Capacity Blocking](#capacity-blocking) - - [InFlightReservation](#inflightreservation) - - [Reservation Controller](#reservation-controller) - - [Info API](#info-api) - - [Change-Commitments API](#change-commitments-api) - - [Quota API](#quota-api) - - [Report-Usage API](#report-usage-api) - - [Report-Capacity API](#report-capacity-api) - - [Syncer Task](#syncer-task) - - [Placement Observability (CRS Evaluation)](#placement-observability-crs-evaluation) - -The CR reservation implementation is located in `internal/scheduling/reservations/commitments/`. Key components include: -- `CommittedResource` controller — acceptance, rejection, child Reservation CRUD (memory) or arithmetic headroom check (cores) -- `Reservation` controller — placement, VM allocation verification -- API endpoints (`api/`) -- Capacity and usage calculation logic -- Syncer for periodic state sync -## Configuration and Observability +## Architecture Overview -**Configuration**: Helm values for intervals, API flags, and pipeline configuration are defined in `helm/bundles/cortex-nova/values.yaml`. Key configuration includes: -- API endpoint toggles (change-commitments, report-usage, report-capacity) — each endpoint can be disabled independently -- Reconciliation intervals (grace period, active monitoring) -- Scheduling pipeline selection per flavor group -- Per-flavor-group resource flags (`handlesCommitments`, `hasCapacity`, `hasQuota`) controlling which resource types are active for each group - -**Metrics and Alerts**: Defined in `helm/bundles/cortex-nova/templates/alerts.yaml` with prefixes: -- `cortex_committed_resource_change_api_*` -- `cortex_committed_resource_usage_api_*` -- `cortex_committed_resource_capacity_api_*` - -## Lifecycle Management - -The system is organized around two CRD types and two controllers. `CommittedResource` CRDs represent customer commitments; `Reservation` CRDs represent individual hypervisor capacity slots. Each has its own controller with a well-defined responsibility boundary. +The system is organized around two CRD types and two controllers. `CommittedResource` CRDs represent customer commitments; `Reservation` CRDs represent individual hypervisor capacity slots held on behalf of a commitment. ```mermaid flowchart LR @@ -80,71 +64,30 @@ flowchart LR ResCtrl -->|update status| Res ``` -### State (CRDs) - -**`CommittedResource` CRD** — primary source of truth for a commitment accepted by Cortex. One CRD per commitment UUID. Spec holds the commitment identity (project, flavor group, resource type, amount, ...). Status holds the acceptance outcome (`Ready` condition with reason `Planned`/`Reserving`/`Rejected`/`Accepted`), the accepted amount, and usage fields populated by the usage reconciler: `AssignedInstances` (VM UUIDs deterministically assigned to this CR), `UsedResources` (total resource consumption of assigned VMs), `LastUsageReconcileAt`, and `UsageObservedGeneration`. - -**`Reservation` CRD** — a single reservation slot on a hypervisor, owned by a `CommittedResource`. One `CommittedResource` may drive multiple `Reservation` CRDs (one per flavor-sized slot). Only memory commitments create Reservation CRDs; cores commitments do not. See [./failover-reservations.md](./failover-reservations.md) for the failover reservation type. - -**`ProjectQuota` CRD** — per-project, per-AZ quota store. One CRD exists per (project × availability zone) pair, named `quota-{projectID}-{az}`. Written by the Quota API when Limes pushes quota (one CRD is created for each AZ in the request). The quota controller reconciles usage into the status: `TotalUsage` and `PaygUsage` are flat `map[string]int64` fields tracking per-resource consumption in that AZ. The controller watches CommittedResource and Hypervisor CRDs to maintain these values via periodic full reconciles, incremental HV diffs, and PaygUsage-only recomputes triggered by CommittedResource status changes. - -**`FlavorGroupCapacity` CRD** — per-flavor-group, per-AZ capacity snapshot maintained by the capacity controller (outside this subsystem). The Report-Capacity endpoint reads these to compute available capacity. - -### CR Commitment Lifecycle - -The CR commitment lifecycle covers everything from a commitment being accepted by Limes through to Cortex confirming or rejecting it. The `CommittedResource` CRD is the entry point; the `CommittedResource` controller owns the acceptance decision. - -**Limes state → Cortex action:** - -| Limes State | Meaning | Cortex action | -|---|---|---| -| `planned` | Future start, no guarantee yet | No capacity reserved | -| `pending` | Limes asking for a yes/no decision now | One-shot acceptance attempt — accept or reject; no retry | -| `guaranteed` / `confirmed` | Capacity must be honoured | Accept and keep in sync; see failure handling below | -| `superseded` / `expired` | Commitment no longer active | Release all held capacity | - -#### Resource types +`FlavorGroupCapacity` CRDs are maintained by the capacity controller (outside this subsystem) and read by the Report-Capacity endpoint. `ProjectQuota` CRDs are written by the Quota API and read by the Report-Usage endpoint. -Cortex handles two resource types for committed resources, with different acceptance mechanisms: +## Limes State → Cortex Action -**Memory (`_ram`)** — Cortex creates and manages `Reservation` CRDs on specific hypervisors. Acceptance means Cortex can place the required number of reservation slots via the scheduling pipeline. If placement is impossible (no hosts with enough free memory), the commitment is rejected or retried depending on the commitment state and `AllowRejection` flag. +| Limes State | Cortex action | +|---|---| +| `planned` | No capacity reserved | +| `pending` | One-shot acceptance attempt — accept or reject; no retry | +| `guaranteed` / `confirmed` | Accept and keep in sync; retry indefinitely unless `AllowRejection=true` | +| `superseded` / `expired` | Release all held capacity | -**CPU cores (`_cores`)** — No `Reservation` CRDs are created. Cortex checks whether sufficient CPU headroom exists by comparing the requested cores against the total CPU capacity for the flavor group and AZ (as reported by the `FlavorGroupCapacity` CRD) minus cores already committed by other active CRs. This is a lightweight arithmetic check that does not interact with the scheduling pipeline. +`AllowRejection` mirrors the request's `RequiresConfirmation` flag. When set, the controller rejects and rolls back on failure rather than retrying. On any rejection, capacity is rolled back to the last successfully accepted amount (or fully released if never accepted). -The two types share the same lifecycle states and the same acceptance/rejection semantics — they differ only in how capacity is verified and held. +## Resource Types -#### CommittedResource Controller +Cortex handles two resource types with different acceptance mechanisms: -The controller accepts or rejects commitments and keeps the allocated capacity in sync with what Limes expects. +**Memory (`_ram`)** — Cortex creates `Reservation` CRDs on specific hypervisors. Acceptance requires the scheduler to place the required slots. If placement fails, the commitment is rejected or retried based on its state and `AllowRejection`. -**`pending`** — Cortex is being asked for a yes/no answer. A single acceptance attempt is made. On failure, the commitment is rejected and all held capacity is released. No retry. - -**`guaranteed` / `confirmed`** — Cortex is expected to honour the commitment indefinitely. The default is to keep retrying on failure (`Ready=False, Reason=Reserving`). Callers that can tolerate rejection set `AllowRejection=true`; the controller then rejects on failure rather than retrying. - -**On rejection** — any capacity held for this CR is rolled back to the last successfully accepted amount (or fully released if never accepted). - -**Reconcile trigger flow:** - -```mermaid -sequenceDiagram - participant API as Change-Commitments API - participant CRCtrl as CR Controller - participant CRCRD as CommittedResource CRD - participant ResCRD as Reservation CRD - participant ResCtrl as Reservation Controller - - API->>CRCRD: write (create/update) - CRCRD-->>CRCtrl: watch fires - CRCtrl->>ResCRD: create/update child slots (memory only) - ResCRD-->>ResCtrl: watch fires - ResCtrl->>ResCRD: update (ObservedParentGeneration, Ready=True/False) - ResCRD-->>CRCtrl: watch fires (Reservation→parent CR lookup) - CRCtrl->>CRCRD: update status (Accepted / Reserving / Rejected) -``` +**CPU cores (`_cores`)** — No `Reservation` CRDs are created. Cortex does an arithmetic headroom check: requested cores vs. total CPU capacity for the flavor group and AZ (from `FlavorGroupCapacity`) minus cores already held by active CRs. Lightweight, no scheduler interaction. -For cores commitments the middle steps (Reservation CRUD, Reservation controller) are skipped — the CR controller updates the `CommittedResource` status directly after the arithmetic check. +The two types share lifecycle states and acceptance/rejection semantics — they differ only in how capacity is verified and held. -**CommittedResource status states:** +## Commitment Lifecycle ```mermaid stateDiagram-v2 @@ -170,22 +113,30 @@ stateDiagram-v2 Planned --> [*] : deleted ``` -### Reservation Lifecycle +The reconcile trigger chain for memory commitments: -*Applies to memory commitments only. Cores commitments do not create Reservations.* +```mermaid +sequenceDiagram + participant API as Change-Commitments API + participant CRCtrl as CR Controller + participant CRCRD as CommittedResource CRD + participant ResCRD as Reservation CRD + participant ResCtrl as Reservation Controller -| Component | Event | Timing | Action | -|-----------|-------|--------|--------| -| **Reservation Controller** | `Reservation` created | Immediate (watch) | Find host via scheduler API, set `TargetHost` | -| **Scheduling Pipeline** | VM Create, Migrate, Resize | Immediate | Add VM to `Spec.Allocations` | -| **Reservation Controller** | Reservation CRD updated | `committedResourceRequeueIntervalGracePeriod` (default: 1 min) | Defer verification for new VMs still spawning; update `Status.Allocations` | -| **Reservation Controller** | Hypervisor CRD updated (VM appeared/disappeared) | Immediate (event-driven) | Verify allocations via Hypervisor CRD; remove gone VMs from `Spec.Allocations` | -| **Reservation Controller** | Periodic safety-net | `committedResourceRequeueIntervalActive` (default: 5 min) | Same as above; catches any missed events | -| **Reservation Controller** | Optimize unused slots | >> minutes | Assign PAYG VMs or re-place reservations | + API->>CRCRD: write (create/update) + CRCRD-->>CRCtrl: watch fires + CRCtrl->>ResCRD: create/update child slots + ResCRD-->>ResCtrl: watch fires + ResCtrl->>ResCRD: update (Ready=True/False) + ResCRD-->>CRCtrl: watch fires + CRCtrl->>CRCRD: update status (Accepted / Reserving / Rejected) +``` -#### VM Lifecycle +## Reservation Lifecycle -VM allocations are tracked within reservations: +*Applies to memory commitments only.* + +A `Reservation` CRD represents one flavor-sized slot on a specific hypervisor. The Reservation controller uses the **Hypervisor CRD as the sole source of truth** for VM presence — no Nova API calls. ```mermaid flowchart LR @@ -201,177 +152,159 @@ flowchart LR C -->|update Spec/Status.Allocations| Res ``` -**Allocation fields**: -- `Spec.Allocations` — Expected VMs (written by the scheduling pipeline on placement) -- `Status.Allocations` — Confirmed VMs (written by the controller after verifying the VM is on the expected host) +VM allocation has two fields with distinct semantics: `Spec.Allocations` (expected — written by the scheduling pipeline) and `Status.Allocations` (confirmed — written by the controller after the VM is verified on the expected hypervisor). New VMs stay in `Spec` only during a grace period to allow for startup time. After the grace period, absence from the Hypervisor CRD removes the VM. -**VM allocation state diagram**: +When a VM is confirmed on a reservation for the first time, the controller proactively removes it from `Spec.Allocations` on all other candidate reservations. This frees phantom capacity blocks immediately rather than waiting for each candidate's grace period to expire. -The controller uses the **Hypervisor CRD** as the sole source of truth for VM allocation verification: +`MaxConcurrentReconciles=1` on the Reservation controller is intentional — parallel reconciles would allow concurrent placements to race and double-book a slot. -```mermaid -stateDiagram-v2 - direction LR - state "Spec only (grace period)" as SpecOnly - state "Spec + Status (on expected host)" as Confirmed - - [*] --> SpecOnly : placement (create, migrate, resize) - SpecOnly --> SpecOnly : within grace period - SpecOnly --> Confirmed : found on HV CRD after grace period - SpecOnly --> [*] : not on HV CRD after grace period - Confirmed --> [*] : not on HV CRD -``` - -**Candidate reservation cleanup**: When a VM is newly confirmed on a reservation (transitions from Spec-only to Spec+Status for the first time), the controller immediately removes that VM's UUID from `Spec.Allocations` on all other candidate reservations that still carry it. This proactive cleanup frees phantom capacity blocks on non-selected hosts immediately rather than waiting for each candidate reservation's own grace period expiry or periodic requeue to detect that the VM landed elsewhere. - -**Note**: VM allocations may not consume all resources of a reservation slot. A reservation with 128 GB may have VMs totaling only 96 GB if that fits the project's needs. Allocations may exceed reservation capacity (e.g., after VM resize). - -#### Capacity Blocking +### Capacity Blocking -**Blocking rules by allocation state:** +Each active Reservation blocks capacity on its target hypervisor so the scheduler cannot double-allocate. The block is recalculated on every reconcile: -| State | In HV Allocation? | Reservation must block? | -|---|---|---| -| No allocations | — | Full `Spec.Resources` | -| Confirmed (Spec + Status) | Yes — already subtracted | No — subtract from reservation block | -| Spec only (not yet running) | No — not yet on host | Yes — must remain in reservation block | - -**Formal calculation (stable state, `Spec.TargetHost == Status.Host`):** +**Stable state (`Spec.TargetHost == Status.Host`):** ``` -confirmed = sum of resources for VMs in both Spec.Allocations and Status.Allocations -spec_only_unblocked = sum of resources for VMs in Spec.Allocations only, NOT having an active pessimistic blocking reservation on this host +confirmed = resources of VMs in both Spec and Status allocations +spec_only_unblocked = resources of Spec-only VMs without an active InFlightReservation on this host remaining = max(0, Spec.Resources - confirmed) block = max(remaining, spec_only_unblocked) ``` -**Interaction with pessimistic blocking reservations:** - -When a VM is in flight (Nova choosing between candidates), a pessimistic blocking reservation exists on each candidate host. For any SpecOnly VM that has such a reservation on the same host, the pessimistic blocking reservation is the authority — the CR reservation must not double-count it. The `spec_only_unblocked` term excludes those VMs. - -See the [InFlightReservation](#inflightreservation) section below for how these reservations are managed. - -**Migration state (`Spec.TargetHost != Status.Host`):** +The `spec_only_unblocked` term exists because an InFlightReservation on the same host already blocks those resources pessimistically — the CR reservation must not double-count them. -When a reservation is being migrated to a new host, block the full `max(Spec.Resources, spec_only_unblocked)` on **both** hosts — no subtraction of confirmed VMs. VMs may be split across hosts mid-migration and the split is not reliably known from reservation data alone; conservatively blocking both hosts prevents overcommit during the transition. The over-blocking resolves once migration completes and `Spec.TargetHost == Status.Host` again. +**Migration state (`Spec.TargetHost != Status.Host`):** Block full `max(Spec.Resources, spec_only_unblocked)` on **both** hosts. VMs may be split across hosts mid-migration; conservative blocking on both prevents overcommit until migration completes. -**Corner cases:** +**Corner cases worth noting:** +- Confirmed VMs exceed reservation size (e.g. after resize): clamp `remaining` to 0, never negative +- Spec-only VM larger than remaining slot: block `spec_only_unblocked` — those resources will land when the VM starts +- Live migration within a reservation: handled implicitly by `hv.Status.Allocation`, which libvirt reports on both source and target during migration; no special logic needed -- **Confirmed VMs exceed reservation size** (e.g., after VM resize): `Spec.Resources - confirmed` goes negative. Clamp to `0` — otherwise the filter would add capacity back to the host. +### InFlightReservation -- **Spec-only VM larger than remaining reservation** (e.g., confirmed VMs have consumed most of the slot, and a new VM awaiting startup is larger than what remains): `remaining < spec_only_unblocked`. Block `spec_only_unblocked` — the VM will consume those resources when it starts, and they are not yet in HV Allocation. +A short-lived `InFlightReservation` CRD is created at the end of each VM placement run, one per candidate host returned to Nova. It pessimistically blocks capacity on every candidate while Nova decides where the VM lands — preventing a second concurrent placement from booking the same slot. -- **VM live migration within a reservation** (VM moves away from the reservation's host): handled implicitly by `hv.Status.Allocation`. Libvirt reports resource consumption on both source and target during live migration, so both hosts' `hv.Status.Allocation` already reflects the in-flight state. No special filter logic needed. The reservation controller will eventually remove the VM from the reservation once it's confirmed on the wrong host past the grace period. +Created by the scheduling pipeline; deleted once the VM is confirmed on a host or after a timeout. Skipped for non-VM-placement runs (reservation scheduling, capacity probes, failover — all set `SkipInflight`). -#### InFlightReservation +## APIs -An `InFlightReservation` is a short-lived Reservation CRD (type `InFlightReservation`) that pessimistically blocks capacity on each candidate host while a VM is being scheduled. It prevents double-booking when multiple scheduling decisions are in flight concurrently. +### Change-Commitments -**Lifecycle:** -- **Created** by the scheduling pipeline at the end of a successful placement run, one per candidate host returned to Nova. Creation is skipped when the `SkipInflight` pipeline option is set (used by reservation scheduling, capacity checks, and failover — any non-VM-placement run). -- **Deleted** once the VM has been confirmed on a host (the in-flight reservation is no longer needed) or after a timeout if the VM never lands. +`POST /commitments/v1/change-commitments` -**Spec fields** (`InFlightReservationSpec`): -- `VMID` — Nova server UUID of the VM being scheduled -- `UserID` — owner of the VM -- `ProjectID` — project/tenant of the VM -- `Intent` — lifecycle operation that triggered the placement (e.g., create, migrate, resize) +**Write-intent, watch-for-outcome**: the handler writes `CommittedResource` CRDs and polls their `Ready` condition until terminal. It does not interact with Reservation CRDs directly. -**Interaction with CR reservations:** When computing how much capacity a CR reservation must block, Spec-only VMs that already have an InFlightReservation on the same host are excluded from the CR reservation's block calculation (the `spec_only_unblocked` term). This avoids double-counting resources that are already blocked by the pessimistic InFlightReservation. +**All-or-nothing semantics**: if any commitment in a batch cannot be fulfilled, the entire request is rolled back. All modified CRDs are restored to their pre-request specs. -#### Reservation Controller +### Quota -The `Reservation` controller watches `Reservation` CRDs and `Hypervisor` CRDs. `MaxConcurrentReconciles=1` prevents overbooking during concurrent placements. +`PUT /commitments/v1/projects/:project_id/quota` -**Placement** — finds hosts for new reservations (calls scheduler API). Placement requests include a `domain_name` scheduler hint resolved from the reservation's `DomainID` via Keystone. This allows the `filter_external_customer` pipeline filter to enforce host restrictions for external customer domains. Domain name resolution uses an in-process cache that stores names indefinitely (domain names are immutable in OpenStack). If the Keystone integration is not configured (`keystoneSecretRef` absent), the hint is omitted and domain-based host restrictions are not enforced. +Persists Limes quota as `ProjectQuota` CRDs (one per project × AZ). The quota controller reconciles actual usage into each CRD's status. Writes are idempotent; concurrent writes are resolved with retry-on-conflict. -**Allocation Verification** — tracks VM lifecycle on reservations. The controller uses the Hypervisor CRD as the sole source of truth, with two triggers: -- New VMs (within `committedResourceAllocationGracePeriod`, default: 15 min): verification deferred — VM may still be spawning; requeued every `committedResourceRequeueIntervalGracePeriod` (default: 1 min) -- Established VMs: verified reactively when the Hypervisor CRD changes (VM appeared or disappeared in `Status.Instances`), with `committedResourceRequeueIntervalActive` (default: 5 min) as a safety-net fallback -- Missing unconfirmed VMs (in `Spec.Allocations` only): removed from `Spec.Allocations` when not found on the Hypervisor CRD after the grace period -- Missing confirmed VMs (already present in `Status.Allocations`): bypass the grace period entirely — their disappearance from the Hypervisor CRD is treated as authoritative and they are removed immediately +### Report-Usage -**Reservation migration is not supported yet.** +`POST /commitments/v1/projects/:project_id/report-usage` -### Info API +Reports current usage per flavor group (ram, cores, instances). VM-to-commitment assignment is **pre-computed** by a background usage reconciler that writes into `CommittedResource.Status` — it is not calculated inline at request time. This assignment is deterministic but may differ from Cortex's internal scheduling assignment. -`GET /commitments/v1/info` — describes the full service to Limes: which flavor groups are active, what resource types each group exposes (ram, cores, instances), their units, LIQUID topologies, and whether each accepts commitments. +For flavor groups with `HandlesCommitments=true`, the response includes per-AZ quota from `ProjectQuota` CRDs. -- RAM resources with `HandlesCommitments=true` use `AZSeparatedTopology` — Limes treats quota as AZ-specific and sends per-AZ breakdowns in quota requests. -- All other resources (cores, instances, and RAM without commitments) use `AZAwareTopology` — no per-AZ quota. +### Report-Capacity -Limes calls this endpoint once on startup and whenever the service description changes. +`POST /commitments/v1/report-capacity` -### Change-Commitments API +Reports available capacity per flavor group and AZ, read from pre-computed `FlavorGroupCapacity` CRDs. If a CRD's `Ready` condition is stale, usage is omitted from the response (capacity is still reported) to avoid underreporting during a controller outage. -The change-commitments API receives batched commitment changes from Limes and applies them using a **write-intent, watch-for-outcome** pattern: the handler creates or updates `CommittedResource` CRDs and polls their `Status.Conditions` until each reaches a terminal state — it does not interact with `Reservation` CRDs directly. +### Capacity Reporting Reference -**Request Semantics**: A request can contain multiple commitment changes across different projects and flavor groups. The semantic is **all-or-nothing** — if any commitment in the batch cannot be fulfilled (e.g., insufficient capacity), the entire request is rejected and rolled back. +This section maps every reporting surface to the values it exposes, the resource dimensions it considers, and the cluster state it reflects. -**Operations**: -1. For each commitment in the batch, create or update a `CommittedResource` CRD. `Spec.AllowRejection` mirrors the request's `RequiresConfirmation` flag: `true` for changes where Limes needs a yes/no answer (new commitments, resizes), `false` for non-confirming changes (deletions, status-only transitions) where Limes doesn't act on the rejection reason -2. Poll `CommittedResource.Status.Conditions[Ready]` until each reaches a terminal state: `Reason=Accepted` (success), `Reason=Planned` (deferred; accepted), or `Reason=Rejected` (failure) — only for confirming changes; non-confirming changes return immediately without polling -3. On any failure or timeout, restore all modified `CommittedResource` CRDs to their pre-request specs (or delete newly-created ones) +#### FlavorGroupCapacity CRD — per-flavor fields -The `CommittedResource` controller handles all downstream work. `AllowRejection=true` tells it to reject and roll back on failure rather than retrying indefinitely. +| Field | Dimensions | Cluster state | Notes | +|---|---|---|---| +| `TotalCapacityVMSlots` | Min(memory, CPU) | Empty datacenter | All reservation types ignored; competing groups not subtracted | +| `TotalCapacityHosts` | Min(memory, CPU) | Empty datacenter | Host count for `TotalCapacityVMSlots` | +| `PlaceableVMs` | Min(memory, CPU) | Current + reservations | If this flavor consumed all remaining capacity; competing groups not subtracted | +| `PlaceableHosts` | Min(memory, CPU) | Current + reservations | Host count for `PlaceableVMs` | -### Quota API +#### FlavorGroupCapacity CRD — group-level fields -`PUT /commitments/v1/projects/:project_id/quota` — receives the project's quota allocation from Limes and persists it as `ProjectQuota` CRDs, one per (project × availability zone) combination, named `quota-{projectID}-{az}`. For flavor groups with `HandlesCommitments=true`, Limes sends per-AZ quota breakdowns; each AZ gets its own CRD with a flat `Quota map[string]int64` holding per-resource quota values for that zone. The quota controller then reconciles usage into each CRD's status (`TotalUsage`, `PaygUsage`). Writes are idempotent; concurrent writes are resolved with retry-on-conflict. +| Field | Dimensions | Cluster state | Notes | +|---|---|---|---| +| `FreeCapacity` | Memory + Cores (separate) | Current + reservations | Raw sum across candidate hosts; may double-count across groups sharing hosts | +| `ExclusivelyFreeCapacity` | Memory + Cores (separate) | Current + reservations | Round-robin split result — sum across all groups never exceeds installed capacity | +| `ExclusivelyFreeSlots` | Min(memory, CPU) → Memory | Current + reservations | `ExclusivelyFreeCapacity[memory] / smallestFlavorMemBytes`; the memory pool is CPU-gated: the round-robin excludes hosts where the flavor doesn't fit on CPU before summing bytes | +| `TotalCapacity` | Memory + Cores (separate) | Empty datacenter | `max(TotalCapacityVMSlots × flavorResources)` over all flavors in the group | +| `CommittedCapacity` | Memory (slot units) | — | Active CR accepted amounts in smallest-flavor slot units | +| `RunningInstances` / `RunningResources` | Memory + Cores | — | Actual running VMs in this group × AZ | -### Report-Usage API +#### Prometheus metrics -`POST /commitments/v1/projects/:project_id/report-usage` — reports current resource usage for a project. +All metrics carry `flavor_group` and `az` labels; per-flavor metrics additionally carry `flavor_name`. -For each flavor group `X` that accepts commitments, Cortex exposes three resource types: -- `hw_version_X_ram` — RAM in units of the smallest flavor in the group (`HandlesCommitments=true`) -- `hw_version_X_cores` — CPU cores (`HandlesCommitments=false`; derived from RAM via fixed ratio where applicable) -- `hw_version_X_instances` — instance count (`HandlesCommitments=false`) +| Metric suffix | Source field | Dimensions | Cluster state | +|---|---|---|---| +| `_vm_slots_empty_datacenter` | `TotalCapacityVMSlots` | Min(memory, CPU) | Empty datacenter | +| `_vm_slots_placeable` | `PlaceableVMs` | Min(memory, CPU) | Current + reservations | +| `_hosts_empty_datacenter` | `TotalCapacityHosts` | Min(memory, CPU) | Empty datacenter | +| `_hosts_placeable` | `PlaceableHosts` | Min(memory, CPU) | Current + reservations | +| `_free_capacity_gib` | `FreeCapacity[memory]` | Memory only | Current + reservations — may overlap across groups | +| `_exclusively_free_capacity_gib` | `ExclusivelyFreeCapacity[memory]` | Memory only | Current + reservations | +| `_exclusively_free_slots` | `ExclusivelyFreeSlots` | Min(memory, CPU) → Memory | Current + reservations | +| `_committed_gib` | `CommittedCapacityBytes` | Memory | — | +| `_committed_reservations` | `CommittedCapacity` | Memory (slot units) | — | +| `_running_instances` | `RunningInstances` | — | — | -For flavor groups with `HandlesCommitments=true`, the response includes per-AZ quota from the `ProjectQuota` CRDs (written by the Quota API). +#### Report-Capacity REST endpoint -VM-to-commitment assignment is read from pre-computed `CommittedResource.Status` fields rather than being calculated inline at request time. A dedicated **usage reconciler** (in `internal/scheduling/reservations/commitments/usage_reconciler.go`) watches `CommittedResource` and `Hypervisor` CRDs and periodically runs the deterministic assignment algorithm, writing `AssignedInstances`, `UsedResources`, `LastUsageReconcileAt`, and `UsageObservedGeneration` into each CommittedResource's status. The Report-Usage endpoint reads these status fields to determine which VMs belong to which commitment. If a CR has not yet been reconciled, its VMs appear as PAYG until the first usage reconcile completes. +Capacity and usage are derived from `FlavorGroupCapacity` CRDs and reported per AZ for three resource types per group: -For each VM, the API reports whether it accounts to a specific commitment or PAYG. This assignment is deterministic and may differ from the actual Cortex internal assignment used for scheduling. +| Resource | Capacity formula | Usage formula | Notes | +|---|---|---|---| +| `_instances` | `runningInstances + ExclusivelyFreeSlots` | `runningInstances` | `ExclusivelyFreeSlots` is CPU-and-memory-gated (round-robin), final slot count via memory division | +| `_ram` (fixed core ratio) | same as `_instances` | `runningInstances` | Slot count stands in for RAM | +| `_ram` (variable) | `(runningMemBytes + ExclusivelyFreeCapacity[memory]) / ramUnitBytes` | `runningMemBytes / ramUnitBytes` | Both in declared units (e.g. GiB); `ramUnitBytes` configured per group | +| `_cores` | `runningCoresCount + ExclusivelyFreeCapacity[cores]` | `runningCoresCount` | CPU-dimension-driven | -### Report-Capacity API +## Syncer Task -`POST /commitments/v1/report-capacity` — reports available hypervisor capacity per flavor group and AZ. Capacity data is pre-computed by the capacity controller and stored in `FlavorGroupCapacity` CRDs; the endpoint aggregates these per-AZ values into the response. If a `FlavorGroupCapacity` CRD is stale (controller behind), the endpoint reports total capacity without subtracting usage to avoid underreporting. +Runs periodically and reconciles local `CommittedResource` CRD state against Limes' view, correcting drift from missed API calls or restarts. Writes `CommittedResource` CRDs only — capacity management remains the controller's responsibility. -### Syncer Task +## Placement Observability -The syncer task runs periodically and syncs local `CommittedResource` CRD state to match Limes' view of commitments, correcting drift from missed API calls or restarts. It writes `CommittedResource` CRDs only — capacity management is the controller's responsibility. - -### Placement Observability (CRS Evaluation) - -The `internal/scheduling/nova/crs/` package provides post-placement classification and Prometheus metrics for committed resource slot utilization. It answers the question: "For each VM placement (or no-host-found failure), what was the CR slot situation?" - -**Prometheus metrics:** +The `internal/scheduling/nova/crs/` package classifies every placement decision by CR slot coverage and emits Prometheus metrics. This answers: "For each VM placement or no-host-found, what was the CR slot situation?" | Metric | Labels | Description | |--------|--------|-------------| | `cortex_nova_no_host_found_total` | `cr_slot`, `flavor_group`, `intent` | No-host-found results classified by CR coverage | | `cortex_nova_placement_total` | `flavor_group`, `intent`, `cr_slot` | Successful placements classified by CR slot outcome | -PAYG placements (flavor not in any configured group) are not counted by either metric. +PAYG placements (flavor not in any configured group) are not counted. -**No-host-found classification (`cr_slot` label on `cortex_nova_no_host_found_total`):** +**`cr_slot` values for no-host-found:** -| Category | Meaning | -|----------|---------| +| Value | Meaning | +|---|---| | `no_cr` | Project has no active CommittedResources for the flavor group | -| `cr_exhausted` | CommittedResources exist but are fully occupied (used >= capacity) | -| `slot_exhausted` | CR has remaining capacity but no input host has a usable reservation slot | -| `slot_blocked` | A usable slot exists on an input host but scheduling constraints excluded all such hosts | +| `cr_exhausted` | CommittedResources exist but are fully occupied | +| `slot_exhausted` | CR has remaining capacity but no candidate host has a usable reservation slot | +| `slot_blocked` | A usable slot exists but scheduling constraints excluded all such hosts | -**Placement classification (`cr_slot` label on `cortex_nova_placement_total`):** +**`cr_slot` values for successful placements:** -| Category | Meaning | -|----------|---------| +| Value | Meaning | +|---|---| | `no_cr` | No active CR or CR capacity fully exhausted | | `slot_missed` | CR has remaining capacity but no candidate host has a slot with remaining memory > 0 | | `slot_used` | CR has remaining capacity and at least one candidate host has a usable slot | -**Slot evaluator:** The `SlotEvaluator` is built once per scheduling request from Hypervisor and Reservation CRDs (no further K8s reads during classification). It computes per-host free memory and indexes ready CR reservation slots by host. `HasUsableSlot` checks whether a host has a slot that can accommodate the VM under the overfill model: `slot.remaining + host.base_free >= vmMemBytes`. +## Configuration and Observability + +**Configuration**: `helm/bundles/cortex-nova/values.yaml` — API endpoint toggles, reconciliation intervals, scheduling pipeline selection, and per-flavor-group resource flags. -**Recorder:** The `Recorder` is called after each placement decision. On success (`slot_used`), it writes the VM UUID into the best-fit reservation slot (`PickSlot` selects the slot that maximises coverage with tightest-fit tiebreaking). On no-host-found, it classifies the failure and increments the counter. +**Metrics and Alerts**: `helm/bundles/cortex-nova/templates/alerts.yaml`, prefixes: +- `cortex_committed_resource_change_api_*` +- `cortex_committed_resource_usage_api_*` +- `cortex_committed_resource_capacity_api_*` diff --git a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml index 952e15722..4102ea447 100644 --- a/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml +++ b/helm/library/cortex/files/crds/cortex.cloud_flavorgroupcapacities.yaml @@ -16,7 +16,7 @@ spec: versions: - additionalPrinterColumns: - jsonPath: .spec.flavorGroup - name: FlavorGroup + name: Group type: string - jsonPath: .spec.availabilityZone name: AZ @@ -24,12 +24,35 @@ spec: - jsonPath: .status.runningInstances name: Running type: integer - - jsonPath: .status.lastReconcileAt - name: LastReconcile - type: date + - jsonPath: .status.exclusivelyFreeSlots + name: Avail + type: integer - jsonPath: .status.conditions[?(@.type=='Ready')].status name: Ready type: string + - jsonPath: .status.lastReconcileAt + name: Reconciled + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + priority: 1 + type: date + - jsonPath: .status.freeCapacity.memory + name: Free_Mem + priority: 1 + type: string + - jsonPath: .status.exclusivelyFreeCapacity.memory + name: Excl_Mem + priority: 1 + type: string + - jsonPath: .status.freeCapacity.cores + name: Free_CPU + priority: 1 + type: string + - jsonPath: .status.exclusivelyFreeCapacity.cores + name: Excl_CPU + priority: 1 + type: string name: v1alpha1 schema: openAPIV3Schema: diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index d3f5c8249..64a87befe 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -31,6 +31,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/pkg/multicluster" + "github.com/go-logr/logr" ) var log = ctrl.Log.WithName("capacity-controller").WithValues("module", "capacity") @@ -191,6 +192,29 @@ type vmUsage struct { fresh bool } +// probeGroupResult holds the outcome of probing all flavors in one (group × AZ). +type probeGroupResult struct { + groupName string + groupData compute.FlavorGroupFeature + flavors []v1alpha1.FlavorCapacityStatus + // allFresh is false if any scheduler probe failed; the group's CRD is left unchanged. + allFresh bool + smallestCandidates []string + committedCapacity int64 +} + +// flavorSlots returns the number of VM slots a resource map can fit for the given flavor. +// It is the binding constraint across both memory and CPU: min(memSlots, cpuSlots). +func flavorSlots(resources map[string]int64, flavorMemBytes, flavorVCPUs int64) int64 { + slots := resources[ResourceMemory] / flavorMemBytes + if flavorVCPUs > 0 { + if cpuSlots := resources[ResourceCores] / flavorVCPUs; cpuSlots < slots { + slots = cpuSlots + } + } + return slots +} + // reconcileAll iterates all AZs, runs the round-robin split per AZ, then writes CRDs. func (c *Reconciler) reconcileAll(ctx context.Context) error { logger := LoggerFromContext(ctx) @@ -222,22 +246,14 @@ func (c *Reconciler) reconcileAll(ctx context.Context) error { usageByKey := c.computeVMUsage(ctx, flavorGroups, hvList.Items) - var succeeded, failed int for _, az := range azs { - if err := c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, usageByKey); err != nil { - logger.Error(err, "failed to reconcile AZ", "az", az) - failed++ - continue - } - succeeded += len(flavorGroups) + c.reconcileAZ(ctx, az, flavorGroups, hvByName, blockedByReservations, usageByKey) } logger.Info("capacity reconcile cycle completed", "flavorGroups", len(flavorGroups), "availabilityZones", len(azs), "hypervisors", len(hvList.Items), - "succeeded", succeeded, - "failed", failed, "duration", time.Since(startTime).String()) return nil } @@ -343,116 +359,100 @@ func hvRemainingResources(hv hv1.Hypervisor, blockedMemBytes int64) map[string]i return result } -// reconcileAZ runs the round-robin capacity split for all flavor groups in one AZ, -// then writes one FlavorGroupCapacity CRD per group that had all probes succeed. -// Groups with failed probes are skipped — their CRDs retain the last good state. -func (c *Reconciler) reconcileAZ( +// probeGroup probes all flavors in a single group for one AZ and returns the result. +// It preserves stale per-flavor values from the existing CRD on individual probe failures. +func (c *Reconciler) probeGroup( ctx context.Context, + groupName string, + groupData compute.FlavorGroupFeature, az string, - flavorGroups map[string]compute.FlavorGroupFeature, hvByName map[string]hv1.Hypervisor, blockedByReservations map[string]int64, - usageByKey map[vmUsageKey]vmUsage, -) error { +) (probeGroupResult, error) { logger := LoggerFromContext(ctx) - type probeResult struct { - groupName string - groupData compute.FlavorGroupFeature - flavors []v1alpha1.FlavorCapacityStatus - // allFresh is false if any scheduler probe failed; the group's CRD is left unchanged. - allFresh bool - smallestCandidates []string - committedCapacity int64 + smallestFlavorBytes := int64(groupData.SmallestFlavor.MemoryMB) * 1024 * 1024 //nolint:gosec + if smallestFlavorBytes <= 0 { + return probeGroupResult{}, fmt.Errorf("smallest flavor %q has invalid memory %d MB", + groupData.SmallestFlavor.Name, groupData.SmallestFlavor.MemoryMB) } - results := make([]probeResult, 0, len(flavorGroups)) - - groupNames := make([]string, 0, len(flavorGroups)) - for name := range flavorGroups { - groupNames = append(groupNames, name) + // Load existing per-flavor data to preserve stale values on probe failure. + crdName := crdNameFor(groupName, az) + var existing v1alpha1.FlavorGroupCapacity + if err := c.client.Get(ctx, types.NamespacedName{Name: crdName}, &existing); err != nil && !apierrors.IsNotFound(err) { + return probeGroupResult{}, fmt.Errorf("failed to get FlavorGroupCapacity %s: %w", crdName, err) + } + existingByName := make(map[string]v1alpha1.FlavorCapacityStatus, len(existing.Status.Flavors)) + for _, f := range existing.Status.Flavors { + existingByName[f.FlavorName] = f } - sort.Strings(groupNames) - for _, groupName := range groupNames { - groupData := flavorGroups[groupName] + // Probe all flavors. Sort for stable CRD output. + flavors := make([]compute.FlavorInGroup, len(groupData.Flavors)) + copy(flavors, groupData.Flavors) + sort.Slice(flavors, func(i, j int) bool { return flavors[i].Name < flavors[j].Name }) - smallestFlavorBytes := int64(groupData.SmallestFlavor.MemoryMB) * 1024 * 1024 //nolint:gosec - if smallestFlavorBytes <= 0 { - logger.Error(fmt.Errorf("smallest flavor %q has invalid memory %d MB", - groupData.SmallestFlavor.Name, groupData.SmallestFlavor.MemoryMB), - "skipping flavor group", "flavorGroup", groupName) - continue - } + allFresh := true + newFlavors := make([]v1alpha1.FlavorCapacityStatus, 0, len(flavors)) + var smallestCandidates []string - // Probe all flavors. Sort for stable CRD output. - flavors := make([]compute.FlavorInGroup, len(groupData.Flavors)) - copy(flavors, groupData.Flavors) - sort.Slice(flavors, func(i, j int) bool { return flavors[i].Name < flavors[j].Name }) + for _, flavor := range flavors { + cur := existingByName[flavor.Name] + cur.FlavorName = flavor.Name - allFresh := true - newFlavors := make([]v1alpha1.FlavorCapacityStatus, 0, len(flavors)) + totalVMSlots, totalHosts, _, totalErr := c.probeScheduler(ctx, flavor, az, c.config.TotalPipeline, hvByName, true, nil) + placeableVMs, placeableHosts, candidates, placeableErr := c.probeScheduler(ctx, flavor, az, c.config.PlaceablePipeline, hvByName, false, blockedByReservations) - // Load existing per-flavor data to preserve stale values on probe failure. - crdName := crdNameFor(groupName, az) - var existing v1alpha1.FlavorGroupCapacity - if err := c.client.Get(ctx, types.NamespacedName{Name: crdName}, &existing); err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to get FlavorGroupCapacity %s: %w", crdName, err) + if totalErr != nil { + allFresh = false + } else { + cur.TotalCapacityVMSlots = totalVMSlots + cur.TotalCapacityHosts = totalHosts } - existingByName := make(map[string]v1alpha1.FlavorCapacityStatus, len(existing.Status.Flavors)) - for _, f := range existing.Status.Flavors { - existingByName[f.FlavorName] = f + if placeableErr != nil { + allFresh = false + } else { + cur.PlaceableVMs = placeableVMs + cur.PlaceableHosts = placeableHosts } + // Capture candidates for the smallest flavor — used as split inputs. + if flavor.Name == groupData.SmallestFlavor.Name && placeableErr == nil { + smallestCandidates = candidates + } + newFlavors = append(newFlavors, cur) + } - var smallestCandidates []string - for _, flavor := range flavors { - cur := existingByName[flavor.Name] - cur.FlavorName = flavor.Name - - totalVMSlots, totalHosts, _, totalErr := c.probeScheduler(ctx, flavor, az, c.config.TotalPipeline, hvByName, true, nil) - placeableVMs, placeableHosts, candidates, placeableErr := c.probeScheduler(ctx, flavor, az, c.config.PlaceablePipeline, hvByName, false, blockedByReservations) + committedCapacity, committedErr := c.sumCommittedCapacity(ctx, groupName, az, smallestFlavorBytes) + if committedErr != nil { + logger.Error(committedErr, "failed to sum committed capacity", "flavorGroup", groupName, "az", az) + committedCapacity = 0 + } - if totalErr != nil { - allFresh = false - } else { - cur.TotalCapacityVMSlots = totalVMSlots - cur.TotalCapacityHosts = totalHosts - } - if placeableErr != nil { - allFresh = false - } else { - cur.PlaceableVMs = placeableVMs - cur.PlaceableHosts = placeableHosts - } - // Capture candidates for the smallest flavor — used as split inputs. - if flavor.Name == groupData.SmallestFlavor.Name && placeableErr == nil { - smallestCandidates = candidates - } - newFlavors = append(newFlavors, cur) - } + return probeGroupResult{ + groupName: groupName, + groupData: groupData, + flavors: newFlavors, + allFresh: allFresh, + smallestCandidates: smallestCandidates, + committedCapacity: committedCapacity, + }, nil +} - committedCapacity, committedErr := c.sumCommittedCapacity(ctx, groupName, az, smallestFlavorBytes) - if committedErr != nil { - logger.Error(committedErr, "failed to sum committed capacity", - "flavorGroup", groupName, "az", az) - committedCapacity = 0 - } +// buildSplitInputs constructs the HostState map and GroupInput slice needed by SplitCapacity. +// Only groups where all probes succeeded are included. +func buildSplitInputs( + results []probeGroupResult, + hvByName map[string]hv1.Hypervisor, + blockedByReservations map[string]int64, + az string, + logger logr.Logger, +) (groupInputs []GroupInput, hosts map[string]HostState) { - results = append(results, probeResult{ - groupName: groupName, - groupData: groupData, - flavors: newFlavors, - allFresh: allFresh, - smallestCandidates: smallestCandidates, - committedCapacity: committedCapacity, - }) - } + hosts = make(map[string]HostState) + groupInputs = make([]GroupInput, 0, len(results)) - // Build HostState and GroupInput for the round-robin split. - // Only include groups where all probes succeeded. - hosts := make(map[string]HostState) - groupInputs := make([]GroupInput, 0, len(results)) for _, r := range results { if !r.allFresh || r.smallestCandidates == nil { continue @@ -469,22 +469,17 @@ func (c *Reconciler) reconcileAZ( continue } remaining := hvRemainingResources(hv, blockedByReservations[h]) - if remaining != nil { - hosts[h] = HostState{Remaining: remaining} - memSlots := remaining[ResourceMemory] / flavorMemBytes - cpuSlots := remaining[ResourceCores] / flavorVCPUs - usableSlots := memSlots - if cpuSlots < usableSlots { - usableSlots = cpuSlots - } - strandedMem := remaining[ResourceMemory] - usableSlots*flavorMemBytes - strandedCPU := remaining[ResourceCores] - usableSlots*flavorVCPUs - logger.V(1).Info("candidate host for capacity split", - "az", az, "flavorGroup", r.groupName, "host", h, - "usableSlots", usableSlots, - "strandedMemoryGiB", strandedMem/(1024*1024*1024), - "strandedCores", strandedCPU) + if remaining == nil { + continue } + hosts[h] = HostState{Remaining: remaining} + usableSlots := flavorSlots(remaining, flavorMemBytes, flavorVCPUs) + strandedMem := remaining[ResourceMemory] - usableSlots*flavorMemBytes + strandedCPU := remaining[ResourceCores] - usableSlots*flavorVCPUs + logger.V(1).Info("candidate host slot details", "az", az, "flavorGroup", r.groupName, "host", h, + "usableSlots", usableSlots, + "strandedMemoryGiB", strandedMem/(1024*1024*1024), + "strandedCores", strandedCPU) } } sort.Strings(candidateHosts) // stable order @@ -497,15 +492,64 @@ func (c *Reconciler) reconcileAZ( CandidateHosts: candidateHosts, }) } + return groupInputs, hosts +} + +// reconcileAZ probes all flavor groups in one AZ, splits capacity across groups, +// and writes one FlavorGroupCapacity CRD per group that had all probes succeed. +func (c *Reconciler) reconcileAZ( + ctx context.Context, + az string, + flavorGroups map[string]compute.FlavorGroupFeature, + hvByName map[string]hv1.Hypervisor, + blockedByReservations map[string]int64, + usageByKey map[vmUsageKey]vmUsage, +) { + + logger := LoggerFromContext(ctx) + + groupNames := make([]string, 0, len(flavorGroups)) + for name := range flavorGroups { + groupNames = append(groupNames, name) + } + sort.Strings(groupNames) + + results := make([]probeGroupResult, 0, len(groupNames)) + for _, groupName := range groupNames { + r, err := c.probeGroup(ctx, groupName, flavorGroups[groupName], az, hvByName, blockedByReservations) + if err != nil { + logger.Error(err, "skipping flavor group", "flavorGroup", groupName, "az", az) + continue + } + results = append(results, r) + } + + groupInputs, hosts := buildSplitInputs(results, hvByName, blockedByReservations, az, logger) + freeResources, exclusiveResources, unassigned, strandedByHost := SplitCapacity(groupInputs, hosts) - freeResources, exclusiveResources, unassigned := SplitCapacity(groupInputs, hosts) if unassigned[ResourceMemory] > 0 || unassigned[ResourceCores] > 0 { + groupNames := make([]string, 0, len(groupInputs)) + hostToGroups := make(map[string][]string) + for _, g := range groupInputs { + groupNames = append(groupNames, g.Name) + for _, h := range g.CandidateHosts { + hostToGroups[h] = append(hostToGroups[h], g.Name) + } + } logger.Info("fragmented capacity not assigned to any group", "az", az, "unassignedMemoryGiB", unassigned[ResourceMemory]/(1024*1024*1024), "unassignedCores", unassigned[ResourceCores], "candidateHosts", len(hosts), - "groups", len(groupInputs)) + "groups", groupNames) + for host, res := range strandedByHost { + logger.V(1).Info("stranded host resources after split", + "az", az, + "host", host, + "strandedMemoryGiB", res[ResourceMemory]/(1024*1024*1024), + "strandedCores", res[ResourceCores], + "eligibleGroups", hostToGroups[host]) + } } // Write one CRD per group. Skip groups with failed probes — their CRDs retain last good state. @@ -523,7 +567,26 @@ func (c *Reconciler) reconcileAZ( "flavorGroup", r.groupName, "az", az) } } - return nil +} + +// computeTotalCapacity returns the maximum memory bytes and CPU cores representable +// by the flavor with the highest slot count in the group (empty-datacenter view). +func computeTotalCapacity(newFlavors []v1alpha1.FlavorCapacityStatus, flavorSpecByName map[string]compute.FlavorInGroup) (maxMemBytes, maxCPUCores int64) { + for _, f := range newFlavors { + spec, ok := flavorSpecByName[f.FlavorName] + if !ok || f.TotalCapacityVMSlots <= 0 { + continue + } + memBytes := f.TotalCapacityVMSlots * int64(spec.MemoryMB) * 1024 * 1024 //nolint:gosec + cpuCores := f.TotalCapacityVMSlots * int64(spec.VCPUs) //nolint:gosec + if memBytes > maxMemBytes { + maxMemBytes = memBytes + } + if cpuCores > maxCPUCores { + maxCPUCores = cpuCores + } + } + return maxMemBytes, maxCPUCores } // writeCRD upserts one FlavorGroupCapacity CRD with fresh computed values. @@ -558,28 +621,11 @@ func (c *Reconciler) writeCRD( return fmt.Errorf("failed to get FlavorGroupCapacity %s: %w", crdName, err) } - // TotalCapacity: for each flavor multiply slot count by its resources; take the max - // across all flavors independently. The flavor best matching the host's resource - // ratio saturates more resources and produces a higher product. flavorSpecByName := make(map[string]compute.FlavorInGroup, len(groupData.Flavors)) for _, f := range groupData.Flavors { flavorSpecByName[f.Name] = f } - var maxMemBytes, maxCPUCores int64 - for _, f := range newFlavors { - spec, ok := flavorSpecByName[f.FlavorName] - if !ok || f.TotalCapacityVMSlots <= 0 { - continue - } - memBytes := f.TotalCapacityVMSlots * int64(spec.MemoryMB) * 1024 * 1024 //nolint:gosec - cpuCores := f.TotalCapacityVMSlots * int64(spec.VCPUs) //nolint:gosec - if memBytes > maxMemBytes { - maxMemBytes = memBytes - } - if cpuCores > maxCPUCores { - maxCPUCores = cpuCores - } - } + maxMemBytes, maxCPUCores := computeTotalCapacity(newFlavors, flavorSpecByName) patch := client.MergeFrom(existing.DeepCopy()) existing.Status.Flavors = newFlavors @@ -598,8 +644,11 @@ func (c *Reconciler) writeCRD( } existing.Status.FreeCapacity = resMapToQuantity(freeRes) existing.Status.ExclusivelyFreeCapacity = resMapToQuantity(exclusiveRes) + var exclusivelyFreeSlots int64 if flavorMemBytes := int64(groupData.SmallestFlavor.MemoryMB) * 1024 * 1024; flavorMemBytes > 0 { //nolint:gosec - existing.Status.ExclusivelyFreeSlots = exclusiveRes[ResourceMemory] / flavorMemBytes + flavorVCPUs := int64(groupData.SmallestFlavor.VCPUs) //nolint:gosec + exclusivelyFreeSlots = flavorSlots(exclusiveRes, flavorMemBytes, flavorVCPUs) + existing.Status.ExclusivelyFreeSlots = exclusivelyFreeSlots } existing.Status.LastReconcileAt = metav1.Now() @@ -633,6 +682,7 @@ func (c *Reconciler) probeScheduler( if flavorBytes <= 0 { return 0, 0, nil, fmt.Errorf("flavor %q has invalid memory %d MB", flavor.Name, flavor.MemoryMB) } + flavorVCPUs := int64(flavor.VCPUs) //nolint:gosec // Build EligibleHosts from all known hypervisors so that novaLimitHostsToRequest // (which filters the response to hosts present in the request) does not zero out @@ -680,7 +730,7 @@ func (c *Reconciler) probeScheduler( if !ok { continue } - var capBytes int64 + var resources map[string]int64 if ignoreAllocations { effCap := hv.Status.EffectiveCapacity if effCap == nil { @@ -693,15 +743,17 @@ func (c *Reconciler) probeScheduler( if !ok { continue } - capBytes = memCap.Value() + resources = map[string]int64{ResourceMemory: memCap.Value()} + if cpuCap, ok := effCap[hv1.ResourceCPU]; ok { + resources[ResourceCores] = cpuCap.Value() + } } else { - remaining := hvRemainingResources(hv, blockedByReservations[hostName]) - if remaining == nil { + resources = hvRemainingResources(hv, blockedByReservations[hostName]) + if resources == nil { continue } - capBytes = remaining[ResourceMemory] } - if slots := capBytes / flavorBytes; slots > 0 { + if slots := flavorSlots(resources, flavorBytes, flavorVCPUs); slots > 0 { capacity += slots candidateHosts = append(candidateHosts, hostName) } diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 0ba4b8f5c..d01110af0 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -226,11 +226,9 @@ func TestReconcileAZ_CreatesCRD(t *testing.T) { } hvByName := map[string]hv1.Hypervisor{"host-1": *hv} - if err := ctrl.reconcileAZ(context.Background(), az, + ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}); err != nil { - t.Fatalf("reconcileAZ failed: %v", err) - } + hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdNameFor(groupName, az)}, &crd); err != nil { @@ -300,11 +298,9 @@ func TestReconcileAZ_SkipsCRDWriteOnSchedulerError(t *testing.T) { Flavors: []compute.FlavorInGroup{smallFlavor}, } - if err := ctrl.reconcileAZ(context.Background(), az, + ctrl.reconcileAZ(context.Background(), az, map[string]compute.FlavorGroupFeature{groupName: groupData}, - map[string]hv1.Hypervisor{}, map[string]int64{}, map[vmUsageKey]vmUsage{}); err != nil { - t.Fatalf("reconcileAZ failed: %v", err) - } + map[string]hv1.Hypervisor{}, map[string]int64{}, map[vmUsageKey]vmUsage{}) // Stale probes → CRD must NOT be written; last good state is preserved. var list v1alpha1.FlavorGroupCapacityList @@ -362,13 +358,9 @@ func TestReconcileAZ_IdempotentUpdate(t *testing.T) { groups := map[string]compute.FlavorGroupFeature{groupName: groupData} // First call - if err := ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}); err != nil { - t.Fatalf("first reconcileAZ failed: %v", err) - } + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) // Second call — should not error on the already-existing CRD. - if err := ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}); err != nil { - t.Fatalf("second reconcileAZ failed: %v", err) - } + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, map[vmUsageKey]vmUsage{}) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { @@ -594,12 +586,9 @@ func TestReconcileAZ_ZeroMemoryFlavorSkipped(t *testing.T) { SmallestFlavor: compute.FlavorInGroup{Name: "bad-flavor", MemoryMB: 0}, } // reconcileAZ logs and skips groups with zero memory; it does not return an error. - err := c.reconcileAZ(context.Background(), "az-a", + c.reconcileAZ(context.Background(), "az-a", map[string]compute.FlavorGroupFeature{"hana-v2": groupData}, nil, nil, nil) - if err != nil { - t.Errorf("reconcileAZ should not return error for zero-memory flavor, got: %v", err) - } // No CRD should have been created. var list v1alpha1.FlavorGroupCapacityList @@ -611,6 +600,136 @@ func TestReconcileAZ_ZeroMemoryFlavorSkipped(t *testing.T) { } } +func TestFlavorSlots(t *testing.T) { + const ( + mem4GiB = 4 * 1024 * 1024 * 1024 + mem8GiB = 8 * 1024 * 1024 * 1024 + mem32GiB = 32 * 1024 * 1024 * 1024 + ) + tests := []struct { + name string + memRemaining int64 + coresRemaining int64 + flavorMem int64 + flavorCPUs int64 + want int64 + }{ + { + name: "memory is binding constraint", + // 8 GiB available, flavor needs 4 GiB and 2 cores; 64 cores available → 2 mem-slots, 32 cpu-slots + memRemaining: mem8GiB, coresRemaining: 64, flavorMem: mem4GiB, flavorCPUs: 2, want: 2, + }, + { + name: "CPU is binding constraint", + // 32 GiB available (fits 8 slots), only 3 cores available (fits 1 slot at 2 vcpus) + memRemaining: mem32GiB, coresRemaining: 3, flavorMem: mem4GiB, flavorCPUs: 2, want: 1, + }, + { + name: "both constraints equal", + memRemaining: mem8GiB, coresRemaining: 4, flavorMem: mem4GiB, flavorCPUs: 2, want: 2, + }, + { + name: "zero VCPUs — CPU dimension ignored", + memRemaining: mem8GiB, coresRemaining: 0, flavorMem: mem4GiB, flavorCPUs: 0, want: 2, + }, + { + name: "not enough memory for even one slot", + memRemaining: mem4GiB - 1, coresRemaining: 64, flavorMem: mem4GiB, flavorCPUs: 2, want: 0, + }, + { + name: "not enough CPU for even one slot", + memRemaining: mem32GiB, coresRemaining: 1, flavorMem: mem4GiB, flavorCPUs: 2, want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resources := map[string]int64{ + ResourceMemory: tt.memRemaining, + ResourceCores: tt.coresRemaining, + } + got := flavorSlots(resources, tt.flavorMem, tt.flavorCPUs) + if got != tt.want { + t.Errorf("flavorSlots() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestComputeTotalCapacity(t *testing.T) { + mb := func(mb int64) int64 { return mb * 1024 * 1024 } + tests := []struct { + name string + flavors []v1alpha1.FlavorCapacityStatus + specs map[string]compute.FlavorInGroup + wantMemBytes int64 + wantCPU int64 + }{ + { + name: "single flavor", + flavors: []v1alpha1.FlavorCapacityStatus{ + {FlavorName: "small", TotalCapacityVMSlots: 10}, + }, + specs: map[string]compute.FlavorInGroup{ + "small": {Name: "small", MemoryMB: 4096, VCPUs: 2}, + }, + wantMemBytes: 10 * mb(4096), + wantCPU: 20, + }, + { + name: "picks flavor with most total memory, not most slots", + // mem: large wins (2×32GiB=64GiB > 10×4GiB=40GiB); CPU: small wins (10×2=20 > 2×8=16) + flavors: []v1alpha1.FlavorCapacityStatus{ + {FlavorName: "small", TotalCapacityVMSlots: 10}, + {FlavorName: "large", TotalCapacityVMSlots: 2}, + }, + specs: map[string]compute.FlavorInGroup{ + "small": {Name: "small", MemoryMB: 4096, VCPUs: 2}, + "large": {Name: "large", MemoryMB: 32768, VCPUs: 8}, + }, + wantMemBytes: 2 * mb(32768), + wantCPU: 20, + }, + { + name: "zero slots excluded", + flavors: []v1alpha1.FlavorCapacityStatus{ + {FlavorName: "small", TotalCapacityVMSlots: 0}, + {FlavorName: "large", TotalCapacityVMSlots: 3}, + }, + specs: map[string]compute.FlavorInGroup{ + "small": {Name: "small", MemoryMB: 4096, VCPUs: 2}, + "large": {Name: "large", MemoryMB: 8192, VCPUs: 4}, + }, + wantMemBytes: 3 * mb(8192), + wantCPU: 12, + }, + { + name: "all zero slots", + flavors: []v1alpha1.FlavorCapacityStatus{{FlavorName: "small", TotalCapacityVMSlots: 0}}, + specs: map[string]compute.FlavorInGroup{"small": {MemoryMB: 4096, VCPUs: 2}}, + wantMemBytes: 0, + wantCPU: 0, + }, + { + name: "empty input", + flavors: nil, + specs: map[string]compute.FlavorInGroup{}, + wantMemBytes: 0, + wantCPU: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotMem, gotCPU := computeTotalCapacity(tt.flavors, tt.specs) + if gotMem != tt.wantMemBytes { + t.Errorf("maxMemBytes = %d, want %d", gotMem, tt.wantMemBytes) + } + if gotCPU != tt.wantCPU { + t.Errorf("maxCPUCores = %d, want %d", gotCPU, tt.wantCPU) + } + }) + } +} + // Verify that the module-level log variable from reservations package doesn't // collide with the one in this package. func TestPackageLogVar(t *testing.T) { @@ -789,9 +908,7 @@ func TestComputeVMUsage_ZerosOutWhenAllVMsRemoved(t *testing.T) { } // Now run reconcileAZ to verify the CRD gets zeroed out. - if err := ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, usageByKey); err != nil { - t.Fatalf("reconcileAZ failed: %v", err) - } + ctrl.reconcileAZ(context.Background(), az, groups, hvByName, map[string]int64{}, usageByKey) var crd v1alpha1.FlavorGroupCapacity if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: crdName}, &crd); err != nil { diff --git a/internal/scheduling/reservations/capacity/split.go b/internal/scheduling/reservations/capacity/split.go index ebeefb5bc..6ba73ef0b 100644 --- a/internal/scheduling/reservations/capacity/split.go +++ b/internal/scheduling/reservations/capacity/split.go @@ -196,27 +196,34 @@ func allocateRoundRobin(states []groupState, hostRes map[string]map[string]int64 } } -// computeUnassigned sums remaining resources on candidate hosts after allocation. +// computeUnassigned sums remaining resources on candidate hosts after allocation +// and returns per-host stranded resources for operator visibility. // Non-candidate hosts are excluded — their leftover is not fragmentation. -func computeUnassigned(groups []GroupInput, hostRes map[string]map[string]int64) map[string]int64 { +func computeUnassigned(groups []GroupInput, hostRes map[string]map[string]int64) (unassigned map[string]int64, strandedByHost map[string]map[string]int64) { candidateSet := make(map[string]struct{}) for _, g := range groups { for _, h := range g.CandidateHosts { candidateSet[h] = struct{}{} } } - unassigned := make(map[string]int64) + unassigned = make(map[string]int64) + strandedByHost = make(map[string]map[string]int64) for h, res := range hostRes { if _, isCandidate := candidateSet[h]; !isCandidate { continue } + hasStranded := false for r, remaining := range res { if remaining > 0 { unassigned[r] += remaining + hasStranded = true } } + if hasStranded { + strandedByHost[h] = res + } } - return unassigned + return } // collectExclusiveResources builds the exclusive allocation map from group assigned counts. @@ -248,12 +255,12 @@ func collectExclusiveResources(states []groupState) map[string]map[string]int64 // // The caller divides exclusiveResources[group][ResourceMemory] by the group's flavor memory // to obtain the slot count meaningful to that group. -func SplitCapacity(groups []GroupInput, hosts map[string]HostState) (freeResources, exclusiveResources map[string]map[string]int64, unassigned map[string]int64) { +func SplitCapacity(groups []GroupInput, hosts map[string]HostState) (freeResources, exclusiveResources map[string]map[string]int64, unassigned map[string]int64, strandedByHost map[string]map[string]int64) { states := initGroupStates(groups, hosts) freeResources = computeFreeResources(groups, hosts) hostRes := copyHostResources(hosts) allocateRoundRobin(states, hostRes) - unassigned = computeUnassigned(groups, hostRes) + unassigned, strandedByHost = computeUnassigned(groups, hostRes) exclusiveResources = collectExclusiveResources(states) - return freeResources, exclusiveResources, unassigned + return } diff --git a/internal/scheduling/reservations/capacity/split_test.go b/internal/scheduling/reservations/capacity/split_test.go index a4b1ff212..d5588d0d7 100644 --- a/internal/scheduling/reservations/capacity/split_test.go +++ b/internal/scheduling/reservations/capacity/split_test.go @@ -234,7 +234,7 @@ func TestComputeUnassigned(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got := computeUnassigned(tc.groups, tc.hostRes) + got, _ := computeUnassigned(tc.groups, tc.hostRes) for r, want := range tc.wantUnassigned { if got[r] != want { t.Errorf("unassigned[%s] = %d, want %d", r, got[r], want) @@ -449,7 +449,7 @@ func TestSplitCapacity(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - free, assigned, unassigned := SplitCapacity(tc.groups, tc.hosts) + free, assigned, unassigned, _ := SplitCapacity(tc.groups, tc.hosts) for groupName, wantMem := range tc.wantAssignedMem { if got := assigned[groupName][ResourceMemory]; got != wantMem { @@ -492,7 +492,7 @@ func TestSplitCapacity_SumNeverExceedsTotal(t *testing.T) { "h3": host(24*GiB, 12), } - _, assigned, _ := SplitCapacity(groups, hosts) + _, assigned, _, _ := SplitCapacity(groups, hosts) var totalInstalled, totalAssigned int64 for _, hs := range hosts { @@ -518,9 +518,9 @@ func TestSplitCapacity_Deterministic(t *testing.T) { "h2": host(8*GiB, 4), } - _, first, firstUnassigned := SplitCapacity(groups, hosts) + _, first, firstUnassigned, _ := SplitCapacity(groups, hosts) for i := range 10 { - _, got, gotUnassigned := SplitCapacity(groups, hosts) + _, got, gotUnassigned, _ := SplitCapacity(groups, hosts) for _, g := range groups { if got[g.Name][ResourceMemory] != first[g.Name][ResourceMemory] { t.Errorf("run %d: assigned[%s][memory] = %d, want %d (non-deterministic)",