diff --git a/CHANGELOG.md b/CHANGELOG.md index 60edd20e6..24ab3d98c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,53 @@ # Changelog +## 2026-07-27 — [#1093](https://github.com/cobaltcore-dev/cortex/pull/1093) + +### cortex v0.3.4 (sha-4f44f3b7) + +Bug fixes: +- Deep-copy capacity map to prevent informer cache corruption — the capacity filter assigned hypervisor Status.Capacity map references directly into a local working map, then mutated those maps during free-resource calculations. Since the maps are shared with the controller-runtime informer cache, each scheduling pass permanently decremented cached capacity values, progressively starving scheduling until all hosts appeared full ([#1065](https://github.com/cobaltcore-dev/cortex/pull/1065)) +- Reset gauge metrics in reservation monitor to prevent stale values — adds `Reset()` calls on `numberOfReservations` and `reservedResources` GaugeVecs before repopulating in `Collect`, preventing stale label combinations from persisting when reservations are deleted or change status ([#1092](https://github.com/cobaltcore-dev/cortex/pull/1092)) +- Stop committed resource controller reconcile storm — `setAccepted` unconditionally wrote a new AcceptedAt timestamp and a time-varying StatusSummary on every call, causing a non-empty status patch even when the CR was already accepted. Adds an early-return guard that skips the patch when Ready=True/Accepted is already set for the current generation ([#1089](https://github.com/cobaltcore-dev/cortex/pull/1089)) + +Non-breaking changes: +- Add History CRD documentation and multicluster guide link ([#1091](https://github.com/cobaltcore-dev/cortex/pull/1091)) +- Update `github.com/prometheus/client_golang` to v1.24.1 ([#1090](https://github.com/cobaltcore-dev/cortex/pull/1090)) +- Update `k8s.io/api`, `k8s.io/apimachinery`, `k8s.io/client-go` to v0.36.3 ([#1086](https://github.com/cobaltcore-dev/cortex/pull/1086)) +- Update `github.com/sapcc/go-bits` ([#1085](https://github.com/cobaltcore-dev/cortex/pull/1085)) +- Update `kube-prometheus-stack` to 87.19.2 ([#1090](https://github.com/cobaltcore-dev/cortex/pull/1090)) + +### cortex-shim v0.1.10 (sha-51c773d9) + +Includes updated image sha-51c773d9 with dependency updates. + +### cortex-nova v0.0.84 + +Includes updated chart cortex v0.3.4. + +### cortex-cinder v0.0.84 + +Includes updated chart cortex v0.3.4. + +### cortex-manila v0.0.84 + +Includes updated chart cortex v0.3.4. + +### cortex-crds v0.0.84 + +Includes updated chart cortex v0.3.4. + +### cortex-ironcore v0.0.84 + +Includes updated chart cortex v0.3.4. + +### cortex-pods v0.0.84 + +Includes updated chart cortex v0.3.4. + +### cortex-placement-shim v0.1.10 + +Includes updated chart cortex-shim v0.1.10. + ## 2026-07-23 — [#1080](https://github.com/cobaltcore-dev/cortex/pull/1080) ### cortex v0.3.3 (sha-4eba8400) diff --git a/README.md b/README.md index a72b1c3f4..195e89d39 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ We as members, contributors, and leaders pledge to make participation in our com ## Licensing -Copyright 2024-2025 SAP SE. Please see our [LICENSE](LICENSE) for copyright and license information. Detailed information including third-party components and their licensing/copyright information is available [via the REUSE tool](https://api.reuse.software/info/github.com/cobaltcore-dev/cortex). +Copyright SAP SE. Please see our [LICENSE](LICENSE) for copyright and license information. Detailed information including third-party components and their licensing/copyright information is available [via the REUSE tool](https://api.reuse.software/info/github.com/cobaltcore-dev/cortex).

Bundesministerium für Wirtschaft und Energie (BMWE)-EU funding logo diff --git a/docs/apis.md b/docs/apis.md index f73753d85..a1edf6a14 100644 --- a/docs/apis.md +++ b/docs/apis.md @@ -22,9 +22,11 @@ graph LR; committedresource(CommittedResource CRD) projectquota(ProjectQuota CRD) flavorgroupcapacity(FlavorGroupCapacity CRD) + history(History CRD) pipeline --> descheduling pipeline --> decision pipeline --> reservation + pipeline --> history committedresource --> reservation committedresource --> projectquota flavorgroupcapacity --> committedresource @@ -104,6 +106,16 @@ Decisions are generated when pipelines are executed with an appropriate request, In its state, decisions reflect the outcome of the pipeline execution, for example the generated weights for each scheduling step. This outcome is reflected back to the caller of the pipeline. In addition, decisions provide a human-readable explanation why the workload was placed at this specific location. +### History + +```bash +kubectl get histories +``` + +History is a cluster-scoped audit trail of scheduling decisions for a given resource (e.g., a Nova instance). Each time a pipeline produces a placement decision, the result — including which pipeline was used, the scheduling intent, the target host, and an explanation — is recorded unless the caller sets `SkipHistory=true` in the pipeline options. The most recent decision lives in `status.current`; older entries are compacted into `status.history`, capped at 10 entries. + +See [`api/v1alpha1/history_types.go`](../api/v1alpha1/history_types.go) for the full struct definition. + ### Reservations ```bash diff --git a/docs/readme.md b/docs/readme.md index 4d9c28cb8..ec994f075 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -7,3 +7,4 @@ - To get started quickly, see [the quickstart guide](quickstart.md). - For details on committed resource reservations, see [committed resource reservations](reservations/committed-resource-reservations.md). - For details on failover reservations, see [failover reservations](reservations/failover-reservations.md). +- For multi-cluster scheduling, see [the multicluster guide](guides/multicluster/readme.md). diff --git a/go.mod b/go.mod index baf3e312b..e5933302a 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,13 @@ require ( github.com/go-gorp/gorp v2.2.0+incompatible github.com/gophercloud/gophercloud/v2 v2.13.0 github.com/ironcore-dev/ironcore v0.4.3 - github.com/prometheus/client_golang v1.24.0 + github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.2 - github.com/sapcc/go-bits v0.0.0-20260721134224-acf388eecf26 + github.com/sapcc/go-bits v0.0.0-20260723170232-89c8670b5841 go.xyrillian.de/gg v1.11.1 - k8s.io/api v0.36.2 - k8s.io/apimachinery v0.36.2 - k8s.io/client-go v0.36.2 + k8s.io/api v0.36.3 + k8s.io/apimachinery v0.36.3 + k8s.io/client-go v0.36.3 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -32,7 +32,7 @@ require ( github.com/gofrs/uuid/v5 v5.4.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - k8s.io/streaming v0.36.2 // indirect + k8s.io/streaming v0.36.3 // indirect ) require ( @@ -85,7 +85,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/poy/onpar v0.3.5 // indirect - github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/sapcc/go-api-declarations v1.24.0 github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/go.sum b/go.sum index 84c655a6e..cfd3b3f69 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,8 @@ github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo79 github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -191,12 +191,12 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v0.3.5 h1:AYOT1AvNtucJhmEK6BSD1KvLvtYXvp/50vL5yaPAljo= github.com/poy/onpar v0.3.5/go.mod h1:Ae6eC73WNjI7NHdcughszLtcw+msOi0pb6eq6LzL97o= -github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= -github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= -github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -204,8 +204,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sapcc/go-api-declarations v1.24.0 h1:sGBvOMVSM1olJlyvNoQSk7NX5uatXHKkztGDBPnTWMs= github.com/sapcc/go-api-declarations v1.24.0/go.mod h1:ZWRTijvgF8o8aHg5stgg7u4DF6jFrd0X97le/uGlZsA= -github.com/sapcc/go-bits v0.0.0-20260721134224-acf388eecf26 h1:H3m0oO3OKPKScPkXxPtSL7WifLvPVyyQZQKfeeQAKwE= -github.com/sapcc/go-bits v0.0.0-20260721134224-acf388eecf26/go.mod h1:McCHf83n0tsTgnTSMxkdyBys84LG/d7uqubQrjsoZ1U= +github.com/sapcc/go-bits v0.0.0-20260723170232-89c8670b5841 h1:xU7PCY7Vcc/0UZhvybOsE+bhh8/RmZlH9UaKIiKzNAY= +github.com/sapcc/go-bits v0.0.0-20260723170232-89c8670b5841/go.mod h1:wUu2ss9a/5/bR4QOcaPIklrZdconXafkbsLgProV0Ro= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= @@ -311,24 +311,24 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -k8s.io/api v0.36.2 h1:TF6YDLIzKfccK7cq9YpTcGX8TJmEkHVRv78DM51fRYY= -k8s.io/api v0.36.2/go.mod h1:F4LbMO4brjZYh7yFkXWhynSvtB7YauxV4c+HHkNRGNg= +k8s.io/api v0.36.3 h1:NxB+05W2UGqXWFXcLO0RB5cnqnUPP5v5sVlaOH0Iz4w= +k8s.io/api v0.36.3/go.mod h1:JzLQKqRHC5+I8RVj/lS3lCg0mg6nWI9Fo/Sk3ElxHzg= k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= -k8s.io/apimachinery v0.36.2 h1:0PE/W/WNy1UX61NLbXY5TMbJ6UwLL6E6lAPkYrKFxbQ= -k8s.io/apimachinery v0.36.2/go.mod h1:fvf/HOLXq9RId0rnDIbN1OEBvHXdQbLMM8nu0LcBUf4= +k8s.io/apimachinery v0.36.3 h1:PkzMRBRG8joFD8EhCuQAtNPvJlxb82FwplP26HIzvAM= +k8s.io/apimachinery v0.36.3/go.mod h1:cTSjBWgPe/6CQyBKzY/hDIRWCQQQeK0mfLbml0UYFHE= k8s.io/apiserver v0.36.0 h1:Jg5OFAENUACByUCg15CmhZAYrr5ZyJ+jodyA1mHl3YE= k8s.io/apiserver v0.36.0/go.mod h1:mHvwdHf+qKEm+1/hYm756SV+oREOKSPnsjagOpx6Vho= -k8s.io/client-go v0.36.2 h1:bfgxmFKc9CgqsgX4xKLAAdmTQlWee7Ob/HlDOrJ5TBI= -k8s.io/client-go v0.36.2/go.mod h1:1vgO4OAlfPnoLcb+Rze2GF5rAr14w8qjrYMoyXJzQj0= +k8s.io/client-go v0.36.3 h1:M4JdVzXxYcZk4fGpfDdYnxSwhLKWCFoQsHW6t+z8Hfg= +k8s.io/client-go v0.36.3/go.mod h1:gcPwr0c87vjjG6HB6pWEqOeuYVoXSsREjzux2j6GF30= k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= -k8s.io/streaming v0.36.2 h1:NSKthPPg9UFSKsRauVJUVGH2Dvn8fhKmY4qrMkw/p98= -k8s.io/streaming v0.36.2/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/streaming v0.36.3 h1:9rAaqBk0C0Pc7+/fqGekj07NV+/Xrew58p647A0JT8w= +k8s.io/streaming v0.36.3/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260507154919-ff6756f316d2 h1:wU4tMEhLGgIbLvXQb1cfN+EcM0wf7zC6CPF+C79jroc= k8s.io/utils v0.0.0-20260507154919-ff6756f316d2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= diff --git a/helm/bundles/cortex-cinder/Chart.yaml b/helm/bundles/cortex-cinder/Chart.yaml index 2aa7b027f..17845a1a4 100644 --- a/helm/bundles/cortex-cinder/Chart.yaml +++ b/helm/bundles/cortex-cinder/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-cinder description: A Helm chart deploying Cortex for Cinder. type: application -version: 0.0.83 +version: 0.0.84 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-crds/Chart.yaml b/helm/bundles/cortex-crds/Chart.yaml index 7c3cadeab..4334a9fb2 100644 --- a/helm/bundles/cortex-crds/Chart.yaml +++ b/helm/bundles/cortex-crds/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-crds description: A Helm chart deploying Cortex CRDs. type: application -version: 0.0.83 +version: 0.0.84 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-ironcore/Chart.yaml b/helm/bundles/cortex-ironcore/Chart.yaml index 7ce01100e..308a117dc 100644 --- a/helm/bundles/cortex-ironcore/Chart.yaml +++ b/helm/bundles/cortex-ironcore/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-ironcore description: A Helm chart deploying Cortex for IronCore. type: application -version: 0.0.83 +version: 0.0.84 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/bundles/cortex-manila/Chart.yaml b/helm/bundles/cortex-manila/Chart.yaml index 146cc82ae..6200dfbbb 100644 --- a/helm/bundles/cortex-manila/Chart.yaml +++ b/helm/bundles/cortex-manila/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-manila description: A Helm chart deploying Cortex for Manila. type: application -version: 0.0.83 +version: 0.0.84 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-nova/Chart.yaml b/helm/bundles/cortex-nova/Chart.yaml index 832cd56c1..8178bc34e 100644 --- a/helm/bundles/cortex-nova/Chart.yaml +++ b/helm/bundles/cortex-nova/Chart.yaml @@ -5,7 +5,7 @@ apiVersion: v2 name: cortex-nova description: A Helm chart deploying Cortex for Nova. type: application -version: 0.0.83 +version: 0.0.84 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-postgres @@ -16,12 +16,12 @@ dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 alias: cortex-knowledge-controllers # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 alias: cortex-scheduling-controllers # Owner info adds a configmap to the kubernetes cluster with information on diff --git a/helm/bundles/cortex-placement-shim/Chart.yaml b/helm/bundles/cortex-placement-shim/Chart.yaml index d58bb6054..5f45c887f 100644 --- a/helm/bundles/cortex-placement-shim/Chart.yaml +++ b/helm/bundles/cortex-placement-shim/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-placement-shim description: A Helm chart deploying the Cortex placement shim. type: application -version: 0.1.9 +version: 0.1.10 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex-shim - name: cortex-shim repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.1.9 + version: 0.1.10 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case # of issues. See: https://github.com/sapcc/helm-charts/pkgs/container/helm-charts%2Fowner-info diff --git a/helm/bundles/cortex-pods/Chart.yaml b/helm/bundles/cortex-pods/Chart.yaml index a08c74900..4cc50ed57 100644 --- a/helm/bundles/cortex-pods/Chart.yaml +++ b/helm/bundles/cortex-pods/Chart.yaml @@ -5,13 +5,13 @@ apiVersion: v2 name: cortex-pods description: A Helm chart deploying Cortex for Pods. type: application -version: 0.0.83 +version: 0.0.84 appVersion: 0.1.0 dependencies: # from: file://../../library/cortex - name: cortex repository: oci://ghcr.io/cobaltcore-dev/cortex/charts - version: 0.3.3 + version: 0.3.4 # Owner info adds a configmap to the kubernetes cluster with information on # the service owner. This makes it easier to find out who to contact in case diff --git a/helm/dev/cortex-prometheus-operator/Chart.yaml b/helm/dev/cortex-prometheus-operator/Chart.yaml index 862d9378f..406d41f58 100644 --- a/helm/dev/cortex-prometheus-operator/Chart.yaml +++ b/helm/dev/cortex-prometheus-operator/Chart.yaml @@ -10,4 +10,4 @@ dependencies: # CRDs of the prometheus operator, such as PrometheusRule, ServiceMonitor, etc. - name: kube-prometheus-stack repository: oci://ghcr.io/prometheus-community/charts - version: 87.19.0 + version: 87.19.2 diff --git a/helm/library/cortex-shim/Chart.yaml b/helm/library/cortex-shim/Chart.yaml index 51f77277f..9d88a4841 100644 --- a/helm/library/cortex-shim/Chart.yaml +++ b/helm/library/cortex-shim/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex-shim description: A Helm chart to distribute cortex shims. type: application -version: 0.1.9 -appVersion: "sha-38f37225" +version: 0.1.10 +appVersion: "sha-51c773d9" icon: "https://example.com/icon.png" dependencies: [] diff --git a/helm/library/cortex/Chart.yaml b/helm/library/cortex/Chart.yaml index 579d7d35a..7049941b9 100644 --- a/helm/library/cortex/Chart.yaml +++ b/helm/library/cortex/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: cortex description: A Helm chart to distribute cortex. type: application -version: 0.3.3 -appVersion: "sha-38f37225" +version: 0.3.4 +appVersion: "sha-68212b99" icon: "https://example.com/icon.png" dependencies: [] diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index e2fadf981..117b032ae 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -82,13 +82,20 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa return nil, err } for _, hv := range hvs.Items { + var sourceMap map[hv1.ResourceName]resource.Quantity if hv.Status.EffectiveCapacity == nil { traceLog.Warn("hypervisor with nil effective capacity, use capacity instead (overprovisioning not considered)", "host", hv.Name) - freeResourcesByHost[hv.Name] = hv.Status.Capacity + sourceMap = hv.Status.Capacity } else { // Start with the total effective capacity which is capacity * overcommit ratio. - freeResourcesByHost[hv.Name] = hv.Status.EffectiveCapacity + sourceMap = hv.Status.EffectiveCapacity } + // Deep-copy the map to avoid mutating the informer cache. + copied := make(map[hv1.ResourceName]resource.Quantity, len(sourceMap)) + for k, v := range sourceMap { + copied[k] = v.DeepCopy() + } + freeResourcesByHost[hv.Name] = copied // Subtract allocated resources (skip when ignoring allocations for empty-datacenter capacity queries). if !ignoreAllocations { diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller.go b/internal/scheduling/reservations/commitments/committed_resource_controller.go index cd0bdc88a..c592bed9c 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller.go @@ -396,6 +396,16 @@ func (r *CommittedResourceController) checkChildReservationStatus(ctx context.Co } func (r *CommittedResourceController) setAccepted(ctx context.Context, cr *v1alpha1.CommittedResource) error { + // Skip the patch when the CR is already accepted for the current generation. + // Without this guard, every reconcile writes a new AcceptedAt timestamp and a + // time-varying StatusSummary, triggering the self-watch and causing a reconcile storm. + cond := meta.FindStatusCondition(cr.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond != nil && + cond.Status == metav1.ConditionTrue && + cond.Reason == v1alpha1.CommittedResourceReasonAccepted && + cond.ObservedGeneration == cr.Generation { + return nil + } now := metav1.Now() old := cr.DeepCopy() specCopy := cr.Spec.DeepCopy() diff --git a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go index 5843bb3d9..293f074a9 100644 --- a/internal/scheduling/reservations/commitments/committed_resource_controller_test.go +++ b/internal/scheduling/reservations/commitments/committed_resource_controller_test.go @@ -1165,3 +1165,52 @@ func TestCheckChildReservationStatus_GenerationGuard(t *testing.T) { }) } } + +func TestCommittedResourceController_SetAcceptedIdempotent(t *testing.T) { + // Reconciling an already-accepted cores CR must not patch the status again. + // Without the idempotency guard, every reconcile writes a new AcceptedAt timestamp, + // which triggers the self-watch and causes a reconcile storm. + scheme := newCRTestScheme(t) + cr := newTestCoresCR("test-cr", v1alpha1.CommitmentStatusConfirmed, 4, false) + cr.Generation = 1 + fgc := newTestFlavorGroupCapacity("test-group", "test-az", 16) + k8sClient := newCRTestClient(scheme, cr, fgc) + + controller := &CommittedResourceController{ + Client: k8sClient, + Scheme: scheme, + Conf: CommittedResourceControllerConfig{RequeueIntervalRetry: metav1.Duration{Duration: 1 * time.Minute}}, + } + + // First reconcile: CR transitions to Accepted and status is written. + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("first reconcile: %v", err) + } + assertCondition(t, k8sClient, cr.Name, metav1.ConditionTrue, v1alpha1.CommittedResourceReasonAccepted) + + var after1 v1alpha1.CommittedResource + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: cr.Name}, &after1); err != nil { + t.Fatalf("get after first reconcile: %v", err) + } + cond := meta.FindStatusCondition(after1.Status.Conditions, v1alpha1.CommittedResourceConditionReady) + if cond == nil { + t.Fatalf("Ready condition not set after first reconcile") + } + if cond.ObservedGeneration != 1 { + t.Errorf("ObservedGeneration: want 1, got %d", cond.ObservedGeneration) + } + rv1 := after1.ResourceVersion + + // Second reconcile: CR is already accepted — must not patch status. + if _, err := controller.Reconcile(context.Background(), reconcileReq(cr.Name)); err != nil { + t.Fatalf("second reconcile: %v", err) + } + + var after2 v1alpha1.CommittedResource + if err := k8sClient.Get(context.Background(), types.NamespacedName{Name: cr.Name}, &after2); err != nil { + t.Fatalf("get after second reconcile: %v", err) + } + if after2.ResourceVersion != rv1 { + t.Errorf("ResourceVersion changed on second reconcile (%s → %s): status was patched unnecessarily", rv1, after2.ResourceVersion) + } +} diff --git a/internal/scheduling/reservations/monitor.go b/internal/scheduling/reservations/monitor.go index 557a87920..6cefa301b 100644 --- a/internal/scheduling/reservations/monitor.go +++ b/internal/scheduling/reservations/monitor.go @@ -6,6 +6,7 @@ package reservations import ( "context" "strings" + "time" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/prometheus/client_golang/prometheus" @@ -53,14 +54,20 @@ func (m *Monitor) Describe(ch chan<- *prometheus.Desc) { func (m *Monitor) Collect(ch chan<- prometheus.Metric) { // Fetch all reservations from kubernetes. var reservations v1alpha1.ReservationList + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() if err := m.List( - context.Background(), + ctx, &reservations, ); err != nil { monitorLog.Error(err, "failed to list reservations") return } + // Reset all gauges so deleted reservations don't linger. + m.numberOfReservations.Reset() + m.reservedResources.Reset() + countByLabels := map[string]uint64{} for _, reservation := range reservations.Items { readyCondition := meta.FindStatusCondition(reservation.Status.Conditions, v1alpha1.ReservationConditionReady)