diff --git a/cmd/manager/main.go b/cmd/manager/main.go index c5d5fa323..72545934b 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -55,6 +55,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/machines" "github.com/cobaltcore-dev/cortex/internal/scheduling/manila" "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" + novafilters "github.com/cobaltcore-dev/cortex/internal/scheduling/nova/plugins/filters" "github.com/cobaltcore-dev/cortex/internal/scheduling/pods" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations/capacity" @@ -385,6 +386,12 @@ func main() { detectorPipelineMonitor := schedulinglib.NewDetectorPipelineMonitor() metrics.Registry.MustRegister(&detectorPipelineMonitor) + // Filter-specific metrics that don't fit the generic per-step monitor (e.g. + // custom labels). Register them globally so they're available wherever the + // filter runs. + novafilters.QuotaEnforcementMetricsSingleton = novafilters.NewQuotaEnforcementMetrics() + metrics.Registry.MustRegister(novafilters.QuotaEnforcementMetricsSingleton) + // Initialize commitments API for LIQUID interface (Postgres-backed usage reporting). commitmentsConfig := conf.GetConfigOrDie[commitments.Config]() var commitmentsUsageDB commitments.UsageDBClient diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index de032b133..6d9572edc 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -109,6 +109,11 @@ spec: If a matching CommittedResource has unused capacity, the request is accepted. Otherwise, PAYG headroom is checked for ram, cores, and instances. Rejects all hosts if neither tier has headroom. + When dryRun is true the filter runs in shadow mode: it logs and emits + the cortex_nova_filter_quota_enforcement_decisions_total metric for + would-be rejects but never actually removes hosts. + params: + - {key: dryRun, boolValue: true} weighers: - name: kvm_prefer_smaller_hosts params: @@ -261,6 +266,11 @@ spec: If a matching CommittedResource has unused capacity, the request is accepted. Otherwise, PAYG headroom is checked for ram, cores, and instances. Rejects all hosts if neither tier has headroom. + When dryRun is true the filter runs in shadow mode: it logs and emits + the cortex_nova_filter_quota_enforcement_decisions_total metric for + would-be rejects but never actually removes hosts. + params: + - {key: dryRun, boolValue: true} weighers: - name: kvm_prefer_smaller_hosts params: diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 18286fce6..fcad4274b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -18,7 +18,14 @@ import ( "k8s.io/apimachinery/pkg/types" ) -type FilterQuotaEnforcementOpts struct{} +type FilterQuotaEnforcementOpts struct { + // DryRun, when true, makes the filter run in shadow mode: it performs the + // full headroom analysis and logs/emits metrics for would-be rejects, but + // never actually removes hosts from the result. Use this for safe rollouts + // and to observe cortex_nova_filter_quota_enforcement_decisions_total in + // shadow mode before flipping to enforce. + DryRun bool `json:"dryRun,omitempty"` +} func (FilterQuotaEnforcementOpts) Validate() error { return nil } @@ -44,6 +51,17 @@ func (FilterQuotaEnforcementOpts) Validate() error { return nil } // On infrastructure errors (e.g. API server unreachable), the filter returns an error // which causes the pipeline to skip it (fail-open). // +// Two modes: +// - DryRun=false (enforce, default zero value): on a no-headroom decision the filter +// clears all host activations to globally reject the request. +// - DryRun=true (shadow): the filter performs the same analysis, logs the would-be +// reject, and emits the same decision metric with mode="shadow" — but does NOT +// remove hosts. Use this to observe rejection volumes before enabling enforcement. +// +// Every accept/reject/skip outcome is recorded as a Prometheus counter: +// cortex_nova_filter_quota_enforcement_decisions_total{mode,decision,resource, +// availability_zone,flavor_group}. +// // Disabled by default; activated by adding "filter_quota_enforcement" to the pipeline config. type FilterQuotaEnforcement struct { lib.BaseFilter[api.ExternalSchedulerRequest, FilterQuotaEnforcementOpts] @@ -52,19 +70,27 @@ type FilterQuotaEnforcement struct { func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + mode := "enforce" + if s.Options.DryRun { + mode = "shadow" + } + // Step 1: Skip intents that don't represent new resource consumption. intent, err := request.GetIntent() if err == nil { switch intent { case api.EvacuateIntent, api.LiveMigrationIntent: traceLog.Info("skipping quota enforcement for non-consuming intent", "intent", intent) + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") return result, nil case api.ReserveForFailoverIntent: traceLog.Info("skipping quota enforcement for failover reservation intent") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") return result, nil case api.ReserveForCommittedResourceIntent: // TODO: revisit whether committed resource reservation scheduling should also be quota-checked traceLog.Info("skipping quota enforcement for committed resource reservation intent") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", "") return result, nil } } @@ -76,14 +102,17 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External if projectID == "" { traceLog.Warn("no project ID in request, skipping quota enforcement") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", az, hwVersion) return result, nil } if az == "" { traceLog.Warn("no availability zone in request, skipping quota enforcement") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", "", hwVersion) return result, nil } if hwVersion == "" { traceLog.Warn("no hw_version in flavor extra specs, skipping quota enforcement") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_skipped", "", az, "") return result, nil } @@ -188,6 +217,7 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External if crHasHeadroom { traceLog.Info("quota enforcement ACCEPT: CR headroom available") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_cr", "ram", az, hwVersion) return result, nil } @@ -200,6 +230,7 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External if apierrors.IsNotFound(err) { traceLog.Info("no ProjectQuota CRD found for project+AZ, skipping enforcement", "projectID", projectID, "az", az) + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_no_quota", "", az, hwVersion) return result, nil } traceLog.Error("failed to get ProjectQuota", "name", pqName, "error", err) @@ -211,14 +242,15 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External // For RAM: paygHeadroom = Quota - sum(CR amounts) - PaygUsage (CRs reserve RAM capacity) // For cores/instances: paygHeadroom = Quota - PaygUsage (no CR deduction) type resourceCheck struct { - name string + name string // liquid resource name, e.g. hw_version__ram + label string // metric label value: "ram" / "cores" / "instances" request int64 crDeduct int64 } checks := []resourceCheck{ - {name: resourceRAM, request: requestRAM, crDeduct: matchingCRAmountGiB}, - {name: resourceCores, request: requestCores, crDeduct: 0}, - {name: resourceInstances, request: requestInstances, crDeduct: 0}, + {name: resourceRAM, label: "ram", request: requestRAM, crDeduct: matchingCRAmountGiB}, + {name: resourceCores, label: "cores", request: requestCores, crDeduct: 0}, + {name: resourceInstances, label: "instances", request: requestInstances, crDeduct: 0}, } for _, check := range checks { @@ -247,9 +279,22 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External ) if headroom < check.request { + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "reject", check.label, az, hwVersion) + if s.Options.DryRun { + traceLog.Info("quota enforcement SHADOW: would reject but dryRun=true", + "projectID", projectID, + "az", az, + "flavorGroup", hwVersion, + "resource", check.name, + "request", check.request, + "headroom", headroom, + ) + return result, nil + } traceLog.Info("quota enforcement REJECT: no PAYG headroom", "projectID", projectID, "az", az, + "flavorGroup", hwVersion, "resource", check.name, "request", check.request, "headroom", headroom, @@ -262,6 +307,7 @@ func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.External } traceLog.Info("quota enforcement ACCEPT: PAYG headroom available") + QuotaEnforcementMetricsSingleton.RecordDecision(mode, "accept_payg", "", az, hwVersion) return result, nil } diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go new file mode 100644 index 000000000..8d8edc17e --- /dev/null +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_metrics.go @@ -0,0 +1,104 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package filters + +import ( + "log/slog" + "sync" + + "github.com/prometheus/client_golang/prometheus" +) + +// QuotaEnforcementMetrics holds Prometheus metrics for the quota-enforcement +// nova filter. The filter runs in either "enforce" mode (rejects when no +// headroom) or "shadow" mode (logs/counts what it would have rejected without +// removing hosts). The decisions counter exposes the same outcomes in both +// modes so operators can compare a shadow rollout against the existing +// generic step metrics (e.g. cortex_filter_weigher_pipeline_step_removed_hosts). +type QuotaEnforcementMetrics struct { + // Decisions is the counter of every accept/reject/skip outcome the filter + // produces. Labels: + // - mode: "enforce" | "shadow" + // - decision: "accept_cr" | "accept_payg" | "accept_no_quota" | + // "accept_skipped" | "reject" + // - resource: "ram" | "cores" | "instances" | "" (empty when + // the decision is not driven by a single resource) + // - availability_zone: AZ string or "" if unknown at decision time + // - flavor_group: hw_version string or "" if unknown at decision time + Decisions *prometheus.CounterVec +} + +// NewQuotaEnforcementMetrics constructs the metrics struct. The returned +// *QuotaEnforcementMetrics implements prometheus.Collector, so the caller is +// expected to register it with a registry (typically metrics.Registry from +// cmd/manager). This matches the Monitor pattern used elsewhere in cortex +// (e.g. db.Monitor, LogMetricsMonitor, PipelineMonitor). +func NewQuotaEnforcementMetrics() *QuotaEnforcementMetrics { + return &QuotaEnforcementMetrics{ + Decisions: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cortex_nova_filter_quota_enforcement_decisions_total", + Help: "Decisions made by the FilterQuotaEnforcement nova filter, " + + "labeled by enforcement mode, decision outcome, the resource " + + "that drove a reject (if any), availability zone, and flavor group.", + }, + []string{"mode", "decision", "resource", "availability_zone", "flavor_group"}, + ), + } +} + +// Describe implements prometheus.Collector by delegating to the underlying +// counter vec. Nil-safe to mirror RecordDecision's defensive guard: a stray +// MustRegister on an uninitialized struct must not crash the manager. +func (m *QuotaEnforcementMetrics) Describe(ch chan<- *prometheus.Desc) { + if m == nil || m.Decisions == nil { + return + } + m.Decisions.Describe(ch) +} + +// Collect implements prometheus.Collector by delegating to the underlying +// counter vec. Nil-safe — Collect runs on every Prometheus scrape, so a nil +// receiver must not panic the metrics endpoint. +func (m *QuotaEnforcementMetrics) Collect(ch chan<- prometheus.Metric) { + if m == nil || m.Decisions == nil { + return + } + m.Decisions.Collect(ch) +} + +// recordDecisionNilOnce ensures the "metrics not initialized" warning is +// emitted at most once per process to keep logs clean while still surfacing +// misconfigurations. A pointer is used so tests can swap in a freshly armed +// once without copying a sync.Once value (which is forbidden after first use). +var recordDecisionNilOnce = &sync.Once{} + +// RecordDecision is a small helper that nil-guards the singleton and increments +// the decisions counter. Filter Run() uses it directly. If the receiver is nil +// (i.e. QuotaEnforcementMetricsSingleton was never initialized — typically a +// missing wiring step in cmd/manager) we log a warn-level message exactly once +// so the misconfiguration is observable both in logs and via +// cortex_log_messages_total{level="warn"}. +func (m *QuotaEnforcementMetrics) RecordDecision(mode, decision, resource, az, flavorGroup string) { + if m == nil || m.Decisions == nil { + recordDecisionNilOnce.Do(func() { + slog.Warn("QuotaEnforcementMetrics is nil; decision metric not recorded "+ + "(is QuotaEnforcementMetricsSingleton initialized in cmd/manager?)", + "mode", mode, + "decision", decision, + "resource", resource, + "availability_zone", az, + "flavor_group", flavorGroup, + ) + }) + return + } + m.Decisions.WithLabelValues(mode, decision, resource, az, flavorGroup).Inc() +} + +// QuotaEnforcementMetricsSingleton is set from cmd/manager/main.go during +// initialization. The filter's Run method reads it via RecordDecision, which +// nil-guards itself, so unit tests that construct a bare FilterQuotaEnforcement +// without setting this singleton stay safe. +var QuotaEnforcementMetricsSingleton *QuotaEnforcementMetrics diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 374192aa5..771728476 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -4,11 +4,16 @@ package filters import ( + "bytes" "log/slog" + "strings" + "sync" "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -17,6 +22,15 @@ import ( ) func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numInstances uint64, hints map[string]any) api.ExternalSchedulerRequest { + flavor := api.NovaFlavor{ + MemoryMB: memoryMB, + VCPUs: 4, + } + if hwVersion != "" { + flavor.ExtraSpecs = map[string]string{"hw_version": hwVersion} + } else { + flavor.ExtraSpecs = map[string]string{} + } return api.ExternalSchedulerRequest{ Spec: api.NovaObject[api.NovaSpec]{ Data: api.NovaSpec{ @@ -24,13 +38,7 @@ func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numI AvailabilityZone: az, NumInstances: numInstances, SchedulerHints: hints, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: memoryMB, - VCPUs: 4, - ExtraSpecs: map[string]string{"hw_version": hwVersion}, - }, - }, + Flavor: api.NovaObject[api.NovaFlavor]{Data: flavor}, }, }, Hosts: []api.ExternalSchedulerHost{ @@ -41,18 +49,42 @@ func makeQuotaEnforcementRequest(projectID, az, hwVersion string, memoryMB, numI } } +// installFreshMetrics swaps in a fresh QuotaEnforcementMetrics for the duration +// of a test and restores the previous singleton afterwards. Each case starts +// from a known-zero state and is fully isolated. +func installFreshMetrics(t *testing.T) *QuotaEnforcementMetrics { + t.Helper() + prev := QuotaEnforcementMetricsSingleton + m := NewQuotaEnforcementMetrics() + QuotaEnforcementMetricsSingleton = m + t.Cleanup(func() { QuotaEnforcementMetricsSingleton = prev }) + return m +} + func TestFilterQuotaEnforcement_Run(t *testing.T) { scheme := runtime.NewScheme() if err := v1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("failed to add v1alpha1 to scheme: %v", err) } - tests := []struct { - name string - objects []client.Object - request api.ExternalSchedulerRequest + type tc struct { + name string + objects []client.Object + request api.ExternalSchedulerRequest + dryRun bool + expectAccept bool - }{ + + // Metric expectations — every case asserts exactly one increment on the + // labeled series and exactly one series in the vector. + expectMode string // "enforce" | "shadow" + expectDecision string // accept_cr | accept_payg | accept_no_quota | accept_skipped | reject + expectResource string // "ram" | "cores" | "instances" | "" + expectAZ string + expectFG string + } + + tests := []tc{ { name: "ACCEPT: CR has headroom", objects: []client.Object{ @@ -67,14 +99,17 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{ - "memory": resource.MustParse("50Gi"), - }, + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("50Gi")}, }, }, }, - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_cr", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "ACCEPT: CR has exact headroom (guaranteed state)", @@ -90,14 +125,17 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{ - "memory": resource.MustParse("90Gi"), - }, + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("90Gi")}, }, }, }, - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_cr", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "ACCEPT: PAYG has headroom (no CR headroom)", @@ -113,9 +151,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("50Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{ - "memory": resource.MustParse("50Gi"), - }, + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("50Gi")}, }, }, &v1alpha1.ProjectQuota{ @@ -134,9 +170,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // PAYG headroom = 200 - 50 - 100 = 50 >= 10 → accept - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_payg", + expectResource: "", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: no CR headroom and no PAYG headroom", @@ -152,9 +192,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("50Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{ - "memory": resource.MustParse("50Gi"), - }, + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("50Gi")}, }, }, &v1alpha1.ProjectQuota{ @@ -169,9 +207,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // PAYG headroom = 100 - 50 - 50 = 0 < 10 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: PAYG headroom negative", @@ -188,80 +230,105 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // No CRs, PAYG headroom = 50 - 0 - 60 = -10 < 10 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, - }, - { - name: "ACCEPT: no ProjectQuota CRD found (skip enforcement)", - objects: []client.Object{}, - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", + }, + { + name: "ACCEPT: no ProjectQuota CRD found (skip enforcement)", + objects: []client.Object{}, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_no_quota", + expectResource: "", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "SKIP: evacuate intent", objects: []client.Object{}, request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, map[string]any{"_nova_check_type": "evacuate"}), - expectAccept: true, + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "", + expectFG: "", }, { name: "SKIP: live migration intent", objects: []client.Object{}, request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, map[string]any{"_nova_check_type": "live_migrate"}), - expectAccept: true, + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "", + expectFG: "", }, { name: "SKIP: reserve_for_failover intent", objects: []client.Object{}, request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, map[string]any{"_nova_check_type": "reserve_for_failover"}), - expectAccept: true, + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "", + expectFG: "", }, { name: "SKIP: reserve_for_committed_resource intent", objects: []client.Object{}, request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, map[string]any{"_nova_check_type": "reserve_for_committed_resource"}), - expectAccept: true, - }, - { - name: "SKIP: no hw_version in flavor", - objects: []client.Object{}, - request: api.ExternalSchedulerRequest{ - Spec: api.NovaObject[api.NovaSpec]{ - Data: api.NovaSpec{ - ProjectID: "project-1", - AvailabilityZone: "az-1", - NumInstances: 1, - Flavor: api.NovaObject[api.NovaFlavor]{ - Data: api.NovaFlavor{ - MemoryMB: 10240, - VCPUs: 4, - ExtraSpecs: map[string]string{}, - }, - }, - }, - }, - Hosts: []api.ExternalSchedulerHost{ - {ComputeHost: "host1"}, - {ComputeHost: "host2"}, - }, - }, - expectAccept: true, - }, - { - name: "SKIP: no project ID", - objects: []client.Object{}, - request: makeQuotaEnforcementRequest("", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, - }, - { - name: "SKIP: no availability zone", - objects: []client.Object{}, - request: makeQuotaEnforcementRequest("project-1", "", "hana_v2", 10240, 1, nil), - expectAccept: true, + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "", + expectFG: "", + }, + { + name: "SKIP: no hw_version in flavor", + objects: []client.Object{}, + request: makeQuotaEnforcementRequest("project-1", "az-1", "", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "az-1", + expectFG: "", + }, + { + name: "SKIP: no project ID", + objects: []client.Object{}, + request: makeQuotaEnforcementRequest("", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "az-1", + expectFG: "hana_v2", + }, + { + name: "SKIP: no availability zone", + objects: []client.Object{}, + request: makeQuotaEnforcementRequest("project-1", "", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "", + expectFG: "hana_v2", }, { name: "REJECT: CR from different project does not provide headroom", @@ -277,9 +344,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{ - "memory": resource.MustParse("0"), - }, + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("0")}, }, }, &v1alpha1.ProjectQuota{ @@ -298,10 +363,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Other-project CR has 100Gi free but belongs to project-OTHER. - // project-1 has no matching CR, PAYG headroom = 5 - 0 - 5 = 0 < 10 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: CR in different AZ doesn't count", @@ -317,9 +385,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{ - "memory": resource.MustParse("0"), - }, + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("0")}, }, }, &v1alpha1.ProjectQuota{ @@ -334,9 +400,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // CR in az-2 doesn't help. PAYG headroom = 5 - 0 - 5 = 0 < 10 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "CR in planned state is ignored", @@ -364,9 +434,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Planned CR is ignored. PAYG headroom = 5 - 0 - 5 = 0 < 10 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "ACCEPT: multiple instances, enough PAYG headroom", @@ -387,10 +461,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // 3 instances * 10240 MB = 30720 MB → ceil(30720/1024) = 30 GiB - // PAYG headroom = 500 - 0 - 100 = 400 >= 30 → accept - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 3, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 3, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_payg", + expectResource: "", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: multiple instances exceed PAYG headroom", @@ -407,10 +484,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // 3 instances * 10240 MB = 30720 MB → ceil(30720/1024) = 30 GiB - // PAYG headroom = 120 - 0 - 100 = 20 < 30 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 3, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 3, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: resize intent is enforced (not skipped)", @@ -420,20 +500,21 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Spec: v1alpha1.ProjectQuotaSpec{ ProjectID: "project-1", AvailabilityZone: "az-1", - Quota: map[string]int64{ - "hw_version_hana_v2_ram": 10, - }, + Quota: map[string]int64{"hw_version_hana_v2_ram": 10}, }, Status: v1alpha1.ProjectQuotaStatus{ PaygUsage: map[string]int64{"hw_version_hana_v2_ram": 10}, }, }, }, - // Resize intent IS enforced (not skipped like evacuate/live-migrate). - // PAYG headroom = 10 - 0 - 10 = 0 < 10 → reject request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, map[string]any{"_nova_check_type": "resize"}), - expectAccept: false, + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "CR cores type is ignored for memory check", @@ -461,9 +542,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Cores CR ignored. PAYG headroom = 5 - 0 - 5 = 0 < 10 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "ACCEPT: different hw_version resource name (vmware_v2)", @@ -484,9 +569,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // PAYG headroom = 500 - 0 - 100 = 400 >= 10 → accept - request: makeQuotaEnforcementRequest("project-1", "az-1", "vmware_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "vmware_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_payg", + expectResource: "", + expectAZ: "az-1", + expectFG: "vmware_v2", }, { name: "ACCEPT: RAM quota has headroom", @@ -503,10 +592,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Only RAM quota set. Headroom = 200 - 0 - 50 = 150 >= 10 → accept. - // Cores/instances not set → not enforced. - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_payg", + expectResource: "", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: RAM quota exceeded", @@ -523,9 +615,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Only RAM quota set. Headroom = 55 - 0 - 50 = 5 < 10 → reject. - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "ACCEPT: cores quota has headroom", @@ -542,10 +638,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Only cores quota set. Headroom = 100 - 10 = 90 >= 4 → accept. - // RAM/instances not set → not enforced. - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_payg", + expectResource: "", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: cores quota exceeded", @@ -562,9 +661,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Only cores quota set. Headroom = 5 - 3 = 2 < 4 → reject. - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "cores", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "ACCEPT: instances quota has headroom", @@ -581,10 +684,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Only instances quota set. Headroom = 10 - 5 = 5 >= 1 → accept. - // RAM/cores not set → not enforced. - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_payg", + expectResource: "", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: instances quota exceeded", @@ -601,9 +707,13 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // Only instances quota set. Headroom = 3 - 3 = 0 < 1 → reject. - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "instances", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "ACCEPT: CR headroom bypasses PAYG cores rejection", @@ -619,9 +729,7 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Amount: resource.MustParse("100Gi"), }, Status: v1alpha1.CommittedResourceStatus{ - UsedResources: map[string]resource.Quantity{ - "memory": resource.MustParse("50Gi"), - }, + UsedResources: map[string]resource.Quantity{"memory": resource.MustParse("50Gi")}, }, }, &v1alpha1.ProjectQuota{ @@ -629,19 +737,20 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { Spec: v1alpha1.ProjectQuotaSpec{ ProjectID: "project-1", AvailabilityZone: "az-1", - Quota: map[string]int64{ - "hw_version_hana_v2_cores": 2, - }, + Quota: map[string]int64{"hw_version_hana_v2_cores": 2}, }, Status: v1alpha1.ProjectQuotaStatus{ PaygUsage: map[string]int64{"hw_version_hana_v2_cores": 2}, }, }, }, - // CR has 50Gi free >= 10 GiB request → CR headroom accept. - // PAYG cores would reject (2 - 2 = 0 < 4), but CR headroom short-circuits. - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), - expectAccept: true, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + expectAccept: true, + expectMode: "enforce", + expectDecision: "accept_cr", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", }, { name: "REJECT: small flavor still rejected when no headroom", @@ -658,14 +767,130 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { }, }, }, - // 2048 MB → ceil(2048/1024) = 2 GiB. PAYG headroom = 1 - 0 - 1 = 0 < 2 → reject - request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 2048, 1, nil), - expectAccept: false, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 2048, 1, nil), + expectAccept: false, + expectMode: "enforce", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", + }, + // Shadow-mode cases. + { + name: "SHADOW: would reject RAM but dryRun preserves activations", + objects: []client.Object{ + &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-1-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-1", + AvailabilityZone: "az-1", + Quota: map[string]int64{"hw_version_hana_v2_ram": 5}, + }, + Status: v1alpha1.ProjectQuotaStatus{ + PaygUsage: map[string]int64{"hw_version_hana_v2_ram": 0}, + }, + }, + }, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + dryRun: true, + expectAccept: true, + expectMode: "shadow", + expectDecision: "reject", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", + }, + { + name: "SHADOW: would reject cores but dryRun preserves activations", + objects: []client.Object{ + &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-1-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-1", + AvailabilityZone: "az-1", + Quota: map[string]int64{"hw_version_hana_v2_cores": 3}, + }, + Status: v1alpha1.ProjectQuotaStatus{ + PaygUsage: map[string]int64{"hw_version_hana_v2_cores": 3}, + }, + }, + }, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + dryRun: true, + expectAccept: true, + expectMode: "shadow", + expectDecision: "reject", + expectResource: "cores", + expectAZ: "az-1", + expectFG: "hana_v2", + }, + { + name: "SHADOW: would reject instances but dryRun preserves activations", + objects: []client.Object{ + &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-1-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-1", + AvailabilityZone: "az-1", + Quota: map[string]int64{"hw_version_hana_v2_instances": 1}, + }, + Status: v1alpha1.ProjectQuotaStatus{ + PaygUsage: map[string]int64{"hw_version_hana_v2_instances": 1}, + }, + }, + }, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + dryRun: true, + expectAccept: true, + expectMode: "shadow", + expectDecision: "reject", + expectResource: "instances", + expectAZ: "az-1", + expectFG: "hana_v2", + }, + { + name: "SHADOW: CR accept records mode=shadow", + objects: []client.Object{ + &v1alpha1.CommittedResource{ + ObjectMeta: metav1.ObjectMeta{Name: "cr-1"}, + Spec: v1alpha1.CommittedResourceSpec{ + ProjectID: "project-1", + AvailabilityZone: "az-1", + FlavorGroupName: "hana_v2", + ResourceType: v1alpha1.CommittedResourceTypeMemory, + State: v1alpha1.CommitmentStatusConfirmed, + Amount: resource.MustParse("100Gi"), + }, + }, + }, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, nil), + dryRun: true, + expectAccept: true, + expectMode: "shadow", + expectDecision: "accept_cr", + expectResource: "ram", + expectAZ: "az-1", + expectFG: "hana_v2", + }, + { + name: "SHADOW: skip intent records mode=shadow", + objects: []client.Object{}, + request: makeQuotaEnforcementRequest("project-1", "az-1", "hana_v2", 10240, 1, + map[string]any{"_nova_check_type": "live_migrate"}), + dryRun: true, + expectAccept: true, + expectMode: "shadow", + expectDecision: "accept_skipped", + expectResource: "", + expectAZ: "", + expectFG: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + m := installFreshMetrics(t) + fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(tt.objects...). @@ -673,9 +898,9 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { filter := &FilterQuotaEnforcement{} filter.Client = fakeClient + filter.Options.DryRun = tt.dryRun - traceLog := slog.Default() - result, err := filter.Run(traceLog, tt.request) + result, err := filter.Run(slog.Default(), tt.request) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -691,6 +916,19 @@ func TestFilterQuotaEnforcement_Run(t *testing.T) { t.Errorf("expected 0 activations (reject), got %d", len(result.Activations)) } } + + // Metric assertions: exactly one increment on the expected label set + // and exactly one series in the vector (no stray increments). + got := testutil.ToFloat64(m.Decisions.WithLabelValues( + tt.expectMode, tt.expectDecision, tt.expectResource, tt.expectAZ, tt.expectFG, + )) + if got != 1 { + t.Errorf("expected 1 increment for mode=%q decision=%q resource=%q az=%q flavor_group=%q; got %v", + tt.expectMode, tt.expectDecision, tt.expectResource, tt.expectAZ, tt.expectFG, got) + } + if n := testutil.CollectAndCount(m.Decisions); n != 1 { + t.Errorf("expected exactly 1 metric series in vector, got %d", n) + } }) } } @@ -701,44 +939,66 @@ func TestQuantityToGiB(t *testing.T) { quantity resource.Quantity expected int64 }{ - { - name: "100Gi exact", - quantity: resource.MustParse("100Gi"), - expected: 100, - }, - { - name: "1Ti = 1024 GiB", - quantity: resource.MustParse("1Ti"), - expected: 1024, - }, - { - name: "512Mi = 1 GiB (ceil)", - quantity: resource.MustParse("512Mi"), - expected: 1, - }, - { - name: "1Gi exact", - quantity: resource.MustParse("1Gi"), - expected: 1, - }, - { - name: "0 bytes", - quantity: resource.MustParse("0"), - expected: 0, - }, - { - name: "1.5Gi = 2 GiB (ceil)", - quantity: resource.MustParse("1536Mi"), - expected: 2, - }, + {name: "100Gi exact", quantity: resource.MustParse("100Gi"), expected: 100}, + {name: "1Ti = 1024 GiB", quantity: resource.MustParse("1Ti"), expected: 1024}, + {name: "512Mi = 1 GiB (ceil)", quantity: resource.MustParse("512Mi"), expected: 1}, + {name: "1Gi exact", quantity: resource.MustParse("1Gi"), expected: 1}, + {name: "0 bytes", quantity: resource.MustParse("0"), expected: 0}, + {name: "1.5Gi = 2 GiB (ceil)", quantity: resource.MustParse("1536Mi"), expected: 2}, } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := quantityToGiB(tt.quantity) - if result != tt.expected { - t.Errorf("quantityToGiB(%v) = %d, want %d", tt.quantity, result, tt.expected) + if got := quantityToGiB(tt.quantity); got != tt.expected { + t.Errorf("quantityToGiB(%v) = %d, want %d", tt.quantity, got, tt.expected) } }) } } + +func TestNewQuotaEnforcementMetrics(t *testing.T) { + m := NewQuotaEnforcementMetrics() + if m == nil || m.Decisions == nil { + t.Fatal("expected non-nil metrics with non-nil Decisions vec") + } + // The constructor no longer registers, so the caller is responsible for + // registering with a registry. Verify *QuotaEnforcementMetrics implements + // prometheus.Collector by registering it. + reg := prometheus.NewRegistry() + reg.MustRegister(m) + // Increment on a label set; verify it lands. + m.RecordDecision("shadow", "reject", "ram", "az-1", "hana_v2") + got := testutil.ToFloat64(m.Decisions.WithLabelValues("shadow", "reject", "ram", "az-1", "hana_v2")) + if got != 1 { + t.Errorf("expected 1 after RecordDecision, got %v", got) + } + // Re-registering the same collector must fail (proves it was registered). + if err := reg.Register(m); err == nil { + t.Error("expected error re-registering already-registered metric") + } +} + +func TestQuotaEnforcementMetrics_RecordDecision_NilWarns(t *testing.T) { + // Capture slog output to confirm the nil-receiver path warns exactly once, + // even when called many times. We freshly arm the package-level sync.Once + // so this test deterministically covers the warn path regardless of test + // ordering. + origOnce := recordDecisionNilOnce + recordDecisionNilOnce = &sync.Once{} + t.Cleanup(func() { recordDecisionNilOnce = origOnce }) + + var buf bytes.Buffer + origDefault := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(origDefault) }) + + var nilMetrics *QuotaEnforcementMetrics + // Must not panic, and must warn exactly once across multiple calls. + nilMetrics.RecordDecision("enforce", "reject", "ram", "az-1", "hana_v2") + nilMetrics.RecordDecision("enforce", "reject", "ram", "az-1", "hana_v2") + + const msg = "QuotaEnforcementMetrics is nil" + if got := strings.Count(buf.String(), msg); got != 1 { + t.Errorf("expected warn message %q to appear exactly once, got %d; output: %q", + msg, got, buf.String()) + } +}