From 5f6be419acee20bc41fdbbb617f93d1f203db04a Mon Sep 17 00:00:00 2001 From: Stephanie Baum Date: Thu, 20 Aug 2026 11:49:30 -0700 Subject: [PATCH] test(bdd): generalize Kubernetes resource assertions Replace resource-specific and raw kubectl checks with explicit table-driven existence and absence assertions. Negative checks use ignore-not-found name output so they do not depend on human-readable errors. Relates to #862 Signed-off-by: Stephanie Baum --- tests/bdd/PLAN.md | 18 +++- tests/bdd/dsl/kubectl.go | 46 ++++++--- tests/bdd/dsl/kubectl_test.go | 51 ++++++++-- .../multi-cluster-eks-helmfile.feature | 5 +- tests/bdd/features/observability-all.feature | 19 ++-- .../features/observability-compute.feature | 23 +++-- .../features/observability-control.feature | 26 +++-- tests/bdd/godog_test.go | 77 ++++++--------- tests/bdd/steps/assertion_steps.go | 72 ++++++++++++-- tests/bdd/steps/steps_test.go | 97 ++++++++++++++++--- 10 files changed, 303 insertions(+), 131 deletions(-) diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index be79b61bb..08048bb61 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -138,7 +138,8 @@ refactor in every consumer; that is a feature. | `Then the rendered manifests in {string} under directories matching {string} should contain:` (table) | Positive rendered-manifest assertion scoped to files below a directory whose name matches the supplied shell pattern, such as `*-nats`. The render directory, directory-name pattern, and table values support `${VAR}` expansion. | | `Then the rendered manifests in {string} should not contain:` (table) | Requires a `text` header and one or more fixed strings. Recursively inspects regular files under the repo-relative directory and fails if any listed string appears. `${VAR}` expansion applies to the path and table values. | | `Then these Helm releases should be deployed using context {string}:` (table) | Requires `name` and `namespace` headers, with an optional `revision` header. Runs one explicit-context, all-namespaces `helm list` and asserts that every listed release has status `deployed`; non-empty revision cells are also matched. | -| `Then these ServiceMonitors should exist in namespace {string} using context {string}:` (table) | Requires a `name` header and one or more names. Runs one `kubectl get` with every named ServiceMonitor; exit code 0 proves every listed resource exists. | +| `Then these Kubernetes resources should exist in namespace {string} using context {string}:` (table) | Requires `kind` and `name` headers. Gets each named resource with the explicit namespace and context, and reports the row whose resource is missing. | +| `Then these Kubernetes resources should not exist in namespace {string} using context {string}:` (table) | Requires `kind` and `name` headers. Gets each named resource with `--ignore-not-found` and requires empty name output, so absence does not depend on human-readable error text. | #### YAML comparison semantics @@ -477,9 +478,18 @@ type HelmReleaseExpectation struct { // JSON output with status deployed and, when provided, the expected revision. func HelmReleasesDeployed(raw string, expected []HelmReleaseExpectation) error -// ServiceMonitorExistenceCommand builds one kubectl get command whose -// successful exit proves every named ServiceMonitor exists. -func ServiceMonitorExistenceCommand(namespace, kubeContext string, names []string) (string, error) +// KubernetesResource identifies one resource by kind and name. +type KubernetesResource struct { + Kind string + Name string +} + +// KubernetesResourceGetCommand builds an explicit-context kubectl get for one +// resource. ignoreNotFound makes a missing resource produce empty name output. +func KubernetesResourceGetCommand(namespace, kubeContext string, resource KubernetesResource, ignoreNotFound bool) (string, error) + +// KubernetesResourceAbsent requires empty output from an ignore-not-found get. +func KubernetesResourceAbsent(raw string, resource KubernetesResource) error ``` #### steps package diff --git a/tests/bdd/dsl/kubectl.go b/tests/bdd/dsl/kubectl.go index 5716cc4dd..c8f4da9da 100644 --- a/tests/bdd/dsl/kubectl.go +++ b/tests/bdd/dsl/kubectl.go @@ -22,36 +22,52 @@ import ( "strings" ) -// ServiceMonitorExistenceCommand builds one kubectl get command whose -// successful exit proves every named ServiceMonitor exists. -func ServiceMonitorExistenceCommand(namespace, kubeContext string, names []string) (string, error) { +// KubernetesResource identifies one resource by kind and name. +type KubernetesResource struct { + Kind string + Name string +} + +// KubernetesResourceGetCommand builds an explicit-context kubectl get for one +// resource. ignoreNotFound makes a missing resource produce empty name output. +func KubernetesResourceGetCommand(namespace, kubeContext string, resource KubernetesResource, ignoreNotFound bool) (string, error) { namespace = strings.TrimSpace(Interpolate(namespace)) kubeContext = strings.TrimSpace(Interpolate(kubeContext)) + kind := strings.TrimSpace(Interpolate(resource.Kind)) + name := strings.TrimSpace(Interpolate(resource.Name)) if namespace == "" { return "", fmt.Errorf("namespace is empty") } if kubeContext == "" { return "", fmt.Errorf("kube context is empty") } - if len(names) == 0 { - return "", fmt.Errorf("ServiceMonitor names are empty") + if kind == "" { + return "", fmt.Errorf("kubernetes resource kind is empty") } - - args := []string{"kubectl", "get"} - for _, rawName := range names { - name := strings.TrimSpace(Interpolate(rawName)) - if name == "" { - return "", fmt.Errorf("ServiceMonitor name is empty") - } - args = append(args, quoteCommandArg("servicemonitor/"+name)) + if name == "" { + return "", fmt.Errorf("kubernetes resource name is empty") } - args = append(args, + + args := []string{ + "kubectl", "get", quoteCommandArg(strings.ToLower(kind) + "/" + name), "--namespace", quoteCommandArg(namespace), "--context", quoteCommandArg(kubeContext), - ) + } + if ignoreNotFound { + args = append(args, "--ignore-not-found") + } + args = append(args, "-o", "name") return strings.Join(args, " "), nil } +// KubernetesResourceAbsent requires empty output from an ignore-not-found get. +func KubernetesResourceAbsent(raw string, resource KubernetesResource) error { + if strings.TrimSpace(raw) != "" { + return fmt.Errorf("kubernetes resource %s/%s exists, want absent", resource.Kind, resource.Name) + } + return nil +} + // KubectlApplyCommand builds a kubectl apply command for a manifest file. // When kubeContext is set, the command always targets that context instead of // relying on the caller's ambient kubeconfig selection. diff --git a/tests/bdd/dsl/kubectl_test.go b/tests/bdd/dsl/kubectl_test.go index 29c5739a9..587df4a03 100644 --- a/tests/bdd/dsl/kubectl_test.go +++ b/tests/bdd/dsl/kubectl_test.go @@ -19,25 +19,58 @@ package dsl import "testing" -func TestServiceMonitorExistenceCommandBuildsSingleExplicitGet(t *testing.T) { - names := []string{ - "nvcf-default-monitors-state-metrics", - "nvcf-default-monitors-grpc-proxy", +func TestKubernetesResourceGetCommandBuildsExplicitExistenceGet(t *testing.T) { + resource := KubernetesResource{Kind: "ServiceMonitor", Name: "nvcf-default-monitors-state-metrics"} + got, err := KubernetesResourceGetCommand("monitoring", "k3d-ncp-local", resource, false) + if err != nil { + t.Fatalf("build command: %v", err) + } + want := "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local -o name" + if got != want { + t.Fatalf("command = %q, want %q", got, want) } +} - got, err := ServiceMonitorExistenceCommand("monitoring", "k3d-ncp-local", names) +func TestKubernetesResourceGetCommandBuildsIgnoreNotFoundGet(t *testing.T) { + resource := KubernetesResource{Kind: "PodMonitor", Name: "nvcf-default-monitors-worker"} + got, err := KubernetesResourceGetCommand("monitoring", "k3d-ncp-local", resource, true) if err != nil { t.Fatalf("build command: %v", err) } - want := "kubectl get servicemonitor/nvcf-default-monitors-state-metrics servicemonitor/nvcf-default-monitors-grpc-proxy --namespace monitoring --context k3d-ncp-local" + want := "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local --ignore-not-found -o name" if got != want { t.Fatalf("command = %q, want %q", got, want) } } -func TestServiceMonitorExistenceCommandRejectsEmptyNames(t *testing.T) { - if _, err := ServiceMonitorExistenceCommand("monitoring", "k3d-ncp-local", nil); err == nil { - t.Fatal("expected empty names error") +func TestKubernetesResourceGetCommandRejectsMissingTargets(t *testing.T) { + tests := []struct { + name string + namespace string + kubeContext string + resource KubernetesResource + }{ + {name: "namespace", kubeContext: "k3d-ncp-local", resource: KubernetesResource{Kind: "Secret", Name: "pull-secret"}}, + {name: "context", namespace: "monitoring", resource: KubernetesResource{Kind: "Secret", Name: "pull-secret"}}, + {name: "kind", namespace: "monitoring", kubeContext: "k3d-ncp-local", resource: KubernetesResource{Name: "pull-secret"}}, + {name: "name", namespace: "monitoring", kubeContext: "k3d-ncp-local", resource: KubernetesResource{Kind: "Secret"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := KubernetesResourceGetCommand(test.namespace, test.kubeContext, test.resource, false); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestKubernetesResourceAbsentRejectsNameOutput(t *testing.T) { + resource := KubernetesResource{Kind: "Secret", Name: "nvcr-pull-secret"} + if err := KubernetesResourceAbsent("secret/nvcr-pull-secret\n", resource); err == nil { + t.Fatal("expected existing resource error") + } + if err := KubernetesResourceAbsent("\n", resource); err != nil { + t.Fatalf("empty output should prove absence: %v", err) } } diff --git a/tests/bdd/features/multi-cluster-eks-helmfile.feature b/tests/bdd/features/multi-cluster-eks-helmfile.feature index 8a8624772..38d760d15 100644 --- a/tests/bdd/features/multi-cluster-eks-helmfile.feature +++ b/tests/bdd/features/multi-cluster-eks-helmfile.feature @@ -355,8 +355,9 @@ Feature: Install a multi-cluster NVCF stack across two pre-provisioned EKS clust # operator propagated it to nvca-system. Asserting propagation here # catches a broken propagation that the node image cache would # otherwise mask under imagePullPolicy IfNotPresent. - When I run command "kubectl --context ${EKS_COMPUTE_CONTEXT} get secret nvcr-pull-secret -n nvca-system" - Then the command exit code should be 0 + Then these Kubernetes resources should exist in namespace "nvca-system" using context "${EKS_COMPUTE_CONTEXT}": + | kind | name | + | Secret | nvcr-pull-secret | Rule: Helmfile-installed multi-cluster NVCF can run workloads diff --git a/tests/bdd/features/observability-all.feature b/tests/bdd/features/observability-all.feature index 68ddfa932..20c104681 100644 --- a/tests/bdd/features/observability-all.feature +++ b/tests/bdd/features/observability-all.feature @@ -122,16 +122,15 @@ Feature: Install local Helmfile observability for both planes Then the command exit code should be 0 And the command output should contain "true" - Then these ServiceMonitors should exist in namespace "monitoring" using context "k3d-ncp-local": - | name | - | nvcf-default-monitors-state-metrics | - | nvcf-default-monitors-grpc-proxy | - | nvcf-default-monitors-llm-api-gateway | - | nvcf-default-monitors-invocation-service | - | nvcf-default-monitors-nvca | - - When I run command "kubectl get podmonitor/nvcf-default-monitors-dcgm podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local" - Then the command exit code should be 0 + Then these Kubernetes resources should exist in namespace "monitoring" using context "k3d-ncp-local": + | kind | name | + | ServiceMonitor | nvcf-default-monitors-state-metrics | + | ServiceMonitor | nvcf-default-monitors-grpc-proxy | + | ServiceMonitor | nvcf-default-monitors-llm-api-gateway | + | ServiceMonitor | nvcf-default-monitors-invocation-service | + | ServiceMonitor | nvcf-default-monitors-nvca | + | PodMonitor | nvcf-default-monitors-dcgm | + | PodMonitor | nvcf-default-monitors-worker | When I run command: """ diff --git a/tests/bdd/features/observability-compute.feature b/tests/bdd/features/observability-compute.feature index 7e2cf102d..be55e928f 100644 --- a/tests/bdd/features/observability-compute.feature +++ b/tests/bdd/features/observability-compute.feature @@ -136,19 +136,18 @@ Feature: Install local Helmfile observability with the compute profile Then the command exit code should be 0 And the command output should contain "true" - Then these ServiceMonitors should exist in namespace "monitoring" using context "k3d-ncp-local-compute-1": - | name | - | nvcf-default-monitors-nvca | + Then these Kubernetes resources should exist in namespace "monitoring" using context "k3d-ncp-local-compute-1": + | kind | name | + | ServiceMonitor | nvcf-default-monitors-nvca | + | PodMonitor | nvcf-default-monitors-dcgm | + | PodMonitor | nvcf-default-monitors-worker | - When I run command "kubectl get podmonitor/nvcf-default-monitors-dcgm podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local-compute-1" - Then the command exit code should be 0 - - When I run command "kubectl get servicemonitor --namespace monitoring --context k3d-ncp-local-compute-1 -o name" - Then the command exit code should be 0 - And the command output should not contain "nvcf-default-monitors-state-metrics" - And the command output should not contain "nvcf-default-monitors-grpc-proxy" - And the command output should not contain "nvcf-default-monitors-llm-api-gateway" - And the command output should not contain "nvcf-default-monitors-invocation-service" + Then these Kubernetes resources should not exist in namespace "monitoring" using context "k3d-ncp-local-compute-1": + | kind | name | + | ServiceMonitor | nvcf-default-monitors-state-metrics | + | ServiceMonitor | nvcf-default-monitors-grpc-proxy | + | ServiceMonitor | nvcf-default-monitors-llm-api-gateway | + | ServiceMonitor | nvcf-default-monitors-invocation-service | When I run command: """ diff --git a/tests/bdd/features/observability-control.feature b/tests/bdd/features/observability-control.feature index 8bc861e90..54ef98d00 100644 --- a/tests/bdd/features/observability-control.feature +++ b/tests/bdd/features/observability-control.feature @@ -65,19 +65,15 @@ Feature: Install local Helmfile observability with the control profile When I successfully run command "kubectl get opentelemetrycollector nvcf-observability -n monitoring --context k3d-ncp-local -o jsonpath='{.spec.targetAllocator.enabled}'" And the command output should contain "true" - Then these ServiceMonitors should exist in namespace "monitoring" using context "k3d-ncp-local": - | name | - | nvcf-default-monitors-state-metrics | - | nvcf-default-monitors-grpc-proxy | - | nvcf-default-monitors-llm-api-gateway | - | nvcf-default-monitors-invocation-service | + Then these Kubernetes resources should exist in namespace "monitoring" using context "k3d-ncp-local": + | kind | name | + | ServiceMonitor | nvcf-default-monitors-state-metrics | + | ServiceMonitor | nvcf-default-monitors-grpc-proxy | + | ServiceMonitor | nvcf-default-monitors-llm-api-gateway | + | ServiceMonitor | nvcf-default-monitors-invocation-service | - When I run command "kubectl get servicemonitor nvcf-default-monitors-nvca -n monitoring --context k3d-ncp-local" - Then the command exit code should be 1 - And the command output should contain "NotFound" - When I run command "kubectl get podmonitor nvcf-default-monitors-dcgm -n monitoring --context k3d-ncp-local" - Then the command exit code should be 1 - And the command output should contain "NotFound" - When I run command "kubectl get podmonitor nvcf-default-monitors-worker -n monitoring --context k3d-ncp-local" - Then the command exit code should be 1 - And the command output should contain "NotFound" + Then these Kubernetes resources should not exist in namespace "monitoring" using context "k3d-ncp-local": + | kind | name | + | ServiceMonitor | nvcf-default-monitors-nvca | + | PodMonitor | nvcf-default-monitors-dcgm | + | PodMonitor | nvcf-default-monitors-worker | diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 85c7b9e35..820a638f3 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -466,19 +466,17 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { // shared releases and monitor resources through explicit local context calls. func TestObservabilityControlFeatureFileWiresToSteps(t *testing.T) { const ( - registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` - serviceMonitorsCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics" + - " servicemonitor/nvcf-default-monitors-grpc-proxy" + - " servicemonitor/nvcf-default-monitors-llm-api-gateway" + - " servicemonitor/nvcf-default-monitors-invocation-service" + - " --namespace monitoring --context k3d-ncp-local" + registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` + serviceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local -o name" + absentPodMonitorCommand = "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local --ignore-not-found -o name" ) t.Setenv("NGC_API_KEY", "test-key") t.Setenv("SAMPLE_NGC_ORG", "test-org") t.Setenv("SAMPLE_NGC_TEAM", "test-team") suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ registryLoginCommand: {ExitCode: 0}, - serviceMonitorsCommand: {ExitCode: 0}, + serviceMonitorCommand: {ExitCode: 0}, + absentPodMonitorCommand: {ExitCode: 0}, "k3d cluster get ncp-local-cp": {ExitCode: 1}, "helm list --all-namespaces --kube-context k3d-ncp-local -o json": { ExitCode: 0, @@ -488,18 +486,6 @@ func TestObservabilityControlFeatureFileWiresToSteps(t *testing.T) { ExitCode: 0, Stdout: "true", }, - "kubectl get servicemonitor nvcf-default-monitors-nvca -n monitoring --context k3d-ncp-local": { - ExitCode: 1, - Stderr: "Error from server (NotFound): servicemonitors.monitoring.coreos.com \"nvcf-default-monitors-nvca\" not found\n", - }, - "kubectl get podmonitor nvcf-default-monitors-dcgm -n monitoring --context k3d-ncp-local": { - ExitCode: 1, - Stderr: "Error from server (NotFound): podmonitors.monitoring.coreos.com \"nvcf-default-monitors-dcgm\" not found\n", - }, - "kubectl get podmonitor nvcf-default-monitors-worker -n monitoring --context k3d-ncp-local": { - ExitCode: 1, - Stderr: "Error from server (NotFound): podmonitors.monitoring.coreos.com \"nvcf-default-monitors-worker\" not found\n", - }, })) seedHelmfileLocalBDDFixture(t, suite.Config.RepoRoot) seedStackSecretsTemplate(t, suite.Config.RepoRoot) @@ -525,7 +511,8 @@ func TestObservabilityControlFeatureFileWiresToSteps(t *testing.T) { runs := suite.Runner.(*fakeRunner).runs for _, command := range []string{ registryLoginCommand, - serviceMonitorsCommand, + serviceMonitorCommand, + absentPodMonitorCommand, } { if !commandRanExactly(runs, command) { t.Fatalf("exact command was never invoked: %s", command) @@ -560,14 +547,12 @@ func observabilityControlHelmListJSON() string { // cluster and that the compute profile verifies its releases and monitors. func TestObservabilityComputeFeatureFileWiresToSteps(t *testing.T) { const ( - registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` - serviceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-nvca" + - " --namespace monitoring --context k3d-ncp-local-compute-1" - podMonitorCommand = "kubectl get podmonitor/nvcf-default-monitors-dcgm" + - " podmonitor/nvcf-default-monitors-worker" + - " --namespace monitoring --context k3d-ncp-local-compute-1" - collectorEnabledCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local-compute-1 -o json | jq -r ".selfManaged.otelCollector.enabled"'` - serviceKeyCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" |` + + registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` + serviceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-nvca --namespace monitoring --context k3d-ncp-local-compute-1 -o name" + podMonitorCommand = "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local-compute-1 -o name" + absentServiceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local-compute-1 --ignore-not-found -o name" + collectorEnabledCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local-compute-1 -o json | jq -r ".selfManaged.otelCollector.enabled"'` + serviceKeyCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" |` + ` kubectl --context k3d-ncp-local-compute-1 create secret generic ngc-service-api-key` + ` --namespace nvca-system --from-file=ngc-service-api-key=/dev/stdin --dry-run=client -o yaml |` + ` kubectl --context k3d-ncp-local-compute-1 apply -f -'` @@ -583,6 +568,7 @@ func TestObservabilityComputeFeatureFileWiresToSteps(t *testing.T) { "k3d cluster get ncp-local": {ExitCode: 1}, serviceMonitorCommand: {ExitCode: 0}, podMonitorCommand: {ExitCode: 0}, + absentServiceMonitorCommand: {ExitCode: 0}, collectorEnabledCommand: {ExitCode: 0, Stdout: "true\n"}, "helm list --all-namespaces --kube-context k3d-ncp-local-compute-1 -o json": { ExitCode: 0, @@ -592,10 +578,6 @@ func TestObservabilityComputeFeatureFileWiresToSteps(t *testing.T) { ExitCode: 0, Stdout: "true", }, - "kubectl get servicemonitor --namespace monitoring --context k3d-ncp-local-compute-1 -o name": { - ExitCode: 0, - Stdout: "servicemonitor.monitoring.coreos.com/nvcf-default-monitors-nvca\n", - }, "helm status function-autoscaler --namespace nvcf --kube-context k3d-ncp-local-compute-1": { ExitCode: 1, Stderr: "Error: release: not found\n", @@ -626,6 +608,11 @@ func TestObservabilityComputeFeatureFileWiresToSteps(t *testing.T) { } runs := suite.Runner.(*fakeRunner).runs + for _, command := range []string{serviceMonitorCommand, podMonitorCommand, absentServiceMonitorCommand} { + if !commandRanExactly(runs, command) { + t.Fatalf("exact command was never invoked: %s", command) + } + } if !commandRanThatContains(runs, "kubectl --context k3d-ncp-local-compute-1 delete pod --namespace nvca-system") { t.Fatal("NVCA restart command was never invoked") } @@ -682,16 +669,9 @@ func observabilityComputeHelmListJSON() string { // local context and verifies that one shared stack serves both monitor sets. func TestObservabilityAllFeatureFileWiresToSteps(t *testing.T) { const ( - registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` - serviceMonitorsCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics" + - " servicemonitor/nvcf-default-monitors-grpc-proxy" + - " servicemonitor/nvcf-default-monitors-llm-api-gateway" + - " servicemonitor/nvcf-default-monitors-invocation-service" + - " servicemonitor/nvcf-default-monitors-nvca" + - " --namespace monitoring --context k3d-ncp-local" - podMonitorsCommand = "kubectl get podmonitor/nvcf-default-monitors-dcgm" + - " podmonitor/nvcf-default-monitors-worker" + - " --namespace monitoring --context k3d-ncp-local" + registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` + serviceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local -o name" + podMonitorCommand = "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local -o name" collectorEnabledCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o json | jq -r ".selfManaged.otelCollector.enabled"'` serviceKeyCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" |` + ` kubectl --context k3d-ncp-local create secret generic ngc-service-api-key` + @@ -707,8 +687,8 @@ func TestObservabilityAllFeatureFileWiresToSteps(t *testing.T) { suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ registryLoginCommand: {ExitCode: 0}, "k3d cluster get ncp-local-cp": {ExitCode: 1}, - serviceMonitorsCommand: {ExitCode: 0}, - podMonitorsCommand: {ExitCode: 0}, + serviceMonitorCommand: {ExitCode: 0}, + podMonitorCommand: {ExitCode: 0}, collectorEnabledCommand: {ExitCode: 0, Stdout: "true\n"}, serviceKeyCommand: {ExitCode: 0}, restartNVCACommand: {ExitCode: 0}, @@ -748,8 +728,8 @@ func TestObservabilityAllFeatureFileWiresToSteps(t *testing.T) { runs := suite.Runner.(*fakeRunner).runs for _, command := range []string{ registryLoginCommand, - serviceMonitorsCommand, - podMonitorsCommand, + serviceMonitorCommand, + podMonitorCommand, collectorEnabledCommand, serviceKeyCommand, restartNVCACommand, @@ -1423,6 +1403,7 @@ func TestMultiClusterEKSHelmfileFeatureFileWiresToSteps(t *testing.T) { "NVCT_BDD_TASK_BACKEND=" + computeClusterName, "tests/bdd/scripts/run-nvct-task-smoke.sh", }, " ") + pullSecretCommand := "kubectl get secret/nvcr-pull-secret --namespace nvca-system --context " + computeContext + " -o name" suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ // @gateway-setup: control-plane gateway address -> EKS_GATEWAY_ADDR. @@ -1444,6 +1425,7 @@ func TestMultiClusterEKSHelmfileFeatureFileWiresToSteps(t *testing.T) { `}, // compute nvca-operator helm list assertion. "helm list --all-namespaces --kube-context " + computeContext + " -o json": {ExitCode: 0, Stdout: helmListNVCAJSON()}, + pullSecretCommand: {ExitCode: 0}, // @function-lifecycle: function invoke returns the echo payload. "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/out/nvcf-cli-eks-bdd-multi.yaml function invoke --request-body '{\"message\":\"bdd-echo\",\"repeats\":1}' --timeout 120 --poll-duration 5": { ExitCode: 0, @@ -1490,6 +1472,9 @@ func TestMultiClusterEKSHelmfileFeatureFileWiresToSteps(t *testing.T) { if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "deploy/stacks/nvcf-compute-plane install") { t.Fatal("compute-plane install make target was never invoked") } + if !commandRanExactly(suite.Runner.(*fakeRunner).runs, pullSecretCommand) { + t.Fatal("compute-plane pull-secret propagation assertion was never invoked") + } if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, "function invoke") { t.Fatal("function invoke CLI command was never invoked") } diff --git a/tests/bdd/steps/assertion_steps.go b/tests/bdd/steps/assertion_steps.go index daba7c55d..c91f704b0 100644 --- a/tests/bdd/steps/assertion_steps.go +++ b/tests/bdd/steps/assertion_steps.go @@ -46,7 +46,8 @@ func registerAssertionSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^the rendered manifests in "([^"]*)" under directories matching "([^"]*)" should contain:$`, sc.renderedManifestsUnderMatchingDirectoriesShouldContain) ctx.Step(`^the rendered manifests in "([^"]*)" should not contain:$`, sc.renderedManifestsShouldNotContain) ctx.Step(`^these Helm releases should be deployed using context "([^"]*)":$`, sc.helmReleasesShouldBeDeployed) - ctx.Step(`^these ServiceMonitors should exist in namespace "([^"]*)" using context "([^"]*)":$`, sc.serviceMonitorsShouldExist) + ctx.Step(`^these Kubernetes resources should exist in namespace "([^"]*)" using context "([^"]*)":$`, sc.kubernetesResourcesShouldExist) + ctx.Step(`^these Kubernetes resources should not exist in namespace "([^"]*)" using context "([^"]*)":$`, sc.kubernetesResourcesShouldNotExist) } func (sc *ScenarioContext) commandExitCodeShouldBe(expected int) error { @@ -245,19 +246,76 @@ func tableToHelmReleaseExpectations(table *godog.Table) ([]dsl.HelmReleaseExpect return expected, nil } -func (sc *ScenarioContext) serviceMonitorsShouldExist(ctx context.Context, namespace, kubeContext string, table *godog.Table) error { - names, err := tableToSingleColumn(table, "name") +func (sc *ScenarioContext) kubernetesResourcesShouldExist(ctx context.Context, namespace, kubeContext string, table *godog.Table) error { + resources, err := tableToKubernetesResources(table) if err != nil { return err } - command, err := dsl.ServiceMonitorExistenceCommand(namespace, kubeContext, names) + for index, resource := range resources { + command, err := dsl.KubernetesResourceGetCommand(namespace, kubeContext, resource, false) + if err != nil { + return fmt.Errorf("row %d (%s/%s): %w", index+1, resource.Kind, resource.Name, err) + } + if err := sc.runAndRecord(ctx, command); err != nil { + return fmt.Errorf("row %d (%s/%s): %w", index+1, resource.Kind, resource.Name, err) + } + if err := sc.commandExitCodeShouldBe(0); err != nil { + return fmt.Errorf("row %d: Kubernetes resource %s/%s should exist: %w", index+1, resource.Kind, resource.Name, err) + } + } + return nil +} + +func (sc *ScenarioContext) kubernetesResourcesShouldNotExist(ctx context.Context, namespace, kubeContext string, table *godog.Table) error { + resources, err := tableToKubernetesResources(table) if err != nil { return err } - if err := sc.runAndRecordWith(ctx, command, sc.Suite.Runner.Run); err != nil { - return err + for index, resource := range resources { + command, err := dsl.KubernetesResourceGetCommand(namespace, kubeContext, resource, true) + if err != nil { + return fmt.Errorf("row %d (%s/%s): %w", index+1, resource.Kind, resource.Name, err) + } + if err := sc.runAndRecord(ctx, command); err != nil { + return fmt.Errorf("row %d (%s/%s): %w", index+1, resource.Kind, resource.Name, err) + } + if err := sc.commandExitCodeShouldBe(0); err != nil { + return fmt.Errorf("row %d: Kubernetes resource %s/%s absence check failed: %w", index+1, resource.Kind, resource.Name, err) + } + if err := dsl.KubernetesResourceAbsent(sc.LastResult.Stdout, resource); err != nil { + return fmt.Errorf("row %d: %w", index+1, err) + } + } + return nil +} + +func tableToKubernetesResources(table *godog.Table) ([]dsl.KubernetesResource, error) { + if table == nil || len(table.Rows) < 2 { + return nil, fmt.Errorf("table must have kind and name headers and at least one data row") + } + headers := table.Rows[0].Cells + if len(headers) != 2 || strings.TrimSpace(headers[0].Value) != "kind" || strings.TrimSpace(headers[1].Value) != "name" { + return nil, fmt.Errorf("table headers must be kind and name") + } + + resources := make([]dsl.KubernetesResource, 0, len(table.Rows)-1) + for index, row := range table.Rows[1:] { + if len(row.Cells) != len(headers) { + return nil, fmt.Errorf("row %d has %d cells, expected %d", index+1, len(row.Cells), len(headers)) + } + resource := dsl.KubernetesResource{ + Kind: strings.TrimSpace(dsl.Interpolate(row.Cells[0].Value)), + Name: strings.TrimSpace(dsl.Interpolate(row.Cells[1].Value)), + } + if resource.Kind == "" { + return nil, fmt.Errorf("row %d has an empty kind", index+1) + } + if resource.Name == "" { + return nil, fmt.Errorf("row %d has an empty name", index+1) + } + resources = append(resources, resource) } - return sc.commandExitCodeShouldBe(0) + return resources, nil } // tableToJSONRows converts a header-first Godog table into a slice of diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index 608fa195a..288ab1b8d 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -546,24 +546,99 @@ func TestSingleClusterBootstrapCachesAcrossCalls(t *testing.T) { } } -func TestServiceMonitorsShouldExistRunsSingleExplicitGet(t *testing.T) { +func TestKubernetesResourcesShouldExistRunsExplicitGets(t *testing.T) { sc, fake := newScenarioContext(t) fake.result = harness.Result{ExitCode: 0} table := docTable(t, [][]string{ - {"name"}, - {"nvcf-default-monitors-state-metrics"}, - {"nvcf-default-monitors-grpc-proxy"}, + {"kind", "name"}, + {"ServiceMonitor", "nvcf-default-monitors-state-metrics"}, + {"PodMonitor", "nvcf-default-monitors-worker"}, }) - if err := sc.serviceMonitorsShouldExist(context.Background(), "monitoring", "k3d-ncp-local", table); err != nil { - t.Fatalf("assert ServiceMonitors: %v", err) + if err := sc.kubernetesResourcesShouldExist(context.Background(), "monitoring", "k3d-ncp-local", table); err != nil { + t.Fatalf("assert Kubernetes resources: %v", err) } - if len(fake.runs) != 1 { - t.Fatalf("runs = %d, want 1", len(fake.runs)) + if len(fake.runs) != 2 { + t.Fatalf("runs = %d, want 2", len(fake.runs)) } - want := "kubectl get servicemonitor/nvcf-default-monitors-state-metrics servicemonitor/nvcf-default-monitors-grpc-proxy --namespace monitoring --context k3d-ncp-local" - if fake.runs[0].command != want { - t.Fatalf("command = %q, want %q", fake.runs[0].command, want) + want := []string{ + "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local -o name", + "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local -o name", + } + for index, run := range fake.runs { + if run.command != want[index] { + t.Fatalf("command %d = %q, want %q", index+1, run.command, want[index]) + } + } +} + +func TestKubernetesResourcesShouldExistNamesFailingRow(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 1} + table := docTable(t, [][]string{ + {"kind", "name"}, + {"ServiceMonitor", "missing-monitor"}, + }) + + err := sc.kubernetesResourcesShouldExist(context.Background(), "monitoring", "k3d-ncp-local", table) + if err == nil || !strings.Contains(err.Error(), "row 1") || !strings.Contains(err.Error(), "ServiceMonitor/missing-monitor should exist") { + t.Fatalf("error = %v", err) + } +} + +func TestKubernetesResourcesShouldNotExistUsesIgnoreNotFound(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0} + table := docTable(t, [][]string{ + {"kind", "name"}, + {"Secret", "nvcr-pull-secret"}, + }) + + if err := sc.kubernetesResourcesShouldNotExist(context.Background(), "nvca-system", "k3d-ncp-local", table); err != nil { + t.Fatalf("assert Kubernetes resource absence: %v", err) + } + want := "kubectl get secret/nvcr-pull-secret --namespace nvca-system --context k3d-ncp-local --ignore-not-found -o name" + if len(fake.runs) != 1 || fake.runs[0].command != want { + t.Fatalf("runs = %#v, want %q", fake.runs, want) + } +} + +func TestKubernetesResourcesShouldNotExistNamesExistingResource(t *testing.T) { + sc, fake := newScenarioContext(t) + fake.result = harness.Result{ExitCode: 0, Stdout: "secret/nvcr-pull-secret\n"} + table := docTable(t, [][]string{ + {"kind", "name"}, + {"Secret", "nvcr-pull-secret"}, + }) + + err := sc.kubernetesResourcesShouldNotExist(context.Background(), "nvca-system", "k3d-ncp-local", table) + if err == nil || !strings.Contains(err.Error(), "row 1") || !strings.Contains(err.Error(), "Secret/nvcr-pull-secret exists") { + t.Fatalf("error = %v", err) + } +} + +func TestKubernetesResourceTableRejectsEmptyFields(t *testing.T) { + for _, row := range [][]string{{"", "name"}, {"Secret", ""}} { + table := docTable(t, [][]string{{"kind", "name"}, row}) + if _, err := tableToKubernetesResources(table); err == nil { + t.Fatalf("expected validation error for row %#v", row) + } + } +} + +func TestKubernetesResourcesValidateAllRowsBeforeRunning(t *testing.T) { + sc, fake := newScenarioContext(t) + table := docTable(t, [][]string{ + {"kind", "name"}, + {"Secret", "valid-secret"}, + {"PodMonitor", ""}, + }) + + if err := sc.kubernetesResourcesShouldExist(context.Background(), "monitoring", "k3d-ncp-local", table); err == nil { + t.Fatal("expected validation error") + } + if len(fake.runs) != 0 { + t.Fatalf("runs = %d, want 0 before all rows validate", len(fake.runs)) } }