test(bdd): generalize Kubernetes resource assertions

tests/bdd/PLAN.md

@@ -138,7 +138,8 @@ refactor in every consumer; that is a feature.
 | `Then the rendered manifests in {string} under directories matching {string} should contain:` (table) | Positive rendered-manifest assertion scoped to files below a directory whose name matches the supplied shell pattern, such as `*-nats`. The render directory, directory-name pattern, and table values support `${VAR}` expansion. |
 | `Then the rendered manifests in {string} should not contain:` (table) | Requires a `text` header and one or more fixed strings. Recursively inspects regular files under the repo-relative directory and fails if any listed string appears. `${VAR}` expansion applies to the path and table values. |
 | `Then these Helm releases should be deployed using context {string}:` (table) | Requires `name` and `namespace` headers, with an optional `revision` header. Runs one explicit-context, all-namespaces `helm list` and asserts that every listed release has status `deployed`; non-empty revision cells are also matched. |
-| `Then these ServiceMonitors should exist in namespace {string} using context {string}:` (table) | Requires a `name` header and one or more names. Runs one `kubectl get` with every named ServiceMonitor; exit code 0 proves every listed resource exists. |
+| `Then these Kubernetes resources should exist in namespace {string} using context {string}:` (table) | Requires `kind` and `name` headers. Gets each named resource with the explicit namespace and context, and reports the row whose resource is missing. |
+| `Then these Kubernetes resources should not exist in namespace {string} using context {string}:` (table) | Requires `kind` and `name` headers. Gets each named resource with `--ignore-not-found` and requires empty name output, so absence does not depend on human-readable error text. |
 
 #### YAML comparison semantics
 
@@ -477,9 +478,18 @@ type HelmReleaseExpectation struct {
 // JSON output with status deployed and, when provided, the expected revision.
 func HelmReleasesDeployed(raw string, expected []HelmReleaseExpectation) error
 
-// ServiceMonitorExistenceCommand builds one kubectl get command whose
-// successful exit proves every named ServiceMonitor exists.
-func ServiceMonitorExistenceCommand(namespace, kubeContext string, names []string) (string, error)
+// KubernetesResource identifies one resource by kind and name.
+type KubernetesResource struct {
+    Kind string
+    Name string
+}
+
+// KubernetesResourceGetCommand builds an explicit-context kubectl get for one
+// resource. ignoreNotFound makes a missing resource produce empty name output.
+func KubernetesResourceGetCommand(namespace, kubeContext string, resource KubernetesResource, ignoreNotFound bool) (string, error)
+
+// KubernetesResourceAbsent requires empty output from an ignore-not-found get.
+func KubernetesResourceAbsent(raw string, resource KubernetesResource) error
 ```
 
 #### steps package

tests/bdd/dsl/kubectl.go

@@ -22,36 +22,52 @@ import (
 	"strings"
 )
 
-// ServiceMonitorExistenceCommand builds one kubectl get command whose
-// successful exit proves every named ServiceMonitor exists.
-func ServiceMonitorExistenceCommand(namespace, kubeContext string, names []string) (string, error) {
+// KubernetesResource identifies one resource by kind and name.
+type KubernetesResource struct {
+	Kind string
+	Name string
+}
+
+// KubernetesResourceGetCommand builds an explicit-context kubectl get for one
+// resource. ignoreNotFound makes a missing resource produce empty name output.
+func KubernetesResourceGetCommand(namespace, kubeContext string, resource KubernetesResource, ignoreNotFound bool) (string, error) {
 	namespace = strings.TrimSpace(Interpolate(namespace))
 	kubeContext = strings.TrimSpace(Interpolate(kubeContext))
+	kind := strings.TrimSpace(Interpolate(resource.Kind))
+	name := strings.TrimSpace(Interpolate(resource.Name))
 	if namespace == "" {
 		return "", fmt.Errorf("namespace is empty")
 	}
 	if kubeContext == "" {
 		return "", fmt.Errorf("kube context is empty")
 	}
-	if len(names) == 0 {
-		return "", fmt.Errorf("ServiceMonitor names are empty")
+	if kind == "" {
+		return "", fmt.Errorf("kubernetes resource kind is empty")
 	}
-
-	args := []string{"kubectl", "get"}
-	for _, rawName := range names {
-		name := strings.TrimSpace(Interpolate(rawName))
-		if name == "" {
-			return "", fmt.Errorf("ServiceMonitor name is empty")
-		}
-		args = append(args, quoteCommandArg("servicemonitor/"+name))
+	if name == "" {
+		return "", fmt.Errorf("kubernetes resource name is empty")
 	}
-	args = append(args,
+
+	args := []string{
+		"kubectl", "get", quoteCommandArg(strings.ToLower(kind) + "/" + name),
 		"--namespace", quoteCommandArg(namespace),
 		"--context", quoteCommandArg(kubeContext),
-	)
+	}
+	if ignoreNotFound {
+		args = append(args, "--ignore-not-found")
+	}
+	args = append(args, "-o", "name")
 	return strings.Join(args, " "), nil
 }
 
+// KubernetesResourceAbsent requires empty output from an ignore-not-found get.
+func KubernetesResourceAbsent(raw string, resource KubernetesResource) error {
+	if strings.TrimSpace(raw) != "" {
+		return fmt.Errorf("kubernetes resource %s/%s exists, want absent", resource.Kind, resource.Name)
+	}
+	return nil
+}
+
 // KubectlApplyCommand builds a kubectl apply command for a manifest file.
 // When kubeContext is set, the command always targets that context instead of
 // relying on the caller's ambient kubeconfig selection.

tests/bdd/dsl/kubectl_test.go

@@ -19,25 +19,58 @@ package dsl
 
 import "testing"
 
-func TestServiceMonitorExistenceCommandBuildsSingleExplicitGet(t *testing.T) {
-	names := []string{
-		"nvcf-default-monitors-state-metrics",
-		"nvcf-default-monitors-grpc-proxy",
+func TestKubernetesResourceGetCommandBuildsExplicitExistenceGet(t *testing.T) {
+	resource := KubernetesResource{Kind: "ServiceMonitor", Name: "nvcf-default-monitors-state-metrics"}
+	got, err := KubernetesResourceGetCommand("monitoring", "k3d-ncp-local", resource, false)
+	if err != nil {
+		t.Fatalf("build command: %v", err)
+	}
+	want := "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local -o name"
+	if got != want {
+		t.Fatalf("command = %q, want %q", got, want)
 	}
+}
 
-	got, err := ServiceMonitorExistenceCommand("monitoring", "k3d-ncp-local", names)
+func TestKubernetesResourceGetCommandBuildsIgnoreNotFoundGet(t *testing.T) {
+	resource := KubernetesResource{Kind: "PodMonitor", Name: "nvcf-default-monitors-worker"}
+	got, err := KubernetesResourceGetCommand("monitoring", "k3d-ncp-local", resource, true)
 	if err != nil {
 		t.Fatalf("build command: %v", err)
 	}
-	want := "kubectl get servicemonitor/nvcf-default-monitors-state-metrics servicemonitor/nvcf-default-monitors-grpc-proxy --namespace monitoring --context k3d-ncp-local"
+	want := "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local --ignore-not-found -o name"
 	if got != want {
 		t.Fatalf("command = %q, want %q", got, want)
 	}
 }
 
-func TestServiceMonitorExistenceCommandRejectsEmptyNames(t *testing.T) {
-	if _, err := ServiceMonitorExistenceCommand("monitoring", "k3d-ncp-local", nil); err == nil {
-		t.Fatal("expected empty names error")
+func TestKubernetesResourceGetCommandRejectsMissingTargets(t *testing.T) {
+	tests := []struct {
+		name        string
+		namespace   string
+		kubeContext string
+		resource    KubernetesResource
+	}{
+		{name: "namespace", kubeContext: "k3d-ncp-local", resource: KubernetesResource{Kind: "Secret", Name: "pull-secret"}},
+		{name: "context", namespace: "monitoring", resource: KubernetesResource{Kind: "Secret", Name: "pull-secret"}},
+		{name: "kind", namespace: "monitoring", kubeContext: "k3d-ncp-local", resource: KubernetesResource{Name: "pull-secret"}},
+		{name: "name", namespace: "monitoring", kubeContext: "k3d-ncp-local", resource: KubernetesResource{Kind: "Secret"}},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if _, err := KubernetesResourceGetCommand(test.namespace, test.kubeContext, test.resource, false); err == nil {
+				t.Fatal("expected validation error")
+			}
+		})
+	}
+}
+
+func TestKubernetesResourceAbsentRejectsNameOutput(t *testing.T) {
+	resource := KubernetesResource{Kind: "Secret", Name: "nvcr-pull-secret"}
+	if err := KubernetesResourceAbsent("secret/nvcr-pull-secret\n", resource); err == nil {
+		t.Fatal("expected existing resource error")
+	}
+	if err := KubernetesResourceAbsent("\n", resource); err != nil {
+		t.Fatalf("empty output should prove absence: %v", err)
 	}
 }
 

tests/bdd/features/multi-cluster-eks-helmfile.feature

@@ -355,8 +355,9 @@ Feature: Install a multi-cluster NVCF stack across two pre-provisioned EKS clust
       # operator propagated it to nvca-system. Asserting propagation here
       # catches a broken propagation that the node image cache would
       # otherwise mask under imagePullPolicy IfNotPresent.
-      When I run command "kubectl --context ${EKS_COMPUTE_CONTEXT} get secret nvcr-pull-secret -n nvca-system"
-      Then the command exit code should be 0
+      Then these Kubernetes resources should exist in namespace "nvca-system" using context "${EKS_COMPUTE_CONTEXT}":
+        | kind   | name             |
+        | Secret | nvcr-pull-secret |
 
   Rule: Helmfile-installed multi-cluster NVCF can run workloads
 

tests/bdd/features/observability-all.feature

@@ -122,16 +122,15 @@ Feature: Install local Helmfile observability for both planes
     Then the command exit code should be 0
     And the command output should contain "true"
 
-    Then these ServiceMonitors should exist in namespace "monitoring" using context "k3d-ncp-local":
-      | name                                             |
-      | nvcf-default-monitors-state-metrics              |
-      | nvcf-default-monitors-grpc-proxy                  |
-      | nvcf-default-monitors-llm-api-gateway             |
-      | nvcf-default-monitors-invocation-service          |
-      | nvcf-default-monitors-nvca                        |
-
-    When I run command "kubectl get podmonitor/nvcf-default-monitors-dcgm podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local"
-    Then the command exit code should be 0
+    Then these Kubernetes resources should exist in namespace "monitoring" using context "k3d-ncp-local":
+      | kind           | name                                             |
+      | ServiceMonitor | nvcf-default-monitors-state-metrics              |
+      | ServiceMonitor | nvcf-default-monitors-grpc-proxy                  |
+      | ServiceMonitor | nvcf-default-monitors-llm-api-gateway             |
+      | ServiceMonitor | nvcf-default-monitors-invocation-service          |
+      | ServiceMonitor | nvcf-default-monitors-nvca                        |
+      | PodMonitor     | nvcf-default-monitors-dcgm                        |
+      | PodMonitor     | nvcf-default-monitors-worker                      |
 
     When I run command:
       """

tests/bdd/features/observability-compute.feature

@@ -136,19 +136,18 @@ Feature: Install local Helmfile observability with the compute profile
     Then the command exit code should be 0
     And the command output should contain "true"
 
-    Then these ServiceMonitors should exist in namespace "monitoring" using context "k3d-ncp-local-compute-1":
-      | name                          |
-      | nvcf-default-monitors-nvca   |
+    Then these Kubernetes resources should exist in namespace "monitoring" using context "k3d-ncp-local-compute-1":
+      | kind           | name                          |
+      | ServiceMonitor | nvcf-default-monitors-nvca    |
+      | PodMonitor     | nvcf-default-monitors-dcgm    |
+      | PodMonitor     | nvcf-default-monitors-worker  |
 
-    When I run command "kubectl get podmonitor/nvcf-default-monitors-dcgm podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local-compute-1"
-    Then the command exit code should be 0
-
-    When I run command "kubectl get servicemonitor --namespace monitoring --context k3d-ncp-local-compute-1 -o name"
-    Then the command exit code should be 0
-    And the command output should not contain "nvcf-default-monitors-state-metrics"
-    And the command output should not contain "nvcf-default-monitors-grpc-proxy"
-    And the command output should not contain "nvcf-default-monitors-llm-api-gateway"
-    And the command output should not contain "nvcf-default-monitors-invocation-service"
+    Then these Kubernetes resources should not exist in namespace "monitoring" using context "k3d-ncp-local-compute-1":
+      | kind           | name                                             |
+      | ServiceMonitor | nvcf-default-monitors-state-metrics              |
+      | ServiceMonitor | nvcf-default-monitors-grpc-proxy                  |
+      | ServiceMonitor | nvcf-default-monitors-llm-api-gateway             |
+      | ServiceMonitor | nvcf-default-monitors-invocation-service          |
 
     When I run command:
       """

tests/bdd/features/observability-control.feature

@@ -65,19 +65,15 @@ Feature: Install local Helmfile observability with the control profile
     When I successfully run command "kubectl get opentelemetrycollector nvcf-observability -n monitoring --context k3d-ncp-local -o jsonpath='{.spec.targetAllocator.enabled}'"
     And the command output should contain "true"
 
-    Then these ServiceMonitors should exist in namespace "monitoring" using context "k3d-ncp-local":
-      | name                                             |
-      | nvcf-default-monitors-state-metrics              |
-      | nvcf-default-monitors-grpc-proxy                  |
-      | nvcf-default-monitors-llm-api-gateway             |
-      | nvcf-default-monitors-invocation-service          |
+    Then these Kubernetes resources should exist in namespace "monitoring" using context "k3d-ncp-local":
+      | kind           | name                                             |
+      | ServiceMonitor | nvcf-default-monitors-state-metrics              |
+      | ServiceMonitor | nvcf-default-monitors-grpc-proxy                  |
+      | ServiceMonitor | nvcf-default-monitors-llm-api-gateway             |
+      | ServiceMonitor | nvcf-default-monitors-invocation-service          |
 
-    When I run command "kubectl get servicemonitor nvcf-default-monitors-nvca -n monitoring --context k3d-ncp-local"
-    Then the command exit code should be 1
-    And the command output should contain "NotFound"
-    When I run command "kubectl get podmonitor nvcf-default-monitors-dcgm -n monitoring --context k3d-ncp-local"
-    Then the command exit code should be 1
-    And the command output should contain "NotFound"
-    When I run command "kubectl get podmonitor nvcf-default-monitors-worker -n monitoring --context k3d-ncp-local"
-    Then the command exit code should be 1
-    And the command output should contain "NotFound"
+    Then these Kubernetes resources should not exist in namespace "monitoring" using context "k3d-ncp-local":
+      | kind           | name                          |
+      | ServiceMonitor | nvcf-default-monitors-nvca    |
+      | PodMonitor     | nvcf-default-monitors-dcgm    |
+      | PodMonitor     | nvcf-default-monitors-worker  |