From 106fd857b500b37f64080df4c26a0c406ea7f72d Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 6 Nov 2026 20:11:02 +0100 Subject: [PATCH 1/9] feat: add native DPF Sentry generation and deployment Consume DPF discovery inventory to resolve a pinned Sentry BF4 profile, render a validated immutable bundle, and deploy or validate explicitly bound management and tenant resources. Preserve ownership and external host maintenance boundaries, and document prerequisites and procedural workflows. Signed-off-by: Alexander Maslennikov --- README.md | 4 +- docs/integrator/automation.md | 12 +- docs/llms.txt | 3 +- docs/reference/cli.md | 46 +- docs/user/dpf-discovery.md | 11 +- docs/user/dpf-sentry.md | 337 ++++++++ docs/user/dpf-workflows.md | 205 ++++- engineering/advanced/targets.md | 7 +- engineering/architecture/overview.md | 2 +- go.mod | 10 +- go.sum | 28 + mkdocs.yml | 3 +- openspec/specs/artifact-handoff/spec.md | 26 +- openspec/specs/workflow-execution/spec.md | 43 ++ openspec/workflows/README.md | 3 +- openspec/workflows/dpf-discovery.json | 194 ++++- pkg/cmd/deploy.go | 15 +- pkg/cmd/generate.go | 16 +- pkg/cmd/schema.go | 25 + pkg/cmd/schema_test.go | 3 +- pkg/cmd/target_cli.go | 7 +- pkg/cmd/target_cli_test.go | 84 +- pkg/cmd/validate.go | 14 +- pkg/configflags/flags.go | 33 +- pkg/configflags/flags_test.go | 78 ++ pkg/dpf/bundle/bundle_test.go | 336 ++++++++ pkg/dpf/bundle/load.go | 185 +++++ pkg/dpf/bundle/publish.go | 180 +++++ pkg/dpf/bundle/rename_darwin.go | 10 + pkg/dpf/bundle/rename_linux.go | 10 + pkg/dpf/bundle/rename_other.go | 12 + pkg/dpf/bundle/scope.go | 46 ++ pkg/dpf/bundle/types.go | 193 +++++ pkg/dpf/bundle/validate.go | 381 +++++++++ pkg/dpf/config/config.go | 26 +- pkg/dpf/config/generation.go | 397 ++++++++++ pkg/dpf/config/generation_test.go | 374 +++++++++ pkg/dpf/config/overrides.go | 74 ++ pkg/dpf/config/overrides_test.go | 56 ++ pkg/dpf/config/resolved.go | 159 ++++ pkg/dpf/config/resolved_test.go | 160 ++++ pkg/dpf/deployment/client.go | 242 ++++++ pkg/dpf/deployment/comparison.go | 430 +++++++++++ pkg/dpf/deployment/comparison_test.go | 84 ++ pkg/dpf/deployment/coverage.go | 194 +++++ pkg/dpf/deployment/coverage_test.go | 115 +++ pkg/dpf/deployment/dns.go | 413 ++++++++++ pkg/dpf/deployment/dns_test.go | 285 +++++++ .../deployment/dpucluster_contract_test.go | 91 +++ pkg/dpf/deployment/engine.go | 482 ++++++++++++ pkg/dpf/deployment/engine_test.go | 564 ++++++++++++++ pkg/dpf/deployment/final_regression_test.go | 184 +++++ pkg/dpf/deployment/flavor_admission_test.go | 115 +++ .../deployment/lifecycle_regression_test.go | 184 +++++ .../deployment/native_serialization_test.go | 146 ++++ pkg/dpf/deployment/preflight.go | 727 ++++++++++++++++++ pkg/dpf/deployment/private.go | 333 ++++++++ pkg/dpf/deployment/private_test.go | 188 +++++ pkg/dpf/deployment/readiness.go | 161 ++++ .../service_account_conversion_test.go | 95 +++ pkg/dpf/deployment/service_stage_test.go | 81 ++ pkg/dpf/deployment/types.go | 85 ++ pkg/dpf/generation/generate.go | 182 +++++ pkg/dpf/generation/generation_test.go | 221 ++++++ pkg/dpf/sentry/catalog.go | 43 ++ pkg/dpf/sentry/projection.go | 129 ++++ pkg/dpf/sentry/render.go | 574 ++++++++++++++ pkg/dpf/sentry/render_test.go | 501 ++++++++++++ pkg/dpf/sentry/schema.go | 194 +++++ .../schemas/26.8.0-BlueFieldSoftware.json | 1 + pkg/dpf/sentry/schemas/26.8.0-DPUFlavor.json | 1 + pkg/dpf/sentry/schemas/26.8.0-DPUService.json | 1 + .../schemas/26.8.0-DPUServiceChain.json | 1 + .../26.8.0-DPUServiceCredentialRequest.json | 1 + .../schemas/26.8.0-DPUServiceInterface.json | 1 + .../sentry/schemas/26.8.0-DPUServiceNAD.json | 1 + pkg/dpf/sentry/schemas/26.8.0-DPUSet.json | 1 + .../26.8.0-rc.1-BlueFieldSoftware.json | 1 + .../sentry/schemas/26.8.0-rc.1-DPUFlavor.json | 1 + .../schemas/26.8.0-rc.1-DPUService.json | 1 + .../schemas/26.8.0-rc.1-DPUServiceChain.json | 1 + ....8.0-rc.1-DPUServiceCredentialRequest.json | 1 + .../26.8.0-rc.1-DPUServiceInterface.json | 1 + .../schemas/26.8.0-rc.1-DPUServiceNAD.json | 1 + .../sentry/schemas/26.8.0-rc.1-DPUSet.json | 1 + pkg/dpf/sentry/schemas/README.md | 14 + pkg/dpf/sentry/schemas/provenance.json | 114 +++ pkg/dpf/sentry/testdata/PROVENANCE.md | 46 ++ .../charts/doca-argus-1.5.1-ai-gateway.8.tgz | Bin 0 -> 9234 bytes .../doca-sentry-policy-controller-0.2.5.tgz | Bin 0 -> 5316 bytes .../doca-sentry-policy-server-0.2.5.tgz | Bin 0 -> 2292 bytes .../testdata/charts/doca-tower-0.2.16.tgz | Bin 0 -> 29710 bytes pkg/dpf/sentry/values.go | 292 +++++++ pkg/dpf/sentry/values_test.go | 181 +++++ .../maintenance_render_test.go | 11 + pkg/options/options.go | 4 + pkg/profiles/profiles.go | 226 +++++- pkg/profiles/target_profile_test.go | 154 ++++ pkg/target/dpf/driver.go | 8 +- pkg/target/dpf/driver_test.go | 22 +- pkg/target/dpf/workflow.go | 166 ++++ profiles/sentry-bf4/10-flavor.yaml | 246 ++++++ .../sentry-bf4/20-passthrough-interfaces.yaml | 61 ++ profiles/sentry-bf4/21-passthrough-chain.yaml | 27 + profiles/sentry-bf4/30-tower-nads.yaml | 27 + profiles/sentry-bf4/31-tower-interfaces.yaml | 45 ++ profiles/sentry-bf4/32-tower.yaml | 101 +++ profiles/sentry-bf4/33-tower-chain.yaml | 26 + profiles/sentry-bf4/40-controller.yaml | 42 + profiles/sentry-bf4/50-argus.yaml | 120 +++ profiles/sentry-bf4/60-policy-rbac.yaml | 28 + profiles/sentry-bf4/61-policy-credential.yaml | 19 + profiles/sentry-bf4/62-policy-server.yaml | 36 + profiles/sentry-bf4/profile.yaml | 19 + skills/k8s-launch-kit-deploy/SKILL.md | 16 + skills/k8s-launch-kit-generate/SKILL.md | 14 + skills/k8s-launch-kit-shared/SKILL.md | 10 +- skills/k8s-launch-kit-validate/SKILL.md | 14 + 118 files changed, 12874 insertions(+), 82 deletions(-) create mode 100644 docs/user/dpf-sentry.md create mode 100644 pkg/dpf/bundle/bundle_test.go create mode 100644 pkg/dpf/bundle/load.go create mode 100644 pkg/dpf/bundle/publish.go create mode 100644 pkg/dpf/bundle/rename_darwin.go create mode 100644 pkg/dpf/bundle/rename_linux.go create mode 100644 pkg/dpf/bundle/rename_other.go create mode 100644 pkg/dpf/bundle/scope.go create mode 100644 pkg/dpf/bundle/types.go create mode 100644 pkg/dpf/bundle/validate.go create mode 100644 pkg/dpf/config/generation.go create mode 100644 pkg/dpf/config/generation_test.go create mode 100644 pkg/dpf/config/overrides.go create mode 100644 pkg/dpf/config/overrides_test.go create mode 100644 pkg/dpf/config/resolved.go create mode 100644 pkg/dpf/config/resolved_test.go create mode 100644 pkg/dpf/deployment/client.go create mode 100644 pkg/dpf/deployment/comparison.go create mode 100644 pkg/dpf/deployment/comparison_test.go create mode 100644 pkg/dpf/deployment/coverage.go create mode 100644 pkg/dpf/deployment/coverage_test.go create mode 100644 pkg/dpf/deployment/dns.go create mode 100644 pkg/dpf/deployment/dns_test.go create mode 100644 pkg/dpf/deployment/dpucluster_contract_test.go create mode 100644 pkg/dpf/deployment/engine.go create mode 100644 pkg/dpf/deployment/engine_test.go create mode 100644 pkg/dpf/deployment/final_regression_test.go create mode 100644 pkg/dpf/deployment/flavor_admission_test.go create mode 100644 pkg/dpf/deployment/lifecycle_regression_test.go create mode 100644 pkg/dpf/deployment/native_serialization_test.go create mode 100644 pkg/dpf/deployment/preflight.go create mode 100644 pkg/dpf/deployment/private.go create mode 100644 pkg/dpf/deployment/private_test.go create mode 100644 pkg/dpf/deployment/readiness.go create mode 100644 pkg/dpf/deployment/service_account_conversion_test.go create mode 100644 pkg/dpf/deployment/service_stage_test.go create mode 100644 pkg/dpf/deployment/types.go create mode 100644 pkg/dpf/generation/generate.go create mode 100644 pkg/dpf/generation/generation_test.go create mode 100644 pkg/dpf/sentry/catalog.go create mode 100644 pkg/dpf/sentry/projection.go create mode 100644 pkg/dpf/sentry/render.go create mode 100644 pkg/dpf/sentry/render_test.go create mode 100644 pkg/dpf/sentry/schema.go create mode 100644 pkg/dpf/sentry/schemas/26.8.0-BlueFieldSoftware.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-DPUFlavor.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-DPUService.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-DPUServiceChain.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-DPUServiceCredentialRequest.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-DPUServiceInterface.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-DPUServiceNAD.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-DPUSet.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-BlueFieldSoftware.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUFlavor.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUService.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceChain.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceCredentialRequest.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceInterface.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceNAD.json create mode 100644 pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUSet.json create mode 100644 pkg/dpf/sentry/schemas/README.md create mode 100644 pkg/dpf/sentry/schemas/provenance.json create mode 100644 pkg/dpf/sentry/testdata/PROVENANCE.md create mode 100644 pkg/dpf/sentry/testdata/charts/doca-argus-1.5.1-ai-gateway.8.tgz create mode 100644 pkg/dpf/sentry/testdata/charts/doca-sentry-policy-controller-0.2.5.tgz create mode 100644 pkg/dpf/sentry/testdata/charts/doca-sentry-policy-server-0.2.5.tgz create mode 100644 pkg/dpf/sentry/testdata/charts/doca-tower-0.2.16.tgz create mode 100644 pkg/dpf/sentry/values.go create mode 100644 pkg/dpf/sentry/values_test.go create mode 100644 pkg/profiles/target_profile_test.go create mode 100644 pkg/target/dpf/workflow.go create mode 100644 profiles/sentry-bf4/10-flavor.yaml create mode 100644 profiles/sentry-bf4/20-passthrough-interfaces.yaml create mode 100644 profiles/sentry-bf4/21-passthrough-chain.yaml create mode 100644 profiles/sentry-bf4/30-tower-nads.yaml create mode 100644 profiles/sentry-bf4/31-tower-interfaces.yaml create mode 100644 profiles/sentry-bf4/32-tower.yaml create mode 100644 profiles/sentry-bf4/33-tower-chain.yaml create mode 100644 profiles/sentry-bf4/40-controller.yaml create mode 100644 profiles/sentry-bf4/50-argus.yaml create mode 100644 profiles/sentry-bf4/60-policy-rbac.yaml create mode 100644 profiles/sentry-bf4/61-policy-credential.yaml create mode 100644 profiles/sentry-bf4/62-policy-server.yaml create mode 100644 profiles/sentry-bf4/profile.yaml diff --git a/README.md b/README.md index dfb32a31..7b328b0d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Kubernetes Launch Kit (`l8k`) helps platform teams deploy NVIDIA networking on K Standard SR-IOV, RDMA-shared and host-device profiles enable NIC tuning by default; opt out with `--deploy-nic-configuration-template=false`. See [NIC configuration](docs/reference/configuration.md#nic-configuration-and-naming) for fabric-specific settings and operator ownership. -The current lifecycle uses the `host` target by default. The `dpf` target provides an [experimental inventory discovery preview](docs/user/dpf-discovery.md); its provisioning phases remain unavailable. +The current lifecycle uses the `host` target by default. The `dpf` target provides [experimental inventory discovery](docs/user/dpf-discovery.md) and a [Sentry workflow for enrolled BlueField-4 devices](docs/user/dpf-sentry.md), with generation, deployment and resource validation against existing DPF infrastructure. DPF inventory can use explicit host/ARM SSH or BMC Redfish sources without an existing management cluster. Inventory success is separate from provisioning readiness. DPF BMC discovery verifies TLS by default; lab endpoints can explicitly opt in to @@ -171,7 +171,7 @@ Available Commands: version Print the version number Target Selection Flags: - --target string Deployment target: host (default) or dpf (experimental inventory discovery) (default "host") + --target string Deployment target: host (default) or dpf (experimental discovery and Sentry workflow) (default "host") Host Target Common Flags: --config-dir string Directory containing optional l8k-config.yaml and presets/ overrides diff --git a/docs/integrator/automation.md b/docs/integrator/automation.md index 89300a45..7c5279c5 100644 --- a/docs/integrator/automation.md +++ b/docs/integrator/automation.md @@ -43,14 +43,20 @@ Output contracts differ by command: | Command | Successful stdout with `--output json` | | --- | --- | -| Root pipeline, `discover`, `generate` | One `JSONResult` envelope, including collected `messages`. | +| Host root pipeline, `discover`, `generate` | One `JSONResult` envelope, including collected `messages`. | | `clean` | One result with a `cleanup` summary. | -| Standalone `deploy` | No finalized success JSON envelope. Use the process exit status; progress is sent to stderr. | -| `validate` | A sequence of JSON objects: manifest/version summary when full validation runs, connectivity when run, and `reportPath` when the HTML report is written. Do not parse the stream as one document. | +| Host standalone `deploy` | No finalized success JSON envelope. Use the process exit status; progress is sent to stderr. | +| Host `validate` | A sequence of JSON objects: manifest/version summary when full validation runs, connectivity when run, and `reportPath` when the HTML report is written. Do not parse the stream as one document. | | `schema` | One capability object; this command always emits JSON. | | `version` | One version/build object. | | `preset list`, `preset update`, `sosreport` | Text output; these commands do not provide a JSON success contract despite accepting the inherited flag. | +DPF `generate`, `deploy` and `validate` emit one final workflow object with +`target`, `phase`, `outcome`, `exitCode`, optional `generation`/`deployment` +results, and `errors`. Inspect each check's state, reason code and explanation; +a nonzero process status or unresolved admission check is not success. DPF +run reports are saved beside the immutable bundle. See [Sentry workflow](../user/dpf-sentry.md). + Lifecycle JSON mode sends human-readable progress to stderr and auto-confirms prompts. Structured error objects may be emitted on failures; invalid command-line arguments can instead write text to stderr. Validation can emit partial results before a failure. Always check the process exit status. diff --git a/docs/llms.txt b/docs/llms.txt index 2880e386..e60f9529 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -19,7 +19,8 @@ This documentation covers the `l8k` CLI for deploying NVIDIA cloud-native networ - Discover cluster hardware: /k8s-launch-kit/user/discovery/ - DPF discovery preview: /k8s-launch-kit/user/dpf-discovery/ -- DPF discovery workflows: /k8s-launch-kit/user/dpf-workflows/ +- DPF workflows: /k8s-launch-kit/user/dpf-workflows/ +- Deploy Sentry with DPF: /k8s-launch-kit/user/dpf-sentry/ - Generate and review manifests: /k8s-launch-kit/advanced/generation/ - Deploy manifests: /k8s-launch-kit/advanced/deployment/ - Deploy on OpenShift: /k8s-launch-kit/user/openshift/ diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 39aadf60..832d92f2 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -32,9 +32,11 @@ with a source filename and document number where applicable. ## Target selection and flag ownership -Omitting `--target` selects `host`. `dpf` provides an -[experimental discovery preview](../user/dpf-discovery.md). Its other phases -are unavailable and return exit `2`; they do not run against the host cluster. +Omitting `--target` selects `host`. `dpf` provides experimental +[inventory discovery](../user/dpf-discovery.md) and the [Sentry BF4 +generate/deploy/validate workflow](../user/dpf-sentry.md) on an existing DPF +management cluster, tenant and enrolled devices. The root DPF pipeline remains +unavailable; it does not fall through into Host execution. Host-only flags are rejected when they are explicitly supplied for another target. Defaults are ignored by this check, including explicit-value flags @@ -45,18 +47,24 @@ flag groups and `l8k schema` for each flag's `targets` list. | --- | --- | --- | --- | | `--target` | discover, generate, deploy, validate, root pipeline | target-agnostic | Target name. Defaults to `host`. | | `--kubeconfig` | root, discover, generate with deploy, deploy, clean, validate, sosreport | host | Path to kubeconfig. Falls back to `$KUBECONFIG` and then `~/.kube/config`. It represents the host workload cluster, not a universal multi-context input. | -| `--user-config` | root, discover, generate, deploy, clean, validate | host | Config file to merge, render, validate against, or use for cleanup namespace and Helm-ownership resolution. | +| `--user-config` | root, discover, generate, deploy, clean, validate | host and dpf where the phase is available | Host configuration or DPF site input. DPF deploy/validate normally consume the bundle's resolved input; an explicit file must preserve generated intent. | | `--config-dir` | all | host | Directory containing optional `l8k-config.yaml` and `presets/` overrides. | | `--network-operator-release` | root, discover, generate | host | Catalogued release line such as `26.1`, `26.4`, `26.7`, or `26.10`. | | `--network-operator-namespace` | root, discover, generate, deploy, clean, validate | host | Override the Network Operator namespace. It is a no-op for discovery. | | `--skip-network-operator-helm` | generate, deploy, validate, root pipeline | host | Skip `values.yaml` generation, Network Operator chart installation, and Helm-specific validation. Custom-resource handling remains enabled. | | `--flavor` | root, discover, generate, deploy, validate, clean | host | `k8s` or `ocp`, overriding config. Clean accepts the flag to reject OpenShift cleanup explicitly. | -| `--output json` | all (inherited) | target-agnostic | Command-specific output; validation emits a stream, preset/sosreport success remains text, and standalone deploy has no success envelope. See [Automation](../integrator/automation.md#json-mode). | +| `--output json` | all (inherited) | target-agnostic | Command-specific output. DPF lifecycle emits one final target/phase result with report paths; Host validation emits a stream, preset/sosreport success remains text, and Host standalone deploy has no success envelope. See [Automation](../integrator/automation.md#json-mode). | | `--yes`, `-y` | root pipeline only | target-agnostic | Auto-confirm root prompts. Subcommands reject this flag; lifecycle JSON mode auto-confirms. | | `--quiet` | root pipeline | target-agnostic | Suppress informational output. | | `--log-level` | all | target-agnostic | Enable `trace`, `debug`, `info`, `warn`, or `error` logging. `debug` shows structured progress; `trace` also shows bounded command output. | | `--log-file` | all | target-agnostic | Write logs to a file instead of `stderr`. | +DPF generation accepts `--dpf-release` and `--dpf-profile` on `generate` only. +Select an explicit generation release and profile from `l8k schema` under +`dpf.generationReleases` and `dpf.profiles`; `dpf.charts` lists the four required +local archive coordinates and hashes. DPF credentials and management context +come from its site/resolved input, never the Host `--kubeconfig` fallback. + The public host config remains the flat `cluster-config.yaml` schema. Generated manifests remain under `deployment/network-operator/`; the exact resolved configuration is stored separately at @@ -76,6 +84,7 @@ available throughout the command tree. | `--kubeconfig`, `--user-config`, `--flavor`, `--network-operator-namespace` | yes | yes | yes | yes | yes | yes | | `--network-operator-release`, `--image-pull-secrets`, profile/Spectrum-X flags | yes | yes | yes | — | — | — | | `--enabled-plugins` | yes | yes | yes | — | — | — | +| `--dpf-release`, `--dpf-profile` | — | — | yes | — | — | — | | `--node-selector` | yes | yes | yes (`--for`) | — | — | — | | `--save-cluster-config`, `--collapse-nic-rails` | yes | yes | — | — | — | — | | `--keep-namespace` | — | yes | — | — | — | — | @@ -114,8 +123,29 @@ resolves missing values from hardware defaults. With `--user-config`, only `clusterConfig` is replaced and explicit flags are the only changes made to the rest of the supplied configuration. +## DPF workflow flags + +| Flag | Command | Meaning | +| --- | --- | --- | +| `--dpf-release` | generate | Explicit installed DPF release; overrides `dpf.release` in the site input. It does not upgrade the operator. | +| `--dpf-profile` | generate | DPF profile; overrides `dpf.profile`. The available profile is `sentry-bf4`. | +| `--save-deployment-files` | generate | Required immutable output directory for the DPF bundle. | +| `--deployment-files` | deploy, validate | Previously generated DPF bundle; retained target metadata must identify DPF. | +| `--deploy` | generate | Deploy the exact generated bundle through the same DPF deployment path. | +| `--dry-run` | deploy or generate with deploy | Server-side admission of native resources; unavailable future APIs may remain unresolved. It does not prove service readiness. | +| `--deploy-timeout` | deploy | Explicit end-to-end budget; use a bounded duration such as `45m`. DPF additionally has a 45-minute default when no explicit budget is supplied. | +| `--wait` | validate | Bound resource validation, for example `10m`. Host connectivity/report flags are not DPF policy tests. | + +DPF generation alone reads local inputs and chart archives. Deploy/validate +connect to the bound management and tenant APIs. The bundle remains unchanged; +run reports are written beside it under `.runs/`. Read the [DPF Sentry +procedure](../user/dpf-sentry.md) for external prerequisites, physical effects, +retry boundaries and separate allow/deny verification. + ## Profile Flags +These flags describe Host profiles; DPF uses the separate workflow flags above. + | Flag | Description | | --- | --- | | `--fabric` | `ethernet` or `infiniband`. | @@ -165,7 +195,7 @@ root pipeline. The remaining profile flags also apply to discovery. | --- | --- | | `--deployment-files` | Directory containing generated manifests. Defaults to `./deployment`. | | `--dry-run` | Use server-side dry run. | -| `--deploy-timeout` | End-to-end deploy timeout, accepted by standalone deploy and the root pipeline. `0` means unbounded. Not available on `generate --deploy`. | +| `--deploy-timeout` | End-to-end deploy timeout, accepted by standalone deploy and the root pipeline. Host `0` means unbounded; DPF retains its separate lifecycle guard. Not available on `generate --deploy`. | | `--overwrite-existing` | Upgrade conflicting Helm chart/values and delete reported stray CRs, including resources without l8k ownership annotations. See the [deletion boundary](../advanced/deployment.md#stray-resource-deletion-boundary). | | `--skip-network-operator-helm` | Skip chart installation and Helm chart-version/values preflight checks; still apply manifests and check component versions and strays. | @@ -203,6 +233,10 @@ full deletion boundary. ## Validate Flags +The following connectivity and HTML/JUnit report behavior describes Host +validation. DPF validation checks resources against its frozen bundle and writes +a separate JSON run report; it does not create a policy-test workload. + When `--deployment-files` contains only user-provided `*example*.yaml` test DaemonSets, `validate` runs connectivity only. Adding `values.yaml` or any non-example YAML manifest selects the full validation pipeline. Connectivity diff --git a/docs/user/dpf-discovery.md b/docs/user/dpf-discovery.md index f5819ed4..933adad5 100644 --- a/docs/user/dpf-discovery.md +++ b/docs/user/dpf-discovery.md @@ -24,8 +24,9 @@ deployment readiness checks do not block inventory collection. Overall deploymen Explicitly select `discovery.probes.mode: inventory`. The default `management` mode is unavailable in this preview; selecting it stops the command before remote access. Inventory reports management topology when a kubeconfig is supplied, -but does not test management-network connectivity. DPF generate/deploy/validate -and the DPF pipeline are unavailable. +but does not test management-network connectivity. The separate experimental +[Sentry BF4 workflow](dpf-sentry.md) provides generation, deployment and resource +validation for an existing DPF environment. The root DPF pipeline remains unavailable. ## Inputs and invocation @@ -80,7 +81,7 @@ remote access. Lists and credentials stay in YAML; no password CLI flag exists. | Observed | Physical-card/system identities, linked host adapters, component firmware, auth/time unknowns, management nodes/version | | Release defaults | Software bill of materials (BOM), minimum versions for supported models, and explicit unknown requirements; these do not establish deployment readiness | | Optional lookup inputs | Explicit ARM/BMC MAC selectors; optional local `dnsmasq-leases` files. Lookup returns address hints, never identity or permission to connect | -| Later phases | Deployment profile, DPU API allocation, DHCP/service networks and controller authentication | +| Separate Sentry workflow | Explicit device/tenant bindings, deployment release, OS/firmware hashes, local charts, service networking and existing controller/Secret access; see [required deployment inputs](dpf-sentry.md#prepare-the-deployment-configuration) | The management kubeconfig must use verified HTTPS and certificate/key or bearer credentials. File and embedded representations of the same credential/trust field are mutually @@ -194,7 +195,9 @@ models `B3210`, `B3220` and `B3240`; generic family names and BlueField-4 do not establish applicability. Desired BFB/component coordinates are not measured versions. Only documented minimum versions are compatibility checks; unsupported firmware requirements remain unknown. Model applicability and software checks do -not qualify hardware or enable DPF deployment profiles. +not qualify hardware. Sentry generation validates its separately selected BF4 +profile, desired cohort and explicit enrollment bindings; this discovery BOM is +not a substitute for those deployment inputs. SSH can report NIC firmware/PSID, MFT version, NVConfig and privilege observations when the existing tools and permissions permit the read. Current and next-boot diff --git a/docs/user/dpf-sentry.md b/docs/user/dpf-sentry.md new file mode 100644 index 00000000..94254b92 --- /dev/null +++ b/docs/user/dpf-sentry.md @@ -0,0 +1,337 @@ + + +# Deploy Sentry on enrolled BlueField-4 devices + +Use the experimental `sentry-bf4` profile to generate, deploy and verify Sentry +resources for explicitly selected Zero Trust BlueField-4 devices. Start with the +[DPF workflows](dpf-workflows.md) for the complete sequence, including discovery. +This procedure consumes an existing DPF management cluster, DPU tenant cluster +and enrolled devices. It does not create VMs, bootstrap Kubernetes or DPF, enroll +devices, or remove another deployment's ownership. + +## Where operations run + +| Location | Operations | +| --- | --- | +| CLI machine during `generate` | Read the saved inventory, desired site settings, profile and four local chart archives; validate and save a bundle. No cluster connection, credential read or download occurs. | +| CLI machine during `deploy` or `validate` | Use the explicitly bound management kubeconfig/context, obtain the selected tenant's kubeconfig through DPF, verify cluster/device/namespace identities, and write a separate run report. | +| Management Kubernetes and DPF controllers during deploy | Accept the provisioning and service resources, download the chosen ISO/firmware, provision the selected devices, and reconcile service charts into the selected tenant or management cluster. | +| DPU OS and tenant Kubernetes | Run the configured DPU flavor, passthrough, Tower, policy controller and Argus workloads. Provisioning can reimage a DPU and require a host cold boot. | +| Management service namespace | Run the Sentry policy REST server. Its credential request supplies tenant policy access. | + +For a Kamaji tenant with a control-plane VIP, DPF hosts tenant DNS on the +management cluster and exposes it through the tenant DNS Service. Deployment +checks the resources on both clusters; functional DNS resolution is a separate +network check. + +Generation, server admission, resource readiness and functional policy enforcement +are separate checks. The profile and selected release do not by themselves +qualify a hardware/software combination for production. + +## Prepare the existing environment + +Before deployment, arrange these prerequisites through the existing cluster and +device administration process: + +- A healthy DPF operator and management services, a healthy selected DPUCluster, + and supported management/tenant API access. Select an available generation + release from `l8k schema` under `dpf.generationReleases`. +- Enrolled DPUDevice and DPUNode objects matching the selected inventory's + physical serials, BMC endpoints, PSIDs and host associations. Devices owned by + another DPUSet or DPUDeployment require an explicitly authorized external + transition before this workflow. Launch Kit does not perform that transition. The selected DPUNode must use the + external reboot method, with no pending reboot or power-off request; arrange + host power handling separately. +- A dedicated Sentry service namespace that already exists and is Active on + **both** management and tenant clusters. It must differ from the existing DPF + enrollment namespace. Launch Kit consumes these namespaces without creating + or adopting them. +- An image-pull Secret in that service namespace on both clusters, containing + `.dockerconfigjson` for the selected registries. +- Argo CD repository credentials for the selected OCI and HTTPS Helm chart + registries, configured in the existing Argo CD namespace. Image-pull Secrets + and the CLI machine's registry login do not provide this chart access. +- The same MITM CA Secret in that service namespace on both clusters, containing + `ca.crt`, `tls.crt` and `tls.key`. Use a site-managed RSA CA suitable for Tower; + distribute only its public certificate to test workloads. +- If LLM inspection is enabled, a tenant Secret containing the real + `INFERENCE_API_KEY`, plus access to the configured inference service. No + placeholder credential is accepted as evidence of working inspection. +- Host IOMMU configuration compatible with Argus. The + [DPF 26.8 Argus requirements](https://networking-docs.nvidia.com/dpf/26.8.0/doca-argus-service) + specify either IOMMU disabled or IOMMU enabled with passthrough for bare-metal + hosts. The site administrator must choose and authorize the mode: these + host-wide settings change DMA isolation and may require host maintenance and + a reboot. Confirm the running host configuration. The DPU flavor's ARM kernel + parameters do not configure the host's boot parameters, and Launch Kit does + not choose or apply a host IOMMU mode. +- An untagged Layer 2 uplink on each selected DPU's `p0`, connected to an + existing IPv4 DHCP server or configured relay. The DHCP scope must supply an + address and router usable with `p0`'s MAC: Tower's egress helper temporarily + borrows that MAC on `net2` to obtain its lease. The router and routing beyond + it must reach the approved upstream test endpoint. Arrange switch/VLAN, DHCP + and routing through the network administrator; Launch Kit does not provision + them. +- Existing DPF BMC credentials, controller client certificates/trust and the + necessary API permissions. Artifact serving, DPU management connectivity, + DNS, time, image pulls and host power handling must work for the chosen cohort. + +For a functional host VF test, also prepare the host's VF allocation, device +resource and network attachment separately. This DPF workflow does not install +Host Network Operator or configure host workload networking. + +Core policy enforcement is the default: `sentry.llmInspection: false`. It uses +Tower's fail-closed authorization path and Argus identity without an inference +credential. The chart retains an optional fail-open processing hook when its +inspection sidecar is disabled; core mode makes **no LLM inspection claim**. +Set `llmInspection: true` and supply `inferenceSecret` only when that additional +inspection path is intended and provisioned. + +## Prepare the deployment configuration + +Copy a successful [discovered configuration](dpf-discovery.md) to +`deployment-site.yaml`, retaining its `inventory`. Add the desired settings +below; replace every angle-bracket placeholder before generating. Do not edit +the inventory to turn an unknown or conflicting observation into a fact. + +| Input class | What belongs here | +| --- | --- | +| Required site intent | Explicit installed DPF release; selected inventory device IDs; existing DPUDevice/DPUNode names; selected tenant name, namespace, resource UID, tenant cluster UID and exact label selector; tenant API server version; ISO and PSID-specific firmware URLs with SHA-256; local chart directory; ingress subnet/address; existing Secret names; reachable management policy-server address. | +| Observed bindings | Device serial, model, BMC endpoint, PSID and unique host association come from the inventory selected by ID. Management cluster UID and API version default from saved management observations; supply them explicitly when absent. Enrollment object names and tenant bindings require explicit inspection and input. | +| Defaultable settings | `dpf.name: sentry`, `dpf.profile: sentry-bf4`, `dpf.namespace: dpf-operator-system`, service namespace `-services`, policy-server NodePort `30080`, and `llmInspection: false`. Explicit empty required values are rejected. | +| Release/profile pins | Chart versions/archive hashes, workload image digests and release-specific API schemas come from Launch Kit. They are separate from measured installed versions and site-selected OS/firmware artifacts. | + +The management kubeconfig file and context are explicit references in the site +file. DPF lifecycle commands do not use `--kubeconfig`, `$KUBECONFIG` or an ambient +current context. Generation records the reference without opening that file; +deploy and validate need access to it. This slice requires self-contained kubeconfigs +with embedded CA and credential data, verified TLS, and no exec/auth-provider +plugins or file-based credential references. Record API **server** versions, rather +than substituting Kubernetes node/kubelet versions. + +This is a configuration fragment to add to the discovered file: + +```yaml +dpf: + name: sentry + namespace: dpf-operator-system +management: + kubeconfigRef: {file: ./credentials/management.kubeconfig} + context: management + # Required here only when the saved management observation is absent: + clusterUID: "" + kubernetesVersion: "" +provisioning: + devices: + - id: "" + dpuDevice: "" + dpuNode: "" + dpuCluster: + name: "" + namespace: "" + resourceUID: "" + clusterUID: "" + kubernetesVersion: "" + selector: {"site.example/tenant": ""} + software: + osISO: + url: "https://artifacts.example/bf4.iso" + sha256: "<64-lowercase-hex-characters>" + pldm: + MT_0000001774: + url: "https://artifacts.example/psid-matched.fwpkg" + sha256: "<64-lowercase-hex-characters>" +sentry: + namespace: sentry-services + chartDirectory: ./sentry-charts + ingressCIDR: 192.168.100.0/24 + ingressIP: 192.168.100.1 + imagePullSecret: sentry-pull + mitmSecret: sentry-mitm-ca + policyServerAddress: 192.0.2.100 + llmInspection: false +``` + +Use the actual selected PSID as the `pldm` key. BlueField-4 consumes ISO/PLDM +artifacts; a BlueField-3 BFB is not interchangeable. If the existing desired +cohort includes separate NIC firmware, retain it under +`provisioning.software.nicFirmware` with its URL and SHA-256. Artifact URLs must +use HTTPS without embedded credentials, query strings or fragments. + +`sentry.ingressCIDR` and `sentry.ingressIP` configure the `net1` ingress +network only. They do not configure `net2`, an egress gateway or a DHCP service. +The current profile obtains its egress IPv4 address and router through the `p0` +DHCP path described above; a management or BMC lease does not establish that path. + +The tenant selector must match exactly the named existing DPUCluster. Arrange an +appropriate label externally if none exists; generation does not add one. +Deployment checks its name, namespace and UID, and checks exact device selector +membership before effects. Reusing a convenient broad selector is insufficient. + +## Obtain and verify the four local charts + +On the CLI machine, inspect the installed binary's catalog: + +```bash +l8k schema > l8k-schema.json +jq '.dpf | {profiles, generationReleases, charts}' l8k-schema.json +``` + +Using authorized registry access and your normal credential store, obtain each +`dpf.charts` entry from its `repository` at its exact `version`. Put the bytes +under the listed `filename` in `sentry.chartDirectory`, and verify SHA-256 against +the catalog. Use the declared repository type: not every chart is available +through an OCI endpoint. Do not put registry passwords in command-line arguments +or site YAML. Generation rejects missing, modified or mismatched archives and +does not download replacements. + +## Generate and review + +Run on the CLI machine: + +```bash +l8k generate --target dpf --user-config deployment-site.yaml \ + --dpf-release 26.8.0 --dpf-profile sentry-bf4 \ + --save-deployment-files ./sentry-bundle --output json \ + > generate.json 2> generate.log +``` + +Choose the release that actually matches the installed operator; the example +does not upgrade DPF. Review the result and process exit status. Inspect the +bundle's selected devices, resolved settings, management/tenant destinations, +resource names, device and tenant selectors, and artifact hashes before deploy. + +The bundle includes native resources, chart archives, effective values and +validation-only chart projections. DPF owns chart deployment; Launch Kit does +not apply the projections a second time. `.l8k/bundle.yaml` identifies the DPF +target and binds the files; `.l8k/resolved-config.yaml` retains the effective +inputs. Preserve the bundle unchanged. To change desired intent, generate a new +bundle directory. Repeating identical generation at the same path is allowed; +different content does not overwrite an existing bundle. + +## Check admission, then deploy + +Server-side dry-run contacts the bound management and tenant APIs, checks live +identity/scope/prerequisites, and submits native resources with dry-run enabled: + +```bash +l8k deploy --target dpf --deployment-files ./sentry-bundle \ + --dry-run --deploy-timeout 10m --output json \ + > admission.json 2> admission.log +``` + +Review every check. An API or admission prerequisite that is not yet served may +be reported as deferred/unresolved. Charts and future chart-created CRDs are +not installed by dry-run, and no runtime readiness is established. Unresolved admission returns an incomplete result and a nonzero exit status. + +After reviewing the concrete plan and arranging any authorized ownership +transition and maintenance window, deploy with a budget that includes artifact downloads, firmware and OS installation, +required host power cycles, and service startup: + +```bash +l8k deploy --target dpf --deployment-files ./sentry-bundle \ + --deploy-timeout 90m --output json > deploy.json 2> deploy.log +``` + +The CLI checks existing operator/tenant health, selected identities and ownership, +service namespaces/Secret keys, then creates the bundle's resources in dependency +order. It verifies the downloaded provisioning bytes before each DPUSet creates +physical effects. DPF performs reimaging, host power coordination and service +reconciliation. External firmware caches must remain unchanged during provisioning. +A host cold boot also affects co-resident devices and workloads; include that +host-wide effect in the maintenance plan. + +After the selected DPU nodes and DNS are ready, Launch Kit submits Tower, the +policy controller and Argus together, then waits for all three. The policy +controller installs the policy API used by Tower, while Tower provides the +policy socket used by the controller. The policy REST server starts afterward +on the management cluster. + +Existing foreign owners or changed desired resources stop the operation. +Launch Kit does not silently adopt, overwrite or delete them. Cancellation or a +timeout stops CLI orchestration; it does not roll back controller work already +started. Preserve the report, inspect actual controller/device state and resolve +the reported cause before retrying the unchanged bundle. Upgrades and teardown +are outside this first workflow. + +`generate --deploy` uses the same generated bundle and deployment path: + +```bash +l8k generate --target dpf --user-config deployment-site.yaml \ + --dpf-release 26.8.0 --dpf-profile sentry-bf4 \ + --save-deployment-files ./sentry-bundle --deploy --output json +``` + +Add `--dry-run` to that form to check admission after local generation. +Generation alone does not need `--dry-run`. Use separate commands when an +explicit `--deploy-timeout` is needed; that flag belongs to standalone deploy. + +## Verify resources and policy behavior + +Check the unchanged bundle against current management and tenant resources: + +```bash +l8k validate --target dpf --deployment-files ./sentry-bundle \ + --wait 10m --output json > validate.json 2> validate.log +``` + +This verifies bound identities, desired resource state and selected DPU coverage. +It does **not** run a Sentry allow/deny traffic test or prove LLM inspection. +Deploy and validate reports are stored outside the immutable bundle, beneath +`sentry-bundle.runs//report.json`; retain these with stdout/stderr. + +The pinned Argus chart has no application readiness probe. A Running/Ready +Argus pod or a successful resource validation does not prove that Argus attached +to the host or can identify its processes. Require successful host attachment, +a responding configured query socket, and the expected executable identity for +the VF test process before claiming functional enforcement. + +Perform a separate functional test using your prepared host VF workload: + +1. Place the test pod on the selected host with its existing VF resource and NAD. + Confirm the real VF MAC, ingress subnet and route; require a working DPU + egress address. Mount only the public MITM CA for HTTPS. +2. With no policy for that MAC, send HTTP and HTTPS through the VF interface to + an approved test endpoint. Expect an attributable authorization denial. +3. Submit a policy for the actual host DPUNode and VF MAC through the policy + server's `PUT /v1/policy`. Permit a known GET path for `/usr/bin/curl` and + deny another. Poll `GET /v1/policy?nodeName=...&mac=...` until `applied`; + writes initially return `202`/`in_progress`. The PUT JSON body contains + `nodeName`, `mac` and `policy` (the policy YAML as a string). +4. Through the same VF, verify that the allowed HTTP/HTTPS request reaches the + test server and the denied request receives the expected policy denial. + Use `curl --interface `, an explicit `--resolve` when VF DNS + is unavailable, and `--cacert ` for HTTPS. An egress `503` + is not a successful positive test. Confirm Argus identifies the requesting + process and the controller programs the intended DPU. +5. Delete the test policy through `DELETE /v1/policy`, poll GET until `404`, + verify baseline denial returns, and remove only the test workload/resources. + DELETE uses a JSON body containing the same `nodeName` and `mac`. + +If LLM inspection was selected, test that path separately with the real service +credentials and inspect its result. Core policy success provides no such result. + +### Distinguish network, Argus and policy failures + +If Tower's `net2` has no IPv4 lease or usable router, inspect the existing egress +helper's DHCP result and ask the network administrator to verify the `p0` link, +untagged VLAN, DHCP scope or relay, MAC policy and return path. Resolve that +prerequisite before testing permitted upstream traffic. Do not start a second +DHCP client alongside the managed helper. + +An attributable authorization denial is a policy result; an upstream connection +failure or egress `503` is not proof of denial or successful enforcement. With a +lease and route present, distinguish upstream reachability from the applied +policy, Argus process identity and controller programming. A positive test must +reach the approved upstream server, and the negative test must produce the +expected authorization denial. + +If Argus is Running/Ready but has no responding query socket or host attachment, +inspect its initialization and DMA errors and verify the chosen host IOMMU mode +against the linked DPF requirement. A DPU ARM passthrough setting does not fix a +host translation failure. Resolve the reported prerequisite through the site +administrator, then verify actual executable identity; do not count pod readiness +alone as an Argus functional pass. diff --git a/docs/user/dpf-workflows.md b/docs/user/dpf-workflows.md index 336c03ad..fda97bbb 100644 --- a/docs/user/dpf-workflows.md +++ b/docs/user/dpf-workflows.md @@ -6,11 +6,11 @@ Generated by scripts/generate-workflow-docs.py from openspec/workflows/dpf-discovery.json. Edit the source, then regenerate. --> -# DPF discovery workflows +# DPF workflows -Use these procedures with the experimental `l8k discover --target dpf` CLI. The DPU estate is separate from Host Network Operator deployment. Start with **inventory mode**; management probe mode is unavailable in this preview. Discovery collects facts and writes a handoff. It does not create VMs, install Kubernetes or DPF, configure DPUServices, change firmware, enroll trust, or start a DPF deployment. +Use these procedures for DPF inventory and the experimental Sentry BF4 generate/deploy/validate workflow. The DPU estate is independent of Host Network Operator deployment. Start with **inventory mode** to collect physical identity and host associations; management probe mode remains unavailable. Deploying Sentry additionally requires an existing DPF management cluster, tenant, enrolled devices and the explicit prerequisites in the [Sentry procedure](dpf-sentry.md). VM creation, Kubernetes/DPF bootstrap, enrollment, upgrades, teardown and implicit transfer from another owner are outside this workflow. -For complete field and error details, see [DPF discovery reference](dpf-discovery.md). Each diagram follows the same steps as its procedure. Commands are examples to run deliberately from a machine with direct access to your selected endpoints. +Each diagram follows its procedure. Run commands from a CLI machine with the required local files and explicitly bound access. Inventory success, local generation, API admission, resource readiness and functional traffic checks establish different outcomes. See [DPF discovery reference](dpf-discovery.md) for collection fields and errors. ## Collect the first inventory {#initial-inventory} @@ -196,7 +196,7 @@ Read each non-success observation's `reasonCode`, `message` and `tool` where pre The resolved software BOM is saved under `inventory.software`, including source references and unknown requirements. Release `26.4.0` has exact BlueField-3 model entries `B3210`, `B3220` and `B3240`; generic family names and BlueField-4 are not sufficient for BOM applicability. The documented NIC/MFT minima can be evaluated when current measurements associated with the correct device exist. BMC, CEC, ATF, UEFI, OS/BSP and golden-image minima remain unknown where the catalog has no supported requirement. A BFB target is a desired artifact, not an observed OS/BSP version. An unknown requirement must remain unresolved; supplying a guessed minimum does not establish compatibility. -Exit zero and `discovery.outcome: complete` mean inventory collection, identity/mapping, cleanup and publication succeeded. `acceptance: not-evaluated` remains explicit. Hardware support, DPF authentication, service connectivity and deployment still need separate assessment; DPF generate/deploy/validate are unavailable in this preview. +Exit zero and `discovery.outcome: complete` mean inventory collection, identity/mapping, cleanup and publication succeeded. `acceptance: not-evaluated` remains explicit. Hardware support, DPF authentication, service connectivity and deployment still need separate assessment; the separate [Sentry BF4 procedure](dpf-sentry.md) provides generation, deployment and resource validation only after its additional prerequisites are met. Next: [Complete inventory reviewed](#initial-inventory-finish); [Required failure or conflict](#initial-inventory-resolve). @@ -354,4 +354,199 @@ Create a new seed from the desired access configuration, without copying old `in ### 6. CLI machine: retain the refreshed files {#refresh-inventory-retain} -Keep the current YAML, JSON/stdout report and stderr log, plus `source-before.yaml` when a complete in-place refresh created it. Discovery completion still means a successful inventory workflow, with deployment `acceptance: not-evaluated`. Unknown software requirements and unverified deployment prerequisites remain unresolved; a refreshed inventory does not enable DPF deployment. +Keep the current YAML, JSON/stdout report and stderr log, plus `source-before.yaml` when a complete in-place refresh created it. Discovery completion still means a successful inventory workflow, with deployment `acceptance: not-evaluated`. Unknown software requirements and unverified deployment prerequisites remain unresolved; a refreshed inventory alone does not establish Sentry deployment readiness. Continue with the [explicit deployment prerequisites](dpf-sentry.md#prepare-the-existing-environment) when that workflow is intended. + +## Generate a Sentry BF4 bundle {#generate-sentry} + +Use this procedure after reviewing physical discovery and selecting an existing DPF environment. Generation runs locally on the CLI machine, reads no deployment credentials and contacts no cluster. It does not establish readiness. + +```mermaid +flowchart TD + n1["CLI operator: select devices and desired site inputs"] + n2["CLI machine: obtain the pinned local charts"] + n3["CLI machine: generate the local bundle"] + n4["CLI operator: review destinations and physical scope"] + n5["CLI operator: preserve the bundle and choose the next action"] + n1 -->|"Bindings and intent reviewed"| n2 + n2 -->|"Four archives verified"| n3 + n3 -->|"Generation completed"| n4 + n4 -->|"Review complete"| n5 +``` + +### 1. CLI operator: select devices and desired site inputs {#generate-sentry-bind-inputs} + +Copy the discovered YAML to `deployment-site.yaml`, retaining its inventory. Choose the exact inventory device IDs and explicitly name their enrolled DPUDevice/DPUNode objects. Bind the existing tenant by name, namespace, selector, resource UID, tenant cluster UID and API server version. Management UID/version may default from saved observations; management kubeconfig/context must be explicit. Supply PSID-matched ISO/PLDM URLs and SHA-256, service networks and existing Secret names. Follow the [configuration fragment and input table](dpf-sentry.md#prepare-the-deployment-configuration). Do not invent missing observations or infer enrollment names. + +Core policy defaults to `llmInspection: false` and makes no LLM inspection claim. Enable it only with the real inference Secret and intended external inference access. + +`ingressCIDR` and `ingressIP` configure only the `net1` ingress network. The current profile obtains `net2` egress IPv4 and its router through DHCP on the untagged `p0` uplink; these inputs do not configure egress, switch VLANs, a DHCP server/relay or upstream routing. + +Next: [Bindings and intent reviewed](#generate-sentry-stage-charts). + +### 2. CLI machine: obtain the pinned local charts {#generate-sentry-stage-charts} + +Run `l8k schema` and inspect `dpf.charts`. Obtain all four exact chart archives using authorized registry access and your credential store, preserving catalog filenames in `sentry.chartDirectory`. Verify their SHA-256. Use each declared repository type; never put registry passwords in CLI arguments or site YAML. Generation does not download or repair missing archives. + +Next: [Four archives verified](#generate-sentry-render). + +### 3. CLI machine: generate the local bundle {#generate-sentry-render} + +Choose a new output directory and run: + +```bash +l8k generate --target dpf --user-config deployment-site.yaml \ + --dpf-release 26.8.0 --dpf-profile sentry-bf4 \ + --save-deployment-files ./sentry-bundle --output json \ + > generate.json 2> generate.log +``` + +Use the installed DPF release, not an intended upgrade. Generation validates exact chart bytes, resource identities, selected scope and release-specific API shapes before publishing. Check the exit status and returned result. + +Next: [Generation completed](#generate-sentry-review). + +### 4. CLI operator: review destinations and physical scope {#generate-sentry-review} + +Inspect the selected-device set, exact device/tenant selectors, resource names, service/provisioning namespaces and artifact hashes. Management provisioning/service CRs and tenant RBAC have separate destinations. Effective chart values and projections are retained for validation; DPF deploys the charts, so do not apply those projections manually. + +The bundle is immutable and includes target metadata and resolved inputs. Repeating identical output is allowed. Corrected intent requires a new bundle directory; do not edit the captured files to force acceptance. A generated bundle does not authorize an ownership transition or physical reimage. + +Next: [Review complete](#generate-sentry-retain). + +### 5. CLI operator: preserve the bundle and choose the next action {#generate-sentry-retain} + +Keep `sentry-bundle`, `generate.json` and `generate.log`. Continue to [deployment](#deploy-sentry) only after external namespaces, Secrets, access, device ownership and the maintenance window are ready. `generate --deploy` invokes the same deployment path after generation; use separate commands to set an explicit standalone deploy timeout. + +## Check admission and deploy the selected Sentry bundle {#deploy-sentry} + +Run from the CLI machine. The existing DPF controllers perform firmware download, DPU provisioning, host power coordination and chart reconciliation. Real deployment can reimage selected devices. No VM/bootstrap/enrollment or implicit foreign-owner transition is performed. + +```mermaid +flowchart TD + n1["Cluster/device administrators: prepare external ownership and access"] + n2["CLI machine and bound APIs: check server admission"] + n3["CLI operator: inspect passed and unresolved admission checks"] + n4["CLI and management controllers: create and reconcile in dependency order"] + n5["CLI operator: inspect the persisted run and actual state"] + n6["Responsible administrator: correct the reported prerequisite or desired intent"] + n7["CLI operator: retain evidence and verify policy behavior"] + n1 -->|"Prerequisites and physical scope authorized"| n2 + n2 -->|"Admission checks returned"| n3 + n3 -->|"Concrete plan reviewed for real deploy"| n4 + n4 -->|"Run completed or stopped"| n5 + n5 -->|"Selected resources ready"| n7 + n5 -->|"Failed or unresolved run"| n6 + n6 -->|"Unchanged intent and cause resolved"| n2 +``` + +### 1. Cluster/device administrators: prepare external ownership and access {#deploy-sentry-prerequisites} + +Arrange the [existing-environment prerequisites](dpf-sentry.md#prepare-the-existing-environment). The dedicated service namespace must be Active on management and tenant clusters, with registry and MITM Secrets in both; optional inference credentials belong on the tenant. Configure Argo CD repository credentials for the selected OCI and HTTPS chart registries in the existing Argo CD namespace; image-pull Secrets and local CLI login do not provide chart access. DPF management and tenant services and BMC controller credentials/trust must already work. If another DPUSet or DPUDeployment owns a selected device, resolve the explicitly authorized transition externally. Do not bypass an ownership error by relabeling or deleting unrelated resources. + +Arrange an untagged Layer 2 `p0` uplink with an existing IPv4 DHCP server or relay that supplies a lease and router acceptable for `p0`'s MAC; Tower temporarily borrows that MAC on `net2`. Require a route to the approved upstream test endpoint. The network administrator owns switch/VLAN, DHCP and routing configuration; Launch Kit does not provision them. A management/BMC lease does not establish this egress path. + +Arrange the host IOMMU mode required by [DPF 26.8 Argus](https://networking-docs.nvidia.com/dpf/26.8.0/doca-argus-service): bare-metal hosts require IOMMU disabled or enabled with passthrough. The site administrator chooses and authorizes this host-wide DMA-isolation setting and any maintenance/reboot. DPU flavor ARM kernel settings do not configure the host; Launch Kit does not select or apply that mode. + +Next: [Prerequisites and physical scope authorized](#deploy-sentry-preview). + +### 2. CLI machine and bound APIs: check server admission {#deploy-sentry-preview} + +Run: + +```bash +l8k deploy --target dpf --deployment-files ./sentry-bundle \ + --dry-run --deploy-timeout 10m --output json \ + > admission.json 2> admission.log +``` + +The management access file/context comes from the resolved bundle; no ambient kubeconfig is used. The CLI checks live identities, selected membership, ownership and external prerequisites, then requests server-side dry-run for native resources. No chart or DPU is provisioned by this preview. + +Next: [Admission checks returned](#deploy-sentry-review-admission). + +### 3. CLI operator: inspect passed and unresolved admission checks {#deploy-sentry-review-admission} + +Read each check and its source/reason. Missing future chart-created APIs or admission prerequisites may remain deferred/unresolved because dry-run does not install them. Projections are validation-only; no runtime service readiness or policy test has occurred. Resolve rejected prerequisites and review deferred work before choosing real deployment. A completion summary is not proof that every API was admitted. + +Next: [Concrete plan reviewed for real deploy](#deploy-sentry-deploy). + +### 4. CLI and management controllers: create and reconcile in dependency order {#deploy-sentry-deploy} + +Use an explicit budget that includes downloads, firmware and OS installation, required host power cycles, and service startup: + +```bash +l8k deploy --target dpf --deployment-files ./sentry-bundle \ + --deploy-timeout 90m --output json > deploy.json 2> deploy.log +``` + +The CLI submits native resources in order. DPF downloads the desired ISO/firmware, and consumed artifact hashes are checked before DPUSet effects. DPF then provisions the selected DPUs. After the selected nodes and DNS are ready, the CLI submits Tower, the policy controller and Argus together, then waits for all three. The controller installs the policy API Tower needs, while Tower provides the controller's policy socket. The policy REST server starts afterward on management. Chart projections are observed, never applied again. Follow the existing DPF host-power procedure when a cold boot is required. Keep firmware caches unchanged during provisioning. + +Next: [Run completed or stopped](#deploy-sentry-inspect). + +### 5. CLI operator: inspect the persisted run and actual state {#deploy-sentry-inspect} + +Read `.runs//report.json` together with stdout/stderr. Confirm selected membership and coverage, and investigate failures, unresolved evidence, ownership changes or readiness timeouts. The immutable bundle is retained. A successful resource run still needs [separate functional verification](dpf-sentry.md#verify-resources-and-policy-behavior). + +Cancellation/timeout stops CLI orchestration; controller work already started is not rolled back. Preserve the report and inspect actual DPF/device state before retrying. Do not start a second run while an earlier controller action remains uncertain. + +Next: [Selected resources ready](#deploy-sentry-complete); [Failed or unresolved run](#deploy-sentry-recover). + +### 6. Responsible administrator: correct the reported prerequisite or desired intent {#deploy-sentry-recover} + +Use the reported source and reason to fix external access, namespace/Secret readiness, physical ownership or the selected artifact. An unchanged bundle can be retried after resolving its cause. Changed desired intent needs a newly generated bundle. Upgrades, teardown and another owner's removal remain separately authorized operations; this procedure does not perform them. + +Next: [Unchanged intent and cause resolved](#deploy-sentry-preview). + +### 7. CLI operator: retain evidence and verify policy behavior {#deploy-sentry-complete} + +Preserve the unchanged bundle and all run reports. Continue to [resource and functional verification](#validate-sentry). Successful resource readiness does not establish an allow/deny or LLM inspection result. + +## Verify resources, then test Sentry policy enforcement {#validate-sentry} + +DPF validation verifies live resources and selected coverage without cluster writes. It reads the frozen bundle and writes a separate report. Functional VF policy traffic and optional LLM inspection require additional deliberate tests. + +```mermaid +flowchart TD + n1["CLI machine: validate the frozen bundle against both APIs"] + n2["CLI operator: assess coverage and unresolved conditions"] + n3["Host workload operator: verify positive and negative VF behavior"] + n4["Test owner: remove only test resources and retain evidence"] + n1 -->|"Resource verification returned"| n2 + n2 -->|"Resource state reviewed"| n3 + n3 -->|"Functional evidence captured"| n4 +``` + +### 1. CLI machine: validate the frozen bundle against both APIs {#validate-sentry-run} + +Run: + +```bash +l8k validate --target dpf --deployment-files ./sentry-bundle \ + --wait 10m --output json > validate.json 2> validate.log +``` + +Access and target identity come from the resolved bundle. Do not substitute Host `--kubeconfig`, connectivity or HTML/JUnit report flags. The command checks current management/tenant resource state, ownership, identity and selected DPU coverage. + +Next: [Resource verification returned](#validate-sentry-inspect). + +### 2. CLI operator: assess coverage and unresolved conditions {#validate-sentry-inspect} + +Inspect the result and `.runs//report.json`. Check that all selected devices and intended workloads are represented and current. Missing reads, stale conditions and unresolved prerequisites are not passes. Resource validation does not send policy-test traffic or claim LLM inspection. Resolve resource failures before relying on the datapath. + +The pinned Argus chart has no application readiness probe. Pod Running/Ready alone does not establish host attachment or a working query service. Inspect Argus initialization/DMA errors and verify the chosen host IOMMU prerequisite if attachment or the configured query socket is unavailable. + +Next: [Resource state reviewed](#validate-sentry-traffic). + +### 3. Host workload operator: verify positive and negative VF behavior {#validate-sentry-traffic} + +Use an already prepared host VF workload on the selected host, with its actual MAC, NAD and advertised resource. Mount only the public MITM CA. Follow the [functional procedure](dpf-sentry.md#verify-resources-and-policy-behavior): baseline denial with no MAC policy; PUT a policy and poll GET until applied; verify an allowed HTTP/HTTPS path reaches the approved server and a denied path receives an attributable authorization denial. Verify Argus identity and intended DPU controller programming. An egress503 does not pass the positive test. + +Use the same VF interface for every request and an explicit DNS resolution when needed. If LLM inspection is selected, test that additional path with real inference credentials; core policy success makes no LLM claim. + +If `net2` lacks an IPv4 lease/router or the managed egress helper reports DHCP failure, resolve the `p0` VLAN, DHCP/relay, MAC policy and return path with the network administrator before permitted-traffic tests. Do not start another DHCP client. An attributable authorization denial is distinct from an upstream connection failure or egress `503`; only reaching the approved upstream server passes the positive test. + +Require successful Argus host attachment, a responding configured query socket, and the expected executable identity for the actual VF test process. Resource readiness alone is insufficient evidence for that identity or functional enforcement. + +Next: [Functional evidence captured](#validate-sentry-cleanup). + +### 4. Test owner: remove only test resources and retain evidence {#validate-sentry-cleanup} + +DELETE the test policy through the policy API, poll GET until404 and confirm baseline denial returns. Remove only the test pod and test-specific public-CA/configuration resources. Keep the generated bundle, run reports, logs and functional results. Do not delete production resources or infer teardown authorization from validation. diff --git a/engineering/advanced/targets.md b/engineering/advanced/targets.md index 79c32390..dfb84114 100644 --- a/engineering/advanced/targets.md +++ b/engineering/advanced/targets.md @@ -29,7 +29,7 @@ Passing `--target host` is an explicit synonym: l8k generate --target host ... ``` -The `dpf` target implements an [experimental inventory discovery preview](../../docs/user/dpf-discovery.md). Its software collects explicit Redfish or host/ARM SSH inventory, optionally with management Kubernetes observations. Inventory acceptance is independent of DPF provisioning readiness; hardware qualification and management probe-image release gates remain explicit. Other DPF phases remain unavailable and cannot fall through to Host execution. +The `dpf` target implements an [experimental inventory discovery preview](../../docs/user/dpf-discovery.md). Its software collects explicit Redfish or host/ARM SSH inventory, optionally with management Kubernetes observations. Inventory acceptance is independent of DPF provisioning readiness; hardware qualification and management probe-image release gates remain explicit. The experimental `sentry-bf4` profile also supports generate/deploy/validate against existing DPF management, tenant and enrolled BF4 resources. Root pipeline and unrelated phases remain unavailable and cannot fall through to Host execution. See [Sentry operations](../../docs/user/dpf-sentry.md). Use `l8k schema` to inspect target and phase availability programmatically. @@ -42,8 +42,9 @@ sound generic: `--target`, `--output`, and logging controls; - host flags describe Network Operator behavior, such as `--fabric`, `--deployment-type`, Spectrum-X settings, and connectivity checks; -- existing path and kubeconfig flags remain host compatibility inputs until a - multi-context environment contract gives them target-independent semantics. +- configuration/artifact paths and lifecycle timeouts have explicit per-command + target support; `--kubeconfig` remains Host-only. DPF access uses the typed site + file and its bound management and tenant identities. Help output separates target selection, host configuration, and target-agnostic output controls. Every target-aware Cobra flag also carries a diff --git a/engineering/architecture/overview.md b/engineering/architecture/overview.md index 1597dfd9..ee347938 100644 --- a/engineering/architecture/overview.md +++ b/engineering/architecture/overview.md @@ -49,7 +49,7 @@ flowchart TB validatesvc["Standalone validate + report service"] hostpaths["Host config, kubeconfig, preset, artifact resolution"] end - dpf["DPF inventory — pkg/target/dpf + pkg/dpf\nexplicit Redfish / host SSH / ARM OOB SSH sources\nprovisioning phases unavailable"] + dpf["DPF — pkg/target/dpf + pkg/dpf\nexplicit inventory sources\nSentry generation / deploy / validation on existing infrastructure"] end subgraph application["Host application orchestration"] diff --git a/go.mod b/go.mod index be089658..beb43d69 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( k8s.io/api v0.37.2 k8s.io/apiextensions-apiserver v0.37.2 k8s.io/apimachinery v0.37.2 + k8s.io/apiserver v0.37.2 k8s.io/client-go v0.37.2 k8s.io/klog/v2 v2.140.0 sigs.k8s.io/controller-runtime v0.25.2 @@ -32,6 +33,7 @@ require ( ) require ( + cel.dev/expr v0.25.1 // indirect dario.cat/mergo v1.0.2 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/BurntSushi/toml v1.6.0 // indirect @@ -41,6 +43,7 @@ require ( github.com/Masterminds/squirrel v1.5.4 // indirect github.com/NVIDIA/k8s-operator-libs v0.0.0-20250708070119-9dc24ccc10ee // indirect github.com/ProtonMail/go-crypto v1.4.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -74,6 +77,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.29.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect @@ -117,9 +121,12 @@ require ( github.com/spf13/cast v1.7.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xlab/treeprint v1.2.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/mod v0.41.0 // indirect golang.org/x/net v0.60.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect @@ -127,10 +134,11 @@ require ( golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.49.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - k8s.io/apiserver v0.37.2 // indirect k8s.io/cli-runtime v0.37.1 // indirect k8s.io/component-base v0.37.2 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect diff --git a/go.sum b/go.sum index f0d1838f..a6a89745 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= @@ -28,6 +30,8 @@ github.com/NVIDIA/k8s-operator-libs v0.0.0-20250708070119-9dc24ccc10ee h1:IW3URI github.com/NVIDIA/k8s-operator-libs v0.0.0-20250708070119-9dc24ccc10ee/go.mod h1:8skFe7Dyub0FRpk5ysCM88ysFqJpIcw/4ZqLBWcevw8= github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -44,6 +48,8 @@ github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnT github.com/chai2010/gettext-go v1.0.3/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -149,8 +155,12 @@ github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.29.2 h1:ZtDxkeiMmz0mxbKDYiNkE5Lk7V5edMRcaaDf2jX002k= +github.com/google/cel-go v0.29.2/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -168,6 +178,10 @@ github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5T github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3 h1:B+8ClL/kCQkRiU82d9xajRPKYMrB7E0MbtzWVi1K4ns= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.3/go.mod h1:NbCUVmiS4foBGBHOYlCT25+YmGpJ32dZPi75pGEUpj4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -195,6 +209,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -317,12 +333,20 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +go.etcd.io/etcd/api/v3 v3.7.0 h1:WZlGK7pRtYGDB8ti8wkrQ5D2oWGMbtNL9VA5e+vF2Fg= +go.etcd.io/etcd/api/v3 v3.7.0/go.mod h1:EcTihnwAQ0BQNh5dfAdaFVFdchuo7EP0HlX7TV3jz/A= +go.etcd.io/etcd/client/pkg/v3 v3.7.0 h1:sW9njJzS3vXKcAJjjLQ4nk+avNUJ12Bcijcx8ehUskE= +go.etcd.io/etcd/client/pkg/v3 v3.7.0/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0= +go.etcd.io/etcd/client/v3 v3.7.0 h1:5MHO37VbPB87VRPKUXEcicjeQWiTSjpPv3Ume8xPx20= +go.etcd.io/etcd/client/v3 v3.7.0/go.mod h1:DJ382WuwjmbowjPDyaaQ0idWXy4dh91XRhe4FOrb9vM= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/prometheus v0.67.0 h1:dkBzNEAIKADEaFnuESzcXvpd09vxvDZsOjx11gjUqLk= go.opentelemetry.io/contrib/bridges/prometheus v0.67.0/go.mod h1:Z5RIwRkZgauOIfnG5IpidvLpERjhTninpP1dTG2jTl4= go.opentelemetry.io/contrib/exporters/autoexport v0.67.0 h1:4fnRcNpc6YFtG3zsFw9achKn3XgmxPxuMuqIL5rE8e8= go.opentelemetry.io/contrib/exporters/autoexport v0.67.0/go.mod h1:qTvIHMFKoxW7HXg02gm6/Wofhq5p3Ib/A/NNt1EoBSQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= @@ -376,6 +400,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= @@ -448,6 +474,8 @@ k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI0 k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= oras.land/oras-go/v2 v2.6.2 h1:N04RXngAp1LJKTG6ifz3xHPipasEkWr+hFmInja5YKo= oras.land/oras-go/v2 v2.6.2/go.mod h1:PlTtg4JTDJkDe8yVHpM2wz7/YDc00GVas+i4jAW2TZ4= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c= sigs.k8s.io/controller-runtime v0.25.2 h1:bEkK3PVOIVK9X8QWLGVhJgmFc++47vfT6wakSzAOLsQ= sigs.k8s.io/controller-runtime v0.25.2/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/mkdocs.yml b/mkdocs.yml index 92177581..353a3bcc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,8 +87,9 @@ nav: - Networking concepts: advanced/concepts.md - Deploy: - Discover cluster hardware: user/discovery.md - - DPF discovery workflows: user/dpf-workflows.md + - DPF workflows: user/dpf-workflows.md - DPF discovery preview: user/dpf-discovery.md + - Deploy Sentry with DPF: user/dpf-sentry.md - Generate and review manifests: advanced/generation.md - Deploy manifests: advanced/deployment.md - Deploy on OpenShift: user/openshift.md diff --git a/openspec/specs/artifact-handoff/spec.md b/openspec/specs/artifact-handoff/spec.md index dc218373..a76d190d 100644 --- a/openspec/specs/artifact-handoff/spec.md +++ b/openspec/specs/artifact-handoff/spec.md @@ -86,9 +86,33 @@ ambiguous inventory SHALL fail rather than guessing a source association. - **THEN** it retries within the existing deployment budget without evaluating readiness until fresh scope inspection succeeds. - **AND** permission denials, invalid source metadata and selector mismatches remain terminal; pre-write inspection failures do not authorize writes. +### Requirement: Closed DPF intent handoff + +DPF Sentry generation SHALL retain a closed, hashed bundle containing the resolved +configuration, native resources, pinned chart archives, effective Helm values and +validation-only workload projections. Consumers SHALL verify the complete bundle +against its resolved profile intent using retained chart bytes before opening +cluster clients. Different desired intent requires generation into a new directory; +explicit access-path relocation SHALL not replace captured deployment intent. + +#### Scenario: Reindexed foreign resource + +- **WHEN** a bundle has consistent hashes but changed resources or stages inconsistent with its resolved profile +- **THEN** deploy and validate reject it before cluster access or writes. + +#### Scenario: Relocated discovery output + +- **WHEN** discovery saves a source config in another directory +- **THEN** chart-directory and credential-file references continue to resolve to their original locations. + +#### Scenario: Workload projections + +- **WHEN** a selected DPF release transforms service Helm inputs +- **THEN** validation projections include that release's effective input transformation and remain separate from resources applied by Launch Kit. + ## Baseline and limits -Structural validation does not prove CRD schema/admission or readiness. Sidecar handling lives outside `pkg/bundle`. Atomic sidecar writing does not promise whole-directory atomicity or cluster transactions. No content hash binds manifests to effective config; explicit override may differ from generated intent. Bundle checks cannot recover artifacts already lost to render-map overwrite; [profile planning](../profile-planning/spec.md) owns that boundary. +Structural validation does not prove CRD schema/admission or readiness. Sidecar handling lives outside `pkg/bundle`. Atomic sidecar writing does not promise whole-directory atomicity or cluster transactions. The legacy Host handoff has no content hash binding manifests to effective config; its explicit override may differ from generated intent. DPF uses the closed intent contract above. Bundle checks cannot recover artifacts already lost to render-map overwrite; [profile planning](../profile-planning/spec.md) owns that boundary. ## Integration and evidence diff --git a/openspec/specs/workflow-execution/spec.md b/openspec/specs/workflow-execution/spec.md index 2fb9a227..abc4544e 100644 --- a/openspec/specs/workflow-execution/spec.md +++ b/openspec/specs/workflow-execution/spec.md @@ -76,6 +76,49 @@ New and changed workflow checks SHALL persist a stable reason code and safe expl - **WHEN** a check depends on an operator-created endpoint before that operator is installed - **THEN** it is deferred with a reason and required phase, not reported as passed or used to fail initial hardware inventory. +### Requirement: Explicit DPF deployment boundary + +The initial Sentry workflow SHALL consume existing management, tenant, enrolled +BlueField-4 and namespace/credential resources. It SHALL verify explicit cluster +identities, exact selected device membership and compatible release contracts. +It SHALL not silently install an operator, enroll hardware, adopt foreign objects, +execute reboot scripts, remove foreign ownership or modify unrelated Host resources. +Selected nodes SHALL use the external reboot method without pending power actions. + +#### Scenario: Existing foreign owner + +- **WHEN** a selected DPU belongs to a different DPUSet or DPUDeployment +- **THEN** deployment fails with a retained reason before creating replacement provisioning resources. + +#### Scenario: Unreviewed reboot automation + +- **WHEN** the selected DPUNode uses a reboot script or has a pending power request +- **THEN** preflight blocks provisioning and identifies the unresolved external transition. + +### Requirement: Honest admission and selected readiness + +Unresolved required admission checks SHALL yield an incomplete result and nonzero +exit status. Readiness SHALL require current generation evidence according to the +kind's actual status contract and exact selected tenant-node coverage. Native +resource comparison SHALL accept declared API defaults while rejecting additional +effectful desired fields. Controller-owned workload projections SHALL accept normal +Kubernetes defaults without ignoring supplied fields or selected membership. + +#### Scenario: Missing freshness + +- **WHEN** a resource is marked Ready but lacks its required current generation evidence +- **THEN** validation remains unsuccessful and retains the reason. + +#### Scenario: Missing future API during admission + +- **WHEN** a dry-run cannot validate a required object because its API is not served +- **THEN** the result records unresolved admission and does not claim complete success. + +#### Scenario: Unexpected hardware setting + +- **WHEN** an existing or admitted DPUSet adds a non-default hardware setting absent from frozen intent +- **THEN** the workflow rejects desired-state drift rather than adopting that setting. + ## Baseline and limits The root pipeline runs discover/generate/deploy, not automatic validation. Exactly-once dispatch is not a promise of global idempotency or a non-reusable operation object. Discovery may mutate supplied config and use a global logger. It performs pre-clean before some worker validation and defers cleanup only after successful setup. See [effects](../managed-effects/spec.md), [artifacts](../artifact-handoff/spec.md) and [outcomes](../outcomes-evidence/spec.md) for those independent responsibilities. diff --git a/openspec/workflows/README.md b/openspec/workflows/README.md index a7ef37e1..261e10b3 100644 --- a/openspec/workflows/README.md +++ b/openspec/workflows/README.md @@ -6,7 +6,8 @@ SPDX-License-Identifier: Apache-2.0 # Procedural documentation sources `dpf-discovery.json` owns the user steps and flow transitions for initial DPF -inventory, address-candidate lookup and refresh. Its generated page is +inventory, address-candidate lookup, refresh, and Sentry BF4 generation, +deployment and resource/functional verification. Its generated page is `docs/user/dpf-workflows.md`; edit the JSON, not the generated Markdown. This is a **repository-owned documentation extension**, not a native executable diff --git a/openspec/workflows/dpf-discovery.json b/openspec/workflows/dpf-discovery.json index b603b7c7..7aa8cf97 100644 --- a/openspec/workflows/dpf-discovery.json +++ b/openspec/workflows/dpf-discovery.json @@ -1,8 +1,8 @@ { "$comment": "Copyright 2026 NVIDIA CORPORATION & AFFILIATES; SPDX-License-Identifier: Apache-2.0", "formatVersion": 1, - "title": "DPF discovery workflows", - "introduction": "Use these procedures with the experimental `l8k discover --target dpf` CLI. The DPU estate is separate from Host Network Operator deployment. Start with **inventory mode**; management probe mode is unavailable in this preview. Discovery collects facts and writes a handoff. It does not create VMs, install Kubernetes or DPF, configure DPUServices, change firmware, enroll trust, or start a DPF deployment.\n\nFor complete field and error details, see [DPF discovery reference](dpf-discovery.md). Each diagram follows the same steps as its procedure. Commands are examples to run deliberately from a machine with direct access to your selected endpoints.", + "title": "DPF workflows", + "introduction": "Use these procedures for DPF inventory and the experimental Sentry BF4 generate/deploy/validate workflow. The DPU estate is independent of Host Network Operator deployment. Start with **inventory mode** to collect physical identity and host associations; management probe mode remains unavailable. Deploying Sentry additionally requires an existing DPF management cluster, tenant, enrolled devices and the explicit prerequisites in the [Sentry procedure](dpf-sentry.md). VM creation, Kubernetes/DPF bootstrap, enrollment, upgrades, teardown and implicit transfer from another owner are outside this workflow.\n\nEach diagram follows its procedure. Run commands from a CLI machine with the required local files and explicitly bound access. Inventory success, local generation, API admission, resource readiness and functional traffic checks establish different outcomes. See [DPF discovery reference](dpf-discovery.md) for collection fields and errors.", "workflows": [ { "id": "initial-inventory", @@ -78,7 +78,7 @@ { "id": "inspect", "title": "CLI machine: review inventory and reasons", - "body": "Inspect `discovery.outcome`, `discovery.checks`, `errors`, `artifacts` and `acceptance` in the JSON report. Review saved `inventory.hosts`, `inventory.dpus`, `inventory.associations` and their evidence before using the handoff. Complete discovery writes `dpf-discovered.yaml`; partial discovery preserves an existing handoff and writes `partial.yaml` and `report.json` under `dpf-discovered.discovery//`. Use the report's artifact paths instead of guessing the run ID.\n\nRead each non-success observation's `reasonCode`, `message` and `tool` where present. A missing `mlxfwmanager` is an unavailable prerequisite, not proof of an incompatible NIC firmware. A denied Redfish firmware collection is different from a component property not being exposed. Conflicting versions or ambiguous activation cannot establish a software minimum.\n\n| Example reason | What to do |\n| --- | --- |\n| `TOOL_NOT_AVAILABLE` / `PERMISSION_DENIED` | Check the named endpoint tool or authorized noninteractive sudo policy; do not treat the unmeasured check as passed. |\n| `ACCESS_DENIED` / `PROPERTY_NOT_EXPOSED` | Distinguish an account permission problem from a property the service does not advertise. |\n| `SOFTWARE_VERSION_CONFLICT` / `SOFTWARE_IDENTITY_UNAVAILABLE` | Reconcile version/activation sources or the rejected physical identity before trusting software comparisons. |\n| `SOFTWARE_BELOW_MINIMUM` | Review the release requirement and plan a separate maintenance action; discovery does not update firmware. |\n| `PUBLICATION_FAILED` | Read the error explanation, resolve the local output/path problem, and rerun; only listed successfully written artifacts exist. |\n\nThe resolved software BOM is saved under `inventory.software`, including source references and unknown requirements. Release `26.4.0` has exact BlueField-3 model entries `B3210`, `B3220` and `B3240`; generic family names and BlueField-4 are not sufficient for BOM applicability. The documented NIC/MFT minima can be evaluated when current measurements associated with the correct device exist. BMC, CEC, ATF, UEFI, OS/BSP and golden-image minima remain unknown where the catalog has no supported requirement. A BFB target is a desired artifact, not an observed OS/BSP version. An unknown requirement must remain unresolved; supplying a guessed minimum does not establish compatibility.\n\nExit zero and `discovery.outcome: complete` mean inventory collection, identity/mapping, cleanup and publication succeeded. `acceptance: not-evaluated` remains explicit. Hardware support, DPF authentication, service connectivity and deployment still need separate assessment; DPF generate/deploy/validate are unavailable in this preview.", + "body": "Inspect `discovery.outcome`, `discovery.checks`, `errors`, `artifacts` and `acceptance` in the JSON report. Review saved `inventory.hosts`, `inventory.dpus`, `inventory.associations` and their evidence before using the handoff. Complete discovery writes `dpf-discovered.yaml`; partial discovery preserves an existing handoff and writes `partial.yaml` and `report.json` under `dpf-discovered.discovery//`. Use the report's artifact paths instead of guessing the run ID.\n\nRead each non-success observation's `reasonCode`, `message` and `tool` where present. A missing `mlxfwmanager` is an unavailable prerequisite, not proof of an incompatible NIC firmware. A denied Redfish firmware collection is different from a component property not being exposed. Conflicting versions or ambiguous activation cannot establish a software minimum.\n\n| Example reason | What to do |\n| --- | --- |\n| `TOOL_NOT_AVAILABLE` / `PERMISSION_DENIED` | Check the named endpoint tool or authorized noninteractive sudo policy; do not treat the unmeasured check as passed. |\n| `ACCESS_DENIED` / `PROPERTY_NOT_EXPOSED` | Distinguish an account permission problem from a property the service does not advertise. |\n| `SOFTWARE_VERSION_CONFLICT` / `SOFTWARE_IDENTITY_UNAVAILABLE` | Reconcile version/activation sources or the rejected physical identity before trusting software comparisons. |\n| `SOFTWARE_BELOW_MINIMUM` | Review the release requirement and plan a separate maintenance action; discovery does not update firmware. |\n| `PUBLICATION_FAILED` | Read the error explanation, resolve the local output/path problem, and rerun; only listed successfully written artifacts exist. |\n\nThe resolved software BOM is saved under `inventory.software`, including source references and unknown requirements. Release `26.4.0` has exact BlueField-3 model entries `B3210`, `B3220` and `B3240`; generic family names and BlueField-4 are not sufficient for BOM applicability. The documented NIC/MFT minima can be evaluated when current measurements associated with the correct device exist. BMC, CEC, ATF, UEFI, OS/BSP and golden-image minima remain unknown where the catalog has no supported requirement. A BFB target is a desired artifact, not an observed OS/BSP version. An unknown requirement must remain unresolved; supplying a guessed minimum does not establish compatibility.\n\nExit zero and `discovery.outcome: complete` mean inventory collection, identity/mapping, cleanup and publication succeeded. `acceptance: not-evaluated` remains explicit. Hardware support, DPF authentication, service connectivity and deployment still need separate assessment; the separate [Sentry BF4 procedure](dpf-sentry.md) provides generation, deployment and resource validation only after its additional prerequisites are met.", "transitions": [ { "to": "finish", @@ -240,7 +240,193 @@ { "id": "retain", "title": "CLI machine: retain the refreshed files", - "body": "Keep the current YAML, JSON/stdout report and stderr log, plus `source-before.yaml` when a complete in-place refresh created it. Discovery completion still means a successful inventory workflow, with deployment `acceptance: not-evaluated`. Unknown software requirements and unverified deployment prerequisites remain unresolved; a refreshed inventory does not enable DPF deployment.", + "body": "Keep the current YAML, JSON/stdout report and stderr log, plus `source-before.yaml` when a complete in-place refresh created it. Discovery completion still means a successful inventory workflow, with deployment `acceptance: not-evaluated`. Unknown software requirements and unverified deployment prerequisites remain unresolved; a refreshed inventory alone does not establish Sentry deployment readiness. Continue with the [explicit deployment prerequisites](dpf-sentry.md#prepare-the-existing-environment) when that workflow is intended.", + "transitions": [] + } + ] + }, + { + "id": "generate-sentry", + "title": "Generate a Sentry BF4 bundle", + "introduction": "Use this procedure after reviewing physical discovery and selecting an existing DPF environment. Generation runs locally on the CLI machine, reads no deployment credentials and contacts no cluster. It does not establish readiness.", + "steps": [ + { + "id": "bind-inputs", + "title": "CLI operator: select devices and desired site inputs", + "body": "Copy the discovered YAML to `deployment-site.yaml`, retaining its inventory. Choose the exact inventory device IDs and explicitly name their enrolled DPUDevice/DPUNode objects. Bind the existing tenant by name, namespace, selector, resource UID, tenant cluster UID and API server version. Management UID/version may default from saved observations; management kubeconfig/context must be explicit. Supply PSID-matched ISO/PLDM URLs and SHA-256, service networks and existing Secret names. Follow the [configuration fragment and input table](dpf-sentry.md#prepare-the-deployment-configuration). Do not invent missing observations or infer enrollment names.\n\nCore policy defaults to `llmInspection: false` and makes no LLM inspection claim. Enable it only with the real inference Secret and intended external inference access.\n\n`ingressCIDR` and `ingressIP` configure only the `net1` ingress network. The current profile obtains `net2` egress IPv4 and its router through DHCP on the untagged `p0` uplink; these inputs do not configure egress, switch VLANs, a DHCP server/relay or upstream routing.", + "transitions": [ + { + "to": "stage-charts", + "label": "Bindings and intent reviewed" + } + ] + }, + { + "id": "stage-charts", + "title": "CLI machine: obtain the pinned local charts", + "body": "Run `l8k schema` and inspect `dpf.charts`. Obtain all four exact chart archives using authorized registry access and your credential store, preserving catalog filenames in `sentry.chartDirectory`. Verify their SHA-256. Use each declared repository type; never put registry passwords in CLI arguments or site YAML. Generation does not download or repair missing archives.", + "transitions": [ + { + "to": "render", + "label": "Four archives verified" + } + ] + }, + { + "id": "render", + "title": "CLI machine: generate the local bundle", + "body": "Choose a new output directory and run:\n\n```bash\nl8k generate --target dpf --user-config deployment-site.yaml \\\n --dpf-release 26.8.0 --dpf-profile sentry-bf4 \\\n --save-deployment-files ./sentry-bundle --output json \\\n > generate.json 2> generate.log\n```\n\nUse the installed DPF release, not an intended upgrade. Generation validates exact chart bytes, resource identities, selected scope and release-specific API shapes before publishing. Check the exit status and returned result.", + "transitions": [ + { + "to": "review", + "label": "Generation completed" + } + ] + }, + { + "id": "review", + "title": "CLI operator: review destinations and physical scope", + "body": "Inspect the selected-device set, exact device/tenant selectors, resource names, service/provisioning namespaces and artifact hashes. Management provisioning/service CRs and tenant RBAC have separate destinations. Effective chart values and projections are retained for validation; DPF deploys the charts, so do not apply those projections manually.\n\nThe bundle is immutable and includes target metadata and resolved inputs. Repeating identical output is allowed. Corrected intent requires a new bundle directory; do not edit the captured files to force acceptance. A generated bundle does not authorize an ownership transition or physical reimage.", + "transitions": [ + { + "to": "retain", + "label": "Review complete" + } + ] + }, + { + "id": "retain", + "title": "CLI operator: preserve the bundle and choose the next action", + "body": "Keep `sentry-bundle`, `generate.json` and `generate.log`. Continue to [deployment](#deploy-sentry) only after external namespaces, Secrets, access, device ownership and the maintenance window are ready. `generate --deploy` invokes the same deployment path after generation; use separate commands to set an explicit standalone deploy timeout.", + "transitions": [] + } + ] + }, + { + "id": "deploy-sentry", + "title": "Check admission and deploy the selected Sentry bundle", + "introduction": "Run from the CLI machine. The existing DPF controllers perform firmware download, DPU provisioning, host power coordination and chart reconciliation. Real deployment can reimage selected devices. No VM/bootstrap/enrollment or implicit foreign-owner transition is performed.", + "steps": [ + { + "id": "prerequisites", + "title": "Cluster/device administrators: prepare external ownership and access", + "body": "Arrange the [existing-environment prerequisites](dpf-sentry.md#prepare-the-existing-environment). The dedicated service namespace must be Active on management and tenant clusters, with registry and MITM Secrets in both; optional inference credentials belong on the tenant. Configure Argo CD repository credentials for the selected OCI and HTTPS chart registries in the existing Argo CD namespace; image-pull Secrets and local CLI login do not provide chart access. DPF management and tenant services and BMC controller credentials/trust must already work. If another DPUSet or DPUDeployment owns a selected device, resolve the explicitly authorized transition externally. Do not bypass an ownership error by relabeling or deleting unrelated resources.\n\nArrange an untagged Layer 2 `p0` uplink with an existing IPv4 DHCP server or relay that supplies a lease and router acceptable for `p0`'s MAC; Tower temporarily borrows that MAC on `net2`. Require a route to the approved upstream test endpoint. The network administrator owns switch/VLAN, DHCP and routing configuration; Launch Kit does not provision them. A management/BMC lease does not establish this egress path.\n\nArrange the host IOMMU mode required by [DPF 26.8 Argus](https://networking-docs.nvidia.com/dpf/26.8.0/doca-argus-service): bare-metal hosts require IOMMU disabled or enabled with passthrough. The site administrator chooses and authorizes this host-wide DMA-isolation setting and any maintenance/reboot. DPU flavor ARM kernel settings do not configure the host; Launch Kit does not select or apply that mode.", + "transitions": [ + { + "to": "preview", + "label": "Prerequisites and physical scope authorized" + } + ] + }, + { + "id": "preview", + "title": "CLI machine and bound APIs: check server admission", + "body": "Run:\n\n```bash\nl8k deploy --target dpf --deployment-files ./sentry-bundle \\\n --dry-run --deploy-timeout 10m --output json \\\n > admission.json 2> admission.log\n```\n\nThe management access file/context comes from the resolved bundle; no ambient kubeconfig is used. The CLI checks live identities, selected membership, ownership and external prerequisites, then requests server-side dry-run for native resources. No chart or DPU is provisioned by this preview.", + "transitions": [ + { + "to": "review-admission", + "label": "Admission checks returned" + } + ] + }, + { + "id": "review-admission", + "title": "CLI operator: inspect passed and unresolved admission checks", + "body": "Read each check and its source/reason. Missing future chart-created APIs or admission prerequisites may remain deferred/unresolved because dry-run does not install them. Projections are validation-only; no runtime service readiness or policy test has occurred. Resolve rejected prerequisites and review deferred work before choosing real deployment. A completion summary is not proof that every API was admitted.", + "transitions": [ + { + "to": "deploy", + "label": "Concrete plan reviewed for real deploy" + } + ] + }, + { + "id": "deploy", + "title": "CLI and management controllers: create and reconcile in dependency order", + "body": "Use an explicit budget that includes downloads, firmware and OS installation, required host power cycles, and service startup:\n\n```bash\nl8k deploy --target dpf --deployment-files ./sentry-bundle \\\n --deploy-timeout 90m --output json > deploy.json 2> deploy.log\n```\n\nThe CLI submits native resources in order. DPF downloads the desired ISO/firmware, and consumed artifact hashes are checked before DPUSet effects. DPF then provisions the selected DPUs. After the selected nodes and DNS are ready, the CLI submits Tower, the policy controller and Argus together, then waits for all three. The controller installs the policy API Tower needs, while Tower provides the controller's policy socket. The policy REST server starts afterward on management. Chart projections are observed, never applied again. Follow the existing DPF host-power procedure when a cold boot is required. Keep firmware caches unchanged during provisioning.", + "transitions": [ + { + "to": "inspect", + "label": "Run completed or stopped" + } + ] + }, + { + "id": "inspect", + "title": "CLI operator: inspect the persisted run and actual state", + "body": "Read `.runs//report.json` together with stdout/stderr. Confirm selected membership and coverage, and investigate failures, unresolved evidence, ownership changes or readiness timeouts. The immutable bundle is retained. A successful resource run still needs [separate functional verification](dpf-sentry.md#verify-resources-and-policy-behavior).\n\nCancellation/timeout stops CLI orchestration; controller work already started is not rolled back. Preserve the report and inspect actual DPF/device state before retrying. Do not start a second run while an earlier controller action remains uncertain.", + "transitions": [ + { + "to": "complete", + "label": "Selected resources ready" + }, + { + "to": "recover", + "label": "Failed or unresolved run" + } + ] + }, + { + "id": "recover", + "title": "Responsible administrator: correct the reported prerequisite or desired intent", + "body": "Use the reported source and reason to fix external access, namespace/Secret readiness, physical ownership or the selected artifact. An unchanged bundle can be retried after resolving its cause. Changed desired intent needs a newly generated bundle. Upgrades, teardown and another owner's removal remain separately authorized operations; this procedure does not perform them.", + "transitions": [ + { + "to": "preview", + "label": "Unchanged intent and cause resolved" + } + ] + }, + { + "id": "complete", + "title": "CLI operator: retain evidence and verify policy behavior", + "body": "Preserve the unchanged bundle and all run reports. Continue to [resource and functional verification](#validate-sentry). Successful resource readiness does not establish an allow/deny or LLM inspection result.", + "transitions": [] + } + ] + }, + { + "id": "validate-sentry", + "title": "Verify resources, then test Sentry policy enforcement", + "introduction": "DPF validation verifies live resources and selected coverage without cluster writes. It reads the frozen bundle and writes a separate report. Functional VF policy traffic and optional LLM inspection require additional deliberate tests.", + "steps": [ + { + "id": "run", + "title": "CLI machine: validate the frozen bundle against both APIs", + "body": "Run:\n\n```bash\nl8k validate --target dpf --deployment-files ./sentry-bundle \\\n --wait 10m --output json > validate.json 2> validate.log\n```\n\nAccess and target identity come from the resolved bundle. Do not substitute Host `--kubeconfig`, connectivity or HTML/JUnit report flags. The command checks current management/tenant resource state, ownership, identity and selected DPU coverage.", + "transitions": [ + { + "to": "inspect", + "label": "Resource verification returned" + } + ] + }, + { + "id": "inspect", + "title": "CLI operator: assess coverage and unresolved conditions", + "body": "Inspect the result and `.runs//report.json`. Check that all selected devices and intended workloads are represented and current. Missing reads, stale conditions and unresolved prerequisites are not passes. Resource validation does not send policy-test traffic or claim LLM inspection. Resolve resource failures before relying on the datapath.\n\nThe pinned Argus chart has no application readiness probe. Pod Running/Ready alone does not establish host attachment or a working query service. Inspect Argus initialization/DMA errors and verify the chosen host IOMMU prerequisite if attachment or the configured query socket is unavailable.", + "transitions": [ + { + "to": "traffic", + "label": "Resource state reviewed" + } + ] + }, + { + "id": "traffic", + "title": "Host workload operator: verify positive and negative VF behavior", + "body": "Use an already prepared host VF workload on the selected host, with its actual MAC, NAD and advertised resource. Mount only the public MITM CA. Follow the [functional procedure](dpf-sentry.md#verify-resources-and-policy-behavior): baseline denial with no MAC policy; PUT a policy and poll GET until applied; verify an allowed HTTP/HTTPS path reaches the approved server and a denied path receives an attributable authorization denial. Verify Argus identity and intended DPU controller programming. An egress503 does not pass the positive test.\n\nUse the same VF interface for every request and an explicit DNS resolution when needed. If LLM inspection is selected, test that additional path with real inference credentials; core policy success makes no LLM claim.\n\nIf `net2` lacks an IPv4 lease/router or the managed egress helper reports DHCP failure, resolve the `p0` VLAN, DHCP/relay, MAC policy and return path with the network administrator before permitted-traffic tests. Do not start another DHCP client. An attributable authorization denial is distinct from an upstream connection failure or egress `503`; only reaching the approved upstream server passes the positive test.\n\nRequire successful Argus host attachment, a responding configured query socket, and the expected executable identity for the actual VF test process. Resource readiness alone is insufficient evidence for that identity or functional enforcement.", + "transitions": [ + { + "to": "cleanup", + "label": "Functional evidence captured" + } + ] + }, + { + "id": "cleanup", + "title": "Test owner: remove only test resources and retain evidence", + "body": "DELETE the test policy through the policy API, poll GET until404 and confirm baseline denial returns. Remove only the test pod and test-specific public-CA/configuration resources. Keep the generated bundle, run reports, logs and functional results. Do not delete production resources or infer teardown authorization from validation.", "transitions": [] } ] diff --git a/pkg/cmd/deploy.go b/pkg/cmd/deploy.go index f50cf11c..ac06c858 100644 --- a/pkg/cmd/deploy.go +++ b/pkg/cmd/deploy.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "github.com/nvidia/k8s-launch-kit/pkg/target" + dpftarget "github.com/nvidia/k8s-launch-kit/pkg/target/dpf" hosttarget "github.com/nvidia/k8s-launch-kit/pkg/target/host" ) @@ -71,7 +72,13 @@ proceeds. Initial device discovery remains unbounded. If /network-operator/ exists (the layout 'l8k generate' produces), that subdirectory is used. Otherwise itself -is used as the manifest directory.`, +is used as the manifest directory. + +For --target dpf, --deployment-files must be an intact generated DPF bundle. +The explicit management and tenant identities come from its resolved configuration. +Deploy verifies existing enrollment and ownership, then creates staged Sentry +resources. It never takes over another DPU deployment. The DPF default deadline +is 45 minutes; --deploy-timeout selects a different budget.`, Example: ` # Apply manifests from ./deployment to the cluster reachable via $KUBECONFIG l8k deploy @@ -81,6 +88,10 @@ is used as the manifest directory.`, # Server-side dry run: validate against the cluster without persisting l8k deploy --dry-run`, Run: func(cmd *cobra.Command, args []string) { + if targetName == string(target.DPF) { + runTargetCommand(cmd, target.Deploy, dpftarget.NewWorkflowAdapter(dpftarget.WorkflowRequest{Phase: target.Deploy, UserConfig: userConfig, DeploymentFiles: deploymentFiles})) + return + } runTargetCommand(cmd, target.Deploy, hosttarget.NewDeployAdapter( hosttarget.DeployRequest{ Flavor: flavor, @@ -114,7 +125,7 @@ func init() { deployCmd.Flags().StringVar(&userConfig, "user-config", "", "Explicit config override; otherwise prefer deployment .l8k/resolved-config.yaml, then adjacent/CWD cluster-config.yaml and default config fallbacks. Resolves release and preflight settings.") deployCmd.Flags().StringVar(&flavor, "flavor", "", "Cluster flavor: k8s or ocp (overrides config)") deployCmd.Flags().BoolVar(&dryRunFlag, "dry-run", false, "Preview the deployment via server-side dry-run without persisting changes") - deployCmd.Flags().DurationVar(&deployTimeout, "deploy-timeout", 0, "Maximum end-to-end wall-clock budget for the deploy phase (e.g. 45m, 2h). 0 (the default) means no deadline; the deploy polls until every manifest reaches a terminal state. Useful for matching a maintenance window when SR-IOV reconciliation on a large cluster can take an hour or more.") + deployCmd.Flags().DurationVar(&deployTimeout, "deploy-timeout", 0, "Maximum end-to-end wall-clock budget for the deploy phase (e.g. 45m, 2h). 0 uses the target default: Host has no deadline, DPF uses 45m; deploy polls until every manifest reaches a terminal state. Useful for matching a maintenance window when SR-IOV reconciliation on a large cluster can take an hour or more.") deployCmd.Flags().BoolVar(&overwriteExistingFlag, "overwrite-existing", false, "Converge the cluster to the rendered manifests when preflight detects drift: helm upgrade the chart on chart-version/values mismatch, delete stray Network Operator CRs in the operator namespace, and rewrite NicClusterPolicy component versions via SSA. Off by default — preflight fails fast and lists what would change.") deployCmd.Flags().BoolVar(&skipNetworkOperatorHelm, "skip-network-operator-helm", false, "Skip Network Operator Helm chart installation and Helm-specific preflight checks") diff --git a/pkg/cmd/generate.go b/pkg/cmd/generate.go index 76c686ca..9790dd6b 100644 --- a/pkg/cmd/generate.go +++ b/pkg/cmd/generate.go @@ -22,6 +22,7 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/configflags" "github.com/nvidia/k8s-launch-kit/pkg/options" "github.com/nvidia/k8s-launch-kit/pkg/target" + dpftarget "github.com/nvidia/k8s-launch-kit/pkg/target/dpf" hosttarget "github.com/nvidia/k8s-launch-kit/pkg/target/host" ) @@ -38,13 +39,18 @@ var ( var generateCmd = &cobra.Command{ Use: "generate", - Short: "Generate deployment manifests for a network profile", + Short: "Generate deployment manifests for a target profile", Long: `Select a deployment profile and generate Kubernetes YAML manifests from a cluster configuration file. Supports SR-IOV, host-device, RDMA-shared, IPoIB, MacVLAN, and Spectrum-X profiles. File-backed configs remain unchanged; the resolved config is saved with the deployment bundle. -Optionally deploy the generated manifests with --deploy.`, +Optionally deploy the generated manifests with --deploy. + +For --target dpf, supply an explicit DPF configuration, pinned local chart +archives and --save-deployment-files. Generation is offline and validates a +Sentry BF4 bundle for existing management and DPU clusters. --dpf-profile and +--dpf-release override the configuration without rewriting the source file.`, Example: ` # SR-IOV Ethernet (most common for GPU clusters) l8k generate --user-config cluster-config.yaml \ --fabric ethernet --deployment-type sriov \ @@ -83,6 +89,10 @@ Optionally deploy the generated manifests with --deploy.`, --save-deployment-files ./output`, Run: func(cmd *cobra.Command, args []string) { opts := mustCollectConfigFlags(cmd, generateConfigOptions) + if targetName == string(target.DPF) { + runTargetCommand(cmd, target.Generate, dpftarget.NewWorkflowAdapter(dpftarget.WorkflowRequest{Phase: target.Generate, UserConfig: userConfig, OutputPath: saveDeploymentFiles, Overrides: opts.ConfigInputs, Deploy: deploy})) + return + } opts.LaunchKitVersion = Version opts.ConfigDir = configDir opts.UserConfig = userConfig @@ -133,6 +143,8 @@ func init() { generateCmd.Flags().BoolVar(&overwriteExistingFlag, "overwrite-existing", false, "Converge the cluster to the rendered manifests when preflight detects drift: helm upgrade the chart on chart-version/values mismatch, delete stray Network Operator CRs in the operator namespace, and rewrite NicClusterPolicy component versions via SSA. Off by default — preflight fails fast and lists what would change.") setFlagGroup(generateCmd, "user-config", GroupCommon) + setFlagGroup(generateCmd, "dpf-release", GroupCommon) + setFlagGroup(generateCmd, "dpf-profile", GroupProfile) setFlagGroup(generateCmd, "flavor", GroupCommon) setFlagGroup(generateCmd, "kubeconfig", GroupCommon) setFlagGroup(generateCmd, "network-operator-namespace", GroupCommon) diff --git a/pkg/cmd/schema.go b/pkg/cmd/schema.go index 9e8c071d..10cb1066 100644 --- a/pkg/cmd/schema.go +++ b/pkg/cmd/schema.go @@ -25,6 +25,8 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/config" "github.com/nvidia/k8s-launch-kit/pkg/configflags" + dpfconfig "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" "github.com/nvidia/k8s-launch-kit/pkg/networkoperatorplugin" "github.com/nvidia/k8s-launch-kit/pkg/networkoperatorplugin/releases" "github.com/nvidia/k8s-launch-kit/pkg/target" @@ -38,6 +40,7 @@ type commandSchema struct { // schema represents the tool's capabilities in a machine-readable format. type schema struct { + DPF dpfSchema `json:"dpf"` Version string `json:"version"` Description string `json:"description"` Commands map[string]commandSchema `json:"commands"` @@ -52,6 +55,27 @@ type schema struct { Flags map[string]flagSchema `json:"flags"` } +type dpfSchema struct { + Profiles []string `json:"profiles"` + GenerationReleases []string `json:"generationReleases"` + Charts []chartSchema `json:"charts"` +} +type chartSchema struct { + Name string `json:"name"` + Version string `json:"version"` + Repository string `json:"repository"` + Filename string `json:"filename"` + SHA256 string `json:"sha256"` +} + +func dpfCapabilitiesSchema() dpfSchema { + s := dpfSchema{Profiles: []string{"sentry-bf4"}, GenerationReleases: dpfconfig.GenerationReleases()} + for _, c := range sentry.Catalog() { + s.Charts = append(s.Charts, chartSchema{Name: c.Name, Version: c.Version, Repository: c.Repository, Filename: c.Filename, SHA256: c.SHA256}) + } + return s +} + type flagSchema struct { Type string `json:"type"` Default string `json:"default,omitempty"` @@ -74,6 +98,7 @@ var schemaCmd = &cobra.Command{ Long: `Output a machine-readable JSON description of l8k's capabilities, flags, and exit codes. Designed for AI agents to programmatically discover what this tool can do.`, Run: func(cmd *cobra.Command, args []string) { s := schema{ + DPF: dpfCapabilitiesSchema(), Version: Version, Description: "CLI tool for deploying NVIDIA cloud-native networking solutions on Kubernetes", Commands: map[string]commandSchema{ diff --git a/pkg/cmd/schema_test.go b/pkg/cmd/schema_test.go index c9862508..27f2bcea 100644 --- a/pkg/cmd/schema_test.go +++ b/pkg/cmd/schema_test.go @@ -38,9 +38,10 @@ func TestSchemaDescribesTargetsAdditively(t *testing.T) { for _, phase := range target.PublicPhases() { assert.Truef(t, host.Phases[string(phase)].Available, "host phase %s", phase) - assert.Equalf(t, phase == target.Discover, dpf.Phases[string(phase)].Available, "dpf phase %s", phase) + assert.Truef(t, dpf.Phases[string(phase)].Available, "dpf phase %s", phase) assert.NotEmptyf(t, dpf.Phases[string(phase)].Reason, "dpf phase %s reason", phase) } + assert.False(t, dpf.Phases[string(target.Pipeline)].Available) } diff --git a/pkg/cmd/target_cli.go b/pkg/cmd/target_cli.go index 1aa92591..2f2aed3e 100644 --- a/pkg/cmd/target_cli.go +++ b/pkg/cmd/target_cli.go @@ -68,7 +68,7 @@ func addTargetFlag(cmd *cobra.Command) { &targetName, "target", string(target.Host), - "Deployment target: host (default) or dpf (experimental inventory discovery)", + "Deployment target: host (default) or dpf (experimental discovery and Sentry workflow)", ) setFlagGroup(cmd, "target", GroupTarget) setFlagTargetScope(cmd, []target.Name{target.Host, target.DPF}, "target") @@ -302,13 +302,14 @@ func markGenerateTargetScopes() { "workload-manifest", "network-operator-namespace", "network-operator-release", "skip-network-operator-helm", "image-pull-secrets", "enabled-plugins", "kubeconfig", "overwrite-existing") markConfigFlagTargetScopes(generateCmd, configflags.ScopeGenerate) + setFlagTargetScope(generateCmd, []target.Name{target.Host, target.DPF}, "user-config", "save-deployment-files") } func markConfigFlagTargetScopes(cmd *cobra.Command, scope configflags.Scope) { for _, definition := range configflags.Definitions() { for _, available := range definition.Scopes { if available == scope { - setFlagTargetScope(cmd, []target.Name{target.Host}, definition.FlagName) + setFlagTargetScope(cmd, []target.Name{definition.Target}, definition.FlagName) break } } @@ -319,6 +320,7 @@ func markDeployTargetScopes() { setFlagTargetScope(deployCmd, []target.Name{target.Host, target.DPF}, "dry-run", "deploy-timeout") setFlagTargetScope(deployCmd, []target.Name{target.Host}, "flavor", "kubeconfig", "deployment-files", "network-operator-namespace", "user-config", "overwrite-existing", "skip-network-operator-helm") + setFlagTargetScope(deployCmd, []target.Name{target.Host, target.DPF}, "deployment-files", "user-config") } func markValidateTargetScopes() { @@ -327,4 +329,5 @@ func markValidateTargetScopes() { "keep", "connectivity-timeout", "validation-mode", "validation-checks", "rdma-rping-iterations", "rdma-ib-write-size", "rdma-ib-write-min-bandwidth-gbps", "wait", "report-path", "junit-path") + setFlagTargetScope(validateCmd, []target.Name{target.Host, target.DPF}, "deployment-files", "user-config", "wait") } diff --git a/pkg/cmd/target_cli_test.go b/pkg/cmd/target_cli_test.go index c80b9099..5e7cfdb0 100644 --- a/pkg/cmd/target_cli_test.go +++ b/pkg/cmd/target_cli_test.go @@ -144,6 +144,8 @@ func TestImportantFlagScopes(t *testing.T) { {command: rootCmd, flag: "fabric", expected: []target.Name{target.Host}}, {command: discoverCmd, flag: "kubeconfig", expected: []target.Name{target.Host}}, {command: generateCmd, flag: "deploy", expected: []target.Name{target.Host, target.DPF}}, + {command: generateCmd, flag: "dpf-release", expected: []target.Name{target.DPF}}, + {command: generateCmd, flag: "dpf-profile", expected: []target.Name{target.DPF}}, {command: deployCmd, flag: "dry-run", expected: []target.Name{target.Host, target.DPF}}, {command: validateCmd, flag: "connectivity", expected: []target.Name{target.Host}}, {command: generateCmd, flag: "skip-network-operator-helm", expected: []target.Name{target.Host}}, @@ -250,7 +252,7 @@ func TestTargetDescriptors(t *testing.T) { assert.Equal(t, target.DPF, descriptors[1].Name) for _, phase := range target.PublicPhases() { assert.True(t, descriptors[0].Capability(phase).Available) - assert.Equal(t, phase == target.Discover, descriptors[1].Capability(phase).Available) + assert.True(t, descriptors[1].Capability(phase).Available) assert.NotEmpty(t, descriptors[1].Capability(phase).Reason) } } @@ -273,7 +275,9 @@ func TestTargetCLIProcess(t *testing.T) { assert.Equal(t, "host", actual.Targets[0].Name) assert.Equal(t, "dpf", actual.Targets[1].Name) assert.True(t, actual.Targets[0].Phases["validate"].Available) - assert.False(t, actual.Targets[1].Phases["validate"].Available) + assert.True(t, actual.Targets[1].Phases["generate"].Available) + assert.True(t, actual.Targets[1].Phases["deploy"].Available) + assert.True(t, actual.Targets[1].Phases["validate"].Available) assert.Equal(t, []string{"host", "dpf"}, actual.Flags["--output"].Targets) assert.Equal(t, []string{"host"}, actual.Flags["--fabric"].Targets) }) @@ -290,17 +294,28 @@ func TestTargetCLIProcess(t *testing.T) { assert.Contains(t, omittedOutput, "cluster config file does not exist") }) - for _, phase := range target.PublicPhases() { - if phase == target.Discover { - continue + t.Run("dpf generate reaches workflow validation", func(t *testing.T) { + dir := dpfProcessTempDir(t) + output, exitCode := runCLIHelper(t, "generate", "--target", "dpf", "--user-config", filepath.Join(dir, "missing.yaml"), "--save-deployment-files", filepath.Join(dir, "bundle"), "--output", "json") + assert.Equal(t, 2, exitCode) + assert.Contains(t, output, `"phase":"generate"`) + assert.Contains(t, output, "reference must be a bounded regular file") + assert.NotContains(t, output, `does not implement phase`) + }) + + t.Run("dpf deploy and validate fail on missing bundles before credential access", func(t *testing.T) { + for _, phase := range []string{"deploy", "validate"} { + t.Run(phase, func(t *testing.T) { + dir := dpfProcessTempDir(t) + missingBundle := filepath.Join(dir, "missing-bundle") + output, exitCode := runCLIHelper(t, phase, "--target", "dpf", "--deployment-files", missingBundle, "--user-config", filepath.Join(dir, "missing-kubeconfig"), "--output", "json") + assert.Equal(t, 2, exitCode) + assert.Contains(t, output, `"phase":"`+phase+`"`) + assert.Contains(t, output, "open DPF bundle") + assert.NotContains(t, output, "kubeconfig") + }) } - t.Run("dpf "+string(phase)+" capability error", func(t *testing.T) { - output, exitCode := runCLIHelper(t, string(phase), "--target", "dpf") - assert.Equal(t, 2, exitCode) - assert.Contains(t, output, `invalid target invocation`) - assert.Contains(t, output, `target "dpf" does not implement phase "`+string(phase)+`"`) - }) - } + }) t.Run("root dpf pipeline cannot enter host path", func(t *testing.T) { output, exitCode := runCLIHelper(t, "--target", "dpf") @@ -308,12 +323,43 @@ func TestTargetCLIProcess(t *testing.T) { assert.Contains(t, output, `target "dpf" does not implement phase "pipeline"`) }) - t.Run("explicit false host flag is rejected for dpf", func(t *testing.T) { + t.Run("DPF-only flags are rejected for host and host-only flags are rejected for DPF", func(t *testing.T) { + for _, flag := range []string{"--dpf-release", "--dpf-profile"} { + output, exitCode := runCLIHelper(t, "generate", flag, "26.8.0", "--user-config", filepath.Join(t.TempDir(), "missing"), "--save-deployment-files", filepath.Join(t.TempDir(), "out")) + assert.Equal(t, 2, exitCode) + assert.Contains(t, output, "not valid for target \"host\"") + } output, exitCode := runCLIHelper(t, "validate", "--target", "dpf", "--connectivity=false") assert.Equal(t, 2, exitCode) assert.Contains(t, output, `--connectivity (targets: host)`) }) + t.Run("invalid DPF config returns one JSON workflow failure without host fallback", func(t *testing.T) { + dir := dpfProcessTempDir(t) + badConfig := filepath.Join(dir, "invalid.yaml") + require.NoError(t, os.WriteFile(badConfig, []byte("dpf: [invalid]\n"), 0o600)) + bundlePath := filepath.Join(dir, "bundle") + output, exitCode := runCLIHelper(t, "generate", "--target", "dpf", "--user-config", badConfig, "--save-deployment-files", bundlePath, "--output", "json") + assert.Equal(t, 2, exitCode) + assert.Contains(t, output, `"target":"dpf"`) + assert.Contains(t, output, `"phase":"generate"`) + assert.NotContains(t, output, "Host configuration") + assert.NoDirExists(t, bundlePath) + }) + + t.Run("explicit empty DPF release returns validation JSON and creates no bundle", func(t *testing.T) { + dir := dpfProcessTempDir(t) + configPath := filepath.Join(dir, "site.yaml") + require.NoError(t, os.WriteFile(configPath, []byte("dpf:\n release: 26.8.0\n"), 0o600)) + bundlePath := filepath.Join(dir, "bundle") + output, exitCode := runCLIHelper(t, "generate", "--target", "dpf", "--user-config", configPath, "--dpf-release=", "--save-deployment-files", bundlePath, "--output", "json") + assert.Equal(t, 2, exitCode) + assert.Contains(t, output, `"target":"dpf"`) + assert.Contains(t, output, "DPF release") + assert.Contains(t, output, "has no Sentry BF4 generation contract") + assert.NoDirExists(t, bundlePath) + }) + t.Run("dpf capability errors preserve json output mode", func(t *testing.T) { output, exitCode := runCLIHelper(t, "discover", "--target", "dpf", "--output", "json") assert.Equal(t, 2, exitCode) @@ -394,3 +440,15 @@ func runCLIHelper(t *testing.T, args ...string) (string, int) { } return string(output), exitErr.ExitCode() } + +func dpfProcessTempDir(t *testing.T) string { + t.Helper() + cwd, err := os.Getwd() + require.NoError(t, err) + repoRoot, err := filepath.Abs(filepath.Join(cwd, "../..")) + require.NoError(t, err) + dir, err := os.MkdirTemp(repoRoot, ".dpf-cli-test-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + return dir +} diff --git a/pkg/cmd/validate.go b/pkg/cmd/validate.go index baa5c7e8..94371476 100644 --- a/pkg/cmd/validate.go +++ b/pkg/cmd/validate.go @@ -22,6 +22,7 @@ import ( "github.com/spf13/cobra" "github.com/nvidia/k8s-launch-kit/pkg/target" + dpftarget "github.com/nvidia/k8s-launch-kit/pkg/target/dpf" hosttarget "github.com/nvidia/k8s-launch-kit/pkg/target/host" ) @@ -43,7 +44,7 @@ var ( var validateCmd = &cobra.Command{ Use: "validate", - Short: "Verify a deployment matches the selected Network Operator release", + Short: "Verify the selected target's deployment resources", Long: `Validate that a previously generated deployment is correctly applied to the cluster. @@ -82,7 +83,12 @@ by the selected checks. Exits non-zero on any missing manifest, version mismatch, or connectivity-matrix failure. A skipped, empty, or incomplete connectivity-only matrix also fails; every selected check family must produce a gating test, and -non-gating cross-rail observations do not count.`, +non-gating cross-rail observations do not count. + +For --target dpf, verify a generated DPF bundle's cluster bindings, selected +devices and Sentry resources without creating them. --wait enables bounded +readiness polling. Resource readiness is separate from a functional Sentry +allow/deny policy test; the Host connectivity workload is not run.`, Example: ` # Full validate (manifest state + connectivity matrix) l8k validate @@ -104,6 +110,10 @@ non-gating cross-rail observations do not count.`, # Agent mode (JSON output) l8k validate --output json 2>/dev/null`, Run: func(cmd *cobra.Command, _ []string) { + if targetName == string(target.DPF) { + runTargetCommand(cmd, target.Validate, dpftarget.NewWorkflowAdapter(dpftarget.WorkflowRequest{Phase: target.Validate, UserConfig: userConfig, DeploymentFiles: deploymentFiles, Wait: validateWait})) + return + } runTargetCommand(cmd, target.Validate, hosttarget.NewValidateAdapter( newHostValidateRequest(cmd), hosttarget.NewValidateRunner(), diff --git a/pkg/configflags/flags.go b/pkg/configflags/flags.go index 8835af65..7f36e4af 100644 --- a/pkg/configflags/flags.go +++ b/pkg/configflags/flags.go @@ -16,7 +16,9 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/config" "github.com/nvidia/k8s-launch-kit/pkg/configinput" + dpfconfig "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" "github.com/nvidia/k8s-launch-kit/pkg/options" + "github.com/nvidia/k8s-launch-kit/pkg/target" ) // Scope identifies the command on which a config-backed flag is available. @@ -33,6 +35,7 @@ type Definition struct { FieldName string FlagName string ValueType string + Target target.Name ConfigPaths []string Resolver string Scopes []Scope @@ -53,6 +56,7 @@ func Definitions() []Definition { FieldName: field.Name, FlagName: flagName, ValueType: valueType(field.Type), + Target: targetForTag(field.Tag.Get("target")), ConfigPaths: splitTag(field.Tag.Get("config")), Resolver: field.Tag.Get("resolve"), Scopes: parseScopes(field.Tag.Get("scopes")), @@ -90,6 +94,9 @@ func ValidateDefinitions() error { if len(definition.Scopes) == 0 { return fmt.Errorf("config flag --%s has no command scope", definition.FlagName) } + if !knownTarget(definition.Target) { + return fmt.Errorf("config flag --%s has unknown target %q", definition.FlagName, definition.Target) + } if definition.Usage == "" { return fmt.Errorf("config flag --%s has no usage text", definition.FlagName) } @@ -104,7 +111,7 @@ func ValidateDefinitions() error { return fmt.Errorf("config flag --%s has unknown resolver %q", definition.FlagName, definition.Resolver) } for _, path := range definition.ConfigPaths { - destinationType, err := configPathType(path) + destinationType, err := configPathType(path, definition.Target) if err != nil { return fmt.Errorf("config flag --%s: %w", definition.FlagName, err) } @@ -222,6 +229,7 @@ func definitionForField(field reflect.StructField) (Definition, bool) { FieldName: field.Name, FlagName: flagName, ValueType: valueType(field.Type), + Target: targetForTag(field.Tag.Get("target")), ConfigPaths: splitTag(field.Tag.Get("config")), Resolver: field.Tag.Get("resolve"), Scopes: parseScopes(field.Tag.Get("scopes")), @@ -406,11 +414,19 @@ func containsScope(scopes []Scope, want Scope) bool { return false } -func configPathType(path string) (reflect.Type, error) { +func configPathType(path string, targetName target.Name) (reflect.Type, error) { if path == "" { return nil, fmt.Errorf("config path must not be empty") } - current := reflect.TypeOf(config.LaunchKitConfig{}) + var current reflect.Type + switch targetName { + case target.Host: + current = reflect.TypeOf(config.LaunchKitConfig{}) + case target.DPF: + current = reflect.TypeOf(dpfconfig.Site{}) + default: + return nil, fmt.Errorf("unknown config target %q", targetName) + } for _, part := range strings.Split(path, ".") { for current.Kind() == reflect.Pointer { current = current.Elem() @@ -427,6 +443,17 @@ func configPathType(path string) (reflect.Type, error) { return current, nil } +func targetForTag(value string) target.Name { + if value == "" { + return target.Host + } + return target.Name(value) +} + +func knownTarget(name target.Name) bool { + return name == target.Host || name == target.DPF +} + func mappingTypesMatch(valueType string, destination reflect.Type) bool { for destination.Kind() == reflect.Pointer { destination = destination.Elem() diff --git a/pkg/configflags/flags_test.go b/pkg/configflags/flags_test.go index 9f47ff18..e0244905 100644 --- a/pkg/configflags/flags_test.go +++ b/pkg/configflags/flags_test.go @@ -13,6 +13,7 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/configinput" "github.com/nvidia/k8s-launch-kit/pkg/options" + "github.com/nvidia/k8s-launch-kit/pkg/target" ) func TestBindAndCollectPreserveExplicitZeroValues(t *testing.T) { @@ -79,3 +80,80 @@ func TestDefinitionsHaveUniqueFlagsAndMappings(t *testing.T) { seen[definition.FlagName] = struct{}{} } } + +func TestTargetOwnershipDefaultsToHostAndRejectsUnknownTarget(t *testing.T) { + require.Equal(t, target.Host, targetForTag("")) + require.True(t, knownTarget(targetForTag(""))) + require.False(t, knownTarget(targetForTag("future"))) + + definitions := map[string]Definition{} + for _, definition := range Definitions() { + definitions[definition.FlagName] = definition + } + assert.Equal(t, target.Host, definitions["fabric"].Target) + assert.Equal(t, target.DPF, definitions["dpf-profile"].Target) + assert.Equal(t, target.DPF, definitions["dpf-release"].Target) + assert.Equal(t, []string{"profile.fabric"}, definitions["fabric"].ConfigPaths) + assert.Equal(t, []string{"dpf.profile"}, definitions["dpf-profile"].ConfigPaths) + assert.Equal(t, []string{"dpf.release"}, definitions["dpf-release"].ConfigPaths) + assert.Empty(t, definitions["dpf-profile"].Resolver) + assert.Empty(t, definitions["dpf-release"].Resolver) + assert.Equal(t, []Scope{ScopeGenerate}, definitions["dpf-profile"].Scopes) + assert.Equal(t, []Scope{ScopeGenerate}, definitions["dpf-release"].Scopes) +} + +func TestConfigPathValidationUsesDefinitionTargetSchema(t *testing.T) { + hostType, err := configPathType("profile.fabric", target.Host) + require.NoError(t, err) + assert.Equal(t, "string", valueType(hostType)) + _, err = configPathType("dpf.profile", target.Host) + require.ErrorContains(t, err, `config path "dpf.profile" does not exist`) + + for _, path := range []string{"dpf.profile", "dpf.release"} { + dpfType, err := configPathType(path, target.DPF) + require.NoError(t, err) + assert.Equal(t, "string", valueType(dpfType)) + } + _, err = configPathType("profile.fabric", target.DPF) + require.ErrorContains(t, err, `config path "profile.fabric" does not exist`) +} + +func TestDPFFlagsAreGenerateOnlyAndPreserveExplicitEmptyValues(t *testing.T) { + for _, scope := range []Scope{ScopeRoot, ScopeDiscover} { + var opts options.Options + flags := pflag.NewFlagSet(string(scope), pflag.ContinueOnError) + require.NoError(t, Bind(flags, &opts, scope)) + assert.Nil(t, flags.Lookup("dpf-profile")) + assert.Nil(t, flags.Lookup("dpf-release")) + } + + var opts options.Options + flags := pflag.NewFlagSet("generate", pflag.ContinueOnError) + require.NoError(t, Bind(flags, &opts, ScopeGenerate)) + require.NotNil(t, flags.Lookup("dpf-profile")) + require.NotNil(t, flags.Lookup("dpf-release")) + require.NoError(t, flags.Parse([]string{"--dpf-profile=", "--dpf-release="})) + require.NoError(t, Collect(flags, &opts)) + assert.Empty(t, opts.DPFProfile) + assert.Empty(t, opts.DPFRelease) + assert.Contains(t, opts.ConfigInputs.Overrides, configinput.Override{ + Flag: "dpf-profile", Path: "dpf.profile", Value: "", + }) + assert.Contains(t, opts.ConfigInputs.Overrides, configinput.Override{ + Flag: "dpf-release", Path: "dpf.release", Value: "", + }) + assert.Empty(t, opts.ConfigInputs.Requests) +} + +func TestHostFlagDefinitionRemainsHostOwned(t *testing.T) { + var fabric Definition + for _, definition := range Definitions() { + if definition.FlagName == "fabric" { + fabric = definition + break + } + } + assert.Equal(t, target.Host, fabric.Target) + assert.Equal(t, []string{"profile.fabric"}, fabric.ConfigPaths) + assert.Equal(t, []Scope{ScopeRoot, ScopeGenerate, ScopeDiscover}, fabric.Scopes) +} diff --git a/pkg/dpf/bundle/bundle_test.go b/pkg/dpf/bundle/bundle_test.go new file mode 100644 index 00000000..05ddd40f --- /dev/null +++ b/pkg/dpf/bundle/bundle_test.go @@ -0,0 +1,336 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/gofrs/flock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBundle(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DPF immutable bundle") +} + +func fixture() (Index, []File) { + return Index{ + FormatVersion: FormatVersion, Target: Target, Phase: Phase, + Profile: "dpf-sentry", Release: "v26.8.0-rc.1", Renderer: "fixture-v1", + SelectedDevices: []string{"device-a"}, BundleDigest: "", + Files: []FileRecord{{Path: ResolvedConfigPath, Role: ResolvedConfig, SHA256: ""}, {Path: "system/object.yaml", Role: Manifest, SHA256: ""}}, + Stages: []Stage{{ID: "system", Destination: Management, Namespace: "dpf-system", Files: []string{"system/object.yaml"}, ProjectionFiles: nil, DependsOn: nil, Requires: nil}}, + References: nil, ExternalPrerequisites: nil, + }, []File{ + {Path: ResolvedConfigPath, Data: []byte("formatVersion: v1alpha1\ntarget: dpf\nconfig:\n explicitFalse: false\n explicitEmpty: []\n")}, + {Path: "system/object.yaml", Data: []byte("# exact comment\napiVersion: v1\nkind: ConfigMap\nmetadata: {name: example-config, namespace: dpf-system}\ndata: {value: original}\n")}, + } +} + +var _ = Describe("validated immutable membership", func() { + It("retains exact bytes and isolates constructor and consumer mutations", func() { + index, files := fixture() + b, err := New(index, files) + Expect(err).NotTo(HaveOccurred()) + original := append([]byte(nil), files[1].Data...) + files[1].Data[0] = '!' + index.Stages[0].Files[0] = "changed" + copy := b.Files() + copy[1].Data[0] = '!' + copyIndex := b.Index() + copyIndex.Stages[0].Files[0] = "changed" + stage, ok := b.Stage("system") + Expect(ok).To(BeTrue()) + doc := stage.Documents()[0] + object := doc.ObjectCopy() + object.SetName("mutated") + Expect(doc.ObjectCopy().GetName()).To(Equal("example-config")) + Expect(b.Files()[1].Data).To(Equal(original)) + Expect(b.Index().Stages[0].Files).To(Equal([]string{"system/object.yaml"})) + Expect(doc.Source().File).To(Equal("system/object.yaml")) + Expect(doc.Source().Document).To(Equal(1)) + }) + + DescribeTable("rejects incomplete or unsupported input", func(mutate func(*Index, *[]File), message string) { + index, files := fixture() + mutate(&index, &files) + _, err := New(index, files) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(message)) + }, + Entry("unknown envelope", func(i *Index, _ *[]File) { i.Target = "host" }, "unsupported"), + Entry("unknown version", func(i *Index, _ *[]File) { i.FormatVersion = "v99" }, "unsupported"), + Entry("no selection", func(i *Index, _ *[]File) { i.SelectedDevices = nil }, "no selected devices"), + Entry("repeated selection", func(i *Index, _ *[]File) { i.SelectedDevices = []string{"device-a", "device-a"} }, "duplicate selected device"), + Entry("unknown destination", func(i *Index, _ *[]File) { i.Stages[0].Destination = "ambient" }, "destination"), + Entry("traversal", func(i *Index, f *[]File) { i.Files[1].Path = "../object.yaml"; (*f)[1].Path = "../object.yaml" }, "invalid relative"), + Entry("path alias", func(i *Index, f *[]File) { i.Files[1].Path = "system/../object.yaml"; (*f)[1].Path = i.Files[1].Path }, "invalid relative"), + Entry("duplicate path", func(_ *Index, f *[]File) { *f = append(*f, (*f)[1]) }, "duplicate DPF bundle file"), + Entry("unindexed bytes", func(_ *Index, f *[]File) { *f = append(*f, File{Path: "extra.txt", Data: []byte("extra")}) }, "unindexed"), + Entry("missing file", func(_ *Index, f *[]File) { *f = (*f)[:1] }, "missing"), + Entry("wrong checksum", func(i *Index, _ *[]File) { i.Files[1].SHA256 = "sha256:wrong" }, "checksum"), + Entry("wrong bundle checksum", func(i *Index, _ *[]File) { i.BundleDigest = "sha256:wrong" }, "bundle digest"), + Entry("wrong sidecar", func(_ *Index, f *[]File) { + (*f)[0].Data = []byte("formatVersion: v1alpha1\ntarget: host\nconfig: {}\n") + }, "envelope"), + Entry("forward stage", func(i *Index, _ *[]File) { i.Stages[0].DependsOn = []string{"later"} }, "earlier stage"), + Entry("self dependency", func(i *Index, _ *[]File) { i.Stages[0].DependsOn = []string{"system"} }, "earlier stage"), + Entry("missing prerequisite declaration", func(i *Index, _ *[]File) { i.Stages[0].Requires = []string{"credential-ready"} }, "undeclared external"), + Entry("missing namespace", func(_ *Index, f *[]File) { + (*f)[1].Data = []byte("apiVersion: v1\nkind: ConfigMap\nmetadata: {name: test}\n") + }, "explicit namespace"), + Entry("wrong namespace", func(i *Index, _ *[]File) { i.Stages[0].Namespace = "elsewhere" }, "explicit namespace"), + Entry("unknown kind", func(_ *Index, f *[]File) { + (*f)[1].Data = []byte("apiVersion: unknown.io/v1\nkind: Unknown\nmetadata: {name: test, namespace: dpf-system}\n") + }, "unsupported resource kind"), + Entry("namespace on cluster kind", func(_ *Index, f *[]File) { + (*f)[1].Data = []byte("apiVersion: v1\nkind: Namespace\nmetadata: {name: test, namespace: dpf-system}\n") + }, "cluster-scoped"), + Entry("malformed final document", func(_ *Index, f *[]File) { (*f)[1].Data = append((*f)[1].Data, []byte("---\nmetadata: [broken\n")...) }, "document 2"), + ) + + It("permits same resource names on different destination clusters but rejects same-cluster duplicates across API versions", func() { + index, files := fixture() + files = append(files, File{Path: "tenant/object.yaml", Data: append([]byte(nil), files[1].Data...)}) + index.Files = append(index.Files, FileRecord{Path: "tenant/object.yaml", Role: Manifest, SHA256: ""}) + index.Stages = append(index.Stages, Stage{ID: "tenant", Destination: DPU, Namespace: "dpf-system", Files: []string{"tenant/object.yaml"}, DependsOn: []string{"system"}}) + _, err := New(index, files) + Expect(err).NotTo(HaveOccurred()) + index.Stages[1].Destination = Management + files[2].Data = []byte(strings.ReplaceAll(string(files[2].Data), "apiVersion: v1", "apiVersion: v2")) + _, err = New(index, files) + Expect(err).To(MatchError(ContainSubstring("duplicate resource"))) + }) + + It("checks projection identities while keeping them out of apply documents", func() { + index, files := fixture() + index.Files = append(index.Files, FileRecord{Path: ".l8k/helm/component/projection.yaml", Role: Projection}) + index.Stages[0].ProjectionFiles = []string{".l8k/helm/component/projection.yaml"} + files = append(files, File{Path: ".l8k/helm/component/projection.yaml", Data: []byte("apiVersion: v1\nkind: ServiceAccount\nmetadata: {name: chart-account, namespace: dpf-system}\n")}) + b, err := New(index, files) + Expect(err).NotTo(HaveOccurred()) + stage, _ := b.Stage("system") + Expect(stage.Documents()).To(HaveLen(1)) + Expect(stage.Projections()).To(HaveLen(1)) + Expect(stage.Projections()[0].IsProjection()).To(BeTrue()) + files[2].Data = append([]byte(nil), files[1].Data...) + _, err = New(index, files) + Expect(err).To(MatchError(ContainSubstring("duplicate resource"))) + }) + + It("validates declared internal references and separates external prerequisites", func() { + index, files := fixture() + ref := ResourceRef{Destination: Management, Group: "", Kind: "ConfigMap", Namespace: "dpf-system", Name: "example-config"} + index.References = []Reference{{From: ref, To: ref}} + index.ExternalPrerequisites = []string{"credential-ready"} + index.Stages[0].Requires = []string{"credential-ready"} + _, err := New(index, files) + Expect(err).NotTo(HaveOccurred()) + index.References[0].To.Name = "absent" + _, err = New(index, files) + Expect(err).To(MatchError(ContainSubstring("target"))) + index.References[0].To = ref + index.References[0].To.Destination = DPU + _, err = New(index, files) + Expect(err).To(MatchError(ContainSubstring("crosses cluster"))) + }) +}) + +var _ = Describe("closed disk loading and publication", func() { + var dir string + var b *Bundle + BeforeEach(func() { + var err error + dir, err = os.MkdirTemp("", "dpf-bundle-") + Expect(err).NotTo(HaveOccurred()) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + index, files := fixture() + b, err = New(index, files) + Expect(err).NotTo(HaveOccurred()) + }) + + It("publishes privately, reloads identically and does not rewrite an identical existing bundle", func() { + dest := filepath.Join(dir, "published") + published, err := b.Publish(context.Background(), dest) + Expect(err).NotTo(HaveOccurred()) + Expect(published.Unchanged).To(BeFalse()) + loaded, err := Load(dest) + Expect(err).NotTo(HaveOccurred()) + Expect(sameBytes(b, loaded)).To(BeTrue()) + st, err := os.Stat(filepath.Join(dest, IndexPath)) + Expect(err).NotTo(HaveOccurred()) + Expect(st.Mode().Perm()).To(Equal(os.FileMode(0600))) + rootInfo, err := os.Stat(dest) + Expect(err).NotTo(HaveOccurred()) + Expect(rootInfo.Mode().Perm()).To(Equal(os.FileMode(0700))) + published, err = b.Publish(context.Background(), dest) + Expect(err).NotTo(HaveOccurred()) + Expect(published.Unchanged).To(BeTrue()) + after, err := os.Stat(filepath.Join(dest, IndexPath)) + Expect(err).NotTo(HaveOccurred()) + Expect(after.ModTime()).To(Equal(st.ModTime())) + }) + + It("rejects tampered bytes, added files, and file or directory symlinks", func() { + dest := filepath.Join(dir, "published") + _, err := b.Publish(context.Background(), dest) + Expect(err).NotTo(HaveOccurred()) + manifest := filepath.Join(dest, "system/object.yaml") + original, err := os.ReadFile(manifest) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(manifest, append(original, '\n'), 0600)).To(Succeed()) + _, err = Load(dest) + Expect(err).To(MatchError(ContainSubstring("checksum"))) + Expect(os.WriteFile(manifest, original, 0600)).To(Succeed()) + extra := filepath.Join(dest, "extra.txt") + Expect(os.WriteFile(extra, []byte("unindexed"), 0600)).To(Succeed()) + _, err = Load(dest) + Expect(err).To(MatchError(ContainSubstring("unindexed"))) + Expect(os.Remove(extra)).To(Succeed()) + Expect(os.Remove(manifest)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "external"), original, 0600)).To(Succeed()) + Expect(os.Symlink(filepath.Join(dir, "external"), manifest)).To(Succeed()) + _, err = Load(dest) + Expect(err).To(MatchError(ContainSubstring("symlink"))) + Expect(os.RemoveAll(filepath.Join(dest, "system"))).To(Succeed()) + Expect(os.Symlink(dir, filepath.Join(dest, "system"))).To(Succeed()) + _, err = Load(dest) + Expect(err).To(MatchError(ContainSubstring("symlink"))) + }) + + It("preserves existing empty, partial and differing destinations", func() { + empty := filepath.Join(dir, "empty") + Expect(os.Mkdir(empty, 0700)).To(Succeed()) + _, err := b.Publish(context.Background(), empty) + Expect(err).To(HaveOccurred()) + entries, err := os.ReadDir(empty) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + partial := filepath.Join(dir, "partial") + Expect(os.Mkdir(partial, 0700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(partial, "keep.txt"), []byte("retained"), 0600)).To(Succeed()) + _, err = b.Publish(context.Background(), partial) + Expect(err).To(HaveOccurred()) + retained, err := os.ReadFile(filepath.Join(partial, "keep.txt")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(retained)).To(Equal("retained")) + dest := filepath.Join(dir, "published") + _, err = b.Publish(context.Background(), dest) + Expect(err).NotTo(HaveOccurred()) + index, files := fixture() + files[1].Data = append(files[1].Data, []byte("# differing observation\n")...) + other, err := New(index, files) + Expect(err).NotTo(HaveOccurred()) + _, err = other.Publish(context.Background(), dest) + Expect(err).To(MatchError(ContainSubstring("differs"))) + loaded, err := Load(dest) + Expect(err).NotTo(HaveOccurred()) + Expect(sameBytes(b, loaded)).To(BeTrue()) + }) + + It("honors cancellation while waiting on the destination lock without publishing", func() { + dest := filepath.Join(dir, "cancelled") + lock := flock.New(dest + ".lock") + Expect(lock.Lock()).To(Succeed()) + defer lock.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + _, err := b.Publish(ctx, dest) + Expect(err).To(HaveOccurred()) + _, err = os.Lstat(dest) + Expect(os.IsNotExist(err)).To(BeTrue()) + staged, err := filepath.Glob(filepath.Join(dir, ".*.staging-*")) + Expect(err).NotTo(HaveOccurred()) + Expect(staged).To(BeEmpty()) + }) + + It("serializes concurrent publishers and returns one verified unchanged result", func() { + dest := filepath.Join(dir, "concurrent") + type result struct { + publication Publication + err error + } + results := make(chan result, 2) + for range 2 { + go func() { p, err := b.Publish(context.Background(), dest); results <- result{p, err} }() + } + unchanged := 0 + for range 2 { + r := <-results + Expect(r.err).NotTo(HaveOccurred()) + if r.publication.Unchanged { + unchanged++ + } + } + Expect(unchanged).To(Equal(1)) + loaded, err := Load(dest) + Expect(err).NotTo(HaveOccurred()) + Expect(sameBytes(b, loaded)).To(BeTrue()) + }) + + It("uses no-replace publication even when the destination is concurrently created empty", func() { + from, to := filepath.Join(dir, "staging"), filepath.Join(dir, "destination") + Expect(os.Mkdir(from, 0700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(from, "complete"), []byte("retained"), 0600)).To(Succeed()) + Expect(os.Mkdir(to, 0700)).To(Succeed()) + Expect(renameNewDirectory(from, to)).NotTo(Succeed()) + entries, err := os.ReadDir(to) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + _, err = os.Stat(filepath.Join(from, "complete")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("removes only its own partial staging when cancellation follows the first write", func() { + dest := filepath.Join(dir, "interrupted") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + interrupted := &cancelAfterStagedIndex{Context: ctx, cancel: cancel, parent: dir} + _, err := b.Publish(interrupted, dest) + Expect(err).To(MatchError(context.Canceled)) + Expect(interrupted.observed).To(BeTrue()) + _, err = os.Stat(dest) + Expect(os.IsNotExist(err)).To(BeTrue()) + staged, err := filepath.Glob(filepath.Join(dir, ".*.staging-*")) + Expect(err).NotTo(HaveOccurred()) + Expect(staged).To(BeEmpty()) + }) + + It("rejects oversized regular files without reading their content", func() { + dest := filepath.Join(dir, "oversized") + _, err := b.Publish(context.Background(), dest) + Expect(err).NotTo(HaveOccurred()) + Expect(os.Truncate(filepath.Join(dest, "system/object.yaml"), maxFileBytes+1)).To(Succeed()) + _, err = Load(dest) + Expect(err).To(MatchError(ContainSubstring("bounded regular file"))) + }) +}) + +// Cancel at the first checkpoint after actual staging I/O, without timing races. +type cancelAfterStagedIndex struct { + context.Context + cancel context.CancelFunc + parent string + observed bool +} + +func (c *cancelAfterStagedIndex) Err() error { + staged, _ := filepath.Glob(filepath.Join(c.parent, ".*.staging-*")) + for _, dir := range staged { + if _, err := os.Stat(filepath.Join(dir, IndexPath)); err == nil { + c.observed = true + c.cancel() + } + } + return c.Context.Err() +} diff --git a/pkg/dpf/bundle/load.go b/pkg/dpf/bundle/load.go new file mode 100644 index 00000000..bd1a0be6 --- /dev/null +++ b/pkg/dpf/bundle/load.go @@ -0,0 +1,185 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// Load validates closed on-disk membership before retaining any desired snapshot. +// It accepts only bounded regular files and rejects symlinks at every path level. +func Load(dir string) (*Bundle, error) { + if err := noSymlinkPath(dir); err != nil { + return nil, err + } + root, err := os.OpenRoot(dir) + if err != nil { + return nil, fmt.Errorf("open DPF bundle: %w", err) + } + defer root.Close() + var files []File + var indexBytes []byte + total := 0 + entries := 0 + err = fs.WalkDir(root.FS(), ".", func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if name == "." { + return nil + } + entries++ + if entries > maxFiles*4 { + return fmt.Errorf("DPF bundle exceeds directory entry limit") + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("DPF bundle path %q is a symlink", name) + } + if entry.IsDir() { + return nil + } + if err := validPath(name); err != nil { + return err + } + if len(files) >= maxFiles+1 { + return fmt.Errorf("DPF bundle exceeds file count limit") + } + data, err := readRegular(root, name) + if err != nil { + return err + } + total += len(data) + if total > maxBundleBytes { + return fmt.Errorf("DPF bundle exceeds content limit") + } + if name == IndexPath { + indexBytes = data + } else { + files = append(files, File{Path: name, Data: data}) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("load DPF bundle: %w", err) + } + if len(indexBytes) == 0 { + return nil, fmt.Errorf("DPF bundle index is missing") + } + var index Index + if err := decodeOne(indexBytes, &index); err != nil { + return nil, fmt.Errorf("decode DPF bundle index: %w", err) + } + b, err := construct(index, files, true) + if err != nil { + return nil, err + } + b.indexBytes = bytes.Clone(indexBytes) + return b, nil +} + +func noSymlinkPath(filename string) error { + abs, err := filepath.Abs(filename) + if err != nil { + return err + } + for current := abs; ; current = filepath.Dir(current) { + info, err := os.Lstat(current) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("inspect DPF path %q: %w", current, err) + } + if err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("DPF path %q is a symlink", current) + } + if filepath.Dir(current) == current { + return nil + } + } +} + +func checkRootPath(root *os.Root, name string) error { + parts := strings.Split(name, "/") + for n := range parts { + info, err := root.Lstat(strings.Join(parts[:n+1], "/")) + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("DPF bundle path %q contains a symlink", name) + } + if n < len(parts)-1 && !info.IsDir() { + return fmt.Errorf("DPF bundle parent for %q is not a directory", name) + } + } + return nil +} + +func readRegular(root *os.Root, name string) ([]byte, error) { + if err := checkRootPath(root, name); err != nil { + return nil, err + } + before, err := root.Lstat(name) + if err != nil { + return nil, err + } + if !before.Mode().IsRegular() || before.Size() > maxFileBytes { + return nil, fmt.Errorf("DPF bundle file %q is not a bounded regular file", name) + } + file, err := root.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + opened, err := file.Stat() + if err != nil { + return nil, err + } + if !os.SameFile(before, opened) || !opened.Mode().IsRegular() { + return nil, fmt.Errorf("DPF bundle file %q changed while opening", name) + } + data, err := io.ReadAll(io.LimitReader(file, maxFileBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxFileBytes { + return nil, fmt.Errorf("DPF bundle file %q exceeds size limit", name) + } + if err := checkRootPath(root, name); err != nil { + return nil, err + } + after, err := root.Lstat(name) + if err != nil { + return nil, err + } + if !os.SameFile(before, after) || before.Size() != after.Size() || !before.ModTime().Equal(after.ModTime()) { + return nil, fmt.Errorf("DPF bundle file %q changed while reading", name) + } + return data, nil +} + +func (b *Bundle) persistedFiles() []File { + files := b.Files() + files = append(files, File{Path: IndexPath, Data: bytes.Clone(b.indexBytes)}) + sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path }) + return files +} + +func sameBytes(a, b *Bundle) bool { + left, right := a.persistedFiles(), b.persistedFiles() + if len(left) != len(right) { + return false + } + for n := range left { + if left[n].Path != right[n].Path || !bytes.Equal(left[n].Data, right[n].Data) { + return false + } + } + return true +} diff --git a/pkg/dpf/bundle/publish.go b/pkg/dpf/bundle/publish.go new file mode 100644 index 00000000..b874ef4b --- /dev/null +++ b/pkg/dpf/bundle/publish.go @@ -0,0 +1,180 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/gofrs/flock" +) + +// Publish writes a complete new directory on Linux/macOS only. An existing +// byte-identical complete bundle is unchanged; every other destination fails. +// The sibling lock coordinates Launch Kit writers. No-replace rename also +// protects a destination created concurrently by a noncooperating writer. +func (b *Bundle) Publish(ctx context.Context, dest string) (Publication, error) { + if b == nil { + return Publication{}, fmt.Errorf("cannot publish a nil DPF bundle") + } + if ctx == nil { + return Publication{}, fmt.Errorf("DPF publication context is nil") + } + if err := ctx.Err(); err != nil { + return Publication{}, err + } + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + return Publication{}, fmt.Errorf("DPF directory publication is unsupported on %s; owner-only permissions and no-replace rename are qualified on Linux/macOS only", runtime.GOOS) + } + if dest == "" { + return Publication{}, fmt.Errorf("DPF publication destination is empty") + } + abs, err := filepath.Abs(dest) + if err != nil { + return Publication{}, err + } + if err := noSymlinkPath(abs); err != nil { + return Publication{}, err + } + parent := filepath.Dir(abs) + info, err := os.Stat(parent) + if err != nil || !info.IsDir() { + return Publication{}, fmt.Errorf("DPF publication parent must already be a directory") + } + lockPath := abs + ".lock" + if err := noSymlinkPath(lockPath); err != nil { + return Publication{}, err + } + if existing, err := os.Lstat(lockPath); err == nil && !existing.Mode().IsRegular() { + return Publication{}, fmt.Errorf("DPF publication lock is not a regular file") + } else if err != nil && !os.IsNotExist(err) { + return Publication{}, err + } + lockFile, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return Publication{}, fmt.Errorf("create DPF publication lock: %w", err) + } + if err := lockFile.Close(); err != nil { + return Publication{}, err + } + lock := flock.New(lockPath) + locked, err := lock.TryLockContext(ctx, 50*time.Millisecond) + if err != nil { + return Publication{}, fmt.Errorf("acquire DPF publication lock: %w", err) + } + if !locked { + return Publication{}, fmt.Errorf("acquire DPF publication lock: %w", ctx.Err()) + } + defer lock.Unlock() + if err := ctx.Err(); err != nil { + return Publication{}, err + } + if err := noSymlinkPath(abs); err != nil { + return Publication{}, err + } + if _, err := os.Lstat(abs); err == nil { + existing, err := Load(abs) + if err != nil { + return Publication{}, fmt.Errorf("existing DPF destination is not a complete bundle: %w", err) + } + if !sameBytes(b, existing) { + return Publication{}, fmt.Errorf("existing DPF destination differs; choose a new directory") + } + return Publication{Path: abs, Unchanged: true}, nil + } else if !os.IsNotExist(err) { + return Publication{}, err + } + staging, err := os.MkdirTemp(parent, "."+filepath.Base(abs)+".staging-") + if err != nil { + return Publication{}, err + } + defer os.RemoveAll(staging) + if err := b.writeStaging(ctx, staging); err != nil { + return Publication{}, err + } + verified, err := Load(staging) + if err != nil { + return Publication{}, fmt.Errorf("verify staged DPF bundle: %w", err) + } + if !sameBytes(b, verified) { + return Publication{}, fmt.Errorf("staged DPF bundle differs from retained snapshot") + } + if err := ctx.Err(); err != nil { + return Publication{}, err + } + if err := noSymlinkPath(parent); err != nil { + return Publication{}, err + } + if err := renameNewDirectory(staging, abs); err != nil { + return Publication{}, fmt.Errorf("publish DPF directory without replacing destination: %w", err) + } + if err := syncDirectory(parent); err != nil { + return Publication{}, fmt.Errorf("sync published DPF parent: %w", err) + } + return Publication{Path: abs}, nil +} + +func (b *Bundle) writeStaging(ctx context.Context, dir string) error { + for _, file := range b.persistedFiles() { + if err := ctx.Err(); err != nil { + return err + } + filename := filepath.Join(dir, filepath.FromSlash(file.Path)) + if err := os.MkdirAll(filepath.Dir(filename), 0700); err != nil { + return err + } + output, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + _, writeErr := output.Write(file.Data) + if writeErr == nil { + writeErr = output.Sync() + } + closeErr := output.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + } + var directories []string + if err := filepath.WalkDir(dir, func(name string, entry os.DirEntry, err error) error { + if err != nil || !entry.IsDir() { + return err + } + directories = append(directories, name) + return nil + }); err != nil { + return err + } + // Sync directories bottom-up before making the root visible. + for n := len(directories) - 1; n >= 0; n-- { + if err := ctx.Err(); err != nil { + return err + } + if err := syncDirectory(directories[n]); err != nil { + return err + } + } + return nil +} + +func syncDirectory(name string) error { + directory, err := os.Open(name) + if err != nil { + return err + } + syncErr := directory.Sync() + closeErr := directory.Close() + if syncErr != nil { + return syncErr + } + return closeErr +} diff --git a/pkg/dpf/bundle/rename_darwin.go b/pkg/dpf/bundle/rename_darwin.go new file mode 100644 index 00000000..410ab283 --- /dev/null +++ b/pkg/dpf/bundle/rename_darwin.go @@ -0,0 +1,10 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import "golang.org/x/sys/unix" + +func renameNewDirectory(from, to string) error { + return unix.RenamexNp(from, to, unix.RENAME_EXCL) +} diff --git a/pkg/dpf/bundle/rename_linux.go b/pkg/dpf/bundle/rename_linux.go new file mode 100644 index 00000000..91bb750f --- /dev/null +++ b/pkg/dpf/bundle/rename_linux.go @@ -0,0 +1,10 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import "golang.org/x/sys/unix" + +func renameNewDirectory(from, to string) error { + return unix.Renameat2(unix.AT_FDCWD, from, unix.AT_FDCWD, to, unix.RENAME_NOREPLACE) +} diff --git a/pkg/dpf/bundle/rename_other.go b/pkg/dpf/bundle/rename_other.go new file mode 100644 index 00000000..6e9909c8 --- /dev/null +++ b/pkg/dpf/bundle/rename_other.go @@ -0,0 +1,12 @@ +//go:build !darwin && !linux + +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import "fmt" + +func renameNewDirectory(_, _ string) error { + return fmt.Errorf("DPF directory publication is unsupported on this platform") +} diff --git a/pkg/dpf/bundle/scope.go b/pkg/dpf/bundle/scope.go new file mode 100644 index 00000000..c9367ed3 --- /dev/null +++ b/pkg/dpf/bundle/scope.go @@ -0,0 +1,46 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +// kindScope is deliberately closed. A renderer must add a reviewed API kind here +// before a new cohort can rely on namespace/collision checks for that kind. +var kindScope = map[string]bool{ + "/Namespace": false, "/Node": false, "/PersistentVolume": false, + "/ConfigMap": true, "/Secret": true, "/Service": true, "/ServiceAccount": true, + "/Pod": true, "/PersistentVolumeClaim": true, "/ResourceQuota": true, "/LimitRange": true, + "apps/Deployment": true, "apps/DaemonSet": true, "apps/StatefulSet": true, + "batch/Job": true, "batch/CronJob": true, + "networking.k8s.io/NetworkPolicy": true, "networking.k8s.io/Ingress": true, + "rbac.authorization.k8s.io/Role": true, "rbac.authorization.k8s.io/RoleBinding": true, + "rbac.authorization.k8s.io/ClusterRole": false, "rbac.authorization.k8s.io/ClusterRoleBinding": false, + "apiextensions.k8s.io/CustomResourceDefinition": false, + "admissionregistration.k8s.io/MutatingWebhookConfiguration": false, + "admissionregistration.k8s.io/ValidatingWebhookConfiguration": false, + "admissionregistration.k8s.io/ValidatingAdmissionPolicy": false, + "admissionregistration.k8s.io/ValidatingAdmissionPolicyBinding": false, + "storage.k8s.io/StorageClass": false, + "cert-manager.io/Certificate": true, "cert-manager.io/CertificateRequest": true, + "cert-manager.io/Issuer": true, "cert-manager.io/ClusterIssuer": false, + "k8s.cni.cncf.io/NetworkAttachmentDefinition": true, + "operator.dpu.nvidia.com/DPFOperatorConfig": true, + "provisioning.dpu.nvidia.com/BFB": true, + "provisioning.dpu.nvidia.com/BlueFieldSoftware": true, + "provisioning.dpu.nvidia.com/DPU": true, + "provisioning.dpu.nvidia.com/DPUCluster": true, + "provisioning.dpu.nvidia.com/DPUDevice": true, + "provisioning.dpu.nvidia.com/DPUDiscovery": true, + "provisioning.dpu.nvidia.com/DPUFlavor": true, + "provisioning.dpu.nvidia.com/DPUFlavorTemplate": true, + "provisioning.dpu.nvidia.com/DPUNode": true, + "provisioning.dpu.nvidia.com/DPUNodeMaintenance": true, + "provisioning.dpu.nvidia.com/DPUSet": true, + "svc.dpu.nvidia.com/DPUDeployment": true, + "svc.dpu.nvidia.com/DPUService": true, + "svc.dpu.nvidia.com/DPUServiceChain": true, + "svc.dpu.nvidia.com/DPUServiceConfiguration": true, + "svc.dpu.nvidia.com/DPUServiceCredentialRequest": true, + "svc.dpu.nvidia.com/DPUServiceInterface": true, + "svc.dpu.nvidia.com/DPUServiceNAD": true, + "svc.dpu.nvidia.com/DPUServiceTemplate": true, +} diff --git a/pkg/dpf/bundle/types.go b/pkg/dpf/bundle/types.go new file mode 100644 index 00000000..a847830a --- /dev/null +++ b/pkg/dpf/bundle/types.go @@ -0,0 +1,193 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package bundle freezes DPF inputs and ordered stages for later consumers. +// Structural validation does not establish API admission or runtime readiness. +package bundle + +import ( + shared "github.com/nvidia/k8s-launch-kit/pkg/bundle" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const ( + FormatVersion = "v1alpha1" + Target = "dpf" + Phase = "generate" + IndexPath = ".l8k/bundle.yaml" + ResolvedConfigPath = ".l8k/resolved-config.yaml" +) + +// Destination names the cluster role, never a kubeconfig or ambient context. +type Destination string + +const ( + Management Destination = "management" + DPU Destination = "dpu" +) + +// Role determines how bytes participate in structural validation. +type Role string + +const ( + Manifest Role = "manifest" + HelmValues Role = "helm-values" + Chart Role = "chart" + Projection Role = "projection" + ResolvedConfig Role = "resolved-config" + Metadata Role = "metadata" +) + +// File is an owned relative path and its exact content. Constructors copy Data. +type File struct { + Path string + Data []byte +} + +// FileRecord closes membership and binds every retained file except the index. +type FileRecord struct { + Path string `yaml:"path" json:"path"` + Role Role `yaml:"role" json:"role"` + SHA256 string `yaml:"sha256" json:"sha256"` +} + +// ResourceRef identifies one resource, ignoring its served API version. +type ResourceRef struct { + Destination Destination `yaml:"destination" json:"destination"` + Group string `yaml:"group" json:"group"` + Kind string `yaml:"kind" json:"kind"` + Namespace string `yaml:"namespace" json:"namespace"` + Name string `yaml:"name" json:"name"` +} + +// Reference is an internal same-cluster edge; external prerequisites use Requires. +type Reference struct { + From ResourceRef `yaml:"from" json:"from"` + To ResourceRef `yaml:"to" json:"to"` +} + +// Stage is a declarative ordered group, not an executable workflow language. +// ProjectionFiles are checked with manifests but must never be applied separately. +type Stage struct { + ID string `yaml:"id" json:"id"` + Destination Destination `yaml:"destination" json:"destination"` + Namespace string `yaml:"namespace" json:"namespace"` + Files []string `yaml:"files" json:"files"` + ProjectionFiles []string `yaml:"projectionFiles,omitempty" json:"projectionFiles,omitempty"` + DependsOn []string `yaml:"dependsOn,omitempty" json:"dependsOn,omitempty"` + Requires []string `yaml:"requires,omitempty" json:"requires,omitempty"` +} + +// Index is the versioned DPF envelope. BundleDigest binds the canonical index +// payload and all exact file hashes, including metadata and observation bytes. +// It makes no promise of stability across refreshed observations. +type Index struct { + FormatVersion string `yaml:"formatVersion" json:"formatVersion"` + Target string `yaml:"target" json:"target"` + Phase string `yaml:"phase" json:"phase"` + Profile string `yaml:"profile" json:"profile"` + Release string `yaml:"release" json:"release"` + Renderer string `yaml:"renderer" json:"renderer"` + BundleDigest string `yaml:"bundleDigest" json:"bundleDigest"` + SelectedDevices []string `yaml:"selectedDevices" json:"selectedDevices"` + Files []FileRecord `yaml:"files" json:"files"` + Stages []Stage `yaml:"stages" json:"stages"` + References []Reference `yaml:"references,omitempty" json:"references,omitempty"` + ExternalPrerequisites []string `yaml:"externalPrerequisites,omitempty" json:"externalPrerequisites,omitempty"` +} + +// Bundle retains one immutable, validated snapshot. +type Bundle struct { + index Index + files []File + indexBytes []byte + stages []StageView +} + +// Document carries source diagnostics and an isolated Kubernetes view. +type Document struct { + ref ResourceRef + source shared.Source + projection bool + object *unstructured.Unstructured +} + +func (d Document) Ref() ResourceRef { return d.ref } +func (d Document) Source() shared.Source { return d.source } +func (d Document) IsProjection() bool { return d.projection } +func (d Document) ObjectCopy() *unstructured.Unstructured { + if d.object == nil { + return nil + } + return d.object.DeepCopy() +} + +// StageView separates apply documents from validation-only projections. +type StageView struct { + stage Stage + documents []Document + projections []Document +} + +func (s StageView) Stage() Stage { return copyStage(s.stage) } +func (s StageView) Documents() []Document { return append([]Document(nil), s.documents...) } +func (s StageView) Projections() []Document { return append([]Document(nil), s.projections...) } + +// Publication identifies a fully validated published snapshot. +type Publication struct { + Path string + Unchanged bool +} + +func (b *Bundle) Index() Index { + if b == nil { + return Index{} + } + return copyIndex(b.index) +} + +func (b *Bundle) Files() []File { + if b == nil { + return nil + } + return copyFiles(b.files) +} + +func (b *Bundle) Stage(id string) (StageView, bool) { + if b != nil { + for _, stage := range b.stages { + if stage.stage.ID == id { + return stage, true + } + } + } + return StageView{}, false +} + +func copyStage(s Stage) Stage { + s.Files = append([]string(nil), s.Files...) + s.ProjectionFiles = append([]string(nil), s.ProjectionFiles...) + s.DependsOn = append([]string(nil), s.DependsOn...) + s.Requires = append([]string(nil), s.Requires...) + return s +} + +func copyIndex(i Index) Index { + i.SelectedDevices = append([]string(nil), i.SelectedDevices...) + i.Files = append([]FileRecord(nil), i.Files...) + i.Stages = append([]Stage(nil), i.Stages...) + for n := range i.Stages { + i.Stages[n] = copyStage(i.Stages[n]) + } + i.References = append([]Reference(nil), i.References...) + i.ExternalPrerequisites = append([]string(nil), i.ExternalPrerequisites...) + return i +} + +func copyFiles(files []File) []File { + out := make([]File, len(files)) + for n, file := range files { + out[n] = File{Path: file.Path, Data: append([]byte(nil), file.Data...)} + } + return out +} diff --git a/pkg/dpf/bundle/validate.go b/pkg/dpf/bundle/validate.go new file mode 100644 index 00000000..bfd690dd --- /dev/null +++ b/pkg/dpf/bundle/validate.go @@ -0,0 +1,381 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package bundle + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "sort" + "strings" + + shared "github.com/nvidia/k8s-launch-kit/pkg/bundle" + "gopkg.in/yaml.v3" +) + +const ( + maxFileBytes = 16 << 20 + maxBundleBytes = 128 << 20 + maxFiles = 4096 +) + +// New validates complete membership, fills omitted file hashes and freezes copies. +// Explicit mismatching hashes or a mismatching BundleDigest are rejected. +func New(index Index, files []File) (*Bundle, error) { + b, err := construct(index, files, false) + if err != nil { + return nil, err + } + b.indexBytes, err = yaml.Marshal(b.index) + if err != nil { + return nil, fmt.Errorf("encode DPF bundle index: %w", err) + } + return b, nil +} + +func construct(index Index, files []File, requireHashes bool) (*Bundle, error) { + index = copyIndex(index) + if index.FormatVersion != FormatVersion || index.Target != Target || index.Phase != Phase { + return nil, fmt.Errorf("unsupported DPF bundle format/target/phase %q/%q/%q", index.FormatVersion, index.Target, index.Phase) + } + if index.Profile == "" || index.Release == "" || index.Renderer == "" { + return nil, fmt.Errorf("DPF bundle profile, release and renderer are required") + } + if len(index.SelectedDevices) == 0 { + return nil, fmt.Errorf("DPF bundle has no selected devices") + } + if err := uniqueStrings("selected device", index.SelectedDevices); err != nil { + return nil, err + } + if len(files) > maxFiles || len(index.Files) > maxFiles { + return nil, fmt.Errorf("DPF bundle exceeds file count limit") + } + fileMap := make(map[string][]byte, len(files)) + total := 0 + for _, file := range files { + if err := validPath(file.Path); err != nil { + return nil, err + } + if file.Path == IndexPath { + return nil, fmt.Errorf("bundle index cannot index itself") + } + if _, exists := fileMap[file.Path]; exists { + return nil, fmt.Errorf("duplicate DPF bundle file %q", file.Path) + } + total += len(file.Data) + if len(file.Data) > maxFileBytes || total > maxBundleBytes { + return nil, fmt.Errorf("DPF bundle exceeds bounded file/content limits at %q", file.Path) + } + fileMap[file.Path] = bytes.Clone(file.Data) + } + records := make(map[string]FileRecord, len(index.Files)) + for n, record := range index.Files { + if err := validPath(record.Path); err != nil { + return nil, err + } + if !validRole(record.Role) { + return nil, fmt.Errorf("unsupported file role %q for %q", record.Role, record.Path) + } + if _, exists := records[record.Path]; exists { + return nil, fmt.Errorf("duplicate indexed file %q", record.Path) + } + data, exists := fileMap[record.Path] + if !exists { + return nil, fmt.Errorf("indexed file %q is missing", record.Path) + } + actual := digest(data) + if record.SHA256 == "" && !requireHashes { + record.SHA256 = actual + } + if record.SHA256 != actual { + return nil, fmt.Errorf("checksum mismatch for %q", record.Path) + } + index.Files[n] = record + records[record.Path] = record + } + if len(records) != len(fileMap) { + return nil, fmt.Errorf("DPF bundle contains unindexed files") + } + for filename := range records { + for parent := path.Dir(filename); parent != "."; parent = path.Dir(parent) { + if _, exists := records[parent]; exists { + return nil, fmt.Errorf("indexed file %q is also a parent directory of %q", parent, filename) + } + } + } + resolved, exists := records[ResolvedConfigPath] + if !exists || resolved.Role != ResolvedConfig { + return nil, fmt.Errorf("DPF bundle requires indexed resolved configuration at %s", ResolvedConfigPath) + } + for _, record := range records { + if record.Role == ResolvedConfig && record.Path != ResolvedConfigPath { + return nil, fmt.Errorf("resolved configuration has wrong path %q", record.Path) + } + } + if err := validateResolved(fileMap[ResolvedConfigPath]); err != nil { + return nil, err + } + sort.Slice(index.Files, func(i, j int) bool { return index.Files[i].Path < index.Files[j].Path }) + b := &Bundle{index: index} + for _, record := range index.Files { + b.files = append(b.files, File{Path: record.Path, Data: fileMap[record.Path]}) + } + if err := b.validateStages(records, fileMap); err != nil { + return nil, err + } + actual, err := indexDigest(b.index) + if err != nil { + return nil, err + } + if index.BundleDigest != "" && index.BundleDigest != actual { + return nil, fmt.Errorf("DPF bundle digest mismatch") + } + if requireHashes && index.BundleDigest == "" { + return nil, fmt.Errorf("DPF bundle digest is missing") + } + b.index.BundleDigest = actual + return b, nil +} + +func validPath(p string) error { + if p == "" || p == "." || len(p) > 1024 || strings.Count(p, "/") > 32 || strings.ContainsAny(p, "\\\x00:") || strings.HasPrefix(p, "/") || path.Clean(p) != p || strings.HasPrefix(p, "../") || p == ".." { + return fmt.Errorf("invalid relative DPF bundle path %q", p) + } + return nil +} + +func validRole(role Role) bool { + switch role { + case Manifest, HelmValues, Chart, Projection, ResolvedConfig, Metadata: + return true + default: + return false + } +} + +func uniqueStrings(label string, values []string) error { + seen := map[string]bool{} + for _, value := range values { + if strings.TrimSpace(value) == "" || seen[value] { + return fmt.Errorf("empty or duplicate %s %q", label, value) + } + seen[value] = true + } + return nil +} + +func digest(data []byte) string { + hash := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(hash[:]) +} + +func indexDigest(index Index) (string, error) { + index.BundleDigest = "" + data, err := json.Marshal(index) + if err != nil { + return "", fmt.Errorf("encode DPF bundle digest payload: %w", err) + } + return digest(data), nil +} + +func decodeOne(data []byte, value any) error { + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(value); err != nil { + return err + } + var extra yaml.Node + if err := decoder.Decode(&extra); err != io.EOF { + if err == nil { + return fmt.Errorf("multiple YAML documents are unsupported") + } + return err + } + return nil +} + +func validateResolved(data []byte) error { + var envelope struct { + FormatVersion string `yaml:"formatVersion"` + Target string `yaml:"target"` + Config yaml.Node `yaml:"config"` + References yaml.Node `yaml:"references"` + } + if err := decodeOne(data, &envelope); err != nil { + return fmt.Errorf("decode DPF resolved configuration: %w", err) + } + if envelope.FormatVersion != FormatVersion || envelope.Target != Target || envelope.Config.Kind != yaml.MappingNode { + return fmt.Errorf("unsupported DPF resolved configuration envelope") + } + return nil +} + +func (b *Bundle) validateStages(records map[string]FileRecord, files map[string][]byte) error { + if len(b.index.Stages) == 0 { + return fmt.Errorf("DPF bundle has no stages") + } + if err := uniqueStrings("external prerequisite", b.index.ExternalPrerequisites); err != nil { + return err + } + external := map[string]bool{} + for _, id := range b.index.ExternalPrerequisites { + external[id] = true + } + seenStages := map[string]bool{} + usedFiles := map[string]bool{} + objects := map[ResourceRef]string{} + for _, stage := range b.index.Stages { + if stage.ID == "" || seenStages[stage.ID] { + return fmt.Errorf("empty or duplicate DPF stage %q", stage.ID) + } + if stage.Destination != Management && stage.Destination != DPU { + return fmt.Errorf("unsupported destination %q in stage %q", stage.Destination, stage.ID) + } + if err := uniqueStrings("stage dependency", stage.DependsOn); err != nil { + return err + } + for _, id := range stage.DependsOn { + if !seenStages[id] { + return fmt.Errorf("stage %q dependency %q is not an earlier stage", stage.ID, id) + } + } + if err := uniqueStrings("stage prerequisite", stage.Requires); err != nil { + return err + } + for _, id := range stage.Requires { + if !external[id] { + return fmt.Errorf("stage %q has undeclared external prerequisite %q", stage.ID, id) + } + } + if len(stage.Files) == 0 && len(stage.ProjectionFiles) == 0 { + return fmt.Errorf("stage %q has no files", stage.ID) + } + view := StageView{stage: copyStage(stage)} + var resources []stageResource + paths := append(append([]string(nil), stage.Files...), stage.ProjectionFiles...) + if err := uniqueStrings("stage file", paths); err != nil { + return err + } + for n, filename := range paths { + record, exists := records[filename] + if !exists { + return fmt.Errorf("stage %q uses unindexed file %q", stage.ID, filename) + } + isProjection := n >= len(stage.Files) + if isProjection != (record.Role == Projection) { + return fmt.Errorf("stage %q has wrong projection role for %q", stage.ID, filename) + } + if record.Role == ResolvedConfig || record.Role == Metadata { + return fmt.Errorf("non-executable file %q cannot belong to stage %q", filename, stage.ID) + } + if usedFiles[filename] { + return fmt.Errorf("file %q belongs to multiple stages", filename) + } + usedFiles[filename] = true + switch record.Role { + case HelmValues: + if _, err := shared.ParseValues(files[filename]); err != nil { + return fmt.Errorf("%s: %w", filename, err) + } + case Manifest, Projection: + resources = append(resources, stageResource{path: filename, data: files[filename], projection: isProjection}) + } + } + docs, err := parseStageResources(stage, resources) + if err != nil { + return err + } + for _, doc := range docs { + if first, exists := objects[doc.ref]; exists { + return fmt.Errorf("duplicate resource %v at %s document %d (first at %s)", doc.ref, doc.source.File, doc.source.Document, first) + } + objects[doc.ref] = fmt.Sprintf("%s document %d", doc.source.File, doc.source.Document) + if doc.projection { + view.projections = append(view.projections, doc) + } else { + view.documents = append(view.documents, doc) + } + } + b.stages = append(b.stages, view) + seenStages[stage.ID] = true + } + for _, record := range records { + if record.Role != Metadata && record.Role != ResolvedConfig && !usedFiles[record.Path] { + return fmt.Errorf("executable file %q has no stage", record.Path) + } + } + for _, reference := range b.index.References { + if reference.From.Destination != reference.To.Destination { + return fmt.Errorf("internal resource reference crosses cluster destinations") + } + if _, exists := objects[reference.From]; !exists { + return fmt.Errorf("reference source %v is missing", reference.From) + } + if _, exists := objects[reference.To]; !exists { + return fmt.Errorf("reference target %v is missing", reference.To) + } + } + return nil +} + +type stageResource struct { + path string + data []byte + projection bool +} + +func parseStageResources(stage Stage, resources []stageResource) ([]Document, error) { + // Adapt nested paths to the shared flat parser once per stage. Numeric flat + // names preserve declared order without inheriting Host filename-role rules. + files := make([]shared.File, 0, len(resources)) + sources := map[string]stageResource{} + for n, resource := range resources { + name := fmt.Sprintf("resource-%08d.yaml", n) + files = append(files, shared.File{Name: name, Content: string(resource.data)}) + sources[name] = resource + } + parsed, err := shared.FromFiles(files) + if err != nil { + var sourceErr *shared.Error + if errors.As(err, &sourceErr) { + source := sourceErr.Source + source.File = sources[source.File].path + reason := sourceErr.Reason + for name, resource := range sources { + reason = strings.ReplaceAll(reason, name, resource.path) + } + return nil, &shared.Error{Source: source, Reason: reason, Err: sourceErr.Err} + } + return nil, err + } + var docs []Document + counts := map[string]int{} + for _, item := range parsed.Documents() { + resource := sources[item.Source().File] + counts[resource.path]++ + ref := item.Ref() + namespaced, known := kindScope[ref.Group+"/"+ref.Kind] + if !known { + return nil, fmt.Errorf("%s document %d: unsupported resource kind %s/%s", resource.path, item.Source().Document, ref.Group, ref.Kind) + } + if namespaced && (ref.Namespace == "" || stage.Namespace != "" && ref.Namespace != stage.Namespace) { + return nil, fmt.Errorf("%s document %d: explicit namespace must match stage %q", resource.path, item.Source().Document, stage.ID) + } + if !namespaced && ref.Namespace != "" { + return nil, fmt.Errorf("%s document %d: cluster-scoped kind %s cannot have a namespace", resource.path, item.Source().Document, ref.Kind) + } + docs = append(docs, Document{ref: ResourceRef{Destination: stage.Destination, Group: ref.Group, Kind: ref.Kind, Namespace: ref.Namespace, Name: ref.Name}, source: shared.Source{File: resource.path, Document: item.Source().Document}, projection: resource.projection, object: item.ObjectCopy()}) + } + for _, resource := range resources { + if counts[resource.path] == 0 { + return nil, fmt.Errorf("manifest %q has no resource documents", resource.path) + } + } + return docs, nil +} diff --git a/pkg/dpf/config/config.go b/pkg/dpf/config/config.go index 96c99ebd..0dc5303a 100644 --- a/pkg/dpf/config/config.go +++ b/pkg/dpf/config/config.go @@ -74,14 +74,21 @@ type Limits struct { } type Site struct { DPF struct { - Release *string `yaml:"release,omitempty"` + Release *string `yaml:"release,omitempty"` + Profile *string `yaml:"profile,omitempty"` + Namespace *string `yaml:"namespace,omitempty"` + Name *string `yaml:"name,omitempty"` } `yaml:"dpf,omitempty"` Management struct { - KubeconfigRef FileRef `yaml:"kubeconfigRef"` - Context *string `yaml:"context,omitempty"` + KubeconfigRef FileRef `yaml:"kubeconfigRef"` + Context *string `yaml:"context,omitempty"` + ClusterUID *string `yaml:"clusterUID,omitempty"` + KubernetesVersion *string `yaml:"kubernetesVersion,omitempty"` } `yaml:"management"` - Credentials map[string]Credential `yaml:"credentials"` - Discovery struct { + Provisioning *Provisioning `yaml:"provisioning,omitempty"` + Sentry *Sentry `yaml:"sentry,omitempty"` + Credentials map[string]Credential `yaml:"credentials"` + Discovery struct { AddressLookup *addresslookup.Config `yaml:"addressLookup,omitempty"` DPUs []Endpoint `yaml:"dpus"` Hosts []Endpoint `yaml:"hosts"` @@ -558,8 +565,8 @@ func rebase(n *yaml.Node, oldBase, newBase, p string) error { if key == "inventory" && p == "" { continue } - isFile := key == "caFile" || key == "knownHostsFile" || key == "file" && (p == "/discovery/addressLookup/sources" || strings.HasSuffix(p, "kubeconfigRef") || strings.HasSuffix(p, "passwordRef") || strings.HasSuffix(p, "privateKeyRef")) - if isFile && value.Kind == yaml.ScalarNode && value.Tag == "!!str" && value.Value != "" { + isReference := next == "/sentry/chartDirectory" || key == "caFile" || key == "knownHostsFile" || key == "file" && (p == "/discovery/addressLookup/sources" || strings.HasSuffix(p, "kubeconfigRef") || strings.HasSuffix(p, "passwordRef") || strings.HasSuffix(p, "privateKeyRef")) + if isReference && value.Kind == yaml.ScalarNode && value.Tag == "!!str" && value.Value != "" { abs := resolvePath(oldBase, value.Value) rel, err := filepath.Rel(newBase, abs) if err != nil { @@ -596,7 +603,10 @@ func validateScalarTypes(n *yaml.Node, p string) error { if p == "/discovery/limits/maxConcurrentBMCs" { expected = "!!int" } - if strings.HasSuffix(p, "/tls/insecureSkipVerify") || strings.HasSuffix(p, "/ssh/sudo") { + if p == "/sentry/policyServerNodePort" { + expected = "!!int" + } + if strings.HasSuffix(p, "/tls/insecureSkipVerify") || strings.HasSuffix(p, "/ssh/sudo") || p == "/sentry/llmInspection" { expected = "!!bool" } if n.Tag != expected { diff --git a/pkg/dpf/config/generation.go b/pkg/dpf/config/generation.go new file mode 100644 index 00000000..55520c47 --- /dev/null +++ b/pkg/dpf/config/generation.go @@ -0,0 +1,397 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "encoding/hex" + "fmt" + "github.com/Masterminds/semver/v3" + "net/netip" + "net/url" + "path/filepath" + "sort" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + yaml "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/util/validation" +) + +// Artifact identifies exact release bytes. Generation never downloads it. +type Artifact struct { + URL string `yaml:"url" json:"url"` + SHA256 string `yaml:"sha256" json:"sha256"` +} + +// GenerationReleases lists the exact inspected API contracts, not production +// qualification claims or the discovery software-minimum catalog. +func GenerationReleases() []string { return []string{"26.8.0-rc.1", "26.8.0"} } + +func generationRelease(value string) bool { + for _, release := range GenerationReleases() { + if release == value { + return true + } + } + return false +} + +// Software separates firmware selected for a release from observed installed software. +type Software struct { + OSISO Artifact `yaml:"osISO" json:"osISO"` + PLDM map[string]Artifact `yaml:"pldm" json:"pldm"` + NICFirmware *Artifact `yaml:"nicFirmware,omitempty" json:"nicFirmware,omitempty"` +} + +// DeviceBinding names existing DPF enrollment objects; it never authorizes adoption +// of a DPU owned by another DPUSet or DPUDeployment. +type DeviceBinding struct { + ID string `yaml:"id" json:"id"` + DPUDevice string `yaml:"dpuDevice" json:"dpuDevice"` + DPUNode string `yaml:"dpuNode" json:"dpuNode"` +} + +type ClusterBinding struct { + KubernetesVersion string `yaml:"kubernetesVersion" json:"kubernetesVersion"` + Name string `yaml:"name" json:"name"` + Namespace string `yaml:"namespace" json:"namespace"` + ClusterUID string `yaml:"clusterUID" json:"clusterUID"` + ResourceUID string `yaml:"resourceUID" json:"resourceUID"` + Selector map[string]string `yaml:"selector" json:"selector"` +} + +type Provisioning struct { + Devices []DeviceBinding `yaml:"devices" json:"devices"` + DPUCluster ClusterBinding `yaml:"dpuCluster" json:"dpuCluster"` + Software Software `yaml:"software" json:"software"` +} + +type Sentry struct { + LLMInspection *bool `yaml:"llmInspection,omitempty" json:"llmInspection,omitempty"` + Namespace *string `yaml:"namespace,omitempty" json:"namespace,omitempty"` + ChartDirectory string `yaml:"chartDirectory" json:"chartDirectory"` + IngressCIDR string `yaml:"ingressCIDR" json:"ingressCIDR"` + IngressIP string `yaml:"ingressIP" json:"ingressIP"` + ImagePullSecret string `yaml:"imagePullSecret" json:"imagePullSecret"` + MITMSecret string `yaml:"mitmSecret" json:"mitmSecret"` + InferenceSecret string `yaml:"inferenceSecret" json:"inferenceSecret"` + PolicyServerAddress string `yaml:"policyServerAddress" json:"policyServerAddress"` + PolicyServerNodePort *int `yaml:"policyServerNodePort,omitempty" json:"policyServerNodePort,omitempty"` +} + +// Generation is a pure, fully resolved input. It contains references, never +// credential contents or a Kubernetes client. Existing DPF management and tenant +// clusters and enrolled devices are prerequisites for this first deployment slice. +type Generation struct { + ManagementKubernetesVersion string `yaml:"managementKubernetesVersion" json:"managementKubernetesVersion"` + Name string `yaml:"name" json:"name"` + Profile string `yaml:"profile" json:"profile"` + Release string `yaml:"release" json:"release"` + Namespace string `yaml:"namespace" json:"namespace"` + ManagementUID string `yaml:"managementUID" json:"managementUID"` + KubeconfigRef FileRef `yaml:"kubeconfigRef" json:"kubeconfigRef"` + Context string `yaml:"context" json:"context"` + Provisioning Provisioning `yaml:"provisioning" json:"provisioning"` + Sentry Sentry `yaml:"sentry" json:"sentry"` + Devices []inventory.Device `yaml:"devices" json:"devices"` + DevicePSIDs map[string]string `yaml:"devicePSIDs" json:"devicePSIDs"` +} + +func requiredDefault(p *string, fallback, field string) (string, error) { + if p == nil { + return fallback, nil + } + if *p == "" { + return "", fmt.Errorf("%s must not be empty", field) + } + return *p, nil +} + +// ResolveGeneration deliberately does not call Resolve: discovery resolves +// passwords and trust files, while local generation must never read them. +func ResolveGeneration(in Input) (Generation, error) { + var g Generation + var err error + if g.Name, err = requiredDefault(in.Site.DPF.Name, "sentry", "/dpf/name"); err != nil { + return g, err + } + if g.Profile, err = requiredDefault(in.Site.DPF.Profile, "sentry-bf4", "/dpf/profile"); err != nil { + return g, err + } + if g.Namespace, err = requiredDefault(in.Site.DPF.Namespace, "dpf-operator-system", "/dpf/namespace"); err != nil { + return g, err + } + if in.Site.DPF.Release == nil { + return g, fmt.Errorf("/dpf/release is required for generation; select the installed DPF release explicitly") + } + g.Release = *in.Site.DPF.Release + if !generationRelease(g.Release) { + return g, fmt.Errorf("DPF release %q has no Sentry BF4 generation contract", g.Release) + } + if g.Profile != "sentry-bf4" { + return g, fmt.Errorf("unknown DPF profile %q", g.Profile) + } + for field, value := range map[string]string{"/dpf/name": g.Name, "/dpf/namespace": g.Namespace} { + if len(validation.IsDNS1123Label(value)) != 0 || len(value) > 40 { + return g, fmt.Errorf("%s must be a DNS label of at most 40 characters", field) + } + } + if in.Site.Provisioning == nil || in.Site.Sentry == nil { + return g, fmt.Errorf("/provisioning and /sentry are required for generation") + } + // Round-trip the typed input to isolate maps, slices and pointer values. + b, err := yaml.Marshal(in.Site.Provisioning) + if err != nil { + return g, err + } + if err = yaml.Unmarshal(b, &g.Provisioning); err != nil { + return g, err + } + b, err = yaml.Marshal(in.Site.Sentry) + if err != nil { + return g, err + } + if err = yaml.Unmarshal(b, &g.Sentry); err != nil { + return g, err + } + g.KubeconfigRef.File = resolvePath(filepath.Dir(in.Path), in.Site.Management.KubeconfigRef.File) + if in.Site.Management.KubeconfigRef.File == "" { + return g, fmt.Errorf("/management/kubeconfigRef/file is required (reference only; not read during generation)") + } + if in.Site.Management.Context == nil || *in.Site.Management.Context == "" { + return g, fmt.Errorf("/management/context is required; ambient current-context is not a deployment binding") + } + g.Context = *in.Site.Management.Context + if in.Site.Management.ClusterUID != nil { + g.ManagementUID = *in.Site.Management.ClusterUID + } + inv := in.Site.Inventory + if inv == nil || inv.FormatVersion != inventory.FormatVersion { + return g, fmt.Errorf("generation requires a supported discovery inventory") + } + if g.ManagementUID == "" && inv.Management != nil { + g.ManagementUID = inv.Management.SystemNamespaceUID + } + if g.ManagementUID == "" { + return g, fmt.Errorf("management cluster UID is unobserved; supply /management/clusterUID from kube-system before generation") + } + if inv.Management != nil && inv.Management.SystemNamespaceUID != "" && inv.Management.SystemNamespaceUID != g.ManagementUID { + return g, fmt.Errorf("management cluster UID conflicts with discovery inventory") + } + if in.Site.Management.KubernetesVersion != nil { + g.ManagementKubernetesVersion = *in.Site.Management.KubernetesVersion + } else if inv.Management != nil { + g.ManagementKubernetesVersion = inv.Management.KubernetesVersion + } + if err = validateKubernetesVersion(g.ManagementKubernetesVersion); err != nil { + return g, fmt.Errorf("management Kubernetes version: %w", err) + } + if err = validateKubernetesVersion(g.Provisioning.DPUCluster.KubernetesVersion); err != nil { + return g, fmt.Errorf("DPUCluster Kubernetes version: %w", err) + } + if err = resolveProvisioning(&g, inv); err != nil { + return g, err + } + if err = resolveSentry(&g.Sentry); err != nil { + return g, err + } + if g.Sentry.Namespace == nil { + value := g.Name + "-services" + g.Sentry.Namespace = &value + } + if !dnsLabel(*g.Sentry.Namespace) || *g.Sentry.Namespace == g.Namespace { + return g, fmt.Errorf("/sentry/namespace must be a dedicated DNS namespace distinct from the DPF enrollment namespace") + } + g.Sentry.ChartDirectory = resolvePath(filepath.Dir(in.Path), g.Sentry.ChartDirectory) + return g, ValidateGeneration(g) +} + +func validateKubernetesVersion(value string) error { + v, err := semver.StrictNewVersion(strings.TrimPrefix(value, "v")) + if err != nil || v.Major() != 1 { + return fmt.Errorf("an explicit observed Kubernetes v1.x.y API server version is required") + } + return nil +} + +func resolveProvisioning(g *Generation, inv *inventory.Inventory) error { + p := &g.Provisioning + if len(p.Devices) == 0 { + return fmt.Errorf("/provisioning/devices must explicitly select at least one inventory device") + } + for field, value := range map[string]string{"dpuCluster/name": p.DPUCluster.Name, "dpuCluster/namespace": p.DPUCluster.Namespace} { + if !dnsLabel(value) { + return fmt.Errorf("/provisioning/%s must be a DNS label", field) + } + } + if p.DPUCluster.ClusterUID == "" || p.DPUCluster.ResourceUID == "" { + return fmt.Errorf("/provisioning/dpuCluster/clusterUID (tenant kube-system UID) and resourceUID (management DPUCluster UID) are required") + } + if len(p.DPUCluster.Selector) == 0 { + return fmt.Errorf("/provisioning/dpuCluster/selector requires labels selecting exactly the intended existing DPUCluster") + } + for key, value := range p.DPUCluster.Selector { + if len(validation.IsQualifiedName(key)) != 0 || len(validation.IsValidLabelValue(value)) != 0 { + return fmt.Errorf("invalid DPUCluster selector") + } + } + seen := map[string]bool{} + names := map[string]bool{} + g.DevicePSIDs = map[string]string{} + for _, binding := range p.Devices { + if binding.ID == "" || seen[binding.ID] { + return fmt.Errorf("/provisioning/devices contains an empty or duplicate inventory id") + } + seen[binding.ID] = true + if !dnsLabel(binding.DPUDevice) || !dnsLabel(binding.DPUNode) { + return fmt.Errorf("device %s requires explicit DPUDevice and DPUNode DNS names", binding.ID) + } + if names[binding.DPUDevice] { + return fmt.Errorf("duplicate DPUDevice binding %s", binding.DPUDevice) + } + names[binding.DPUDevice] = true + matches := []inventory.Device{} + for _, d := range inv.DPUs { + if d.ID == binding.ID { + matches = append(matches, d) + } + } + if len(matches) != 1 || matches[0].SerialNumber == "" { + return fmt.Errorf("device %s does not resolve to exactly one observed serial", binding.ID) + } + d := matches[0] + if d.ObservationState != "observed" || d.Model != "BlueField-4" { + return fmt.Errorf("device %s requires observed BlueField-4 identity", binding.ID) + } + associations := []inventory.Association{} + for _, a := range inv.Associations { + if a.DPUID == d.ID { + associations = append(associations, a) + } + } + if len(associations) != 1 || associations[0].SerialNumber != d.SerialNumber { + return fmt.Errorf("device %s requires one unambiguous host association", binding.ID) + } + psids := map[string]bool{} + collectPSIDs := func(device inventory.Device) { + if device.Runtime == nil || device.Runtime.Hardware == nil { + return + } + for _, card := range device.Runtime.Hardware.Cards { + if card.SerialNumber == d.SerialNumber && card.PSID.State == "observed" { + for _, v := range card.PSID.Values { + if v != "" { + psids[v] = true + } + } + } + } + } + collectPSIDs(d) + hosts := 0 + for _, h := range inv.Hosts { + if h.ID == associations[0].HostID { + hosts++ + collectPSIDs(h) + } + } + if hosts != 1 { + return fmt.Errorf("device %s host association does not resolve to one inventory host", binding.ID) + } + if len(psids) != 1 { + return fmt.Errorf("device %s requires one observed, nonconflicting PSID", binding.ID) + } + for psid := range psids { + if _, ok := p.Software.PLDM[psid]; !ok { + return fmt.Errorf("device %s PSID %s has no selected PLDM artifact", binding.ID, psid) + } + g.DevicePSIDs[d.ID] = psid + } + if d.BMCEndpoint == "" { + return fmt.Errorf("device %s has no observed BMC endpoint", binding.ID) + } + if _, err := Origin(d.BMCEndpoint); err != nil { + return fmt.Errorf("device %s has invalid BMC endpoint", binding.ID) + } + b, err := yaml.Marshal(d) + if err != nil { + return err + } + var copy inventory.Device + if err = yaml.Unmarshal(b, ©); err != nil { + return err + } + g.Devices = append(g.Devices, copy) + } + sort.Slice(p.Devices, func(i, j int) bool { return p.Devices[i].ID < p.Devices[j].ID }) + sort.Slice(g.Devices, func(i, j int) bool { return g.Devices[i].ID < g.Devices[j].ID }) + if err := validateArtifact(p.Software.OSISO); err != nil { + return fmt.Errorf("/provisioning/software/osISO: %w", err) + } + if len(p.Software.PLDM) == 0 { + return fmt.Errorf("/provisioning/software/pldm requires PSID-specific artifacts") + } + for psid, a := range p.Software.PLDM { + if !strings.HasPrefix(psid, "MT_") || len(psid) > 32 { + return fmt.Errorf("invalid PLDM PSID %q", psid) + } + if err := validateArtifact(a); err != nil { + return fmt.Errorf("PLDM %s: %w", psid, err) + } + } + if p.Software.NICFirmware != nil { + return validateArtifact(*p.Software.NICFirmware) + } + return nil +} + +func validateArtifact(a Artifact) error { + u, err := url.Parse(a.URL) + if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return fmt.Errorf("artifact requires HTTPS URL without credentials, query or fragment") + } + h, err := hex.DecodeString(a.SHA256) + if err != nil || len(h) != 32 || a.SHA256 != strings.ToLower(a.SHA256) { + return fmt.Errorf("artifact requires lowercase SHA-256 digest") + } + return nil +} + +func resolveSentry(s *Sentry) error { + if s.LLMInspection == nil { + value := false + s.LLMInspection = &value + } + if s.ChartDirectory == "" { + return fmt.Errorf("/sentry/chartDirectory requires local pinned Helm chart archives") + } + prefix, err := netip.ParsePrefix(s.IngressCIDR) + if err != nil || !prefix.Addr().Is4() || prefix != prefix.Masked() || prefix.Bits() > 30 || prefix.Bits() < 8 { + return fmt.Errorf("/sentry/ingressCIDR requires a canonical IPv4 subnet (/8 through /30)") + } + ip, err := netip.ParseAddr(s.IngressIP) + if err != nil || !prefix.Contains(ip) || ip == prefix.Addr() || !prefix.Contains(ip.Next()) { + return fmt.Errorf("/sentry/ingressIP must be a usable address inside ingressCIDR") + } + for field, name := range map[string]string{"imagePullSecret": s.ImagePullSecret, "mitmSecret": s.MITMSecret} { + if !dnsLabel(name) { + return fmt.Errorf("/sentry/%s requires an existing Secret name", field) + } + } + if *s.LLMInspection && !dnsLabel(s.InferenceSecret) { + return fmt.Errorf("/sentry/inferenceSecret requires an existing Secret when llmInspection is enabled") + } + ip, err = netip.ParseAddr(s.PolicyServerAddress) + if err != nil || !ip.Is4() || ip.IsUnspecified() || ip.IsMulticast() { + return fmt.Errorf("/sentry/policyServerAddress requires a management-node IPv4 address reachable from the DPU") + } + if s.PolicyServerNodePort == nil { + v := 30080 + s.PolicyServerNodePort = &v + } + if *s.PolicyServerNodePort < 30000 || *s.PolicyServerNodePort > 32767 { + return fmt.Errorf("/sentry/policyServerNodePort must be within 30000-32767") + } + return nil +} diff --git a/pkg/dpf/config/generation_test.go b/pkg/dpf/config/generation_test.go new file mode 100644 index 00000000..35f4b111 --- /dev/null +++ b/pkg/dpf/config/generation_test.go @@ -0,0 +1,374 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package config + +import ( + "path/filepath" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + yaml "gopkg.in/yaml.v3" +) + +const generationPSID = "MT_0000000010" + +var _ = Describe("DPF generation configuration", func() { + It("preserves the chart directory through a relocated discovery output", func() { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Sentry.ChartDirectory = "charts/sentry" + in = decodeGenerationInput(in) + before, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + destination := filepath.Join(filepath.Dir(in.Path), "refreshed", "site.yaml") + data, err := Saved(in, in.Site.Inventory, destination) + Expect(err).NotTo(HaveOccurred()) + moved, err := Decode(data, destination) + Expect(err).NotTo(HaveOccurred()) + after, err := ResolveGeneration(moved) + Expect(err).NotTo(HaveOccurred()) + Expect(after.Sentry.ChartDirectory).To(Equal(before.Sentry.ChartDirectory)) + }) + + It("resolves explicit release, context, management identity and only selected observed BF4 devices", func() { + in := validGenerationInput(GinkgoT().TempDir()) + // Extra inventory records are not implicitly selected or required to be usable. + in.Site.Inventory.DPUs = append(in.Site.Inventory.DPUs, inventory.Device{ + ID: "unselected-bf3", Model: "BlueField-3", ObservationState: "failed", + }) + + resolved, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Release).To(Equal("26.8.0")) + Expect(resolved.Name).To(Equal("sentry")) + Expect(resolved.Profile).To(Equal("sentry-bf4")) + Expect(resolved.Namespace).To(Equal("dpf-operator-system")) + Expect(resolved.ManagementUID).To(Equal("management-uid-1")) + Expect(resolved.ManagementKubernetesVersion).To(Equal("v1.35.4")) + Expect(resolved.Context).To(Equal("management-context")) + Expect(resolved.Provisioning.DPUCluster.Selector).To(Equal(map[string]string{"site.example.com/tenant": "sentry"})) + Expect(resolved.Provisioning.DPUCluster.ResourceUID).To(Equal("resource-uid-1")) + Expect(resolved.Provisioning.DPUCluster.KubernetesVersion).To(Equal("v1.35.6")) + Expect(resolved.DevicePSIDs).To(Equal(map[string]string{"dpu-1": generationPSID})) + Expect(resolved.Devices).To(HaveLen(1)) + Expect(resolved.Devices[0].ID).To(Equal("dpu-1")) + Expect(resolved.Sentry.PolicyServerNodePort).NotTo(BeNil()) + Expect(*resolved.Sentry.PolicyServerNodePort).To(Equal(30080)) + Expect(resolved.Sentry.Namespace).NotTo(BeNil()) + Expect(*resolved.Sentry.Namespace).To(Equal("sentry-services")) + Expect(resolved.KubeconfigRef.File).To(Equal(filepath.Join(filepath.Dir(in.Path), "missing", "management.kubeconfig"))) + Expect(resolved.Sentry.ChartDirectory).To(Equal(filepath.Join(filepath.Dir(in.Path), "charts", "sentry"))) + }) + + It("does not read kubeconfig, credential, or known-host references during local generation", func() { + in := validGenerationInput(GinkgoT().TempDir()) + missing := filepath.Join(filepath.Dir(in.Path), "definitely-not-present") + passwordPath := missing + ".password" + knownHostsPath := missing + ".known_hosts" + in.Site.Management.KubeconfigRef.File = "definitely-not-present/kubeconfig" + in.Site.Credentials = map[string]Credential{ + "remote": {Username: "user", PasswordRef: PasswordRef{File: &passwordPath}}, + } + in.Site.Discovery.Hosts = []Endpoint{{ + CredentialRef: "remote", + SSH: &SSH{Address: "192.0.2.12", KnownHostsFile: knownHostsPath}, + }} + + resolved, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.KubeconfigRef.File).To(Equal(filepath.Join(filepath.Dir(in.Path), "definitely-not-present", "kubeconfig"))) + Expect(resolved.Sentry.ChartDirectory).To(Equal(filepath.Join(filepath.Dir(in.Path), "charts", "sentry"))) + }) + + It("keeps omitted values distinct from explicit empty or zero values", func() { + in := validGenerationInput(GinkgoT().TempDir()) + encoded, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + decoded, err := Decode(encoded, in.Path) + Expect(err).NotTo(HaveOccurred()) + resolved, err := ResolveGeneration(decoded) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Name).To(Equal("sentry")) + Expect(resolved.Profile).To(Equal("sentry-bf4")) + Expect(resolved.Namespace).To(Equal("dpf-operator-system")) + Expect(*resolved.Sentry.PolicyServerNodePort).To(Equal(30080)) + + for _, tc := range []struct { + name string + edit func(*Input) + want string + }{ + {name: "explicit empty name", edit: func(i *Input) { v := ""; i.Site.DPF.Name = &v }, want: "/dpf/name must not be empty"}, + {name: "explicit empty profile", edit: func(i *Input) { v := ""; i.Site.DPF.Profile = &v }, want: "/dpf/profile must not be empty"}, + {name: "explicit empty namespace", edit: func(i *Input) { v := ""; i.Site.DPF.Namespace = &v }, want: "/dpf/namespace must not be empty"}, + {name: "omitted release", edit: func(i *Input) { i.Site.DPF.Release = nil }, want: "/dpf/release is required"}, + {name: "explicit empty release", edit: func(i *Input) { v := ""; i.Site.DPF.Release = &v }, want: `DPF release "" has no Sentry BF4 generation contract`}, + {name: "zero node port", edit: func(i *Input) { v := 0; i.Site.Sentry.PolicyServerNodePort = &v }, want: "/sentry/policyServerNodePort must be within 30000-32767"}, + {name: "empty chart path", edit: func(i *Input) { i.Site.Sentry.ChartDirectory = "" }, want: "/sentry/chartDirectory requires local pinned Helm chart archives"}, + {name: "empty Sentry namespace", edit: func(i *Input) { v := ""; i.Site.Sentry.Namespace = &v }, want: "/sentry/namespace must be a dedicated DNS namespace"}, + {name: "Sentry namespace collides with DPF namespace", edit: func(i *Input) { v := "dpf-operator-system"; i.Site.Sentry.Namespace = &v }, want: "/sentry/namespace must be a dedicated DNS namespace"}, + } { + By(tc.name) + copy := validGenerationInput(GinkgoT().TempDir()) + tc.edit(©) + copy = decodeGenerationInput(copy) + _, err := ResolveGeneration(copy) + Expect(err).To(MatchError(ContainSubstring(tc.want))) + } + + in = validGenerationInput(GinkgoT().TempDir()) + encoded, err = yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + withNullRelease := strings.Replace(string(encoded), "release: 26.8.0", "release: null", 1) + decoded, err = Decode([]byte(withNullRelease), in.Path) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Site.DPF.Release).To(BeNil()) + _, err = ResolveGeneration(decoded) + Expect(err).To(MatchError(ContainSubstring("/dpf/release is required"))) + }) + + It("strictly decodes generation fields and rejects unknown configuration keys", func() { + in := validGenerationInput(GinkgoT().TempDir()) + encoded, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + _, err = Decode(append(encoded, []byte("unexpectedGenerationField: true\n")...), in.Path) + Expect(err).To(HaveOccurred()) + }) + + It("rejects unsupported release lines and non-BF4 selected hardware", func() { + for _, tc := range []struct { + name string + setup func(*Input) + want string + }{ + {name: "26.4 release", setup: func(i *Input) { v := "26.4.0"; i.Site.DPF.Release = &v }, want: `DPF release "26.4.0" has no Sentry BF4 generation contract`}, + {name: "BF3 device", setup: func(i *Input) { i.Site.Inventory.DPUs[0].Model = "BlueField-3" }, want: "requires observed BlueField-4 identity"}, + } { + By(tc.name) + in := validGenerationInput(GinkgoT().TempDir()) + tc.setup(&in) + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring(tc.want))) + } + }) + + It("requires explicit management context and either explicit or observed management UID", func() { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Management.Context = nil + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("/management/context is required"))) + + in = validGenerationInput(GinkgoT().TempDir()) + in.Site.Management.Context = stringPointer("") + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("/management/context is required"))) + + in = validGenerationInput(GinkgoT().TempDir()) + in.Site.Management.ClusterUID = nil + in.Site.Inventory.Management = nil + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("management cluster UID is unobserved"))) + + in = validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.Management = &inventory.Management{SystemNamespaceUID: "other-uid"} + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("management cluster UID conflicts"))) + }) + + It("uses explicit or observed management Kubernetes version and rejects missing or invalid versions", func() { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Management.KubernetesVersion = nil + resolved, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.ManagementKubernetesVersion).To(Equal("v1.35.4")) + + in = validGenerationInput(GinkgoT().TempDir()) + in.Site.Management.KubernetesVersion = stringPointer("v1.36.2") + resolved, err = ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.ManagementKubernetesVersion).To(Equal("v1.36.2")) + + in = validGenerationInput(GinkgoT().TempDir()) + in.Site.Management.KubernetesVersion = nil + in.Site.Inventory.Management.KubernetesVersion = "" + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("management Kubernetes version"))) + + for _, tc := range []struct { + name string + setup func(*Input) + }{ + {name: "invalid management version", setup: func(i *Input) { i.Site.Management.KubernetesVersion = stringPointer("v2.0.0") }}, + {name: "missing tenant version", setup: func(i *Input) { i.Site.Provisioning.DPUCluster.KubernetesVersion = "" }}, + {name: "invalid tenant version", setup: func(i *Input) { i.Site.Provisioning.DPUCluster.KubernetesVersion = "v1.35" }}, + } { + By(tc.name) + in := validGenerationInput(GinkgoT().TempDir()) + tc.setup(&in) + _, err := ResolveGeneration(in) + Expect(err).To(HaveOccurred()) + } + }) + + It("requires one association whose serial and host mapping match the selected device", func() { + for _, tc := range []struct { + name string + setup func(*Input) + want string + }{ + {name: "duplicate association", setup: func(i *Input) { + i.Site.Inventory.Associations = append(i.Site.Inventory.Associations, i.Site.Inventory.Associations[0]) + }, want: "one unambiguous host association"}, + {name: "association serial mismatch", setup: func(i *Input) { i.Site.Inventory.Associations[0].SerialNumber = "different-serial" }, want: "one unambiguous host association"}, + {name: "missing host mapping", setup: func(i *Input) { i.Site.Inventory.Associations[0].HostID = "missing-host" }, want: "host association does not resolve to one inventory host"}, + {name: "duplicate host mapping", setup: func(i *Input) { i.Site.Inventory.Hosts = append(i.Site.Inventory.Hosts, i.Site.Inventory.Hosts[0]) }, want: "host association does not resolve to one inventory host"}, + } { + By(tc.name) + in := validGenerationInput(GinkgoT().TempDir()) + tc.setup(&in) + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring(tc.want))) + } + }) + + It("rejects absent or conflicting observed PSID mappings", func() { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.DPUs[0].Runtime.Hardware.Cards[0].PSID = inventory.Observation{State: "unavailable"} + resolved, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.DevicePSIDs).To(Equal(map[string]string{"dpu-1": generationPSID})) + + in = validGenerationInput(GinkgoT().TempDir()) + delete(in.Site.Provisioning.Software.PLDM, generationPSID) + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("has no selected PLDM artifact"))) + + in = validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.Hosts[0].Runtime.Hardware.Cards[0].PSID.Values = []string{"MT_0000000011"} + in.Site.Provisioning.Software.PLDM["MT_0000000011"] = testArtifact("host-conflict.pldm") + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("one observed, nonconflicting PSID"))) + }) + + It("requires a syntactically valid explicit DPUCluster label selector", func() { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Provisioning.DPUCluster.Selector = nil + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("selector requires labels"))) + + in = validGenerationInput(GinkgoT().TempDir()) + in.Site.Provisioning.DPUCluster.Selector = map[string]string{"Invalid Key": "sentry"} + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("invalid DPUCluster selector"))) + }) + + It("rejects duplicate selected serials even when they have different inventory IDs", func() { + in := validGenerationInput(GinkgoT().TempDir()) + duplicate := in.Site.Inventory.DPUs[0] + duplicate.ID = "dpu-2" + duplicate.BMCEndpoint = "https://192.0.2.22" + in.Site.Inventory.DPUs = append(in.Site.Inventory.DPUs, duplicate) + in.Site.Inventory.Associations = append(in.Site.Inventory.Associations, inventory.Association{ + DPUID: "dpu-2", HostID: "host-1", SerialNumber: duplicate.SerialNumber, + }) + in.Site.Provisioning.Devices = append(in.Site.Provisioning.Devices, DeviceBinding{ + ID: "dpu-2", DPUDevice: "dpu-device-2", DPUNode: "dpu-node-2", + }) + + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("resolved device identities or bindings conflict"))) + }) + + It("rejects ingress network and broadcast addresses", func() { + for _, address := range []string{"10.70.0.0", "10.70.0.255"} { + By(address) + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Sentry.IngressIP = address + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("usable address inside ingressCIDR"))) + } + }) + + It("isolates caller configuration, inventory and mutable resolved values", func() { + in := validGenerationInput(GinkgoT().TempDir()) + before, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + resolved, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + after, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + Expect(string(after)).To(Equal(string(before))) + + resolved.Provisioning.Devices[0].DPUNode = "changed-node" + resolved.Provisioning.Software.PLDM[generationPSID] = testArtifact("changed.pldm") + resolved.Sentry.PolicyServerNodePort = intPointer(12345) + resolved.Devices[0].Runtime.Hardware.Cards[0].PSID.Values[0] = "changed-psid" + after, err = yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + Expect(string(after)).To(Equal(string(before))) + Expect(in.Site.Inventory.DPUs[0].Runtime.Hardware.Cards[0].PSID.Values).To(Equal([]string{generationPSID})) + }) +}) + +func validGenerationInput(base string) Input { + psid := inventory.Observation{State: "observed", Values: []string{generationPSID}} + dpu := inventory.Device{ + ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "BlueField-4", ObservationState: "observed", + BMCEndpoint: "https://192.0.2.21", + Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{ + SerialNumber: "BF4-SERIAL-1", PSID: psid, + }}}}, + } + host := inventory.Device{ + ID: "host-1", + Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{ + SerialNumber: "BF4-SERIAL-1", PSID: psid, + }}}}, + } + site := Site{} + site.DPF.Release = stringPointer("26.8.0") + site.Management.KubeconfigRef = FileRef{File: "missing/management.kubeconfig"} + site.Management.Context = stringPointer("management-context") + site.Management.ClusterUID = stringPointer("management-uid-1") + site.Management.KubernetesVersion = stringPointer("v1.35.4") + site.Provisioning = &Provisioning{ + Devices: []DeviceBinding{{ID: "dpu-1", DPUDevice: "dpu-device-1", DPUNode: "dpu-node-1"}}, + DPUCluster: ClusterBinding{ + KubernetesVersion: "v1.35.6", Name: "tenant-one", Namespace: "dpf-operator-system", ClusterUID: "tenant-cluster-uid", ResourceUID: "resource-uid-1", + Selector: map[string]string{"site.example.com/tenant": "sentry"}, + }, + Software: Software{ + OSISO: testArtifact("dpf-os.iso"), + PLDM: map[string]Artifact{generationPSID: testArtifact("bf4.pldm")}, + }, + } + site.Sentry = &Sentry{ + ChartDirectory: "charts/sentry", IngressCIDR: "10.70.0.0/24", IngressIP: "10.70.0.20", + ImagePullSecret: "sentry-pull", MITMSecret: "sentry-mitm", InferenceSecret: "sentry-inference", + PolicyServerAddress: "192.0.2.40", + } + site.Inventory = &inventory.Inventory{ + FormatVersion: inventory.FormatVersion, + Management: &inventory.Management{SystemNamespaceUID: "management-uid-1", KubernetesVersion: "v1.35.4"}, + DPUs: []inventory.Device{dpu}, + Hosts: []inventory.Device{host}, + Associations: []inventory.Association{{DPUID: "dpu-1", HostID: "host-1", SerialNumber: "BF4-SERIAL-1"}}, + } + return Input{Path: filepath.Join(base, "site.yaml"), Site: site} +} + +func testArtifact(name string) Artifact { + return Artifact{URL: "https://artifacts.example.invalid/" + name, SHA256: strings.Repeat("a", 64)} +} + +func decodeGenerationInput(in Input) Input { + encoded, err := yaml.Marshal(in.Site) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + decoded, err := Decode(encoded, in.Path) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return decoded +} + +func stringPointer(value string) *string { return &value } +func intPointer(value int) *int { return &value } diff --git a/pkg/dpf/config/overrides.go b/pkg/dpf/config/overrides.go new file mode 100644 index 00000000..052aa650 --- /dev/null +++ b/pkg/dpf/config/overrides.go @@ -0,0 +1,74 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "fmt" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/configinput" + "github.com/nvidia/k8s-launch-kit/pkg/yamlinput" + yaml "gopkg.in/yaml.v3" +) + +// WithOverrides patches a detached YAML tree using the shared flag metadata's +// paths. The normal strict DPF decoder validates the result. Source bytes and +// the caller's document are retained unchanged. +func WithOverrides(in Input, values configinput.Values) (Input, error) { + if len(values.Requests) > 0 { + return Input{}, fmt.Errorf("host configuration resolvers are not valid for DPF") + } + if values.Empty() { + return in, nil + } + doc, err := yamlinput.Expand(in.Document) + if err != nil { + return Input{}, err + } + if doc == nil || len(doc.Content) != 1 { + return Input{}, fmt.Errorf("missing DPF source document") + } + for _, o := range values.Overrides { + parts := strings.Split(o.Path, ".") + if len(parts) < 2 { + return Input{}, fmt.Errorf("invalid DPF override path %q", o.Path) + } + n := doc.Content[0] + for _, key := range parts[:len(parts)-1] { + if key == "" || n.Kind != yaml.MappingNode { + return Input{}, fmt.Errorf("invalid DPF override path %q", o.Path) + } + var child *yaml.Node + for i := 0; i < len(n.Content); i += 2 { + if n.Content[i].Value == key { + child = n.Content[i+1] + break + } + } + if child == nil || child.Tag == "!!null" { + child = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + yamlinput.Set(n, key, child) + } + n = child + } + if n.Kind != yaml.MappingNode || parts[len(parts)-1] == "" { + return Input{}, fmt.Errorf("invalid DPF override path %q", o.Path) + } + var value yaml.Node + if err = value.Encode(o.Value); err != nil { + return Input{}, err + } + yamlinput.Set(n, parts[len(parts)-1], &value) + } + b, err := yamlinput.Encode(doc) + if err != nil { + return Input{}, err + } + out, err := Decode(b, in.Path) + if err != nil { + return Input{}, err + } + out.Source = append([]byte(nil), in.Source...) + return out, nil +} diff --git a/pkg/dpf/config/overrides_test.go b/pkg/dpf/config/overrides_test.go new file mode 100644 index 00000000..39fabc19 --- /dev/null +++ b/pkg/dpf/config/overrides_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/configinput" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + yaml "gopkg.in/yaml.v3" +) + +var _ = Describe("DPF config-backed overrides", func() { + It("applies explicit shared CLI values on an isolated copy and preserves the source", func() { + in := validGenerationInput(GinkgoT().TempDir()) + in = decodeGenerationInput(in) + source := append([]byte(nil), in.Source...) + original, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + + out, err := WithOverrides(in, configinput.Values{Overrides: []configinput.Override{ + {Flag: "--dpf-release", Path: "dpf.release", Value: "26.8.0-rc.1"}, + {Flag: "--dpf-profile", Path: "dpf.profile", Value: "sentry-bf4"}, + }}) + Expect(err).NotTo(HaveOccurred()) + Expect(*out.Site.DPF.Release).To(Equal("26.8.0-rc.1")) + Expect(*out.Site.DPF.Profile).To(Equal("sentry-bf4")) + Expect(out.Source).To(Equal(source)) + after, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + Expect(after).To(Equal(original)) + }) + + It("keeps explicitly empty CLI values and rejects them through normal config validation", func() { + in := decodeGenerationInput(validGenerationInput(GinkgoT().TempDir())) + out, err := WithOverrides(in, configinput.Values{Overrides: []configinput.Override{{ + Flag: "--dpf-release", Path: "dpf.release", Value: "", + }}}) + Expect(err).NotTo(HaveOccurred()) + Expect(out.Site.DPF.Release).NotTo(BeNil()) + Expect(*out.Site.DPF.Release).To(BeEmpty()) + _, err = ResolveGeneration(out) + Expect(err).To(MatchError(ContainSubstring("has no Sentry BF4 generation contract"))) + }) + + It("rejects Host resolver requests and malformed override paths", func() { + in := decodeGenerationInput(validGenerationInput(GinkgoT().TempDir())) + _, err := WithOverrides(in, configinput.Values{Requests: []configinput.Request{{Flag: "--network-operator-release", Kind: configinput.ResolverNetworkOperatorRelease, Value: "26.8"}}}) + Expect(err).To(MatchError(ContainSubstring("host configuration resolvers are not valid for DPF"))) + for _, path := range []string{"release", "dpf.", "dpf..release"} { + _, err = WithOverrides(in, configinput.Values{Overrides: []configinput.Override{{Flag: "--x", Path: path, Value: "x"}}}) + Expect(err).To(HaveOccurred(), "path %q", path) + } + }) + +}) diff --git a/pkg/dpf/config/resolved.go b/pkg/dpf/config/resolved.go new file mode 100644 index 00000000..891d31d1 --- /dev/null +++ b/pkg/dpf/config/resolved.go @@ -0,0 +1,159 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "bytes" + "fmt" + "io" + "path/filepath" + "reflect" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + yaml "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/util/validation" +) + +// Resolved is deployment metadata, not another user input/defaulting path. +type Resolved struct { + FormatVersion string `yaml:"formatVersion"` + Target string `yaml:"target"` + Config Generation `yaml:"config"` +} + +func EncodeResolved(g Generation) ([]byte, error) { + if err := ValidateGeneration(g); err != nil { + return nil, err + } + return yaml.Marshal(Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: g}) +} + +func DecodeResolved(b []byte) (Generation, error) { + var r Resolved + if len(b) > 16<<20 { + return r.Config, fmt.Errorf("resolved DPF configuration exceeds 16 MiB") + } + var syntax yaml.Node + if err := yaml.Unmarshal(b, &syntax); err != nil { + return r.Config, fmt.Errorf("invalid resolved DPF YAML") + } + if err := validateKeys(&syntax, map[*yaml.Node]bool{}, map[*yaml.Node]bool{}, 0); err != nil { + return r.Config, err + } + d := yaml.NewDecoder(bytes.NewReader(b)) + d.KnownFields(true) + if err := d.Decode(&r); err != nil { + return r.Config, fmt.Errorf("invalid resolved DPF configuration: %w", err) + } + var extra any + if err := d.Decode(&extra); err != io.EOF { + return r.Config, fmt.Errorf("resolved DPF configuration must be one document") + } + if r.Target != "dpf" || r.FormatVersion != "v1alpha1" { + return r.Config, fmt.Errorf("unsupported resolved DPF format or target") + } + return r.Config, ValidateGeneration(r.Config) +} + +// ValidateGeneration checks the resolved deployment contract without defaulting, +// filesystem access, live discovery or credential resolution. +func ValidateGeneration(g Generation) error { + if err := validateKubernetesVersion(g.ManagementKubernetesVersion); err != nil { + return err + } + if err := validateKubernetesVersion(g.Provisioning.DPUCluster.KubernetesVersion); err != nil { + return err + } + if g.Profile != "sentry-bf4" || !generationRelease(g.Release) { + return fmt.Errorf("unsupported resolved DPF profile/release") + } + if !dnsLabel(g.Name) || len(g.Name) > 40 || !dnsLabel(g.Namespace) || len(g.Namespace) > 40 { + return fmt.Errorf("invalid resolved DPF name or namespace") + } + if g.ManagementUID == "" || g.Context == "" || !filepath.IsAbs(g.KubeconfigRef.File) { + return fmt.Errorf("resolved management identity and explicit absolute kubeconfig/context are required") + } + if g.Sentry.Namespace == nil || !dnsLabel(*g.Sentry.Namespace) || *g.Sentry.Namespace == g.Namespace { + return fmt.Errorf("resolved Sentry namespace must be dedicated") + } + if g.Sentry.PolicyServerNodePort == nil || g.Sentry.LLMInspection == nil || !filepath.IsAbs(g.Sentry.ChartDirectory) { + return fmt.Errorf("resolved Sentry port and absolute chart directory are required") + } + s := g.Sentry + if err := resolveSentry(&s); err != nil { + return err + } + if g.Provisioning.DPUCluster.ResourceUID == "" || g.Provisioning.DPUCluster.ClusterUID == "" || !dnsLabel(g.Provisioning.DPUCluster.Name) || !dnsLabel(g.Provisioning.DPUCluster.Namespace) || len(g.Provisioning.DPUCluster.Selector) == 0 { + return fmt.Errorf("resolved DPUCluster identity/selector is incomplete") + } + for key, value := range g.Provisioning.DPUCluster.Selector { + if len(validation.IsQualifiedName(key)) != 0 || len(validation.IsValidLabelValue(value)) != 0 { + return fmt.Errorf("invalid resolved DPUCluster selector") + } + } + for psid, artifact := range g.Provisioning.Software.PLDM { + if !strings.HasPrefix(psid, "MT_") || len(psid) > 32 { + return fmt.Errorf("invalid resolved PLDM PSID") + } + if err := validateArtifact(artifact); err != nil { + return err + } + } + if len(g.Devices) == 0 || len(g.Devices) != len(g.Provisioning.Devices) || len(g.DevicePSIDs) != len(g.Devices) { + return fmt.Errorf("resolved selected-device membership is incomplete") + } + ids, serials, names := map[string]bool{}, map[string]bool{}, map[string]bool{} + for i, d := range g.Devices { + binding := g.Provisioning.Devices[i] + if d.ID == "" || d.ID != binding.ID || ids[d.ID] || d.SerialNumber == "" || serials[d.SerialNumber] || names[binding.DPUDevice] || !dnsLabel(binding.DPUDevice) || !dnsLabel(binding.DPUNode) { + return fmt.Errorf("resolved device identities or bindings conflict") + } + ids[d.ID], serials[d.SerialNumber], names[binding.DPUDevice] = true, true, true + if d.Model != "BlueField-4" || d.ObservationState != "observed" { + return fmt.Errorf("resolved device %s has no observed BF4 identity", d.ID) + } + if _, err := Origin(d.BMCEndpoint); err != nil { + return err + } + psid := g.DevicePSIDs[d.ID] + if psid == "" { + return fmt.Errorf("resolved device %s has no PSID", d.ID) + } + a, ok := g.Provisioning.Software.PLDM[psid] + if !ok { + return fmt.Errorf("resolved device %s has no matching PLDM artifact", d.ID) + } + if err := validateArtifact(a); err != nil { + return err + } + } + if err := validateArtifact(g.Provisioning.Software.OSISO); err != nil { + return err + } + if g.Provisioning.Software.NICFirmware != nil { + return validateArtifact(*g.Provisioning.Software.NICFirmware) + } + return nil +} + +// SameIntent allows an explicit config to win only when its effective desired +// deployment still matches the generated bundle. A changed intent needs generate, +// rather than silently applying manifests for the previous input. +func SameIntent(a, b Generation) bool { + // Fresh observation timestamps and diagnostics are not desired configuration. + identities := func(devices []inventory.Device) []inventory.Device { + out := make([]inventory.Device, len(devices)) + for i, d := range devices { + out[i] = inventory.Device{ID: d.ID, SerialNumber: d.SerialNumber, BMCEndpoint: d.BMCEndpoint, Model: d.Model, ObservationState: d.ObservationState} + } + return out + } + a.Devices, b.Devices = identities(a.Devices), identities(b.Devices) + // Local access paths can move without changing resource intent; deploy uses the + // explicit path only after its cluster identity has been checked live. + a.KubeconfigRef, b.KubeconfigRef = FileRef{}, FileRef{} + a.Sentry.ChartDirectory, b.Sentry.ChartDirectory = "", "" + return reflect.DeepEqual(a, b) +} diff --git a/pkg/dpf/config/resolved_test.go b/pkg/dpf/config/resolved_test.go new file mode 100644 index 00000000..2fdf8012 --- /dev/null +++ b/pkg/dpf/config/resolved_test.go @@ -0,0 +1,160 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package config + +import ( + "path/filepath" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("resolved DPF generation", func() { + It("round-trips a validated deployment config and returns independent values", func() { + generation := resolvedGeneration() + encoded, err := EncodeResolved(generation) + Expect(err).NotTo(HaveOccurred()) + + decoded, err := DecodeResolved(encoded) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded).To(Equal(generation)) + + decoded.Provisioning.Software.PLDM[generationPSID] = testArtifact("mutated.pldm") + decoded.Provisioning.Devices[0].DPUNode = "mutated-node" + decoded.Devices[0].SerialNumber = "mutated-serial" + Expect(generation.Provisioning.Software.PLDM[generationPSID]).To(Equal(testArtifact("bf4.pldm"))) + Expect(generation.Provisioning.Devices[0].DPUNode).To(Equal("dpu-node-1")) + Expect(generation.Devices[0].SerialNumber).To(Equal("BF4-SERIAL-1")) + }) + + It("strictly checks the resolved format, fields, duplicate keys and document count", func() { + encoded, err := EncodeResolved(resolvedGeneration()) + Expect(err).NotTo(HaveOccurred()) + document := string(encoded) + for _, tc := range []struct { + name string + edit func(string) string + want string + }{ + {name: "unknown format", edit: func(s string) string { return strings.Replace(s, "formatVersion: v1alpha1", "formatVersion: v9", 1) }, want: "unsupported resolved DPF format or target"}, + {name: "wrong target", edit: func(s string) string { return strings.Replace(s, "target: dpf", "target: host", 1) }, want: "unsupported resolved DPF format or target"}, + {name: "unknown field", edit: func(s string) string { return s + "unexpectedField: true\n" }, want: "field unexpectedField not found"}, + {name: "duplicate key", edit: func(s string) string { return strings.Replace(s, "target: dpf\n", "target: dpf\ntarget: host\n", 1) }, want: "duplicate"}, + {name: "extra document", edit: func(s string) string { return s + "---\n{}\n" }, want: "one document"}, + } { + By(tc.name) + _, err := DecodeResolved([]byte(tc.edit(document))) + Expect(err).To(MatchError(ContainSubstring(tc.want))) + } + }) + + It("validates without reading local reference paths or mutating resolved defaults", func() { + generation := resolvedGeneration() + generation.KubeconfigRef.File = filepath.Join("/missing", "management.kubeconfig") + generation.Sentry.ChartDirectory = filepath.Join("/missing", "charts", "sentry") + before := cloneResolvedGeneration(generation) + + Expect(ValidateGeneration(generation)).To(Succeed()) + Expect(generation).To(Equal(before)) + }) + + It("rejects incomplete selected-device membership and duplicate selected serials", func() { + generation := resolvedGeneration() + generation.Provisioning.Devices = nil + Expect(ValidateGeneration(generation)).To(MatchError(ContainSubstring("selected-device membership is incomplete"))) + _, err := EncodeResolved(generation) + Expect(err).To(MatchError(ContainSubstring("selected-device membership is incomplete"))) + + generation = resolvedGeneration() + generation.Devices = append(generation.Devices, generation.Devices[0]) + generation.Devices[1].ID = "dpu-2" + generation.Provisioning.Devices = append(generation.Provisioning.Devices, DeviceBinding{ + ID: "dpu-2", DPUDevice: "dpu-device-2", DPUNode: "dpu-node-2", + }) + generation.DevicePSIDs["dpu-2"] = generationPSID + Expect(ValidateGeneration(generation)).To(MatchError(ContainSubstring("resolved device identities or bindings conflict"))) + }) + + It("rejects invalid persisted DPUCluster selector labels", func() { + generation := resolvedGeneration() + generation.Provisioning.DPUCluster.Selector = map[string]string{"Invalid Key": "sentry"} + encoded, err := EncodeResolved(generation) + if err == nil { + _, err = DecodeResolved(encoded) + } + Expect(err).To(MatchError(ContainSubstring("selector"))) + }) + + It("ignores fresh observation diagnostics while comparing desired intent", func() { + original := resolvedGeneration() + fresh := cloneResolvedGeneration(original) + fresh.Devices[0].Message = "fresh read diagnostic" + fresh.Devices[0].ReasonCode = "NEW_OBSERVATION" + fresh.Devices[0].Firmware = append(fresh.Devices[0].Firmware, inventoryFirmwareFixture()) + fresh.Devices[0].EvidenceRefs = []string{"new-evidence"} + fresh.Devices[0].Time.DateTime = "2099-01-02T03:04:05Z" + fresh.Devices[0].Runtime.Hardware.Cards[0].Firmware = inventoryObservationFixture() + Expect(SameIntent(original, fresh)).To(BeTrue()) + }) + + It("detects changes to target identity, cluster identity, names, software and namespaces", func() { + for _, tc := range []struct { + name string + change func(*Generation) + }{ + {name: "selected serial", change: func(g *Generation) { g.Devices[0].SerialNumber = "BF4-SERIAL-2" }}, + {name: "selected PSID", change: func(g *Generation) { g.DevicePSIDs["dpu-1"] = "MT_0000000011" }}, + {name: "management UID", change: func(g *Generation) { g.ManagementUID = "other-management-uid" }}, + {name: "management Kubernetes version", change: func(g *Generation) { g.ManagementKubernetesVersion = "v1.36.2" }}, + {name: "tenant cluster UID", change: func(g *Generation) { g.Provisioning.DPUCluster.ClusterUID = "other-tenant-uid" }}, + {name: "tenant Kubernetes version", change: func(g *Generation) { g.Provisioning.DPUCluster.KubernetesVersion = "v1.36.2" }}, + {name: "DPUCluster resource UID", change: func(g *Generation) { g.Provisioning.DPUCluster.ResourceUID = "other-resource-uid" }}, + {name: "generation name", change: func(g *Generation) { g.Name = "other-sentry" }}, + {name: "DPU resource name", change: func(g *Generation) { g.Provisioning.Devices[0].DPUDevice = "other-device" }}, + {name: "DPU node name", change: func(g *Generation) { g.Provisioning.Devices[0].DPUNode = "other-node" }}, + {name: "Sentry namespace", change: func(g *Generation) { g.Sentry.Namespace = stringPointer("other-services") }}, + {name: "DPF namespace", change: func(g *Generation) { g.Namespace = "other-operator" }}, + {name: "software digest", change: func(g *Generation) { g.Provisioning.Software.OSISO.SHA256 = strings.Repeat("b", 64) }}, + } { + By(tc.name) + original := resolvedGeneration() + changed := cloneResolvedGeneration(original) + tc.change(&changed) + Expect(SameIntent(original, changed)).To(BeFalse()) + } + }) + + It("allows a kubeconfig path move only while the management cluster identity stays fixed", func() { + original := resolvedGeneration() + moved := cloneResolvedGeneration(original) + moved.KubeconfigRef.File = "/new/location/management.kubeconfig" + Expect(SameIntent(original, moved)).To(BeTrue()) + + moved.ManagementUID = "different-management-uid" + Expect(SameIntent(original, moved)).To(BeFalse()) + }) +}) + +func resolvedGeneration() Generation { + generation, err := ResolveGeneration(validGenerationInput(GinkgoT().TempDir())) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return generation +} + +func cloneResolvedGeneration(g Generation) Generation { + encoded, err := EncodeResolved(g) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + cloned, err := DecodeResolved(encoded) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return cloned +} + +func inventoryFirmwareFixture() inventory.Firmware { + return inventory.Firmware{Component: "bf4", State: "observed", RawVersion: "new-version", EvidenceRefs: []string{"fresh-firmware"}} +} + +func inventoryObservationFixture() inventory.Observation { + return inventory.Observation{State: "observed", Values: []string{"fresh-psid-observation"}} +} diff --git a/pkg/dpf/deployment/client.go b/pkg/dpf/deployment/client.go new file mode 100644 index 00000000..1a2ae9fa --- /dev/null +++ b/pkg/dpf/deployment/client.go @@ -0,0 +1,242 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "os" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" + typedcore "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +// NativeClients resolves only explicit config bindings, never ambient kubeconfig. +type NativeClients struct{} + +type nativeClient struct { + dynamic dynamic.Interface + mapper meta.RESTMapper + core typedcore.CoreV1Interface +} + +func (c *nativeClient) ServerVersion(ctx context.Context) (string, error) { + requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + data, err := c.core.RESTClient().Get().AbsPath("/version").Do(requestCtx).Raw() + if err != nil { + return "", fmt.Errorf("cannot query bound Kubernetes server version") + } + var version struct { + GitVersion string `json:"gitVersion"` + } + if json.Unmarshal(data, &version) != nil || version.GitVersion == "" { + return "", fmt.Errorf("kubernetes server version response is invalid") + } + return version.GitVersion, nil +} + +func (NativeClients) Management(_ context.Context, g config.Generation) (Client, error) { + if g.KubeconfigRef.File == "" || g.Context == "" { + return nil, fmt.Errorf("explicit management kubeconfig and context are required") + } + st, err := os.Lstat(g.KubeconfigRef.File) + if err != nil || !st.Mode().IsRegular() || st.Size() > 4<<20 { + return nil, fmt.Errorf("management kubeconfig must be a bounded regular file") + } + data, err := os.ReadFile(g.KubeconfigRef.File) + if err != nil { + return nil, fmt.Errorf("cannot read explicit management kubeconfig") + } + return clientFromBytes(data, g.Context) +} + +func (NativeClients) Tenant(ctx context.Context, management Client, binding config.ClusterBinding) (Client, error) { + client, ok := management.(*nativeClient) + if !ok { + return nil, fmt.Errorf("native tenant resolution requires native management access") + } + cluster, err := client.Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUCluster", binding.Namespace, binding.Name)) + if err != nil { + return nil, err + } + if string(cluster.GetUID()) != binding.ResourceUID { + return nil, fmt.Errorf("DPUCluster UID changed before tenant resolution") + } + secret := text(cluster, "spec", "kubeconfig") + if secret == "" { + return nil, fmt.Errorf("DPUCluster has no explicit kubeconfig Secret reference") + } + // super-admin.conf is the exact rc.1 controller contract; admin.conf is the + // explicitly observed Kamaji compatibility key. Never scan arbitrary values. + object, err := client.Get(ctx, ref(bundle.Management, "", "Secret", binding.Namespace, secret)) + if err != nil { + return nil, fmt.Errorf("cannot read bound tenant kubeconfig Secret") + } + key := "super-admin.conf" + if text(object, "data", key) == "" { + key = "admin.conf" + } + data, err := client.secretData(ctx, binding.Namespace, secret, key) + if err != nil { + return nil, err + } + return clientFromBytes(data, "") +} + +func clientFromBytes(data []byte, selectedContext string) (Client, error) { + cfg, err := clientcmd.Load(data) + if err != nil { + return nil, fmt.Errorf("cannot decode explicit kubeconfig") + } + if selectedContext == "" { + selectedContext = cfg.CurrentContext + } + selected, exists := cfg.Contexts[selectedContext] + if !exists { + return nil, fmt.Errorf("explicit kubeconfig context is absent") + } + cluster, exists := cfg.Clusters[selected.Cluster] + if !exists || cluster.Server == "" || cluster.InsecureSkipTLSVerify { + return nil, fmt.Errorf("kubeconfig requires a verified TLS cluster binding") + } + server, err := url.Parse(cluster.Server) + if err != nil || server.Scheme != "https" || server.Host == "" || server.User != nil || server.RawQuery != "" || server.Fragment != "" { + return nil, fmt.Errorf("kubeconfig requires an explicit HTTPS API server origin") + } + auth, exists := cfg.AuthInfos[selected.AuthInfo] + if !exists || auth.Exec != nil || auth.AuthProvider != nil { + return nil, fmt.Errorf("kubeconfig external credential execution is unsupported by the native DPF slice") + } + // Tenant kubeconfig Secret data must be self-contained; file references could + // accidentally resolve against workstation ambient paths. + if auth.ClientCertificate != "" || auth.ClientKey != "" || auth.TokenFile != "" || cluster.CertificateAuthority != "" { + return nil, fmt.Errorf("DPF kubeconfig must contain embedded credentials and CA data, not file references") + } + minimal := clientcmdapi.Config{Clusters: map[string]*clientcmdapi.Cluster{selected.Cluster: cluster}, AuthInfos: map[string]*clientcmdapi.AuthInfo{selected.AuthInfo: auth}, Contexts: map[string]*clientcmdapi.Context{selectedContext: selected}, CurrentContext: selectedContext} + restConfig, err := clientcmd.NewNonInteractiveClientConfig(minimal, selectedContext, nil, nil).ClientConfig() + if err != nil { + return nil, fmt.Errorf("cannot resolve explicit kubeconfig") + } + restConfig.Timeout = 30 * time.Second + return newNativeClient(restConfig) +} + +func newNativeClient(cfg *rest.Config) (Client, error) { + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("cannot construct native dynamic client") + } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("cannot construct native discovery client") + } + streamConfig := rest.CopyConfig(cfg) + streamConfig.Timeout = 0 + coreClient, err := typedcore.NewForConfig(streamConfig) + if err != nil { + return nil, fmt.Errorf("cannot construct bound Kubernetes stream client") + } + return &nativeClient{dynamic: dynamicClient, mapper: restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(discoveryClient)), core: coreClient}, nil +} + +func (c *nativeClient) resource(reference bundle.ResourceRef, version string, allowAllNamespaces bool) (dynamic.ResourceInterface, error) { + versions := []string{} + if version != "" { + versions = append(versions, version) + } + mapping, err := c.mapper.RESTMapping(schema.GroupKind{Group: reference.Group, Kind: reference.Kind}, versions...) + if err != nil { + return nil, err + } + resource := c.dynamic.Resource(mapping.Resource) + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + if reference.Namespace == "" && !allowAllNamespaces { + return nil, fmt.Errorf("served API requires explicit namespace") + } + return resource.Namespace(reference.Namespace), nil + } + if reference.Namespace != "" { + return nil, fmt.Errorf("served API reports cluster scope for a namespaced bundle identity") + } + return resource, nil +} + +func (c *nativeClient) Get(ctx context.Context, reference bundle.ResourceRef) (*unstructured.Unstructured, error) { + resource, err := c.resource(reference, "", false) + if err != nil { + return nil, err + } + return resource.Get(ctx, reference.Name, metav1.GetOptions{}) +} + +func (c *nativeClient) List(ctx context.Context, reference bundle.ResourceRef, selector string) ([]*unstructured.Unstructured, error) { + resource, err := c.resource(reference, "", true) + if err != nil { + return nil, err + } + listed, err := resource.List(ctx, metav1.ListOptions{LabelSelector: selector, Limit: 1000}) + if err != nil { + return nil, err + } + if listed.GetContinue() != "" { + return nil, fmt.Errorf("DPF lifecycle list exceeds bounded membership limit") + } + items := make([]*unstructured.Unstructured, 0, len(listed.Items)) + for n := range listed.Items { + items = append(items, listed.Items[n].DeepCopy()) + } + return items, nil +} + +func (c *nativeClient) Create(ctx context.Context, object *unstructured.Unstructured, dryRun bool) (*unstructured.Unstructured, error) { + gvk := object.GroupVersionKind() + resource, err := c.resource(ref("", gvk.Group, gvk.Kind, object.GetNamespace(), object.GetName()), gvk.Version, false) + if err != nil { + return nil, err + } + opts := metav1.CreateOptions{FieldManager: "k8s-launch-kit-dpf"} + if dryRun { + opts.DryRun = []string{metav1.DryRunAll} + } + return resource.Create(ctx, object.DeepCopy(), opts) +} + +func (c *nativeClient) secretData(ctx context.Context, namespace, name, key string) ([]byte, error) { + secret, err := c.Get(ctx, ref(bundle.Management, "", "Secret", namespace, name)) + if err != nil { + return nil, fmt.Errorf("cannot read required private Secret %s/%s", namespace, name) + } + raw := text(secret, "data", key) + data, err := base64.StdEncoding.DecodeString(raw) + if err != nil || len(data) == 0 || len(data) > 4<<20 { + return nil, fmt.Errorf("required Secret %s/%s key %s is missing or invalid", namespace, name, key) + } + return data, nil +} + +func (c *nativeClient) SecretKeys(ctx context.Context, namespace, name string, keys []string) error { + for _, key := range keys { + if _, err := c.secretData(ctx, namespace, name, key); err != nil { + return err + } + } + return nil +} diff --git a/pkg/dpf/deployment/comparison.go b/pkg/dpf/deployment/comparison.go new file mode 100644 index 00000000..f72b1928 --- /dev/null +++ b/pkg/dpf/deployment/comparison.go @@ -0,0 +1,430 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "bytes" + "encoding/json" + "reflect" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +// Native objects have closed meaningful content. DPF normalization uses only +// pinned CRD defaults and listed typed-serialization zeros; supported Kubernetes +// kinds use their actual typed API and defaults. Metadata/status remain API/controller observations, while supplied +// labels and annotations must still match. No arbitrary native kind is accepted. +func nativeMatches(release string, desired, current *unstructured.Unstructured) bool { + if desired == nil || current == nil || desired.GroupVersionKind() != current.GroupVersionKind() { + return false + } + want, ok := nativeBody(release, desired) + if !ok { + return false + } + actual, ok := nativeBody(release, current) + if !ok || !jsonEqual(want, actual) { + return false + } + return contains(current.Object["metadata"], desired.Object["metadata"]) +} + +func nativeBody(release string, object *unstructured.Unstructured) (map[string]any, bool) { + if object == nil { + return nil, false + } + copy := object.DeepCopy() + group := copy.GroupVersionKind().Group + if group == "provisioning.dpu.nvidia.com" || group == "svc.dpu.nvidia.com" { + defaulted, err := sentry.DefaultedCopy(release, copy) + if err != nil { + return nil, false + } + copy = defaulted + if !normalizeNativeSerialization(copy) { + return nil, false + } + } else { + // API-owned observations are not part of declarative RBAC/ServiceAccount + // content. Strict decoding below rejects every unknown meaningful field. + delete(copy.Object, "metadata") + delete(copy.Object, "status") + data, err := json.Marshal(copy.Object) + if err != nil { + return nil, false + } + var typed any + switch copy.GetAPIVersion() + "/" + copy.GetKind() { + case "v1/ServiceAccount": + typed = &corev1.ServiceAccount{} // v1.35 has no ServiceAccount API defaulter. + case "rbac.authorization.k8s.io/v1/Role": + typed = &rbacv1.Role{} + case "rbac.authorization.k8s.io/v1/RoleBinding": + typed = &rbacv1.RoleBinding{} + default: + return nil, false + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if decoder.Decode(typed) != nil { + return nil, false + } + if binding, ok := typed.(*rbacv1.RoleBinding); ok { + // Kubernetes v1.35 pkg/apis/rbac/v1/defaults.go. + if binding.RoleRef.APIGroup == "" { + binding.RoleRef.APIGroup = rbacv1.GroupName + } + for i := range binding.Subjects { + subject := &binding.Subjects[i] + if subject.APIGroup == "" && (subject.Kind == "User" || subject.Kind == "Group") { + subject.APIGroup = rbacv1.GroupName + } + } + } + data, err = json.Marshal(typed) + copy.Object = nil + if err != nil || json.Unmarshal(data, ©.Object) != nil { + return nil, false + } + } + for _, key := range []string{"apiVersion", "kind", "metadata", "status"} { + delete(copy.Object, key) + } + return copy.Object, true +} + +// Only pinned non-pointer value fields whose zero value is serialized by Go +// are listed here. In particular, optional pointer objects (such as security. +// spiffe) are presence-gated configuration and must never be synthesized. +// rc.1 and GA have identical definitions at these paths. +type nativeSerializationZero struct { + path []string + value any +} + +var nativeSerializationZeros = map[string][]nativeSerializationZero{ + "DPUFlavor": { // provisioning/v1alpha1/dpuflavor_types.go:38,41,56,75 + {[]string{"spec", "grub"}, map[string]any{}}, + {[]string{"spec", "sysctl"}, map[string]any{}}, + {[]string{"spec", "ovs"}, map[string]any{}}, + {[]string{"spec", "containerdConfig"}, map[string]any{}}, + }, + "DPUServiceInterface": { // dpuserviceinterface_types.go:98; serviceinterfaceset_types.go:88 + {[]string{"spec", "template", "metadata"}, map[string]any{}}, + {[]string{"spec", "template", "spec", "template", "metadata"}, map[string]any{}}, + }, + "DPUServiceChain": { // dpuservicechain_types.go:91; servicechainset_types.go:81 + {[]string{"spec", "template", "metadata"}, map[string]any{}}, + {[]string{"spec", "template", "spec", "template", "metadata"}, map[string]any{}}, + }, + "DPUService": { // dpuservice_types.go:348 (optional without omitempty) + {[]string{"spec", "helmChart", "source", "path"}, ""}, + }, +} + +// Fill absent zero-value fields only under existing required parents. Never +// roundtrip a DPF spec through a typed struct: that would discard unknown fields. +func normalizeNativeSerialization(object *unstructured.Unstructured) bool { + for _, zero := range nativeSerializationZeros[object.GetKind()] { + parent, found, err := unstructured.NestedFieldNoCopy(object.Object, zero.path[:len(zero.path)-1]...) + fields, isObject := parent.(map[string]any) + if err != nil || !found || !isObject { + return false + } + if _, exists := fields[zero.path[len(zero.path)-1]]; !exists { + if unstructured.SetNestedField(object.Object, zero.value, zero.path...) != nil { + return false + } + } + } + return true +} + +// Projections preserve ordered array membership recursively. For the two +// supported Sentry workload kinds, compare the entire typed, defaulted spec so +// injected containers, security settings, scheduling or volumes cannot hide in +// additional map keys. API metadata/status are observed separately. +func desiredMatches(desired, current *unstructured.Unstructured) bool { + if desired == nil || current == nil { + return false + } + want, actual := desired.DeepCopy(), current.DeepCopy() + if desired.GroupVersionKind().Group == "apps" && (desired.GetKind() == "Deployment" || desired.GetKind() == "DaemonSet") { + a, ok := workloadSpec(want) + if !ok { + return false + } + b, ok := workloadSpec(actual) + if !ok || !jsonEqual(a, b) { + return false + } + delete(want.Object, "spec") + } + return contains(actual.Object, want.Object) +} + +func contains(actual, desired any) bool { + switch expected := desired.(type) { + case map[string]any: + found, ok := actual.(map[string]any) + if !ok { + return false + } + for key, value := range expected { + if !contains(found[key], value) { + return false + } + } + return true + case []any: + found, ok := actual.([]any) + if !ok || len(found) != len(expected) { + return false + } + for i := range expected { + if !contains(found[i], expected[i]) { + return false + } + } + return true + default: + return jsonEqual(actual, desired) + } +} +func jsonEqual(a, b any) bool { + x, e1 := json.Marshal(a) + y, e2 := json.Marshal(b) + return e1 == nil && e2 == nil && bytes.Equal(x, y) +} + +// These bounded typed defaults follow Kubernetes v1.35 pkg/apis/apps/v1 and +// pkg/apis/core/v1 defaults.go, PodSpec conversion.go alias handling, and their +// generated PodTemplate traversal. They +// cover the pinned Sentry Deployment/DaemonSet charts. New/feature-gated defaults +// are intentionally not discarded; a changed API contract requires review. +// Strict typed decoding rejects unknown fields instead of silently dropping them. +func workloadSpec(o *unstructured.Unstructured) (any, bool) { + data, err := json.Marshal(o.Object["spec"]) + if err != nil { + return nil, false + } + decode := func(value any) bool { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + return decoder.Decode(value) == nil + } + switch o.GetKind() { + case "Deployment": + var spec appsv1.DeploymentSpec + if !decode(&spec) { + return nil, false + } + if spec.Replicas == nil { + spec.Replicas = ptr.To(int32(1)) + } + if spec.RevisionHistoryLimit == nil { + spec.RevisionHistoryLimit = ptr.To(int32(10)) + } + if spec.ProgressDeadlineSeconds == nil { + spec.ProgressDeadlineSeconds = ptr.To(int32(600)) + } + if spec.Strategy.Type == "" { + spec.Strategy.Type = appsv1.RollingUpdateDeploymentStrategyType + } + if spec.Strategy.Type == appsv1.RollingUpdateDeploymentStrategyType { + if spec.Strategy.RollingUpdate == nil { + spec.Strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{} + } + if spec.Strategy.RollingUpdate.MaxUnavailable == nil { + spec.Strategy.RollingUpdate.MaxUnavailable = ptr.To(intstr.FromString("25%")) + } + if spec.Strategy.RollingUpdate.MaxSurge == nil { + spec.Strategy.RollingUpdate.MaxSurge = ptr.To(intstr.FromString("25%")) + } + } + if !normalizePodSpec(&spec.Template.Spec) { + return nil, false + } + return spec, true + case "DaemonSet": + var spec appsv1.DaemonSetSpec + if !decode(&spec) { + return nil, false + } + if spec.RevisionHistoryLimit == nil { + spec.RevisionHistoryLimit = ptr.To(int32(10)) + } + if spec.UpdateStrategy.Type == "" { + spec.UpdateStrategy.Type = appsv1.RollingUpdateDaemonSetStrategyType + } + if spec.UpdateStrategy.Type == appsv1.RollingUpdateDaemonSetStrategyType { + if spec.UpdateStrategy.RollingUpdate == nil { + spec.UpdateStrategy.RollingUpdate = &appsv1.RollingUpdateDaemonSet{} + } + if spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil { + spec.UpdateStrategy.RollingUpdate.MaxUnavailable = ptr.To(intstr.FromInt32(1)) + } + if spec.UpdateStrategy.RollingUpdate.MaxSurge == nil { + spec.UpdateStrategy.RollingUpdate.MaxSurge = ptr.To(intstr.FromInt32(0)) + } + } + if !normalizePodSpec(&spec.Template.Spec) { + return nil, false + } + return spec, true + } + return nil, false +} + +func normalizePodSpec(p *corev1.PodSpec) bool { + // Kubernetes v1.35 core/v1/conversion.go copies serviceAccountName to + // serviceAccount on output, and accepts the deprecated alias on input. + // Reject conflicting explicit aliases rather than discard supplied intent. + if p.ServiceAccountName != "" && p.DeprecatedServiceAccount != "" && p.ServiceAccountName != p.DeprecatedServiceAccount { + return false + } + if p.ServiceAccountName == "" { + p.ServiceAccountName = p.DeprecatedServiceAccount + } + p.DeprecatedServiceAccount = p.ServiceAccountName + if p.DNSPolicy == "" { + p.DNSPolicy = corev1.DNSClusterFirst + } + if p.RestartPolicy == "" { + p.RestartPolicy = corev1.RestartPolicyAlways + } + if p.SecurityContext == nil { + p.SecurityContext = &corev1.PodSecurityContext{} + } + if p.TerminationGracePeriodSeconds == nil { + p.TerminationGracePeriodSeconds = ptr.To(int64(30)) + } + if p.SchedulerName == "" { + p.SchedulerName = corev1.DefaultSchedulerName + } + for i := range p.Containers { + defaultContainer(&p.Containers[i]) + } + for i := range p.InitContainers { + defaultContainer(&p.InitContainers[i]) + } + for i := range p.Volumes { + v := &p.Volumes[i] + if reflect.DeepEqual(v.VolumeSource, corev1.VolumeSource{}) { + v.EmptyDir = &corev1.EmptyDirVolumeSource{} + } + if v.HostPath != nil && v.HostPath.Type == nil { + v.HostPath.Type = ptr.To(corev1.HostPathUnset) + } + if v.Secret != nil && v.Secret.DefaultMode == nil { + v.Secret.DefaultMode = ptr.To(int32(420)) + } + if v.ConfigMap != nil && v.ConfigMap.DefaultMode == nil { + v.ConfigMap.DefaultMode = ptr.To(int32(420)) + } + if v.DownwardAPI != nil { + if v.DownwardAPI.DefaultMode == nil { + v.DownwardAPI.DefaultMode = ptr.To(int32(420)) + } + for j := range v.DownwardAPI.Items { + defaultFieldRef(v.DownwardAPI.Items[j].FieldRef) + } + } + if v.Projected != nil { + if v.Projected.DefaultMode == nil { + v.Projected.DefaultMode = ptr.To(int32(420)) + } + for j := range v.Projected.Sources { + source := &v.Projected.Sources[j] + if source.ServiceAccountToken != nil && source.ServiceAccountToken.ExpirationSeconds == nil { + source.ServiceAccountToken.ExpirationSeconds = ptr.To(int64(3600)) + } + if source.DownwardAPI != nil { + for k := range source.DownwardAPI.Items { + defaultFieldRef(source.DownwardAPI.Items[k].FieldRef) + } + } + } + } + } + return true +} +func defaultFieldRef(r *corev1.ObjectFieldSelector) { + if r != nil && r.APIVersion == "" { + r.APIVersion = "v1" + } +} +func defaultContainer(c *corev1.Container) { + if c.ImagePullPolicy == "" { + name := strings.SplitN(c.Image, "@", 2)[0] + tag := name[strings.LastIndex(name, "/")+1:] + if !strings.Contains(c.Image, "@") && (!strings.Contains(tag, ":") || strings.HasSuffix(tag, ":latest")) { + c.ImagePullPolicy = corev1.PullAlways + } else { + c.ImagePullPolicy = corev1.PullIfNotPresent + } + } + if c.TerminationMessagePath == "" { + c.TerminationMessagePath = corev1.TerminationMessagePathDefault + } + if c.TerminationMessagePolicy == "" { + c.TerminationMessagePolicy = corev1.TerminationMessageReadFile + } + for i := range c.Ports { + if c.Ports[i].Protocol == "" { + c.Ports[i].Protocol = corev1.ProtocolTCP + } + } + for i := range c.Env { + if c.Env[i].ValueFrom != nil { + defaultFieldRef(c.Env[i].ValueFrom.FieldRef) + } + } + for _, p := range []*corev1.Probe{c.LivenessProbe, c.ReadinessProbe, c.StartupProbe} { + if p == nil { + continue + } + if p.TimeoutSeconds == 0 { + p.TimeoutSeconds = 1 + } + if p.PeriodSeconds == 0 { + p.PeriodSeconds = 10 + } + if p.SuccessThreshold == 0 { + p.SuccessThreshold = 1 + } + if p.FailureThreshold == 0 { + p.FailureThreshold = 3 + } + defaultHTTP(p.HTTPGet) + if p.GRPC != nil && p.GRPC.Service == nil { + p.GRPC.Service = ptr.To("") + } + } + if c.Lifecycle != nil { + if c.Lifecycle.PostStart != nil { + defaultHTTP(c.Lifecycle.PostStart.HTTPGet) + } + if c.Lifecycle.PreStop != nil { + defaultHTTP(c.Lifecycle.PreStop.HTTPGet) + } + } +} +func defaultHTTP(h *corev1.HTTPGetAction) { + if h != nil { + if h.Path == "" { + h.Path = "/" + } + if h.Scheme == "" { + h.Scheme = corev1.URISchemeHTTP + } + } +} diff --git a/pkg/dpf/deployment/comparison_test.go b/pkg/dpf/deployment/comparison_test.go new file mode 100644 index 00000000..714a4547 --- /dev/null +++ b/pkg/dpf/deployment/comparison_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "encoding/json" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var _ = Describe("closed desired state comparison", func() { + It("accepts the API-owned metadata and status of native resources", func() { + b, g, _, _ := lifecycleFixture() + for _, kind := range []string{"BlueFieldSoftware", "DPUSet"} { + doc := resourcesInBundle(b, kind)[0] + desired := doc.ObjectCopy() + actual := mark(desired, b.Index().BundleDigest) + actual.SetUID("new-uid") + actual.SetGeneration(1) + actual.SetResourceVersion("2") + setReady(actual) + a, err := sentry.DefaultedCopy(g.Release, desired) + Expect(err).NotTo(HaveOccurred()) + c, err := sentry.DefaultedCopy(g.Release, actual) + Expect(err).NotTo(HaveOccurred()) + Expect(a.Object["spec"]).To(Equal(c.Object["spec"])) + delete(a.Object, "spec") + delete(a.Object, "status") + aa, _ := json.Marshal(a.Object) + cc, _ := json.Marshal(c.Object) + Expect(contains(c.Object, a.Object)).To(BeTrue(), "want=%s actual=%s", aa, cc) + Expect(nativeMatches(g.Release, desired, actual)).To(BeTrue()) + } + }) + It("accepts pinned CRD defaults while rejecting additional physical spec fields", func() { + b, g, _, _ := lifecycleFixture() + desired := resourcesInBundle(b, "DPUSet")[0].ObjectCopy() + actual, err := sentry.DefaultedCopy(g.Release, mark(desired, b.Index().BundleDigest)) + Expect(err).NotTo(HaveOccurred()) + Expect(owns(actual, desired, b.Index().BundleDigest, g.Release)).To(BeTrue()) + _ = unstructured.SetNestedField(actual.Object, true, "spec", "dpuTemplate", "spec", "secureBoot") + Expect(owns(actual, desired, b.Index().BundleDigest, g.Release)).To(BeFalse()) + }) + It("compares nested ordered arrays without allowing changed or added members", func() { + want := map[string]any{"items": []any{map[string]any{"name": "first", "value": "keep"}}} + Expect(contains(map[string]any{"items": []any{map[string]any{"name": "first", "value": "keep", "apiDefault": "value"}}}, want)).To(BeTrue()) + Expect(contains(map[string]any{"items": []any{map[string]any{"name": "first", "value": "changed"}}}, want)).To(BeFalse()) + Expect(contains(map[string]any{"items": []any{map[string]any{"name": "first", "value": "keep"}, map[string]any{"name": "extra"}}}, want)).To(BeFalse()) + }) + It("normalizes real nested workload defaults while rejecting extra security fields", func() { + desired := testObject("apps/v1", "DaemonSet", "sentry-system", "tower", map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"app": "tower"}}, "template": map[string]any{"metadata": map[string]any{"labels": map[string]any{"app": "tower"}}, "spec": map[string]any{"containers": []any{map[string]any{"name": "server", "image": "example.invalid/server:v1", "ports": []any{map[string]any{"containerPort": int64(8080)}}}}}}}) + + // API-server defaults captured from the Kubernetes v1.35 apps/core + // defaulting contract; do not construct these with the tested normalizer. + actual := desired.DeepCopy() + Expect(unstructured.SetNestedMap(actual.Object, map[string]any{"type": "RollingUpdate", "rollingUpdate": map[string]any{"maxUnavailable": int64(1), "maxSurge": int64(0)}}, "spec", "updateStrategy")).To(Succeed()) + Expect(unstructured.SetNestedField(actual.Object, int64(10), "spec", "revisionHistoryLimit")).To(Succeed()) + Expect(unstructured.SetNestedField(actual.Object, "ClusterFirst", "spec", "template", "spec", "dnsPolicy")).To(Succeed()) + Expect(unstructured.SetNestedField(actual.Object, "Always", "spec", "template", "spec", "restartPolicy")).To(Succeed()) + Expect(unstructured.SetNestedField(actual.Object, "default-scheduler", "spec", "template", "spec", "schedulerName")).To(Succeed()) + Expect(unstructured.SetNestedField(actual.Object, int64(30), "spec", "template", "spec", "terminationGracePeriodSeconds")).To(Succeed()) + Expect(unstructured.SetNestedMap(actual.Object, map[string]any{}, "spec", "template", "spec", "securityContext")).To(Succeed()) + expectedContainers, _, _ := unstructured.NestedSlice(actual.Object, "spec", "template", "spec", "containers") + container := expectedContainers[0].(map[string]any) + container["imagePullPolicy"] = "IfNotPresent" + container["terminationMessagePath"] = "/dev/termination-log" + container["terminationMessagePolicy"] = "File" + container["resources"] = map[string]any{} + container["ports"].([]any)[0].(map[string]any)["protocol"] = "TCP" + Expect(unstructured.SetNestedSlice(actual.Object, expectedContainers, "spec", "template", "spec", "containers")).To(Succeed()) + Expect(desiredMatches(desired, actual)).To(BeTrue()) + containers, _, _ := unstructured.NestedSlice(actual.Object, "spec", "template", "spec", "containers") + containers[0].(map[string]any)["securityContext"] = map[string]any{"privileged": true} + Expect(unstructured.SetNestedSlice(actual.Object, containers, "spec", "template", "spec", "containers")).To(Succeed()) + Expect(desiredMatches(desired, actual)).To(BeFalse()) + containers = append(containers, map[string]any{"name": "extra", "image": "example.invalid/extra:v1"}) + Expect(unstructured.SetNestedSlice(actual.Object, containers, "spec", "template", "spec", "containers")).To(Succeed()) + Expect(desiredMatches(desired, actual)).To(BeFalse()) + }) +}) diff --git a/pkg/dpf/deployment/coverage.go b/pkg/dpf/deployment/coverage.go new file mode 100644 index 00000000..ab0ac5c7 --- /dev/null +++ b/pkg/dpf/deployment/coverage.go @@ -0,0 +1,194 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" +) + +// Re-evaluate the complete tenant Node domain, including unselected nodes. A +// scope label copied onto another node would broaden every service in the scope. +func (s *run) tenantScope(ctx context.Context, selected map[string]map[string]string) ([]*unstructured.Unstructured, error) { + nodes, err := s.clients[bundle.DPU].List(ctx, ref(bundle.DPU, "", "Node", "", ""), "") + if err != nil { + return nil, fail("TENANT_NODE_SCOPE_UNAVAILABLE", "cannot enumerate tenant Nodes") + } + scopes := map[string]bool{} + for _, expected := range selected { + scopes[expected[sentry.ScopeLabel]] = true + } + found := map[string]bool{} + for _, node := range nodes { + expected, isSelected := selected[node.GetName()] + if !isSelected { + if scopes[node.GetLabels()[sentry.ScopeLabel]] { + return nil, fail("UNSELECTED_TENANT_NODE_MATCH", "Sentry scope label also selects unrequested tenant Node "+node.GetName()) + } + continue + } + found[node.GetName()] = true + for key, value := range expected { + if node.GetLabels()[key] != value { + return nil, fail("TENANT_NODE_LABEL_DRIFT", node.GetName()+" no longer has the exact DPUSet node labels") + } + } + } + if len(found) != len(selected) { + return nil, fail("TENANT_NODE_SCOPE_CHANGED", "a selected tenant Node disappeared") + } + return nodes, nil +} + +// Positive DaemonSet counters alone do not prove coverage of selected devices. +// Require exact scheduling membership and one Ready pod owned by this DaemonSet +// on every selected node. A replacement pod may be pending; extra ready or +// unselected owned pods remain a conflict, including during a rollout. +func (s *run) daemonSetCoverage(ctx context.Context, object *unstructured.Unstructured) (bool, string, error) { + if len(s.selectedNodes) != len(s.g.Provisioning.Devices) || len(s.selectedNodes) == 0 { + return false, "selected tenant Node scope is not established", nil + } + nodes, err := s.tenantScope(ctx, s.selectedNodes) + if err != nil { + return false, "", err + } + var ds appsv1.DaemonSet + data, err := json.Marshal(object.Object) + if err != nil || json.Unmarshal(data, &ds) != nil || ds.UID == "" { + return false, "", fail("DAEMONSET_IDENTITY_UNAVAILABLE", "cannot decode service DaemonSet identity") + } + matched := 0 + for _, node := range nodes { + matches, err := podMatchesNode(&ds.Spec.Template.Spec, node) + if err != nil { + return false, "", fail("DAEMONSET_SELECTOR_UNSUPPORTED", err.Error()) + } + _, selected := s.selectedNodes[node.GetName()] + if matches && !selected { + return false, "", fail("DAEMONSET_SCOPE_BROADENED", "service DaemonSet schedules on unselected Node "+node.GetName()) + } + if matches { + matched++ + } + } + if matched != len(s.selectedNodes) { + return false, "service DaemonSet scheduling does not cover every selected tenant Node", nil + } + selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + if err != nil || ds.Spec.Selector == nil { + return false, "", fail("DAEMONSET_SELECTOR_INVALID", "service DaemonSet selector is invalid") + } + pods, err := s.clients[bundle.DPU].List(ctx, ref(bundle.DPU, "", "Pod", object.GetNamespace(), ""), selector.String()) + if err != nil { + return false, "", fail("DAEMONSET_PODS_UNAVAILABLE", "cannot inspect service DaemonSet pods") + } + coverage := map[string]int{} + for _, pod := range pods { + owned := false + for _, owner := range pod.GetOwnerReferences() { + if owner.Kind == "DaemonSet" && owner.UID == ds.UID && owner.Controller != nil && *owner.Controller { + owned = true + } + } + if !owned { + continue + } + node := text(pod, "spec", "nodeName") + if node == "" { + continue + } + if _, selected := s.selectedNodes[node]; !selected { + return false, "", fail("DAEMONSET_UNSELECTED_POD", "service DaemonSet owns a pod on an unselected tenant Node") + } + ready, _, _ := Readiness(pod) + if ready { + coverage[node]++ + } + } + for node := range s.selectedNodes { + if coverage[node] != 1 { + return false, "service DaemonSet requires exactly one Ready owned pod on selected Node " + node, nil + } + } + if number(object, "status", "desiredNumberScheduled") != int64(len(s.selectedNodes)) { + return false, "service DaemonSet counters do not match exact selected node count", nil + } + return true, "service DaemonSet has exact selected-node scheduling and Ready owned pod coverage", nil +} + +// Scheduling predicates are explicit inputs. Required node-affinity terms use +// the Kubernetes OR-of-ANDs contract; unsupported MatchFields fail closed. +func podMatchesNode(p *corev1.PodSpec, node *unstructured.Unstructured) (bool, error) { + if p.NodeName != "" && p.NodeName != node.GetName() { + return false, nil + } + if !labels.SelectorFromSet(p.NodeSelector).Matches(labels.Set(node.GetLabels())) { + return false, nil + } + if p.Affinity == nil || p.Affinity.NodeAffinity == nil || p.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + return true, nil + } + required := p.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + for _, term := range required.NodeSelectorTerms { + if len(term.MatchExpressions) == 0 && len(term.MatchFields) == 0 { + continue + } + matches, err := matchesRequirements(term.MatchExpressions, labels.Set(node.GetLabels())) + if err != nil { + return false, err + } + fields := labels.Set{"metadata.name": node.GetName()} + for _, field := range term.MatchFields { + if field.Key != "metadata.name" { + return false, fmt.Errorf("unsupported node affinity field %s", field.Key) + } + } + fieldMatches, err := matchesRequirements(term.MatchFields, fields) + if err != nil { + return false, err + } + if matches && fieldMatches { + return true, nil + } + } + return false, nil +} +func matchesRequirements(requirements []corev1.NodeSelectorRequirement, values labels.Set) (bool, error) { + selector := labels.NewSelector() + for _, requirement := range requirements { + var op selection.Operator + switch requirement.Operator { + case corev1.NodeSelectorOpIn: + op = selection.In + case corev1.NodeSelectorOpNotIn: + op = selection.NotIn + case corev1.NodeSelectorOpExists: + op = selection.Exists + case corev1.NodeSelectorOpDoesNotExist: + op = selection.DoesNotExist + case corev1.NodeSelectorOpGt: + op = selection.GreaterThan + case corev1.NodeSelectorOpLt: + op = selection.LessThan + default: + return false, fmt.Errorf("unsupported node affinity operator %s", requirement.Operator) + } + r, err := labels.NewRequirement(requirement.Key, op, requirement.Values) + if err != nil { + return false, err + } + selector = selector.Add(*r) + } + return selector.Matches(values), nil +} diff --git a/pkg/dpf/deployment/coverage_test.go b/pkg/dpf/deployment/coverage_test.go new file mode 100644 index 00000000..1bbefabe --- /dev/null +++ b/pkg/dpf/deployment/coverage_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/ptr" +) + +var _ = Describe("service DaemonSet exact device coverage", func() { + fixture := func() (*run, *fakeClient, *unstructured.Unstructured) { + _, g, f, _ := lifecycleFixture() + second := g.Provisioning.Devices[0] + second.ID = "physical-b" + second.DPUDevice = "device-b" + second.DPUNode = "node-b" + g.Provisioning.Devices = append(g.Provisioning.Devices, second) + selected := map[string]map[string]string{} + for _, name := range []string{"selected-a", "selected-b"} { + node := testObject("v1", "Node", "", name, nil) + node.SetLabels(map[string]string{sentry.ScopeLabel: "fixture"}) + setReady(node) + f.tenant.put(node) + selected[name] = node.GetLabels() + } + ds := testObject("apps/v1", "DaemonSet", "sentry-system", "tower", map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"app": "tower"}}, "template": map[string]any{"spec": map[string]any{"nodeSelector": map[string]any{sentry.ScopeLabel: "fixture"}, "containers": []any{map[string]any{"name": "tower", "image": "example.invalid/tower:v1"}}}}}) + setReady(ds) + for _, field := range []string{"desiredNumberScheduled", "numberReady", "updatedNumberScheduled"} { + Expect(unstructured.SetNestedField(ds.Object, int64(2), "status", field)).To(Succeed()) + } + return &run{g: g, clients: map[bundle.Destination]Client{bundle.DPU: f.tenant}, selectedNodes: selected}, f.tenant, ds + } + pod := func(ds *unstructured.Unstructured, node, name string) *unstructured.Unstructured { + p := testObject("v1", "Pod", ds.GetNamespace(), name, map[string]any{"nodeName": node}) + p.SetLabels(map[string]string{"app": "tower"}) + setReady(p) + Expect(unstructured.SetNestedField(p.Object, "Running", "status", "phase")).To(Succeed()) + p.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "DaemonSet", Name: ds.GetName(), UID: ds.GetUID(), Controller: ptr.To(true)}}) + return p + } + It("requires a Ready owned pod on each selected node despite positive rollout counters", func() { + s, c, ds := fixture() + c.put(pod(ds, "selected-a", "only-a")) + ready, message, err := s.daemonSetCoverage(context.Background(), ds) + Expect(err).NotTo(HaveOccurred()) + Expect(ready).To(BeFalse()) + Expect(message).To(ContainSubstring("selected-b")) + c.put(pod(ds, "selected-b", "also-b")) + ready, _, err = s.daemonSetCoverage(context.Background(), ds) + Expect(err).NotTo(HaveOccurred()) + Expect(ready).To(BeTrue()) + }) + It("does not count a Ready pod owned by a different DaemonSet UID", func() { + s, c, ds := fixture() + c.put(pod(ds, "selected-a", "a")) + foreign := pod(ds, "selected-b", "foreign") + owners := foreign.GetOwnerReferences() + owners[0].UID = "foreign" + foreign.SetOwnerReferences(owners) + c.put(foreign) + ready, _, err := s.daemonSetCoverage(context.Background(), ds) + Expect(err).NotTo(HaveOccurred()) + Expect(ready).To(BeFalse()) + }) + It("rejects owned pods scheduled on unselected nodes", func() { + s, c, ds := fixture() + c.put(pod(ds, "selected-a", "a")) + c.put(pod(ds, "selected-b", "b")) + c.put(pod(ds, "unselected", "extra")) + _, _, err := s.daemonSetCoverage(context.Background(), ds) + Expect(err).To(MatchError(ContainSubstring("DAEMONSET_UNSELECTED_POD"))) + }) + It("rejects scheduling selectors that broaden beyond the selected nodes", func() { + s, c, ds := fixture() + node := testObject("v1", "Node", "", "unselected", nil) + setReady(node) + c.put(node) + unstructured.RemoveNestedField(ds.Object, "spec", "template", "spec", "nodeSelector") + _, _, err := s.daemonSetCoverage(context.Background(), ds) + Expect(err).To(MatchError(ContainSubstring("DAEMONSET_SCOPE_BROADENED"))) + }) + It("does not pass duplicate Ready owned pods on one selected node", func() { + s, c, ds := fixture() + c.put(pod(ds, "selected-a", "a")) + c.put(pod(ds, "selected-a", "duplicate-a")) + c.put(pod(ds, "selected-b", "b")) + ready, _, err := s.daemonSetCoverage(context.Background(), ds) + Expect(err).NotTo(HaveOccurred()) + Expect(ready).To(BeFalse()) + }) + It("honors OR terms and AND requirements in required node affinity", func() { + p := &corev1.PodSpec{Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{ + {MatchExpressions: []corev1.NodeSelectorRequirement{{Key: sentry.ScopeLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{"fixture"}}, {Key: "role", Operator: corev1.NodeSelectorOpIn, Values: []string{"tower"}}}}, + {MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{"explicit-node"}}}}, + }}}}} + node := testObject("v1", "Node", "", "other", nil) + node.SetLabels(map[string]string{sentry.ScopeLabel: "fixture"}) + matched, err := podMatchesNode(p, node) + Expect(err).NotTo(HaveOccurred()) + Expect(matched).To(BeFalse()) + node.SetName("explicit-node") + matched, err = podMatchesNode(p, node) + Expect(err).NotTo(HaveOccurred()) + Expect(matched).To(BeTrue()) + }) +}) diff --git a/pkg/dpf/deployment/dns.go b/pkg/dpf/deployment/dns.go new file mode 100644 index 00000000..981e2973 --- /dev/null +++ b/pkg/dpf/deployment/dns.go @@ -0,0 +1,413 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "encoding/json" + "fmt" + "net/netip" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" +) + +type dnsObservation struct { + ready bool + code, message, source string +} + +func dnsPending(code, message string) dnsObservation { + return dnsObservation{code: code, message: message, source: "management/tenant DNS resource chain"} +} + +// DNS is a resource-readiness prerequisite, not an active DNS query. rc.1 +// Kamaji+keepalived publishes management-hosted CoreDNS through a selectorless +// tenant Service. Other tenants retain their local DNS Deployment contract. +func (s *run) observeDNS(ctx context.Context) (dnsObservation, error) { + if err := s.cluster(ctx); err != nil { + return dnsObservation{}, err + } + binding := s.g.Provisioning.DPUCluster + cluster, err := s.clients[bundle.Management].Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUCluster", binding.Namespace, binding.Name)) + if err != nil { + return dnsObservation{}, dnsReadError(err, "bound DPUCluster") + } + if string(cluster.GetUID()) != binding.ResourceUID || cluster.GetGeneration() != s.clusterGeneration { + return dnsObservation{}, fail("DPU_CLUSTER_GENERATION_DRIFT", "DPUCluster changed while selecting its DNS topology") + } + vip := text(cluster, "spec", "clusterEndpoint", "keepalived", "vip") + if text(cluster, "spec", "type") == "kamaji" && vip != "" { + return s.hostedDNS(ctx, cluster, vip) + } + objects, err := s.clients[bundle.DPU].List(ctx, ref(bundle.DPU, "apps", "Deployment", "kube-system", ""), "k8s-app=kube-dns") + if err != nil { + return dnsObservation{}, dnsReadError(err, "tenant DNS Deployment") + } + ready := len(objects) > 0 + for _, object := range objects { + available, _, _ := Readiness(object) + ready = ready && available + } + if !ready { + return dnsPending("TENANT_DNS_NOT_READY", "tenant DNS Deployment is absent or not Ready"), nil + } + return dnsObservation{ready: true, code: "TENANT_DNS_READY", source: "tenant:Deployment k8s-app=kube-dns", message: "tenant DNS Deployment resource readiness; DNS reachability is not tested"}, nil +} + +func dnsReadError(err error, subject string) error { + if apierrors.IsNotFound(err) { + return fail("DNS_RESOURCE_MISSING", subject+" is absent") + } + if apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) { + return fail("DNS_ACCESS_DENIED", "cannot read "+subject) + } + return fail("DNS_RESOURCE_UNAVAILABLE", "cannot inspect "+subject) +} +func decodeDNS(o *unstructured.Unstructured, target any) error { + if o == nil { + return fmt.Errorf("DNS resource is absent") + } + data, err := json.Marshal(o.Object) + if err != nil { + return err + } + return json.Unmarshal(data, target) +} +func liveDNSObject(o *unstructured.Unstructured) bool { + return o != nil && o.GetUID() != "" && o.GetDeletionTimestamp().IsZero() +} + +func (s *run) hostedDNS(ctx context.Context, cluster *unstructured.Unstructured, vip string) (dnsObservation, error) { + address, err := netip.ParseAddr(vip) + if err != nil || !address.Is4() { + return dnsPending("DNS_TOPOLOGY_UNSUPPORTED", "managed DNS requires the source-supported IPv4 keepalived VIP"), nil + } + // GetCoreDNSWorkloadLabels in the pinned inventory source. Names are hashed. + selector := labels.SelectorFromSet(map[string]string{"app.kubernetes.io/name": "coredns", "dpu.nvidia.com/cluster": cluster.GetName(), "dpu.nvidia.com/cluster-namespace": cluster.GetNamespace()}).String() + management := s.clients[bundle.Management] + deployments, err := management.List(ctx, ref(bundle.Management, "apps", "Deployment", s.g.Namespace, ""), selector) + if err != nil { + return dnsObservation{}, dnsReadError(err, "management CoreDNS Deployments") + } + services, err := management.List(ctx, ref(bundle.Management, "", "Service", s.g.Namespace, ""), selector) + if err != nil { + return dnsObservation{}, dnsReadError(err, "management CoreDNS Services") + } + if len(deployments) == 0 || len(services) == 0 { + return dnsPending("DNS_HOST_RESOURCES_MISSING", "management CoreDNS Deployment/Service for the bound tenant is absent"), nil + } + if len(deployments) != 1 || len(services) != 1 { + return dnsPending("DNS_BINDING_AMBIGUOUS", "bound tenant labels must identify exactly one management CoreDNS Deployment and Service"), nil + } + deployment, service := deployments[0], services[0] + ready, _, _ := Readiness(deployment) + if !liveDNSObject(deployment) || !liveDNSObject(service) || !ready { + return dnsPending("DNS_HOST_ROLLOUT_NOT_READY", "management CoreDNS must have a current fully available rollout"), nil + } + var dep appsv1.Deployment + var svc corev1.Service + if decodeDNS(deployment, &dep) != nil || decodeDNS(service, &svc) != nil { + return dnsPending("DNS_RESOURCE_INVALID", "cannot decode management CoreDNS resources"), nil + } + if dep.Spec.Selector == nil || len(dep.Spec.Selector.MatchExpressions) > 0 || len(svc.Spec.Selector) == 0 || !jsonEqual(dep.Spec.Selector.MatchLabels, svc.Spec.Selector) { + return dnsPending("DNS_SELECTOR_MISMATCH", "management DNS Service selector differs from its Deployment"), nil + } + if svc.Spec.Type != corev1.ServiceTypeNodePort || svc.Spec.ExternalTrafficPolicy == corev1.ServiceExternalTrafficPolicyLocal { + return dnsPending("DNS_TOPOLOGY_MISMATCH", "management DNS must expose cluster-routed NodePorts"), nil + } + nodePorts, ok := dnsServicePorts(svc.Spec.Ports, true) + if !ok { + return dnsPending("DNS_PORT_MISMATCH", "management DNS requires named UDP/TCP53 NodePorts targeting their DNS container ports"), nil + } + keepalivedNodes, observation, err := s.keepalivedDNS(ctx, cluster, vip, dep.Spec.Template.Spec.NodeSelector) + if err != nil || !observation.ready { + return observation, err + } + backingPorts := map[string]dnsPort{} + for _, container := range dep.Spec.Template.Spec.Containers { + for _, port := range container.Ports { + if port.Name == "dns" || port.Name == "dns-tcp" { + backingPorts[port.Name] = dnsPort{protocol: port.Protocol, port: port.ContainerPort} + } + } + } + if backingPorts["dns"].protocol != corev1.ProtocolUDP || backingPorts["dns-tcp"].protocol != corev1.ProtocolTCP || backingPorts["dns"].port < 1 || backingPorts["dns-tcp"].port < 1 { + return dnsPending("DNS_PORT_MISMATCH", "management CoreDNS container DNS ports are missing or inconsistent"), nil + } + observation, err = s.dnsBackends(ctx, deployment, service, &dep, &svc, backingPorts, keepalivedNodes) + if err != nil || !observation.ready { + return observation, err + } + observation, err = s.tenantDNSPublication(ctx, vip, nodePorts) + if err != nil || !observation.ready { + return observation, err + } + return dnsObservation{ready: true, code: "MANAGED_DNS_RESOURCES_READY", source: "bound DPUCluster + management CoreDNS/keepalived + tenant kube-dns Service/EndpointSlice", message: "exact management-hosted DNS publication, current rollout and Ready owned backing pods verified; DNS reachability is not tested"}, nil +} + +type dnsPort struct { + protocol corev1.Protocol + port int32 +} + +func dnsServicePorts(ports []corev1.ServicePort, nodePort bool) (map[string]dnsPort, bool) { + out := map[string]dnsPort{} + for _, port := range ports { + if port.Name != "dns" && port.Name != "dns-tcp" { + continue + } + want := corev1.ProtocolUDP + if port.Name == "dns-tcp" { + want = corev1.ProtocolTCP + } + if _, duplicate := out[port.Name]; duplicate || port.Protocol != want || port.Port != 53 { + return nil, false + } + if nodePort { + if port.NodePort < 1 || port.NodePort > 65535 || port.TargetPort.Type != intstr.String || port.TargetPort.StrVal != port.Name { + return nil, false + } + out[port.Name] = dnsPort{protocol: want, port: port.NodePort} + } else { + if port.TargetPort.Type != intstr.Int || port.TargetPort.IntVal != 53 { + return nil, false + } + out[port.Name] = dnsPort{protocol: want, port: 53} + } + } + return out, len(out) == 2 +} +func dnsEndpointPorts(ports []discoveryv1.EndpointPort, expected map[string]dnsPort) bool { + found := map[string]bool{} + for _, port := range ports { + if port.Name == nil { + continue + } + want, needed := expected[*port.Name] + if !needed { + continue + } + if found[*port.Name] || port.Protocol == nil || *port.Protocol != want.protocol || port.Port == nil || *port.Port != want.port { + return false + } + found[*port.Name] = true + } + return len(found) == len(expected) +} +func dnsOwner(o *unstructured.Unstructured, kind, name string, uid types.UID, controller bool) bool { + for _, owner := range o.GetOwnerReferences() { + if owner.Kind == kind && owner.Name == name && owner.UID == uid && (!controller || owner.Controller != nil && *owner.Controller) { + return true + } + } + return false +} + +func (s *run) keepalivedDNS(ctx context.Context, cluster *unstructured.Unstructured, vip string, dnsNodeSelector map[string]string) (map[string]bool, dnsObservation, error) { + c := s.clients[bundle.Management] + object, err := c.Get(ctx, ref(bundle.Management, "apps", "DaemonSet", cluster.GetNamespace(), cluster.GetName()+"-keepalived")) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, dnsPending("DNS_KEEPALIVED_NOT_READY", "bound keepalived DaemonSet is absent"), nil + } + return nil, dnsObservation{}, dnsReadError(err, "bound keepalived DaemonSet") + } + ready, _, _ := Readiness(object) + if !liveDNSObject(object) || !ready || !dnsOwner(object, "DPUCluster", cluster.GetName(), cluster.GetUID(), false) { + return nil, dnsPending("DNS_KEEPALIVED_NOT_READY", "keepalived must be current, Ready and owned by the bound DPUCluster UID"), nil + } + var ds appsv1.DaemonSet + if decodeDNS(object, &ds) != nil { + return nil, dnsPending("DNS_RESOURCE_INVALID", "cannot decode keepalived"), nil + } + expected, _, err := unstructured.NestedStringMap(cluster.Object, "spec", "clusterEndpoint", "keepalived", "nodeSelector") + if err != nil || !labels.Equals(expected, ds.Spec.Template.Spec.NodeSelector) || !labels.Equals(expected, dnsNodeSelector) || !ds.Spec.Template.Spec.HostNetwork { + return nil, dnsPending("DNS_KEEPALIVED_BINDING_MISMATCH", "CoreDNS and keepalived must follow the bound keepalived node selector"), nil + } + interfaceName := text(cluster, "spec", "clusterEndpoint", "keepalived", "interface") + boundVIP, boundInterface := "", "" + for _, container := range ds.Spec.Template.Spec.InitContainers { + if container.Name == "create-conf" { + for _, env := range container.Env { + if env.Name == "VIRTUAL_IP" { + boundVIP = env.Value + } + if env.Name == "INTERFACE" { + boundInterface = env.Value + } + } + } + } + if boundVIP != vip || interfaceName == "" || boundInterface != interfaceName { + return nil, dnsPending("DNS_KEEPALIVED_BINDING_MISMATCH", "keepalived rendered VIP/interface differs from its DPUCluster"), nil + } + if ds.Spec.Selector == nil { + return nil, dnsPending("DNS_SELECTOR_MISMATCH", "keepalived selector is missing"), nil + } + selector, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector) + if err != nil { + return nil, dnsPending("DNS_SELECTOR_MISMATCH", "keepalived selector is invalid"), nil + } + pods, err := c.List(ctx, ref(bundle.Management, "", "Pod", object.GetNamespace(), ""), selector.String()) + if err != nil { + return nil, dnsObservation{}, dnsReadError(err, "keepalived backing pods") + } + nodes := map[string]bool{} + for _, pod := range pods { + if !dnsOwner(pod, "DaemonSet", object.GetName(), object.GetUID(), true) { + continue + } + ready, _, _ := Readiness(pod) + if !ready { + continue + } + name := text(pod, "spec", "nodeName") + if name == "" || nodes[name] { + return nil, dnsPending("DNS_KEEPALIVED_NOT_READY", "keepalived Ready pods do not have unique node membership"), nil + } + node, err := c.Get(ctx, ref(bundle.Management, "", "Node", "", name)) + if err != nil { + return nil, dnsObservation{}, dnsReadError(err, "keepalived management Node") + } + nodeReady, _, _ := Readiness(node) + if !nodeReady || !labels.SelectorFromSet(expected).Matches(labels.Set(node.GetLabels())) { + return nil, dnsPending("DNS_KEEPALIVED_NOT_READY", "keepalived is not backed by Ready selected management Nodes"), nil + } + nodes[name] = true + } + if int64(len(nodes)) != number(object, "status", "desiredNumberScheduled") { + return nil, dnsPending("DNS_KEEPALIVED_NOT_READY", "keepalived Ready owned pod membership differs from its rollout"), nil + } + return nodes, dnsObservation{ready: true}, nil +} + +func (s *run) dnsBackends(ctx context.Context, deployment, service *unstructured.Unstructured, dep *appsv1.Deployment, svc *corev1.Service, ports map[string]dnsPort, keepalivedNodes map[string]bool) (dnsObservation, error) { + c := s.clients[bundle.Management] + pods, err := c.List(ctx, ref(bundle.Management, "", "Pod", service.GetNamespace(), ""), labels.SelectorFromSet(svc.Spec.Selector).String()) + if err != nil { + return dnsObservation{}, dnsReadError(err, "management CoreDNS backing pods") + } + readyPods := map[types.UID]*corev1.Pod{} + for _, object := range pods { + ready, _, _ := Readiness(object) + if !ready { + continue + } + var pod corev1.Pod + if decodeDNS(object, &pod) != nil || pod.UID == "" || !keepalivedNodes[pod.Spec.NodeName] { + return dnsPending("DNS_BACKEND_BINDING_MISMATCH", "DNS pod is not on a Ready bound VIP management node"), nil + } + var rsOwner *metav1.OwnerReference + for _, owner := range object.GetOwnerReferences() { + if owner.Kind == "ReplicaSet" && owner.Controller != nil && *owner.Controller { + copy := owner + rsOwner = © + } + } + if rsOwner == nil { + return dnsPending("DNS_BACKEND_BINDING_MISMATCH", "DNS pod has no ReplicaSet controller identity"), nil + } + rs, err := c.Get(ctx, ref(bundle.Management, "apps", "ReplicaSet", pod.Namespace, rsOwner.Name)) + if err != nil { + return dnsObservation{}, dnsReadError(err, "CoreDNS ReplicaSet") + } + if !liveDNSObject(rs) || rs.GetUID() != rsOwner.UID || !dnsOwner(rs, "Deployment", deployment.GetName(), deployment.GetUID(), true) { + return dnsPending("DNS_BACKEND_BINDING_MISMATCH", "DNS pod ReplicaSet is not owned by the bound DNS Deployment UID"), nil + } + var replicaSet appsv1.ReplicaSet + if decodeDNS(rs, &replicaSet) != nil || !dnsTemplateEqual(&replicaSet.Spec.Template, &dep.Spec.Template) { + return dnsPending("DNS_HOST_ROLLOUT_NOT_READY", "Ready DNS backend belongs to a previous Deployment pod template"), nil + } + readyPods[pod.UID] = &pod + } + slices, err := c.List(ctx, ref(bundle.Management, "discovery.k8s.io", "EndpointSlice", service.GetNamespace(), ""), labels.SelectorFromSet(map[string]string{discoveryv1.LabelServiceName: service.GetName()}).String()) + if err != nil { + return dnsObservation{}, dnsReadError(err, "management DNS EndpointSlices") + } + backed := map[types.UID]bool{} + for _, object := range slices { + var slice discoveryv1.EndpointSlice + if !liveDNSObject(object) || decodeDNS(object, &slice) != nil || !dnsOwner(object, "Service", service.GetName(), service.GetUID(), true) || slice.AddressType != discoveryv1.AddressTypeIPv4 || !dnsEndpointPorts(slice.Ports, ports) { + return dnsPending("DNS_BACKEND_BINDING_MISMATCH", "DNS EndpointSlice owner, address family or ports differ from its bound Service/Deployment"), nil + } + for _, endpoint := range slice.Endpoints { + if endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready || endpoint.Conditions.Serving != nil && !*endpoint.Conditions.Serving || endpoint.Conditions.Terminating != nil && *endpoint.Conditions.Terminating { + continue + } + target := endpoint.TargetRef + if target == nil || target.Kind != "Pod" || target.Namespace != service.GetNamespace() { + return dnsPending("DNS_BACKEND_BINDING_MISMATCH", "Ready DNS endpoint has no bound Pod identity"), nil + } + pod := readyPods[target.UID] + if pod == nil || pod.Name != target.Name || len(endpoint.Addresses) != 1 || endpoint.Addresses[0] != pod.Status.PodIP { + return dnsPending("DNS_BACKEND_BINDING_MISMATCH", "Ready DNS endpoint does not match an owned Ready DNS pod IP/UID"), nil + } + backed[pod.UID] = true + } + } + want := int32(1) + if dep.Spec.Replicas != nil { + want = *dep.Spec.Replicas + } + if want < 1 || len(backed) != len(readyPods) || len(backed) != int(want) { + return dnsPending("DNS_BACKENDS_NOT_READY", "management DNS requires the current replica count of Ready owned endpoint-backed pods"), nil + } + return dnsObservation{ready: true}, nil +} + +func (s *run) tenantDNSPublication(ctx context.Context, vip string, nodePorts map[string]dnsPort) (dnsObservation, error) { + c := s.clients[bundle.DPU] + object, err := c.Get(ctx, ref(bundle.DPU, "", "Service", "kube-system", "kube-dns")) + if err != nil { + if apierrors.IsNotFound(err) { + return dnsPending("TENANT_DNS_NOT_READY", "tenant kube-dns Service is absent"), nil + } + return dnsObservation{}, dnsReadError(err, "tenant kube-dns Service") + } + var svc corev1.Service + if !liveDNSObject(object) || decodeDNS(object, &svc) != nil { + return dnsPending("DNS_RESOURCE_INVALID", "tenant DNS Service identity is unavailable"), nil + } + address, err := netip.ParseAddr(svc.Spec.ClusterIP) + _, portsOK := dnsServicePorts(svc.Spec.Ports, false) + if err != nil || !address.Is4() || svc.Spec.Type != corev1.ServiceTypeClusterIP || len(svc.Spec.Selector) != 0 || !portsOK || len(svc.Spec.Ports) != 2 || svc.Labels["k8s-app"] != "kube-dns" || svc.Labels["kubernetes.io/name"] != "CoreDNS" { + return dnsPending("DNS_PUBLICATION_MISMATCH", "tenant kube-dns must be the selectorless ClusterIP/UDP+TCP53 Service managed by Kamaji"), nil + } + slices, err := c.List(ctx, ref(bundle.DPU, "discovery.k8s.io", "EndpointSlice", "kube-system", ""), labels.SelectorFromSet(map[string]string{discoveryv1.LabelServiceName: "kube-dns"}).String()) + if err != nil { + return dnsObservation{}, dnsReadError(err, "tenant DNS EndpointSlices") + } + if len(slices) != 1 { + return dnsPending("DNS_PUBLICATION_MISMATCH", "tenant DNS requires exactly its single managed kube-dns EndpointSlice"), nil + } + var slice discoveryv1.EndpointSlice + if !liveDNSObject(slices[0]) || decodeDNS(slices[0], &slice) != nil || slice.Name != "kube-dns" || slice.Labels[discoveryv1.LabelManagedBy] != "kamaji-cluster-manager" || slice.AddressType != discoveryv1.AddressTypeIPv4 || len(slice.Ports) != 2 || !dnsEndpointPorts(slice.Ports, nodePorts) || len(slice.Endpoints) != 1 { + return dnsPending("DNS_PUBLICATION_MISMATCH", "tenant DNS slice provenance, ports or membership differs from the managed mapping"), nil + } + endpoint := slice.Endpoints[0] + if len(endpoint.Addresses) != 1 || endpoint.Addresses[0] != vip || endpoint.Conditions.Ready == nil || !*endpoint.Conditions.Ready || endpoint.Conditions.Serving != nil && !*endpoint.Conditions.Serving || endpoint.Conditions.Terminating != nil && *endpoint.Conditions.Terminating { + return dnsPending("DNS_PUBLICATION_MISMATCH", "tenant DNS must publish the exact Ready keepalived VIP"), nil + } + return dnsObservation{ready: true}, nil +} + +// The Deployment controller adds only pod-template-hash when deriving its +// ReplicaSet template. Follow its EqualIgnoreHash association rather than +// revision annotations, which can change during a rollback. +func dnsTemplateEqual(replicaSet, deployment *corev1.PodTemplateSpec) bool { + left, right := replicaSet.DeepCopy(), deployment.DeepCopy() + delete(left.Labels, appsv1.DefaultDeploymentUniqueLabelKey) + delete(right.Labels, appsv1.DefaultDeploymentUniqueLabelKey) + return apiequality.Semantic.DeepEqual(left, right) +} diff --git a/pkg/dpf/deployment/dns_test.go b/pkg/dpf/deployment/dns_test.go new file mode 100644 index 00000000..b868ea49 --- /dev/null +++ b/pkg/dpf/deployment/dns_test.go @@ -0,0 +1,285 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" +) + +// Anonymous fixture follows rc.1 kamaji/dns.go and its CoreDNS/keepalived chart. +// Operator namespace and tenant control-plane namespace deliberately differ. +func hostedDNSFixture() (*bundle.Bundle, config.Generation, *fakeFactory, *fakeArtifacts) { + b, g, f, v := lifecycleFixture() + old := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUCluster", g.Provisioning.DPUCluster.Namespace, g.Provisioning.DPUCluster.Name))] + delete(f.management.objects, keyFor(old)) + g.Provisioning.DPUCluster.Namespace = "tenant-control" + old.SetNamespace("tenant-control") + selector := map[string]string{"node-role.kubernetes.io/control-plane": ""} + Expect(unstructured.SetNestedMap(old.Object, map[string]any{"keepalived": map[string]any{"vip": "192.0.2.10", "interface": "eth-test", "nodeSelector": map[string]any{"node-role.kubernetes.io/control-plane": ""}}}, "spec", "clusterEndpoint")).To(Succeed()) + f.management.put(old) + dnsLabels := map[string]string{"app.kubernetes.io/name": "coredns", "app.kubernetes.io/instance": "dns-tenant", "dpu.nvidia.com/cluster": "tenant", "dpu.nvidia.com/cluster-namespace": "tenant-control"} + match := map[string]any{"app.kubernetes.io/name": "coredns", "app.kubernetes.io/instance": "dns-tenant"} + dep := readyDeployment("dpf-system", "dns-tenant") + dep.SetLabels(dnsLabels) + Expect(unstructured.SetNestedMap(dep.Object, map[string]any{"matchLabels": match}, "spec", "selector")).To(Succeed()) + Expect(unstructured.SetNestedMap(dep.Object, map[string]any{"metadata": map[string]any{"labels": match}, "spec": map[string]any{"nodeSelector": map[string]any{"node-role.kubernetes.io/control-plane": ""}, "containers": []any{map[string]any{"name": "coredns", "ports": []any{map[string]any{"name": "dns", "containerPort": int64(5353), "protocol": "UDP"}, map[string]any{"name": "dns-tcp", "containerPort": int64(5353), "protocol": "TCP"}}}}}}, "spec", "template")).To(Succeed()) + f.management.put(dep) + hostSvc := testObject("v1", "Service", "dpf-system", "dns-tenant", map[string]any{"type": "NodePort", "selector": match, "externalTrafficPolicy": "Cluster", "ports": []any{map[string]any{"name": "dns", "protocol": "UDP", "port": int64(53), "targetPort": "dns", "nodePort": int64(32001)}, map[string]any{"name": "dns-tcp", "protocol": "TCP", "port": int64(53), "targetPort": "dns-tcp", "nodePort": int64(32002)}}}) + hostSvc.SetLabels(dnsLabels) + f.management.put(hostSvc) + node := testObject("v1", "Node", "", "manager-a", nil) + node.SetLabels(selector) + setReady(node) + f.management.put(node) + ds := testObject("apps/v1", "DaemonSet", "tenant-control", "tenant-keepalived", map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"app.kubernetes.io/name": "tenant-keepalived"}}, "template": map[string]any{"spec": map[string]any{"hostNetwork": true, "nodeSelector": map[string]any{"node-role.kubernetes.io/control-plane": ""}, "initContainers": []any{map[string]any{"name": "create-conf", "env": []any{map[string]any{"name": "VIRTUAL_IP", "value": "192.0.2.10"}, map[string]any{"name": "INTERFACE", "value": "eth-test"}}}}}}}) + ds.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: old.GetAPIVersion(), Kind: "DPUCluster", Name: old.GetName(), UID: old.GetUID()}}) + setReady(ds) + for _, field := range []string{"desiredNumberScheduled", "numberReady", "updatedNumberScheduled"} { + Expect(unstructured.SetNestedField(ds.Object, int64(1), "status", field)).To(Succeed()) + } + f.management.put(ds) + keepPod := testObject("v1", "Pod", "tenant-control", "keep-a", map[string]any{"nodeName": "manager-a", "hostNetwork": true}) + keepPod.SetLabels(map[string]string{"app.kubernetes.io/name": "tenant-keepalived"}) + keepPod.SetOwnerReferences([]metav1.OwnerReference{{Kind: "DaemonSet", Name: ds.GetName(), UID: ds.GetUID(), Controller: ptr.To(true)}}) + setReady(keepPod) + Expect(unstructured.SetNestedField(keepPod.Object, "Running", "status", "phase")).To(Succeed()) + f.management.put(keepPod) + rs := testObject("apps/v1", "ReplicaSet", "dpf-system", "dns-rs", nil) + template, _, err := unstructured.NestedMap(dep.Object, "spec", "template") + Expect(err).NotTo(HaveOccurred()) + Expect(unstructured.SetNestedMap(rs.Object, template, "spec", "template")).To(Succeed()) + rs.SetOwnerReferences([]metav1.OwnerReference{{Kind: "Deployment", Name: dep.GetName(), UID: dep.GetUID(), Controller: ptr.To(true)}}) + f.management.put(rs) + dnsPod := testObject("v1", "Pod", "dpf-system", "dns-pod", map[string]any{"nodeName": "manager-a"}) + dnsPod.SetLabels(dnsLabels) + dnsPod.SetOwnerReferences([]metav1.OwnerReference{{Kind: "ReplicaSet", Name: rs.GetName(), UID: rs.GetUID(), Controller: ptr.To(true)}}) + setReady(dnsPod) + Expect(unstructured.SetNestedField(dnsPod.Object, "Running", "status", "phase")).To(Succeed()) + Expect(unstructured.SetNestedField(dnsPod.Object, "198.51.100.10", "status", "podIP")).To(Succeed()) + f.management.put(dnsPod) + hostSlice := testObject("discovery.k8s.io/v1", "EndpointSlice", "dpf-system", "dns-backends", nil) + hostSlice.SetLabels(map[string]string{"kubernetes.io/service-name": "dns-tenant", "endpointslice.kubernetes.io/managed-by": "endpointslice-controller.k8s.io"}) + hostSlice.SetOwnerReferences([]metav1.OwnerReference{{Kind: "Service", Name: hostSvc.GetName(), UID: hostSvc.GetUID(), Controller: ptr.To(true)}}) + hostSlice.Object["addressType"] = "IPv4" + hostSlice.Object["ports"] = []any{map[string]any{"name": "dns", "port": int64(5353), "protocol": "UDP"}, map[string]any{"name": "dns-tcp", "port": int64(5353), "protocol": "TCP"}} + hostSlice.Object["endpoints"] = []any{map[string]any{"addresses": []any{"198.51.100.10"}, "conditions": map[string]any{"ready": true}, "targetRef": map[string]any{"kind": "Pod", "name": dnsPod.GetName(), "namespace": dnsPod.GetNamespace(), "uid": string(dnsPod.GetUID())}}} + f.management.put(hostSlice) + tenantSvc := testObject("v1", "Service", "kube-system", "kube-dns", map[string]any{"type": "ClusterIP", "clusterIP": "203.0.113.53", "ports": []any{map[string]any{"name": "dns", "protocol": "UDP", "port": int64(53), "targetPort": int64(53)}, map[string]any{"name": "dns-tcp", "protocol": "TCP", "port": int64(53), "targetPort": int64(53)}}}) + tenantSvc.SetLabels(map[string]string{"k8s-app": "kube-dns", "kubernetes.io/name": "CoreDNS"}) + f.tenant.put(tenantSvc) + tenantSlice := testObject("discovery.k8s.io/v1", "EndpointSlice", "kube-system", "kube-dns", nil) + tenantSlice.SetLabels(map[string]string{"kubernetes.io/service-name": "kube-dns", "endpointslice.kubernetes.io/managed-by": "kamaji-cluster-manager"}) + tenantSlice.Object["addressType"] = "IPv4" + tenantSlice.Object["ports"] = []any{map[string]any{"name": "dns", "port": int64(32001), "protocol": "UDP"}, map[string]any{"name": "dns-tcp", "port": int64(32002), "protocol": "TCP"}} + tenantSlice.Object["endpoints"] = []any{map[string]any{"addresses": []any{"192.0.2.10"}, "conditions": map[string]any{"ready": true}}} + f.tenant.put(tenantSlice) + delete(f.tenant.objects, objectKey(ref("", "apps", "Deployment", "kube-system", "coredns"))) + return b, g, f, v +} + +func dnsTestRunner(f *fakeFactory, v *fakeArtifacts) Runner { + dir, err := os.MkdirTemp("", "dpf-dns-topology-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + return Runner{Clients: f, Artifacts: v, EvidenceDir: dir, PollInterval: time.Millisecond, Timeout: 10 * time.Second} +} + +var _ = Describe("bound tenant DNS resource topology", func() { + It("converges through management-hosted DNS without a tenant DNS Deployment", func() { + b, g, f, v := hostedDNSFixture() + r := dnsTestRunner(f, v) + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(result.Checks).To(ContainElement(And(HaveField("ReasonCode", "MANAGED_DNS_RESOURCES_READY"), HaveField("Message", ContainSubstring("reachability is not tested"))))) + Expect(f.tenant.creates).To(BeEmpty()) + result, err = r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + }) + + It("rejects a mixed revision rollout and passes only after the current revision converges", func() { + b, g, f, v := hostedDNSFixture() + r := dnsTestRunner(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + dep := f.management.objects[objectKey(ref("", "apps", "Deployment", "dpf-system", "dns-tenant"))] + dep.SetGeneration(2) + dep.SetAnnotations(map[string]string{"deployment.kubernetes.io/revision": "2"}) + Expect(unstructured.SetNestedField(dep.Object, int64(2), "status", "observedGeneration")).To(Succeed()) + Expect(unstructured.SetNestedField(dep.Object, int64(2), "status", "replicas")).To(Succeed()) + containers, _, _ := unstructured.NestedSlice(dep.Object, "spec", "template", "spec", "containers") + containers[0].(map[string]any)["image"] = "example.invalid/coredns:current" + Expect(unstructured.SetNestedSlice(dep.Object, containers, "spec", "template", "spec", "containers")).To(Succeed()) + oldRS := f.management.objects[objectKey(ref("", "apps", "ReplicaSet", "dpf-system", "dns-rs"))] + oldRS.SetAnnotations(map[string]string{"deployment.kubernetes.io/revision": "1"}) + newRS := oldRS.DeepCopy() + newRS.SetName("dns-rs-current") + newRS.SetUID("dns-rs-current-uid") + newRS.SetAnnotations(map[string]string{"deployment.kubernetes.io/revision": "2"}) + template, _, _ := unstructured.NestedMap(dep.Object, "spec", "template") + Expect(unstructured.SetNestedMap(newRS.Object, template, "spec", "template")).To(Succeed()) + Expect(unstructured.SetNestedField(newRS.Object, "current-hash", "spec", "template", "metadata", "labels", "pod-template-hash")).To(Succeed()) + f.management.put(newRS) + oldPod := f.management.objects[objectKey(ref("", "", "Pod", "dpf-system", "dns-pod"))] + newPod := oldPod.DeepCopy() + newPod.SetName("dns-pod-current") + newPod.SetUID("dns-pod-current-uid") + newPod.SetOwnerReferences([]metav1.OwnerReference{{Kind: "ReplicaSet", Name: newRS.GetName(), UID: newRS.GetUID(), Controller: ptr.To(true)}}) + Expect(unstructured.SetNestedField(newPod.Object, "Pending", "status", "phase")).To(Succeed()) + f.management.put(newPod) + _, err = r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("DNS_HOST_ROLLOUT_NOT_READY"))) + + // Even misleading completed counters must not let the previous template's + // Ready endpoint stand in for a current-template backend. + Expect(unstructured.SetNestedField(dep.Object, int64(1), "status", "replicas")).To(Succeed()) + _, err = r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("DNS_HOST_ROLLOUT_NOT_READY"))) + + delete(f.management.objects, keyFor(oldPod)) + Expect(unstructured.SetNestedField(newPod.Object, "Running", "status", "phase")).To(Succeed()) + f.management.put(newPod) + slice := f.management.objects[objectKey(ref("", "discovery.k8s.io", "EndpointSlice", "dpf-system", "dns-backends"))] + endpoints, _, _ := unstructured.NestedSlice(slice.Object, "endpoints") + endpoints[0].(map[string]any)["targetRef"] = map[string]any{"kind": "Pod", "name": newPod.GetName(), "namespace": newPod.GetNamespace(), "uid": string(newPod.GetUID())} + Expect(unstructured.SetNestedSlice(slice.Object, endpoints, "endpoints")).To(Succeed()) + result, err := r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + }) + DescribeTable("requires every completed-rollout counter", func(field string, value int64) { + _, _, f, _ := hostedDNSFixture() + object := f.management.objects[objectKey(ref("", "apps", "Deployment", "dpf-system", "dns-tenant"))] + Expect(unstructured.SetNestedField(object.Object, value, "status", field)).To(Succeed()) + ready, _, _ := Readiness(object) + Expect(ready).To(BeFalse()) + }, Entry("old replicas remain", "replicas", int64(2)), Entry("a current pod is not Ready", "readyReplicas", int64(0)), Entry("a replica is not updated", "updatedReplicas", int64(0)), Entry("a replica is not available", "availableReplicas", int64(0))) + DescribeTable("does not pass drifted, stale or missing hosted DNS chain inputs", func(change, reason string) { + b, g, f, v := hostedDNSFixture() + r := dnsTestRunner(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + hostSlice := f.management.objects[objectKey(ref("", "discovery.k8s.io", "EndpointSlice", "dpf-system", "dns-backends"))] + tenantSlice := f.tenant.objects[objectKey(ref("", "discovery.k8s.io", "EndpointSlice", "kube-system", "kube-dns"))] + switch change { + case "stale": + f.management.objects[objectKey(ref("", "apps", "Deployment", "dpf-system", "dns-tenant"))].SetGeneration(2) + case "vip": + endpoints, _, _ := unstructured.NestedSlice(tenantSlice.Object, "endpoints") + endpoints[0].(map[string]any)["addresses"] = []any{"192.0.2.99"} + Expect(unstructured.SetNestedSlice(tenantSlice.Object, endpoints, "endpoints")).To(Succeed()) + case "port": + ports, _, _ := unstructured.NestedSlice(tenantSlice.Object, "ports") + ports[0].(map[string]any)["port"] = int64(32099) + Expect(unstructured.SetNestedSlice(tenantSlice.Object, ports, "ports")).To(Succeed()) + case "extra-slice": + extra := tenantSlice.DeepCopy() + extra.SetName("foreign-dns") + f.tenant.put(extra) + case "foreign-service-owner": + owners := hostSlice.GetOwnerReferences() + owners[0].UID = "wrong-service" + hostSlice.SetOwnerReferences(owners) + case "foreign-pod": + endpoints, _, _ := unstructured.NestedSlice(hostSlice.Object, "endpoints") + endpoints[0].(map[string]any)["targetRef"].(map[string]any)["uid"] = "foreign-pod" + Expect(unstructured.SetNestedSlice(hostSlice.Object, endpoints, "endpoints")).To(Succeed()) + case "foreign-rs": + rs := f.management.objects[objectKey(ref("", "apps", "ReplicaSet", "dpf-system", "dns-rs"))] + owners := rs.GetOwnerReferences() + owners[0].UID = "foreign-deployment" + rs.SetOwnerReferences(owners) + case "missing-service": + delete(f.tenant.objects, objectKey(ref("", "", "Service", "kube-system", "kube-dns"))) + case "missing-host-service": + delete(f.management.objects, objectKey(ref("", "", "Service", "dpf-system", "dns-tenant"))) + case "endpoint-not-serving": + endpoints, _, _ := unstructured.NestedSlice(hostSlice.Object, "endpoints") + endpoints[0].(map[string]any)["conditions"].(map[string]any)["serving"] = false + Expect(unstructured.SetNestedSlice(hostSlice.Object, endpoints, "endpoints")).To(Succeed()) + case "keepalived-vip": + ds := f.management.objects[objectKey(ref("", "apps", "DaemonSet", "tenant-control", "tenant-keepalived"))] + containers, _, _ := unstructured.NestedSlice(ds.Object, "spec", "template", "spec", "initContainers") + containers[0].(map[string]any)["env"].([]any)[0].(map[string]any)["value"] = "192.0.2.99" + Expect(unstructured.SetNestedSlice(ds.Object, containers, "spec", "template", "spec", "initContainers")).To(Succeed()) + case "missing-pods": + delete(f.management.objects, objectKey(ref("", "", "Pod", "dpf-system", "dns-pod"))) + case "keepalived-owner": + ds := f.management.objects[objectKey(ref("", "apps", "DaemonSet", "tenant-control", "tenant-keepalived"))] + owners := ds.GetOwnerReferences() + owners[0].UID = "foreign-cluster" + ds.SetOwnerReferences(owners) + case "other-cluster-label": + dep := f.management.objects[objectKey(ref("", "apps", "Deployment", "dpf-system", "dns-tenant"))] + l := dep.GetLabels() + l["dpu.nvidia.com/cluster-namespace"] = "other-tenant" + dep.SetLabels(l) + } + before := len(f.management.creates) + result, err := r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring(reason))) + Expect(result.Outcome).To(Equal("failed")) + Expect(f.management.creates).To(HaveLen(before)) + Expect(f.tenant.creates).To(BeEmpty()) + }, Entry("stale management rollout", "stale", "DNS_HOST_ROLLOUT_NOT_READY"), Entry("VIP drift", "vip", "DNS_PUBLICATION_MISMATCH"), Entry("NodePort drift", "port", "DNS_PUBLICATION_MISMATCH"), Entry("additional foreign publication", "extra-slice", "DNS_PUBLICATION_MISMATCH"), Entry("stale Service UID", "foreign-service-owner", "DNS_BACKEND_BINDING_MISMATCH"), Entry("foreign Pod UID", "foreign-pod", "DNS_BACKEND_BINDING_MISMATCH"), Entry("foreign ReplicaSet ownership", "foreign-rs", "DNS_BACKEND_BINDING_MISMATCH"), Entry("missing tenant Service", "missing-service", "TENANT_DNS_NOT_READY"), Entry("no Ready backing pods", "missing-pods", "DNS_BACKEND_BINDING_MISMATCH"), Entry("foreign keepalived owner", "keepalived-owner", "DNS_KEEPALIVED_NOT_READY"), Entry("wrong tenant namespace label", "other-cluster-label", "DNS_HOST_RESOURCES_MISSING"), Entry("missing host Service", "missing-host-service", "DNS_HOST_RESOURCES_MISSING"), Entry("non-serving endpoint", "endpoint-not-serving", "DNS_BACKENDS_NOT_READY"), Entry("keepalived VIP drift", "keepalived-vip", "DNS_KEEPALIVED_BINDING_MISMATCH")) + It("reports denied EndpointSlice reads instead of interpreting inaccessible data as Ready", func() { + b, g, f, v := hostedDNSFixture() + r := dnsTestRunner(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + f.management.listErrors = map[string]error{"EndpointSlice": apierrors.NewForbidden(schema.GroupResource{Group: "discovery.k8s.io", Resource: "endpointslices"}, "", nil)} + result, err := r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("DNS_ACCESS_DENIED"))) + Expect(result.Outcome).To(Equal("failed")) + }) +}) + +var _ = Describe("controller canonical selected DPU node labels", func() { + It("adds only the three reserved labels without mutating frozen user intent", func() { + b, g, _, _ := lifecycleFixture() + set := resourcesInBundle(b, "DPUSet")[0].ObjectCopy() + expected, err := expectedDPUNodeLabels(set, "node-a", "dpf-system", g.Release) + Expect(err).NotTo(HaveOccurred()) + Expect(expected).To(HaveLen(4)) + Expect(expected["operator.dpu.nvidia.com/dpf-version"]).To(Equal("v26.8.0-rc.1")) + raw, _, _ := unstructured.NestedStringMap(set.Object, "spec", "dpuTemplate", "spec", "cluster", "nodeLabels") + Expect(raw).To(HaveLen(1)) + }) + It("rejects reserved labels that contradict frozen device intent", func() { + b, g, _, _ := lifecycleFixture() + set := resourcesInBundle(b, "DPUSet")[0].ObjectCopy() + Expect(unstructured.SetNestedField(set.Object, "foreign", "spec", "dpuTemplate", "spec", "cluster", "nodeLabels", "provisioning.dpu.nvidia.com/dpunode-name")).To(Succeed()) + _, err := expectedDPUNodeLabels(set, "node-a", "dpf-system", g.Release) + Expect(err).To(MatchError(ContainSubstring("DPU_NODE_LABEL_INTENT_CONFLICT"))) + }) + DescribeTable("rejects scope-changing or unreviewed label additions", func(key, value string) { + b, g, f, v := lifecycleFixture() + r := dnsTestRunner(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + dpu := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPU", "dpf-system", "dpu-child"))] + nodeLabels, _, _ := unstructured.NestedStringMap(dpu.Object, "spec", "cluster", "nodeLabels") + nodeLabels[key] = value + Expect(unstructured.SetNestedStringMap(dpu.Object, nodeLabels, "spec", "cluster", "nodeLabels")).To(Succeed()) + result, err := r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("DPU_NODE_SCOPE_MISMATCH"))) + Expect(result.Outcome).To(Equal("failed")) + }, Entry("scope label changes", "nvidia.kubernetes-launch-kit.dpf-instance", "foreign"), Entry("node association changes", "provisioning.dpu.nvidia.com/dpunode-name", "other-node"), Entry("namespace changes", "provisioning.dpu.nvidia.com/dpunode-namespace", "other-namespace"), Entry("release changes", "operator.dpu.nvidia.com/dpf-version", "v0.0.0"), Entry("extra workload label", "foreign.example/selected", "yes")) +}) diff --git a/pkg/dpf/deployment/dpucluster_contract_test.go b/pkg/dpf/deployment/dpucluster_contract_test.go new file mode 100644 index 00000000..2a44982f --- /dev/null +++ b/pkg/dpf/deployment/dpucluster_contract_test.go @@ -0,0 +1,91 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var _ = Describe("actual rc.1 and GA DPUCluster readiness contract", func() { + runnerFor := func(f *fakeFactory, v *fakeArtifacts) Runner { + dir, err := os.MkdirTemp("", "dpf-cluster-contract-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + return Runner{Clients: f, Artifacts: v, EvidenceDir: dir, PollInterval: time.Millisecond, Timeout: 10 * time.Second} + } + It("accepts source-realistic Ready clusters without observation fields and records the limitation", func() { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + cluster := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUCluster", "dpf-system", "tenant"))] + _, exists, _ := unstructured.NestedFieldNoCopy(cluster.Object, "status", "observedGeneration") + Expect(exists).To(BeFalse()) + conditions, _, _ := unstructured.NestedSlice(cluster.Object, "status", "conditions") + Expect(conditions[0].(map[string]any)).NotTo(HaveKey("observedGeneration")) + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(result.Checks).To(ContainElement(And(HaveField("State", "diagnostic"), HaveField("ReasonCode", "FRESHNESS_NOT_EXPOSED")))) + conditions[0].(map[string]any)["observedGeneration"] = int64(0) + Expect(unstructured.SetNestedSlice(cluster.Object, conditions, "status", "conditions")).To(Succeed()) + result, err = r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + }) + DescribeTable("requires both actual Ready phase and Ready condition", func(phase, status string) { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + cluster := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUCluster", "dpf-system", "tenant"))] + cluster.Object["status"] = map[string]any{"phase": phase, "conditions": []any{map[string]any{"type": "Ready", "status": status}}} + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("DPU_CLUSTER_NOT_READY"))) + Expect(f.management.creates).To(BeEmpty()) + }, Entry("Ready condition with Pending phase", "Pending", "True"), Entry("stale True while NotReady", "NotReady", "True"), Entry("Ready phase with false condition", "Ready", "False"), Entry("failed phase", "Failed", "True")) + DescribeTable("rejects in-run tenant spec drift before persistent DPUSet creation", func(when string) { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + cluster := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUCluster", "dpf-system", "tenant"))] + change := func() { + cluster.SetGeneration(cluster.GetGeneration() + 1) + Expect(unstructured.SetNestedField(cluster.Object, int64(2000), "spec", "maxNodes")).To(Succeed()) + } + if when == "cache" { + v.afterVerify = change + } else { + f.management.admit = func(object *unstructured.Unstructured, dryRun bool) { + if object.GetKind() == "DPUSet" && dryRun { + change() + } + } + } + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("DPU_CLUSTER_GENERATION_DRIFT"))) + Expect(result.Outcome).To(Equal("failed")) + Expect(v.calls).To(Equal(1)) + Expect(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", "dpf-system", "selected-set"))]).To(BeNil()) + if when == "cache" { + Expect(f.management.creates).To(Equal([]string{"BlueFieldSoftware/software"})) + } else { + Expect(f.management.dryRuns).To(Equal([]bool{false, true})) + } + }, Entry("after cache hashing", "cache"), Entry("during admission precheck", "admission")) + + It("retains direct tenant UID verification when controller freshness is unavailable", func() { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + tenantNamespace := f.tenant.objects[objectKey(ref("", "", "Namespace", "", "kube-system"))] + tenantNamespace.SetUID("replacement-tenant") + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("CLUSTER_IDENTITY_MISMATCH"))) + Expect(f.management.creates).To(BeEmpty()) + }) +}) diff --git a/pkg/dpf/deployment/engine.go b/pkg/dpf/deployment/engine.go new file mode 100644 index 00000000..978b319e --- /dev/null +++ b/pkg/dpf/deployment/engine.go @@ -0,0 +1,482 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/artifacts" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const bundleAnnotation = "nvidia.kubernetes-launch-kit.dpf-bundle-digest" +const desiredAnnotation = "nvidia.kubernetes-launch-kit.dpf-object-digest" + +type run struct { + runner Runner + result Result + bundle *bundle.Bundle + g config.Generation + clients map[bundle.Destination]Client + namespaceUIDs map[bundle.Destination]map[string]string + selectedNodes map[string]map[string]string + clusterGeneration int64 +} + +// Run operates only on a frozen bundle and verified explicit cluster bindings. +// DryRun uses Kubernetes server-side dry-run for native documents; projections +// are validated/waited separately and never submitted as independent resources. +func (r Runner) Run(ctx context.Context, b *bundle.Bundle, g config.Generation, mode Mode) (Result, error) { + if ctx == nil || b == nil { + return Result{}, fmt.Errorf("DPF lifecycle requires context and a validated bundle") + } + if mode != Deploy && mode != Validate && mode != DryRun { + return Result{}, fmt.Errorf("unsupported DPF lifecycle mode %q", mode) + } + if r.Timeout <= 0 { + r.Timeout = 45 * time.Minute + } + ctx, cancel := context.WithTimeout(ctx, r.Timeout) + defer cancel() + if r.EvidenceDir == "" { + return Result{}, fmt.Errorf("DPF lifecycle requires an explicit evidence directory outside the bundle") + } + if ctx.Err() != nil { + return Result{}, ctx.Err() + } + if err := os.MkdirAll(r.EvidenceDir, 0700); err != nil { + return Result{}, err + } + runDir, err := os.MkdirTemp(r.EvidenceDir, "dpf-"+string(mode)+"-") + if err != nil { + return Result{}, err + } + if r.Clients == nil { + r.Clients = NativeClients{} + } + if r.Artifacts == nil { + r.Artifacts = CachedArtifacts{} + } + if r.PollInterval <= 0 { + r.PollInterval = 2 * time.Second + } + s := &run{runner: r, bundle: b, g: g, result: Result{Mode: mode, BundleDigest: b.Index().BundleDigest, Outcome: "running", Checks: []Check{}, Events: []Event{}, ReportPath: filepath.Join(runDir, "report.json")}, clients: map[bundle.Destination]Client{}, namespaceUIDs: map[bundle.Destination]map[string]string{}} + if err := s.persist(); err != nil { + return s.result, err + } + err = s.execute(ctx) + if err == nil && mode == DryRun { + for _, check := range s.result.Checks { + if check.State == "unresolved" { + err = fail("DRY_RUN_INCOMPLETE", "one or more native API admission checks remain unresolved") + break + } + } + } + if ctx.Err() != nil { + err = ctx.Err() + } + if err != nil { + s.result.Outcome = "failed" + if reason(err) == "DRY_RUN_INCOMPLETE" { + s.result.Outcome = "incomplete" + } + if ctx.Err() != nil { + s.result.Outcome = "cancelled" + } + s.result.Checks = append(s.result.Checks, Check{ID: "execution", Subject: "bundle", Source: bundle.IndexPath, State: "failed", ReasonCode: reason(err), Message: err.Error()}) + } else { + s.result.Outcome = "complete" + } + if persistErr := s.persist(); persistErr != nil { + return s.result, fmt.Errorf("persist DPF lifecycle evidence: %w", persistErr) + } + return s.result, err +} + +func (s *run) persist() error { + data, err := json.MarshalIndent(s.result, "", " ") + if err != nil { + return err + } + return artifacts.AtomicWrite(s.result.ReportPath, append(data, '\n')) +} + +func (s *run) check(id, subject, source, state, code, message string) error { + s.result.Checks = append(s.result.Checks, Check{ID: id, Subject: subject, Source: source, State: state, ReasonCode: code, Message: message}) + return s.persist() +} + +func (s *run) event(stage, action string, reference bundle.ResourceRef, object *unstructured.Unstructured) error { + e := Event{At: time.Now().UTC(), Stage: stage, Action: action, Resource: reference} + if object != nil { + e.UID = string(object.GetUID()) + e.ResourceVersion = object.GetResourceVersion() + } + s.result.Events = append(s.result.Events, e) + return s.persist() +} + +func (s *run) execute(ctx context.Context) error { + i := s.bundle.Index() + if i.Profile != s.g.Profile || strings.TrimPrefix(i.Release, "v") != s.g.Release { + return fail("BUNDLE_INTENT_MISMATCH", "bundle profile/release differs from resolved intent") + } + want := map[string]bool{} + for _, binding := range s.g.Provisioning.Devices { + want[binding.ID] = true + } + if len(want) != len(i.SelectedDevices) { + return fail("BUNDLE_SCOPE_MISMATCH", "bundle selected-device count differs") + } + for _, id := range i.SelectedDevices { + if !want[id] { + return fail("BUNDLE_SCOPE_MISMATCH", "bundle selected-device identity differs") + } + } + management, err := s.runner.Clients.Management(ctx, s.g) + if err != nil { + return fail("MANAGEMENT_ACCESS_FAILED", err.Error()) + } + s.clients[bundle.Management] = management + if err = s.identity(ctx, management, s.g.ManagementUID, "management"); err != nil { + return err + } + if err = s.management(ctx); err != nil { + return err + } + if err = s.cluster(ctx); err != nil { + return err + } + tenant, err := s.runner.Clients.Tenant(ctx, management, s.g.Provisioning.DPUCluster) + if err != nil { + return fail("TENANT_ACCESS_FAILED", err.Error()) + } + s.clients[bundle.DPU] = tenant + if err = s.identity(ctx, tenant, s.g.Provisioning.DPUCluster.ClusterUID, "tenant"); err != nil { + return err + } + if err = s.selection(ctx); err != nil { + return err + } + if err = s.externalSecrets(ctx); err != nil { + return err + } + if s.result.Mode == DryRun { + if err := s.check("runtime-readiness", "tenant DNS and selected DPU Nodes", "runtime", "not-evaluated", "RUNTIME_READINESS_NOT_EVALUATED", "server-side admission only; workload readiness requires persistent provisioning"); err != nil { + return err + } + } + for _, stage := range i.Stages { + if err := ctx.Err(); err != nil { + return err + } + for _, id := range stage.Requires { + switch id { + case "existing-management", "existing-tenant", "enrolled-devices", "selected-scope", "external-transition-complete", "existing-secrets": + case "software-integrity": // Enforced immediately before every DPUSet below. + default: + return fail("EXTERNAL_PREREQUISITE_UNSUPPORTED", "unknown prerequisite "+id) + } + } + view, _ := s.bundle.Stage(stage.ID) + for _, doc := range view.Documents() { + if err := s.document(ctx, stage, doc); err != nil { + return err + } + } + if s.result.Mode != DryRun { + for _, doc := range append(view.Documents(), view.Projections()...) { + if err := s.wait(ctx, stage.ID, doc); err != nil { + return err + } + } + for _, doc := range view.Documents() { + if doc.Ref().Kind == "DPUSet" { + if err := s.selectedReady(ctx); err != nil { + return err + } + if err := s.baseWorkloads(ctx); err != nil { + return err + } + break + } + } + } + } + if s.result.Mode != DryRun { + if err := s.selectedReady(ctx); err != nil { + return err + } + if err := s.baseWorkloads(ctx); err != nil { + return err + } + // DNS may take time to converge; sample node transitions again afterward. + return s.selectedReady(ctx) + } + return nil +} + +func (s *run) identity(ctx context.Context, c Client, expected, subject string) error { + if expected == "" { + return fail("CLUSTER_IDENTITY_MISSING", subject+" kube-system UID is unresolved") + } + object, err := c.Get(ctx, ref("", "", "Namespace", "", "kube-system")) + if err != nil { + return fail("CLUSTER_IDENTITY_UNAVAILABLE", subject+" kube-system cannot be read") + } + if string(object.GetUID()) != expected { + return fail("CLUSTER_IDENTITY_MISMATCH", subject+" kube-system UID differs") + } + expectedVersion := s.g.ManagementKubernetesVersion + if subject == "tenant" || subject == string(bundle.DPU) { + expectedVersion = s.g.Provisioning.DPUCluster.KubernetesVersion + } + version, err := c.ServerVersion(ctx) + if err != nil || expectedVersion == "" { + return fail("KUBERNETES_VERSION_UNOBSERVED", subject+" API server version must be bound") + } + if strings.TrimPrefix(version, "v") != strings.TrimPrefix(expectedVersion, "v") { + return fail("KUBERNETES_VERSION_DRIFT", subject+" API server version differs from frozen Helm projection; regenerate") + } + return s.check("cluster-identity", subject, "kubernetes:Namespace/kube-system", "passed", "IDENTITY_MATCH", "") +} + +func (s *run) document(ctx context.Context, stage bundle.Stage, doc bundle.Document) error { + c := s.clients[stage.Destination] + object := doc.ObjectCopy() + reference := doc.Ref() + if reference.Kind == "DPUDevice" || reference.Kind == "DPUNode" || reference.Kind == "DPUDiscovery" || reference.Kind == "DPFOperatorConfig" || reference.Kind == "DPUCluster" { + return fail("EXTERNAL_OWNERSHIP_REQUIRED", "first lifecycle slice cannot create enrollment/operator/tenant resources") + } + if _, ok := nativeBody(s.g.Release, object); !ok { + return fail("NATIVE_KIND_UNSUPPORTED", fmt.Sprintf("no native comparison contract for %s %s", object.GetAPIVersion(), object.GetKind())) + } + current, err := c.Get(ctx, reference) + if err == nil { + if !owns(current, object, s.result.BundleDigest, s.g.Release) { + return fail("RESOURCE_OWNERSHIP_CONFLICT", fmt.Sprintf("existing %s/%s is not this bundle's unchanged object", reference.Kind, reference.Name)) + } + return s.event(stage.ID, "unchanged", reference, current) + } + if !apierrors.IsNotFound(err) { + if s.result.Mode == DryRun && meta.IsNoMatchError(err) { + return s.check("admission", reference.Kind+"/"+reference.Name, doc.Source().File, "unresolved", "API_NOT_SERVED", "server-side dry-run deferred until the declared API exists") + } + return fail("RESOURCE_READ_FAILED", fmt.Sprintf("cannot inspect %s/%s", reference.Kind, reference.Name)) + } + if s.result.Mode == Validate { + if s.runner.Wait { + return nil + } + return fail("RESOURCE_MISSING", fmt.Sprintf("desired %s/%s is absent", reference.Kind, reference.Name)) + } + if reference.Kind == "DPUSet" && s.result.Mode != DryRun { + if err := s.selection(ctx); err != nil { + return err + } + if err := s.provisioningGate(ctx, object); err != nil { + return err + } + } + marked := mark(object, s.result.BundleDigest) + expectedUID := s.g.ManagementUID + if stage.Destination == bundle.DPU { + expectedUID = s.g.Provisioning.DPUCluster.ClusterUID + } + if err := s.identity(ctx, c, expectedUID, string(stage.Destination)); err != nil { + return err + } + if reference.Namespace != "" { + if pinned := s.namespaceUIDs[stage.Destination][reference.Namespace]; pinned != "" { + namespace, err := c.Get(ctx, ref(stage.Destination, "", "Namespace", "", reference.Namespace)) + if err != nil || string(namespace.GetUID()) != pinned { + return fail("NAMESPACE_IDENTITY_MISMATCH", "namespace changed before mutation") + } + } + } + if s.result.Mode == Deploy && len(s.selectedNodes) > 0 { + if _, err := s.tenantScope(ctx, s.selectedNodes); err != nil { + return err + } + clear, diagnostic, err := s.selectedTransitionsClear(ctx) + if err != nil { + return err + } + if !clear { + return fail("NODE_TRANSITION_PENDING", diagnostic) + } + } + if reference.Kind == "DPUSet" && s.result.Mode == Deploy { + // Admission may introduce additional physical settings. Inspect the + // admitted spec without persistence before requesting a real DPUSet. + if err := s.event(stage.ID, "admission-precheck-requested", reference, nil); err != nil { + return err + } + admitted, err := c.Create(ctx, marked, true) + if err != nil { + return fail("ADMISSION_PRECHECK_FAILED", fmt.Sprintf("server-side admission precheck rejected %s/%s: %s", reference.Kind, reference.Name, reason(err))) + } + if !nativeMatches(s.g.Release, object, admitted) { + return fail("ADMISSION_CHANGED_DESIRED_STATE", "admission introduced unexpected DPUSet desired state before persistent creation") + } + if err := s.event(stage.ID, "admission-precheck-passed", reference, admitted); err != nil { + return err + } + // Neither cache verification nor API admission is instantaneous. + // Retain the DPUCluster spec snapshot at the physical create boundary. + if err := s.cluster(ctx); err != nil { + return err + } + } + action := "create-requested" + if s.result.Mode == DryRun { + action = "dry-run-requested" + } + if err := s.event(stage.ID, action, reference, nil); err != nil { + return err + } + created, err := c.Create(ctx, marked, s.result.Mode == DryRun) + if err != nil { + if s.result.Mode == DryRun && apierrors.IsNotFound(err) { + return s.check("admission", reference.Kind+"/"+reference.Name, doc.Source().File, "unresolved", "ADMISSION_PREREQUISITE_UNRESOLVED", "server-side dry-run requires an API or namespace that is not yet served") + } + return fail("RESOURCE_CREATE_FAILED", fmt.Sprintf("API rejected %s/%s: %s", reference.Kind, reference.Name, reason(err))) + } + if !nativeMatches(s.g.Release, object, created) { + return fail("ADMISSION_CHANGED_DESIRED_STATE", fmt.Sprintf("admission changed supplied fields on %s/%s", reference.Kind, reference.Name)) + } + action = "created" + if s.result.Mode == DryRun { + action = "dry-run-admitted" + } + return s.event(stage.ID, action, reference, created) +} + +func (s *run) wait(ctx context.Context, stage string, doc bundle.Document) error { + for { + object, err := s.clients[doc.Ref().Destination].Get(ctx, doc.Ref()) + if err != nil && !apierrors.IsNotFound(err) { + return fail("READINESS_READ_FAILED", fmt.Sprintf("cannot read %s/%s readiness", doc.Ref().Kind, doc.Ref().Name)) + } + if object != nil { + if doc.IsProjection() { + if !desiredMatches(doc.ObjectCopy(), object) { + return fail("PROJECTION_DRIFT", doc.Ref().Kind+"/"+doc.Ref().Name+" differs from its frozen workload projection") + } + } else if !owns(object, doc.ObjectCopy(), s.result.BundleDigest, s.g.Release) { + return fail("RESOURCE_OWNERSHIP_CONFLICT", doc.Ref().Kind+"/"+doc.Ref().Name+" changed while waiting") + } + } + ready, terminal, message := Readiness(object) + if ready && doc.IsProjection() && doc.Ref().Destination == bundle.DPU && doc.Ref().Kind == "DaemonSet" { + ready, message, err = s.daemonSetCoverage(ctx, object) + if err != nil { + return err + } + } + if ready { + return s.check("readiness", doc.Ref().Kind+"/"+doc.Ref().Name, doc.Source().File, "passed", "READY", message) + } + if terminal { + return fail("RESOURCE_NOT_READY", message) + } + if doc.Ref().Kind == "DPUSet" { + diagnostics, err := s.selectedTransitionDiagnostics(ctx) + if err != nil { + return err + } + message += "; " + diagnostics + } + if s.result.Mode == Validate && !s.runner.Wait { + return fail("RESOURCE_NOT_READY", doc.Ref().Kind+"/"+doc.Ref().Name+": "+message) + } + if err := s.check("readiness", doc.Ref().Kind+"/"+doc.Ref().Name, doc.Source().File, "pending", "WAITING_FOR_RECONCILIATION", message); err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(s.runner.PollInterval): + } + } +} + +func mark(object *unstructured.Unstructured, digest string) *unstructured.Unstructured { + copy := object.DeepCopy() + annotations := copy.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[bundleAnnotation] = digest + annotations[desiredAnnotation] = objectDigest(object) + copy.SetAnnotations(annotations) + return copy +} + +func objectDigest(object *unstructured.Unstructured) string { + data, _ := json.Marshal(object.Object) + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func owns(current, desired *unstructured.Unstructured, digest, release string) bool { + return current != nil && desired != nil && current.GetAnnotations()[bundleAnnotation] == digest && current.GetAnnotations()[desiredAnnotation] == objectDigest(desired) && nativeMatches(release, desired, current) +} + +type failure struct{ code, message string } + +func (e *failure) Error() string { return e.code + ": " + e.message } +func fail(code, message string) error { return &failure{code: code, message: message} } +func reason(err error) string { + if f, ok := err.(*failure); ok { + return f.code + } + if apierrors.IsForbidden(err) { + return "ACCESS_DENIED" + } + if apierrors.IsAlreadyExists(err) { + return "ALREADY_EXISTS" + } + if apierrors.IsNotFound(err) { + return "NOT_FOUND" + } + if err == context.Canceled { + return "CANCELLED" + } + if err == context.DeadlineExceeded { + return "TIMEOUT" + } + return "OPERATION_FAILED" +} + +func ref(destination bundle.Destination, group, kind, namespace, name string) bundle.ResourceRef { + return bundle.ResourceRef{Destination: destination, Group: group, Kind: kind, Namespace: namespace, Name: name} +} +func text(o *unstructured.Unstructured, fields ...string) string { + if o == nil { + return "" + } + value, _, _ := unstructured.NestedString(o.Object, fields...) + return value +} +func number(o *unstructured.Unstructured, fields ...string) int64 { + if o == nil { + return 0 + } + value, _, _ := unstructured.NestedInt64(o.Object, fields...) + return value +} diff --git a/pkg/dpf/deployment/engine_test.go b/pkg/dpf/deployment/engine_test.go new file mode 100644 index 00000000..24d69a4e --- /dev/null +++ b/pkg/dpf/deployment/engine_test.go @@ -0,0 +1,564 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/yaml" +) + +func TestDeployment(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DPF existing infrastructure lifecycle") +} + +type fakeClient struct { + objects map[string]*unstructured.Unstructured + creates []string + dryRuns []bool + bmcErr error + secretErr error + tenant *fakeClient + noChildren bool + version string + tenantBinding config.ClusterBinding + listErrors map[string]error + afterCreate func(*unstructured.Unstructured) + onGet func(bundle.ResourceRef) + onList func(bundle.ResourceRef) + getErrors map[string]error + admit func(*unstructured.Unstructured, bool) +} + +func (f *fakeClient) ServerVersion(context.Context) (string, error) { return f.version, nil } + +func objectKey(r bundle.ResourceRef) string { + return r.Group + "/" + r.Kind + "/" + r.Namespace + "/" + r.Name +} +func keyFor(o *unstructured.Unstructured) string { + gvk := o.GroupVersionKind() + return objectKey(ref("", gvk.Group, gvk.Kind, o.GetNamespace(), o.GetName())) +} +func (f *fakeClient) put(o *unstructured.Unstructured) { f.objects[keyFor(o)] = o.DeepCopy() } +func (f *fakeClient) Get(_ context.Context, r bundle.ResourceRef) (*unstructured.Unstructured, error) { + if f.onGet != nil { + f.onGet(r) + } + if err := f.getErrors[r.Kind]; err != nil { + return nil, err + } + if o := f.objects[objectKey(r)]; o != nil { + return o.DeepCopy(), nil + } + return nil, apierrors.NewNotFound(schema.GroupResource{Group: r.Group, Resource: r.Kind}, r.Name) +} +func (f *fakeClient) List(_ context.Context, r bundle.ResourceRef, selector string) ([]*unstructured.Unstructured, error) { + if f.onList != nil { + f.onList(r) + } + if err := f.listErrors[r.Kind]; err != nil { + return nil, err + } + parsed, err := labels.Parse(selector) + if err != nil { + return nil, err + } + out := []*unstructured.Unstructured{} + for _, o := range f.objects { + if o.GroupVersionKind().Group == r.Group && o.GetKind() == r.Kind && (r.Namespace == "" || o.GetNamespace() == r.Namespace) && parsed.Matches(labels.Set(o.GetLabels())) { + out = append(out, o.DeepCopy()) + } + } + return out, nil +} +func (f *fakeClient) Create(_ context.Context, o *unstructured.Unstructured, dryRun bool) (*unstructured.Unstructured, error) { + f.creates = append(f.creates, o.GetKind()+"/"+o.GetName()) + f.dryRuns = append(f.dryRuns, dryRun) + created := o.DeepCopy() + created.SetUID(types.UID("created-" + o.GetName())) + created.SetGeneration(1) + created.SetResourceVersion("1") + if f.admit != nil { + f.admit(created, dryRun) + } + if !dryRun { + setReady(created) + if created.GetKind() == "BlueFieldSoftware" { + _ = unstructured.SetNestedField(created.Object, "Ready", "status", "phase") + } + f.put(created) + if f.afterCreate != nil { + f.afterCreate(created) + } + if created.GetKind() == "DPUSet" && !f.noChildren { + dpu := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPU", o.GetNamespace(), "dpu-child", map[string]any{"dpuDeviceName": "device-a", "blueFieldSoftware": "software", "cluster": map[string]any{"name": f.tenantBinding.Name, "namespace": f.tenantBinding.Namespace}}) + scope, _, _ := unstructured.NestedStringMap(o.Object, "spec", "dpuTemplate", "spec", "cluster", "nodeLabels") + // Source-realistic augmentation from buildDPUClusterNodeLabels. + scope["provisioning.dpu.nvidia.com/dpunode-name"] = "node-a" + scope["provisioning.dpu.nvidia.com/dpunode-namespace"] = o.GetNamespace() + scope["operator.dpu.nvidia.com/dpf-version"] = "v26.8.0-rc.1" + _ = unstructured.SetNestedStringMap(dpu.Object, scope, "spec", "cluster", "nodeLabels") + if flavor := text(o, "spec", "dpuTemplate", "spec", "dpuFlavor"); flavor != "" { + _ = unstructured.SetNestedField(dpu.Object, flavor, "spec", "dpuFlavor") + } + dpu.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: "provisioning.dpu.nvidia.com/v1alpha1", Kind: "DPUSet", Name: o.GetName(), UID: created.GetUID()}}) + setReady(dpu) + _ = unstructured.SetNestedField(dpu.Object, "Ready", "status", "phase") + f.put(dpu) + node := testObject("v1", "Node", "", "dpu-child", nil) + node.SetLabels(scope) + setReady(node) + f.tenant.put(node) + } + } + return created, nil +} +func (f *fakeClient) SecretKeys(context.Context, string, string, []string) error { return f.secretErr } +func (f *fakeClient) VerifyBMC(context.Context, config.Generation, config.DeviceBinding, inventory.Device, string) error { + return f.bmcErr +} + +type fakeFactory struct { + management, tenant *fakeClient + opens int +} + +func (f *fakeFactory) Management(context.Context, config.Generation) (Client, error) { + f.opens++ + return f.management, nil +} +func (f *fakeFactory) Tenant(_ context.Context, _ Client, binding config.ClusterBinding) (Client, error) { + f.management.tenantBinding = binding + return f.tenant, nil +} + +type fakeArtifacts struct { + calls int + err error + afterVerify func() +} + +func (f *fakeArtifacts) Cached(context.Context, Client, config.Generation, *unstructured.Unstructured) error { + f.calls++ + if f.afterVerify != nil { + f.afterVerify() + } + return f.err +} + +func testObject(api, kind, ns, name string, spec map[string]any) *unstructured.Unstructured { + o := &unstructured.Unstructured{Object: map[string]any{"apiVersion": api, "kind": kind, "metadata": map[string]any{"name": name, "namespace": ns}}} + if ns == "" { + o.SetNamespace("") + } + if spec != nil { + o.Object["spec"] = spec + } + o.SetGeneration(1) + o.SetUID(types.UID(name + "-uid")) + o.SetResourceVersion("1") + return o +} +func setReady(o *unstructured.Unstructured) { + o.Object["status"] = map[string]any{"observedGeneration": o.GetGeneration(), "conditions": []any{map[string]any{"type": "Ready", "status": "True", "observedGeneration": o.GetGeneration()}}, "phase": "Ready"} +} +func readyDeployment(ns, name string) *unstructured.Unstructured { + o := testObject("apps/v1", "Deployment", ns, name, map[string]any{"replicas": int64(1)}) + setReady(o) + _ = unstructured.SetNestedField(o.Object, int64(1), "status", "replicas") + _ = unstructured.SetNestedField(o.Object, int64(1), "status", "readyReplicas") + _ = unstructured.SetNestedField(o.Object, int64(1), "status", "updatedReplicas") + _ = unstructured.SetNestedField(o.Object, int64(1), "status", "availableReplicas") + return o +} + +func lifecycleFixture() (*bundle.Bundle, config.Generation, *fakeFactory, *fakeArtifacts) { + namespace := "sentry-system" + g := config.Generation{Name: "fixture", Profile: "sentry-bf4", Release: "26.8.0-rc.1", Namespace: "dpf-system", ManagementUID: "management-uid", KubeconfigRef: config.FileRef{File: "unread-by-fake"}, Context: "explicit", Provisioning: config.Provisioning{Devices: []config.DeviceBinding{{ID: "physical-a", DPUDevice: "device-a", DPUNode: "node-a"}}, DPUCluster: config.ClusterBinding{Name: "tenant", Namespace: "dpf-system", ClusterUID: "tenant-uid", ResourceUID: "tenant-resource-uid", Selector: map[string]string{"tenant": "first"}}}, Sentry: config.Sentry{Namespace: &namespace, ImagePullSecret: "pull", MITMSecret: "mitm", InferenceSecret: "inference"}, Devices: []inventory.Device{{ID: "physical-a", SerialNumber: "SERIAL-A", BMCEndpoint: "https://192.0.2.10"}}, DevicePSIDs: map[string]string{"physical-a": "TEST-PSID"}} + g.ManagementKubernetesVersion = "1.35.4" + g.Provisioning.DPUCluster.KubernetesVersion = "1.35.6" + management := &fakeClient{objects: map[string]*unstructured.Unstructured{}, version: "v1.35.4"} + tenant := &fakeClient{objects: map[string]*unstructured.Unstructured{}, version: "v1.35.6"} + management.tenant = tenant + for _, entry := range []struct { + client *fakeClient + uid string + }{{management, "management-uid"}, {tenant, "tenant-uid"}} { + ns := testObject("v1", "Namespace", "", "kube-system", nil) + ns.SetUID(types.UID(entry.uid)) + entry.client.put(ns) + ns = testObject("v1", "Namespace", "", namespace, nil) + setReady(ns) + _ = unstructured.SetNestedField(ns.Object, "Active", "status", "phase") + entry.client.put(ns) + } + operator := testObject("operator.dpu.nvidia.com/v1alpha1", "DPFOperatorConfig", "dpf-system", "operator", map[string]any{"deploymentMode": "zero-trust", "provisioningController": map[string]any{"installInterface": map[string]any{"installViaRedfish": map[string]any{}}}}) + operator.Object["status"] = map[string]any{"version": "v26.8.0-rc.1"} + management.put(operator) + for _, name := range []string{"dpf-operator-controller-manager", "dpf-provisioning-controller-manager", "dpuservice-controller-manager", "kamaji-cluster-manager"} { + deployment := readyDeployment("dpf-system", name) + deployment.SetLabels(map[string]string{"dpu.nvidia.com/component": name}) + management.put(deployment) + } + dns := readyDeployment("kube-system", "coredns") + dns.SetLabels(map[string]string{"k8s-app": "kube-dns"}) + tenant.put(dns) + for _, component := range []string{"cni-installer", "multus", "sriov-device-plugin", "sfc-controller", "nvidia-k8s-ipam", "flannel", "kube-proxy"} { + ns := "dpf-system" + podLabels := map[string]string{"dpu.nvidia.com/component": component} + if component == "flannel" { + podLabels = map[string]string{"app": "flannel", "tier": "node"} + } + if component == "kube-proxy" { + ns = "kube-system" + podLabels = map[string]string{"k8s-app": "kube-proxy"} + } + pod := testObject("v1", "Pod", ns, "base-"+component, map[string]any{"nodeName": "dpu-child"}) + pod.SetLabels(podLabels) + setReady(pod) + _ = unstructured.SetNestedField(pod.Object, "Running", "status", "phase") + tenant.put(pod) + } + cluster := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUCluster", "dpf-system", "tenant", map[string]any{"type": "kamaji", "kubeconfig": "tenant-kubeconfig"}) + cluster.SetUID("tenant-resource-uid") + cluster.SetLabels(map[string]string{"tenant": "first"}) + // Actual rc.1/GA DPUCluster controller uses cutil.NewCondition, which + // publishes neither top-level nor per-condition observedGeneration. + cluster.Object["status"] = map[string]any{"phase": "Ready", "version": "v1.35.6", "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "HealthCheckPassed"}}} + management.put(cluster) + device := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUDevice", "dpf-system", "device-a", map[string]any{"serialNumber": "SERIAL-A", "bmcIp": "192.0.2.10"}) + device.SetLabels(map[string]string{"provisioning.dpu.nvidia.com/dpunode-name": "node-a", "provisioning.dpu.nvidia.com/dpudevice-bmc-ip": "192.0.2.10"}) + setReady(device) + _ = unstructured.SetNestedField(device.Object, "TEST-PSID", "status", "psid") + for _, name := range []string{"NodeAttached", "Initialized", "Discovered"} { + conditions, _, _ := unstructured.NestedSlice(device.Object, "status", "conditions") + conditions = append(conditions, map[string]any{"type": name, "status": "True"}) + _ = unstructured.SetNestedSlice(device.Object, conditions, "status", "conditions") + } + management.put(device) + node := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUNode", "dpf-system", "node-a", map[string]any{"dpus": []any{map[string]any{"name": "device-a"}}, "nodeRebootMethod": map[string]any{"external": map[string]any{}}}) + setReady(node) + _ = unstructured.SetNestedField(node.Object, "redfish", "status", "dpuInstallInterface") + management.put(node) + software := testObject("provisioning.dpu.nvidia.com/v1alpha1", "BlueFieldSoftware", "dpf-system", "software", map[string]any{"osIso": "https://example.invalid/image.iso"}) + software.SetUID("") + software.SetResourceVersion("") + software.SetGeneration(0) + set := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUSet", "dpf-system", "selected-set", map[string]any{"dpuDeviceSelector": map[string]any{"matchLabels": map[string]any{"provisioning.dpu.nvidia.com/dpunode-name": "node-a", "provisioning.dpu.nvidia.com/dpudevice-bmc-ip": "192.0.2.10"}}, "dpuTemplate": map[string]any{"spec": map[string]any{"blueFieldSoftware": map[string]any{"name": "software"}, "cluster": map[string]any{"nodeLabels": map[string]any{"nvidia.kubernetes-launch-kit.dpf-instance": "fixture"}}}}}) + set.SetUID("") + set.SetResourceVersion("") + set.SetGeneration(0) + files := []bundle.File{{Path: bundle.ResolvedConfigPath, Data: []byte("formatVersion: v1alpha1\ntarget: dpf\nconfig: {}\n")}} + records := []bundle.FileRecord{{Path: bundle.ResolvedConfigPath, Role: bundle.ResolvedConfig}} + for i, o := range []*unstructured.Unstructured{software, set} { + data, _ := yaml.Marshal(o.Object) + filename := fmt.Sprintf("objects/%d.yaml", i) + files = append(files, bundle.File{Path: filename, Data: data}) + records = append(records, bundle.FileRecord{Path: filename, Role: bundle.Manifest}) + } + b, err := bundle.New(bundle.Index{FormatVersion: bundle.FormatVersion, Target: bundle.Target, Phase: bundle.Phase, Profile: g.Profile, Release: g.Release, Renderer: "fixture", SelectedDevices: []string{"physical-a"}, Files: records, Stages: []bundle.Stage{{ID: "software", Destination: bundle.Management, Namespace: "dpf-system", Files: []string{"objects/0.yaml"}}, {ID: "provision", Destination: bundle.Management, Namespace: "dpf-system", Files: []string{"objects/1.yaml"}, DependsOn: []string{"software"}}}}, files) + Expect(err).NotTo(HaveOccurred()) + return b, g, &fakeFactory{management: management, tenant: tenant}, &fakeArtifacts{} +} + +var _ = Describe("existing infrastructure native lifecycle", func() { + var dir string + var b *bundle.Bundle + var g config.Generation + var factory *fakeFactory + var verifier *fakeArtifacts + var runner Runner + BeforeEach(func() { + var err error + dir, err = os.MkdirTemp("", "dpf-lifecycle-") + Expect(err).NotTo(HaveOccurred()) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + b, g, factory, verifier = lifecycleFixture() + runner = Runner{Clients: factory, Artifacts: verifier, EvidenceDir: dir, PollInterval: time.Millisecond, Timeout: 10 * time.Second} + }) + It("creates only native stages after all gates and persists recovery evidence", func() { + result, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(verifier.calls).To(Equal(1)) + Expect(factory.management.creates).To(Equal([]string{"BlueFieldSoftware/software", "DPUSet/selected-set", "DPUSet/selected-set"})) + Expect(factory.tenant.creates).To(BeEmpty()) + data, err := os.ReadFile(result.ReportPath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring("create-requested")) + Expect(string(data)).To(ContainSubstring("GLOBAL_OPERATOR_NOT_READY")) + Expect(string(data)).NotTo(ContainSubstring("unread-by-fake")) + _, err = runner.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(factory.management.creates).To(HaveLen(3)) + }) + It("rejects API server version drift before writes", func() { + factory.tenant.version = "v1.36.0" + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("KUBERNETES_VERSION_DRIFT"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("rejects management UID mismatch before any mutation", func() { + g.ManagementUID = "elsewhere" + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("CLUSTER_IDENTITY_MISMATCH"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("does not let an unhealthy unselected sibling block healthy selected-node networking", func() { + pod := testObject("v1", "Pod", "dpf-system", "sibling-multus", map[string]any{"nodeName": "unselected-sibling"}) + pod.SetLabels(map[string]string{"dpu.nvidia.com/component": "multus"}) + factory.tenant.put(pod) + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + }) + It("does not pass a selected node with a missing networking component", func() { + delete(factory.tenant.objects, objectKey(ref("", "", "Pod", "dpf-system", "base-multus"))) + runner.Timeout = 40 * time.Millisecond + result, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(context.DeadlineExceeded)) + Expect(result.Outcome).To(Equal("cancelled")) + }) + It("rejects existing foreign Sentry controllers in the dedicated namespace before writes", func() { + factory.tenant.put(readyDeployment("sentry-system", "foreign-policy-controller")) + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("FOREIGN_SERVICE_WORKLOAD"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("waits for a chart projection without applying it as an independent object", func() { + projection := readyDeployment("sentry-system", "projected-workload") + desired := projection.DeepCopy() + delete(desired.Object, "status") + desired.SetUID("") + desired.SetResourceVersion("") + desired.SetGeneration(0) + data, err := yaml.Marshal(desired.Object) + Expect(err).NotTo(HaveOccurred()) + index := b.Index() + index.BundleDigest = "" + index.Files = append(index.Files, bundle.FileRecord{Path: ".l8k/helm/chart/projection.yaml", Role: bundle.Projection}) + index.Stages = append(index.Stages, bundle.Stage{ID: "projection", Destination: bundle.DPU, Namespace: "sentry-system", ProjectionFiles: []string{".l8k/helm/chart/projection.yaml"}, DependsOn: []string{"provision"}}) + files := b.Files() + files = append(files, bundle.File{Path: ".l8k/helm/chart/projection.yaml", Data: data}) + b, err = bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + factory.management.afterCreate = func(o *unstructured.Unstructured) { + if o.GetKind() == "DPUSet" { + factory.tenant.put(projection) + } + } + _, err = runner.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(factory.tenant.creates).To(BeEmpty()) + }) + It("rejects tenant UID mismatch before any mutation", func() { + g.Provisioning.DPUCluster.ClusterUID = "elsewhere" + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("CLUSTER_IDENTITY_MISMATCH"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("requires the actual service-controller role rather than three similarly named Deployments", func() { + delete(factory.management.objects, objectKey(ref("", "apps", "Deployment", "dpf-system", "dpuservice-controller-manager"))) + factory.management.put(readyDeployment("dpf-system", "dpf-unrelated-controller-manager")) + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("MANAGEMENT_CONTROLLER_MISSING"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("rejects an extra selector-matching tenant in a different namespace", func() { + other := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUCluster", "other-system", "other", nil) + other.SetLabels(map[string]string{"tenant": "first"}) + setReady(other) + factory.management.put(other) + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("DPU_CLUSTER_SELECTION_MISMATCH"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("requires a healthy Kamaji cluster manager for a Kamaji tenant", func() { + controller := factory.management.objects[objectKey(ref("", "apps", "Deployment", "dpf-system", "kamaji-cluster-manager"))] + _ = unstructured.SetNestedField(controller.Object, int64(0), "status", "availableReplicas") + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("KAMAJI_CONTROLLER_NOT_READY"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("rejects an extra co-resident DPU before any mutation", func() { + key := objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", "dpf-system", "node-a")) + _ = unstructured.SetNestedSlice(factory.management.objects[key].Object, []any{map[string]any{"name": "device-a"}, map[string]any{"name": "device-unselected"}}, "spec", "dpus") + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("EFFECT_DOMAIN_MISMATCH"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("rejects foreign selected DPU ownership and selectors", func() { + foreign := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUSet", "dpf-system", "foreign", map[string]any{"dpuDeviceSelector": map[string]any{}}) + factory.management.put(foreign) + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("FOREIGN_DPUSET_SELECTOR"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("rejects an orphan child before recreating a missing provisioning owner", func() { + dpu := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPU", "dpf-system", "orphan", map[string]any{"dpuDeviceName": "device-a"}) + dpu.SetOwnerReferences([]metav1.OwnerReference{{Kind: "DPUSet", Name: "selected-set", UID: "old-owner-uid"}}) + factory.management.put(dpu) + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("STALE_DPU_OWNER"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("prevents DPUSet creation when actual cached artifact bytes do not match", func() { + verifier.err = fmt.Errorf("cached payload hash mismatch") + result, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("SOFTWARE_INTEGRITY_FAILED"))) + Expect(factory.management.creates).To(Equal([]string{"BlueFieldSoftware/software"})) + Expect(result.Events).NotTo(BeEmpty()) + Expect(result.ReportPath).To(BeAnExistingFile()) + }) + It("does not adopt an existing unmarked object even if its desired fields match", func() { + view, _ := b.Stage("software") + factory.management.put(view.Documents()[0].ObjectCopy()) + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("RESOURCE_OWNERSHIP_CONFLICT"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("performs only server-side dry-run creates without artifact checks or persistent resources", func() { + result, err := runner.Run(context.Background(), b, g, DryRun) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(verifier.calls).To(BeZero()) + Expect(factory.management.dryRuns).To(Equal([]bool{true, true})) + _, err = factory.management.Get(context.Background(), ref("", "provisioning.dpu.nvidia.com", "DPUSet", "dpf-system", "selected-set")) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + It("validates without writes and reports missing desired resources", func() { + _, err := runner.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("RESOURCE_MISSING"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("does not poll validation when waiting was not requested", func() { + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + software := factory.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "BlueFieldSoftware", "dpf-system", "software"))] + _ = unstructured.SetNestedField(software.Object, "Downloading", "status", "phase") + _ = unstructured.SetNestedSlice(software.Object, []any{map[string]any{"type": "Ready", "status": "False"}}, "status", "conditions") + reads := 0 + factory.management.onGet = func(r bundle.ResourceRef) { + if r.Kind == "BlueFieldSoftware" { + reads++ + } + } + _, err = runner.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("RESOURCE_NOT_READY"))) + Expect(reads).To(Equal(2)) + Expect(factory.management.creates).To(HaveLen(3)) + }) + It("polls requested validation against refreshed live state without writing", func() { + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + software := factory.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "BlueFieldSoftware", "dpf-system", "software"))] + _ = unstructured.SetNestedField(software.Object, "Downloading", "status", "phase") + _ = unstructured.SetNestedSlice(software.Object, []any{map[string]any{"type": "Ready", "status": "False"}}, "status", "conditions") + reads := 0 + factory.management.onGet = func(r bundle.ResourceRef) { + if r.Kind == "BlueFieldSoftware" { + reads++ + if reads == 3 { + setReady(software) + } + } + } + runner.Wait = true + _, err = runner.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(reads).To(Equal(3)) + Expect(factory.management.creates).To(HaveLen(3)) + }) + It("cancels requested validation while preserving the pending report", func() { + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + software := factory.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "BlueFieldSoftware", "dpf-system", "software"))] + _ = unstructured.SetNestedField(software.Object, "Downloading", "status", "phase") + _ = unstructured.SetNestedSlice(software.Object, []any{map[string]any{"type": "Ready", "status": "False"}}, "status", "conditions") + runner.Wait = true + runner.Timeout = 30 * time.Millisecond + result, err := runner.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(context.DeadlineExceeded)) + Expect(result.Outcome).To(Equal("cancelled")) + Expect(result.ReportPath).To(BeAnExistingFile()) + Expect(factory.management.creates).To(HaveLen(3)) + }) + It("never treats zero selected DPU children as success and retains timeout evidence", func() { + factory.management.noChildren = true + runner.Timeout = 30 * time.Millisecond + result, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(HaveOccurred()) + Expect(result.Outcome).To(Equal("cancelled")) + Expect(result.ReportPath).To(BeAnExistingFile()) + }) + It("rejects BMC TLS/identity failure and absent external keys before any mutation", func() { + factory.management.bmcErr = fmt.Errorf("certificate mismatch") + _, err := runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("BMC_IDENTITY_UNVERIFIED"))) + Expect(factory.management.creates).To(BeEmpty()) + factory.management.bmcErr = nil + factory.tenant.secretErr = fmt.Errorf("missing inference key") + _, err = runner.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("EXTERNAL_SECRET_MISSING"))) + Expect(factory.management.creates).To(BeEmpty()) + }) + It("does not construct ambient clients for a pre-cancelled operation", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := runner.Run(ctx, b, g, Deploy) + Expect(err).To(MatchError(context.Canceled)) + Expect(factory.opens).To(BeZero()) + }) + It("fails native access without explicit kubeconfig even when ambient configuration exists", func() { + DeferCleanup(os.Unsetenv, "KUBECONFIG") + Expect(os.Setenv("KUBECONFIG", "ambient-must-not-be-read")).To(Succeed()) + _, err := (NativeClients{}).Management(context.Background(), config.Generation{}) + Expect(err).To(HaveOccurred()) + Expect(strings.ToLower(err.Error())).To(ContainSubstring("explicit")) + }) +}) + +var _ = Describe("completed Deployment readiness", func() { + It("does not treat disjoint old available and new updated replicas as a completed rollout", func() { + deployment := readyDeployment("example", "controller") + deployment.SetGeneration(2) + Expect(unstructured.SetNestedField(deployment.Object, int64(2), "status", "observedGeneration")).To(Succeed()) + Expect(unstructured.SetNestedField(deployment.Object, int64(2), "status", "replicas")).To(Succeed()) + ready, terminal, _ := Readiness(deployment) + Expect(ready).To(BeFalse()) + Expect(terminal).To(BeFalse()) + Expect(unstructured.SetNestedField(deployment.Object, int64(1), "status", "replicas")).To(Succeed()) + ready, terminal, _ = Readiness(deployment) + Expect(ready).To(BeTrue()) + Expect(terminal).To(BeFalse()) + }) + It("uses the API replica default while requiring positive explicit replicas", func() { + deployment := readyDeployment("example", "controller") + unstructured.RemoveNestedField(deployment.Object, "spec", "replicas") + ready, _, _ := Readiness(deployment) + Expect(ready).To(BeTrue()) + Expect(unstructured.SetNestedField(deployment.Object, int64(0), "spec", "replicas")).To(Succeed()) + ready, _, _ = Readiness(deployment) + Expect(ready).To(BeFalse()) + }) +}) diff --git a/pkg/dpf/deployment/final_regression_test.go b/pkg/dpf/deployment/final_regression_test.go new file mode 100644 index 00000000..e15704f4 --- /dev/null +++ b/pkg/dpf/deployment/final_regression_test.go @@ -0,0 +1,184 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "bytes" + "context" + "os" + "path/filepath" + "text/template" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Use the actual profile RBAC document so this end-to-end fixture cannot omit +// the top-level rules/subjects/roleRef fields which differ from DPF spec APIs. +func policyRBACBundle(b *bundle.Bundle) *bundle.Bundle { + data, err := os.ReadFile("../../../profiles/sentry-bf4/60-policy-rbac.yaml") + Expect(err).NotTo(HaveOccurred()) + parsed, err := template.New("policy-rbac").Parse(string(data)) + Expect(err).NotTo(HaveOccurred()) + var output bytes.Buffer + Expect(parsed.Execute(&output, map[string]any{"Names": map[string]string{"Server": "policy-server"}, "Namespace": "sentry-system"})).To(Succeed()) + account := []byte("apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: policy-server\n namespace: sentry-system\nimagePullSecrets:\n - name: pull\nautomountServiceAccountToken: false\n") + index := b.Index() + index.BundleDigest = "" + index.Files = append(index.Files, bundle.FileRecord{Path: "objects/policy-account.yaml", Role: bundle.Manifest}, bundle.FileRecord{Path: "objects/policy-rbac.yaml", Role: bundle.Manifest}) + index.Stages = append(index.Stages, bundle.Stage{ID: "policy-rbac", Destination: bundle.DPU, Namespace: "sentry-system", Files: []string{"objects/policy-account.yaml", "objects/policy-rbac.yaml"}, DependsOn: []string{"provision"}}) + index.References = append(index.References, bundle.Reference{From: ref(bundle.DPU, "rbac.authorization.k8s.io", "RoleBinding", "sentry-system", "policy-server"), To: ref(bundle.DPU, "rbac.authorization.k8s.io", "Role", "sentry-system", "policy-server")}) + files := append(b.Files(), bundle.File{Path: "objects/policy-account.yaml", Data: account}, bundle.File{Path: "objects/policy-rbac.yaml", Data: output.Bytes()}) + result, err := bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + return result +} + +var _ = Describe("full native lifecycle convergence regressions", func() { + runnerFor := func(f *fakeFactory, v *fakeArtifacts) Runner { + dir, err := os.MkdirTemp("", "dpf-final-regression-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + return Runner{Clients: f, Artifacts: v, EvidenceDir: dir, PollInterval: time.Millisecond, Timeout: 10 * time.Second} + } + It("creates and resumes actual native policy RBAC and explicit ServiceAccount intent", func() { + b, g, f, v := lifecycleFixture() + b = policyRBACBundle(b) + r := runnerFor(f, v) + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.tenant.creates).To(Equal([]string{"ServiceAccount/policy-server", "Role/policy-server", "RoleBinding/policy-server"})) + role := f.tenant.objects[objectKey(ref("", "rbac.authorization.k8s.io", "Role", "sentry-system", "policy-server"))] + rules, _, err := unstructured.NestedSlice(role.Object, "rules") + Expect(err).NotTo(HaveOccurred()) + Expect(rules).To(HaveLen(2)) + Expect(rules[0].(map[string]any)["resources"]).To(Equal([]any{"docatowerpolicies"})) + result, err = r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.tenant.creates).To(HaveLen(3)) + result, err = r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.tenant.creates).To(HaveLen(3)) + }) + It("accepts only documented native RBAC defaults and rejects privilege or identity changes", func() { + b, _, _, _ := lifecycleFixture() + b = policyRBACBundle(b) + for _, kind := range []string{"ServiceAccount", "Role", "RoleBinding"} { + desired := resourcesInBundle(b, kind)[0].ObjectCopy() + actual := desired.DeepCopy() + if kind == "RoleBinding" { + unstructured.RemoveNestedField(desired.Object, "roleRef", "apiGroup") + subjects, _, _ := unstructured.NestedSlice(actual.Object, "subjects") + subjects[0].(map[string]any)["apiGroup"] = "" + Expect(unstructured.SetNestedSlice(actual.Object, subjects, "subjects")).To(Succeed()) + } + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeTrue(), kind) + switch kind { + case "Role": + rules, _, _ := unstructured.NestedSlice(actual.Object, "rules") + rules[0].(map[string]any)["verbs"] = []any{"*"} + Expect(unstructured.SetNestedSlice(actual.Object, rules, "rules")).To(Succeed()) + case "RoleBinding": + Expect(unstructured.SetNestedField(actual.Object, "foreign-role", "roleRef", "name")).To(Succeed()) + case "ServiceAccount": + Expect(unstructured.SetNestedField(actual.Object, true, "automountServiceAccountToken")).To(Succeed()) + } + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeFalse(), kind) + } + }) + It("rejects additional RoleBinding subjects and unsupported native kinds", func() { + b, _, _, _ := lifecycleFixture() + b = policyRBACBundle(b) + desired := resourcesInBundle(b, "RoleBinding")[0].ObjectCopy() + actual := desired.DeepCopy() + subjects, _, _ := unstructured.NestedSlice(actual.Object, "subjects") + subjects = append(subjects, map[string]any{"kind": "ServiceAccount", "name": "other", "namespace": "sentry-system"}) + Expect(unstructured.SetNestedSlice(actual.Object, subjects, "subjects")).To(Succeed()) + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeFalse()) + arbitrary := testObject("v1", "ConfigMap", "sentry-system", "unsupported", nil) + Expect(nativeMatches("26.8.0-rc.1", arbitrary, arbitrary.DeepCopy())).To(BeFalse()) + }) + DescribeTable("blocks late DPUNode transitions after observing an otherwise Ready child", func(change string) { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + dpuNode := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", "dpf-system", "node-a"))] + f.tenant.onGet = func(reference bundle.ResourceRef) { + if reference.Kind != "Node" { + return + } + switch change { + case "annotation": + dpuNode.SetAnnotations(map[string]string{"provisioning.dpu.nvidia.com/dpunode-external-reboot-required": "SystemLevelReset"}) + case "flag": + Expect(unstructured.SetNestedField(dpuNode.Object, true, "status", "rebootInProgress")).To(Succeed()) + case "method": + Expect(unstructured.SetNestedMap(dpuNode.Object, map[string]any{"script": map[string]any{"name": "late-executor"}}, "spec", "nodeRebootMethod")).To(Succeed()) + } + } + result, err := r.Run(context.Background(), b, g, Validate) + Expect(err).To(HaveOccurred()) + Expect(result.Outcome).NotTo(Equal("complete")) + if change == "method" { + Expect(err).To(MatchError(ContainSubstring("NODE_REBOOT_METHOD_UNSUPPORTED"))) + } else { + Expect(err).To(MatchError(ContainSubstring("NODE_TRANSITION_PENDING"))) + } + }, Entry("external marker", "annotation"), Entry("active reboot flag", "flag"), Entry("executor changed", "method")) + It("provisions first tenant workers before requiring DNS and polls DNS before completion", func() { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + dns := f.tenant.objects[objectKey(ref("", "apps", "Deployment", "kube-system", "coredns"))] + Expect(unstructured.SetNestedField(dns.Object, int64(0), "status", "availableReplicas")).To(Succeed()) + nodes, err := f.tenant.List(context.Background(), ref("", "", "Node", "", ""), "") + Expect(err).NotTo(HaveOccurred()) + Expect(nodes).To(BeEmpty()) + reads := 0 + f.tenant.onList = func(reference bundle.ResourceRef) { + if reference.Kind != "Deployment" || reference.Namespace != "kube-system" { + return + } + reads++ + Expect(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", "dpf-system", "selected-set"))]).NotTo(BeNil()) + Expect(f.tenant.objects[objectKey(ref("", "", "Node", "", "dpu-child"))]).NotTo(BeNil()) + if reads >= 2 { + Expect(unstructured.SetNestedField(dns.Object, int64(1), "status", "availableReplicas")).To(Succeed()) + } + } + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(reads).To(BeNumerically(">=", 2)) + Expect(result.Checks).To(ContainElement(HaveField("ReasonCode", "TENANT_DNS_NOT_READY"))) + }) + It("does not complete validation while DNS remains unavailable after provisioning", func() { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + dns := f.tenant.objects[objectKey(ref("", "apps", "Deployment", "kube-system", "coredns"))] + Expect(unstructured.SetNestedField(dns.Object, int64(0), "status", "availableReplicas")).To(Succeed()) + result, err := r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("TENANT_DNS_NOT_READY"))) + Expect(result.Outcome).To(Equal("failed")) + }) + It("keeps dry-run admission independent of runtime DNS availability with an explicit non-evaluation", func() { + b, g, f, v := lifecycleFixture() + r := runnerFor(f, v) + delete(f.tenant.objects, objectKey(ref("", "apps", "Deployment", "kube-system", "coredns"))) + result, err := r.Run(context.Background(), b, g, DryRun) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(result.Checks).To(ContainElement(And(HaveField("State", "not-evaluated"), HaveField("ReasonCode", "RUNTIME_READINESS_NOT_EVALUATED")))) + }) +}) diff --git a/pkg/dpf/deployment/flavor_admission_test.go b/pkg/dpf/deployment/flavor_admission_test.go new file mode 100644 index 00000000..c0bc8961 --- /dev/null +++ b/pkg/dpf/deployment/flavor_admission_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +var _ = Describe("pinned DPUFlavor webhook typed serialization", func() { + flavor := func() *unstructured.Unstructured { + return testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUFlavor", "dpf-system", "flavor", map[string]any{ + "dpuMode": "zero-trust", "grub": map[string]any{"kernelParameters": []any{"console=hvc0"}}, "sysctl": map[string]any{"parameters": []any{"net.ipv4.conf.all.arp_ignore=1"}}, "ovs": map[string]any{"rawConfigScript": "ovs-vsctl --may-exist add-br br-sfc\n"}, + "configFiles": []any{map[string]any{"path": "/etc/mellanox/mlnx-sf.conf", "raw": "", "permissions": "0644", "operation": "override"}}, + }) + } + DescribeTable("accepts the observed empty value-object addition without changing content", func(release string) { + desired := flavor() + actual := desired.DeepCopy() + // This is the exact spec delta in the saved server-side dry-run response. + Expect(unstructured.SetNestedMap(actual.Object, map[string]any{}, "spec", "containerdConfig")).To(Succeed()) + Expect(nativeMatches(release, desired, actual)).To(BeTrue()) + for _, field := range []string{"grub", "sysctl", "ovs", "containerdConfig"} { + omitted := flavor() + unstructured.RemoveNestedField(omitted.Object, "spec", field) + zero := omitted.DeepCopy() + Expect(unstructured.SetNestedMap(zero.Object, map[string]any{}, "spec", field)).To(Succeed()) + Expect(nativeMatches(release, omitted, zero)).To(BeTrue(), field) + } + }, Entry("installed rc.1", "26.8.0-rc.1"), Entry("GA equivalent types", "26.8.0")) + DescribeTable("rejects additional meaningful or unknown admitted fields", func(field string, value any) { + desired := flavor() + actual := desired.DeepCopy() + Expect(unstructured.SetNestedMap(actual.Object, map[string]any{field: value}, "spec", "containerdConfig")).To(Succeed()) + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeFalse()) + }, Entry("different containerd endpoint", "registryEndpoint", "https://unreviewed.invalid"), Entry("unknown containerd configuration", "unreviewed", true)) + It("retains exact script, configuration file and unknown top-level spec comparisons", func() { + desired := flavor() + for _, change := range []func(*unstructured.Unstructured){ + func(o *unstructured.Unstructured) { + Expect(unstructured.SetNestedField(o.Object, "changed\n", "spec", "ovs", "rawConfigScript")).To(Succeed()) + }, + func(o *unstructured.Unstructured) { + files, _, _ := unstructured.NestedSlice(o.Object, "spec", "configFiles") + files[0].(map[string]any)["raw"] = "unreviewed content" + Expect(unstructured.SetNestedSlice(o.Object, files, "spec", "configFiles")).To(Succeed()) + }, + func(o *unstructured.Unstructured) { + Expect(unstructured.SetNestedMap(o.Object, map[string]any{}, "spec", "unreviewed")).To(Succeed()) + }, + } { + actual := desired.DeepCopy() + change(actual) + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeFalse()) + } + }) + It("admits and resumes a native flavor stage with the captured serialization delta", func() { + b, g, f, v := lifecycleFixture() + desired := flavor() + desired.SetGeneration(0) + desired.SetUID("") + desired.SetResourceVersion("") + data, err := yaml.Marshal(desired.Object) + Expect(err).NotTo(HaveOccurred()) + index := b.Index() + index.BundleDigest = "" + index.Files = append(index.Files, bundle.FileRecord{Path: "objects/flavor.yaml", Role: bundle.Manifest}) + index.Stages[0].Files = append(index.Stages[0].Files, "objects/flavor.yaml") + files := append(b.Files(), bundle.File{Path: "objects/flavor.yaml", Data: data}) + set := resourcesInBundle(b, "DPUSet")[0].ObjectCopy() + Expect(unstructured.SetNestedField(set.Object, "flavor", "spec", "dpuTemplate", "spec", "dpuFlavor")).To(Succeed()) + setData, err := yaml.Marshal(set.Object) + Expect(err).NotTo(HaveOccurred()) + for i := range files { + if files[i].Path == "objects/1.yaml" { + files[i].Data = setData + } + } + for i := range index.Files { + if index.Files[i].Path == "objects/1.yaml" { + index.Files[i].SHA256 = "" + } + } + b, err = bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + f.management.admit = func(o *unstructured.Unstructured, _ bool) { + if o.GetKind() == "DPUFlavor" { + Expect(unstructured.SetNestedMap(o.Object, map[string]any{}, "spec", "containerdConfig")).To(Succeed()) + } + } + dir, err := os.MkdirTemp("", "dpf-flavor-admission-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + r := Runner{Clients: f, Artifacts: v, EvidenceDir: dir, PollInterval: time.Millisecond, Timeout: 10 * time.Second} + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + created := len(f.management.creates) + result, err = r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.management.creates).To(HaveLen(created)) + }) +}) diff --git a/pkg/dpf/deployment/lifecycle_regression_test.go b/pkg/dpf/deployment/lifecycle_regression_test.go new file mode 100644 index 00000000..60d90d2b --- /dev/null +++ b/pkg/dpf/deployment/lifecycle_regression_test.go @@ -0,0 +1,184 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("lifecycle safety regressions", func() { + newRunner := func(factory *fakeFactory, verifier *fakeArtifacts) Runner { + dir, err := os.MkdirTemp("", "dpf-regression-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + return Runner{Clients: factory, Artifacts: verifier, EvidenceDir: dir, PollInterval: time.Millisecond, Timeout: 10 * time.Second} + } + DescribeTable("rejects reboot methods outside explicit external:{} before writes", func(method map[string]any) { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", "dpf-system", "node-a"))] + if method == nil { + unstructured.RemoveNestedField(node.Object, "spec", "nodeRebootMethod") + } else { + Expect(unstructured.SetNestedMap(node.Object, method, "spec", "nodeRebootMethod")).To(Succeed()) + } + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("NODE_REBOOT_METHOD_UNSUPPORTED"))) + Expect(f.management.creates).To(BeEmpty()) + }, Entry("default host-agent", nil), Entry("arbitrary script", map[string]any{"script": map[string]any{"name": "unreviewed"}}), Entry("gNOI", map[string]any{"gNOI": map[string]any{}}), Entry("hostless none", map[string]any{"none": map[string]any{}}), Entry("mixed methods", map[string]any{"external": map[string]any{}, "hostAgent": map[string]any{}}), Entry("unexpected external fields", map[string]any{"external": map[string]any{"reboot": "now"}})) + DescribeTable("rejects a pending prior node transition before writes", func(marker string) { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", "dpf-system", "node-a"))] + switch marker { + case "flag": + Expect(unstructured.SetNestedField(node.Object, true, "status", "rebootInProgress")).To(Succeed()) + case "annotation": + node.SetAnnotations(map[string]string{"provisioning.dpu.nvidia.com/dpunode-external-reboot-required": ""}) + default: + conditions, _, _ := unstructured.NestedSlice(node.Object, "status", "conditions") + conditions = append(conditions, map[string]any{"type": marker, "status": "True"}) + Expect(unstructured.SetNestedSlice(node.Object, conditions, "status", "conditions")).To(Succeed()) + } + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("NODE_TRANSITION_PENDING"))) + Expect(f.management.creates).To(BeEmpty()) + }, Entry("reboot flag", "flag"), Entry("external annotation even empty", "annotation"), Entry("active reboot", "DPUNodeRebootInProgress"), Entry("active DPU update", "DPUUpdateInProgress"), Entry("active node effect", "DPUNodeNodeEffectInProgress")) + It("accepts a completed reset method without an active flag", func() { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", "dpf-system", "node-a"))] + Expect(unstructured.SetNestedField(node.Object, "SystemLevelReset", "status", "rebootMethod")).To(Succeed()) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + }) + It("returns an incomplete non-success result for deferred server-side admission", func() { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + f.management.getErrors = map[string]error{"BlueFieldSoftware": &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "provisioning.dpu.nvidia.com", Kind: "BlueFieldSoftware"}}} + result, err := r.Run(context.Background(), b, g, DryRun) + Expect(err).To(MatchError(ContainSubstring("DRY_RUN_INCOMPLETE"))) + Expect(result.Outcome).To(Equal("incomplete")) + Expect(result.Checks).To(ContainElement(HaveField("ReasonCode", "API_NOT_SERVED"))) + Expect(f.management.dryRuns).To(Equal([]bool{true})) + Expect(v.calls).To(BeZero()) + }) + It("rejects retained object annotations when physical spec fields were added", func() { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + set := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", "dpf-system", "selected-set"))] + Expect(unstructured.SetNestedField(set.Object, true, "spec", "dpuTemplate", "spec", "secureBoot")).To(Succeed()) + before := len(f.management.creates) + _, err = r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("RESOURCE_OWNERSHIP_CONFLICT"))) + Expect(f.management.creates).To(HaveLen(before)) + }) + It("rejects effectful DPUSet admission changes before persistent creation", func() { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + f.management.admit = func(o *unstructured.Unstructured, dryRun bool) { + if o.GetKind() == "DPUSet" { + Expect(dryRun).To(BeTrue()) + Expect(unstructured.SetNestedField(o.Object, true, "spec", "dpuTemplate", "spec", "secureBoot")).To(Succeed()) + } + } + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("ADMISSION_CHANGED_DESIRED_STATE"))) + Expect(f.management.dryRuns).To(Equal([]bool{false, true})) + Expect(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", "dpf-system", "selected-set"))]).To(BeNil()) + }) + + It("does not pass selected Ready nodes with missing scope labels", func() { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + f.tenant.objects[objectKey(ref("", "", "Node", "", "dpu-child"))].SetLabels(nil) + result, err := r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("SELECTED_DPUS_NOT_READY"))) + Expect(result.Outcome).To(Equal("failed")) + }) + It("rejects an unselected tenant Node carrying the instance scope label", func() { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + sibling := testObject("v1", "Node", "", "unselected", nil) + setReady(sibling) + sibling.SetLabels(map[string]string{"nvidia.kubernetes-launch-kit.dpf-instance": "fixture"}) + f.tenant.put(sibling) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("UNSELECTED_TENANT_NODE_MATCH"))) + }) + It("retains actual DPU and external reboot diagnostics while waiting", func() { + b, g, f, v := lifecycleFixture() + r := newRunner(f, v) + _, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + set := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", "dpf-system", "selected-set"))] + Expect(unstructured.SetNestedSlice(set.Object, []any{map[string]any{"type": "Ready", "status": "False", "reason": "Pending", "observedGeneration": set.GetGeneration()}}, "status", "conditions")).To(Succeed()) + child := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPU", "dpf-system", "dpu-child"))] + Expect(unstructured.SetNestedField(child.Object, "DPURebooting", "status", "phase")).To(Succeed()) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", "dpf-system", "node-a"))] + // Simulate a transition beginning after preflight rather than an old pending operation. + reads := 0 + f.management.onGet = func(r bundle.ResourceRef) { + if r.Kind == "DPUNode" { + reads++ + if reads > 1 { + node.SetAnnotations(map[string]string{"provisioning.dpu.nvidia.com/dpunode-external-reboot-required": "SystemLevelReset"}) + } + } + } + _, err = r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(And(ContainSubstring("DPURebooting"), ContainSubstring("SystemLevelReset")))) + }) +}) + +var _ = Describe("DPF generation freshness", func() { + DescribeTable("requires exact top-level observed generations", func(kind string) { + o := testObject("provisioning.dpu.nvidia.com/v1alpha1", kind, "dpf-system", "test", nil) + o.SetGeneration(2) + setReady(o) + for _, observed := range []int64{0, 1, 3} { + Expect(unstructured.SetNestedField(o.Object, observed, "status", "observedGeneration")).To(Succeed()) + ready, _, _ := Readiness(o) + Expect(ready).To(BeFalse(), "observed=%d", observed) + } + unstructured.RemoveNestedField(o.Object, "status", "observedGeneration") + ready, _, _ := Readiness(o) + Expect(ready).To(BeFalse()) + Expect(unstructured.SetNestedField(o.Object, int64(2), "status", "observedGeneration")).To(Succeed()) + ready, _, _ = Readiness(o) + Expect(ready).To(BeTrue()) + }, Entry("software", "BlueFieldSoftware"), Entry("DPU", "DPU"), Entry("set", "DPUSet"), Entry("service", "DPUService"), Entry("interface", "DPUServiceInterface"), Entry("chain", "DPUServiceChain")) + DescribeTable("uses Ready condition freshness for APIs without top-level observation", func(kind string) { + o := testObject("svc.dpu.nvidia.com/v1alpha1", kind, "dpf-system", "test", nil) + o.SetGeneration(2) + setReady(o) + unstructured.RemoveNestedField(o.Object, "status", "observedGeneration") + ready, _, _ := Readiness(o) + Expect(ready).To(BeTrue()) + for _, observed := range []int64{0, 1, 3} { + o.Object["status"] = map[string]any{"conditions": []any{map[string]any{"type": "Ready", "status": "True", "observedGeneration": observed}}} + ready, _, _ = Readiness(o) + Expect(ready).To(BeFalse()) + } + o.Object["status"] = map[string]any{"conditions": []any{map[string]any{"type": "Ready", "status": "True"}}} + ready, _, _ = Readiness(o) + Expect(ready).To(BeFalse()) + }, Entry("NAD", "DPUServiceNAD"), Entry("credentials", "DPUServiceCredentialRequest")) +}) diff --git a/pkg/dpf/deployment/native_serialization_test.go b/pkg/dpf/deployment/native_serialization_test.go new file mode 100644 index 00000000..b77396f3 --- /dev/null +++ b/pkg/dpf/deployment/native_serialization_test.go @@ -0,0 +1,146 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +// These anonymous fixtures model the pinned API's value-typed fields, independently +// of the comparator's normalization registry. Optional pointer fields stay absent. +func serializationObject(kind string) *unstructured.Unstructured { + switch kind { + case "DPUFlavor": + return testObject("provisioning.dpu.nvidia.com/v1alpha1", kind, "sentry-system", "flavor", map[string]any{}) + case "DPUService": + return testObject("svc.dpu.nvidia.com/v1alpha1", kind, "sentry-system", "service", map[string]any{"helmChart": map[string]any{"source": map[string]any{"repoURL": "oci://example.invalid/charts", "chart": "example", "version": "1.0.0"}}, "security": map[string]any{"privileged": false}}) + case "DPUServiceInterface": + return testObject("svc.dpu.nvidia.com/v1alpha1", kind, "sentry-system", "interface", map[string]any{"template": map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{"interfaceType": "physical", "physical": map[string]any{"interfaceName": "p0"}}}}}}) + case "DPUServiceChain": + return testObject("svc.dpu.nvidia.com/v1alpha1", kind, "sentry-system", "chain", map[string]any{"template": map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{"switches": []any{map[string]any{"ports": []any{map[string]any{"serviceInterface": map[string]any{"matchLabels": map[string]any{"example.invalid/port": "p0"}}}}}}}}}}}) + } + Fail("unsupported serialization fixture: " + kind) + return nil +} + +func serializedValueFields(object *unstructured.Unstructured) { + switch object.GetKind() { + case "DPUFlavor": + for _, field := range []string{"grub", "sysctl", "ovs", "containerdConfig"} { + Expect(unstructured.SetNestedMap(object.Object, map[string]any{}, "spec", field)).To(Succeed()) + } + case "DPUService": + Expect(unstructured.SetNestedField(object.Object, "", "spec", "helmChart", "source", "path")).To(Succeed()) + case "DPUServiceInterface", "DPUServiceChain": + Expect(unstructured.SetNestedMap(object.Object, map[string]any{}, "spec", "template", "metadata")).To(Succeed()) + Expect(unstructured.SetNestedMap(object.Object, map[string]any{}, "spec", "template", "spec", "template", "metadata")).To(Succeed()) + } +} + +var _ = Describe("pinned native value serialization", func() { + DescribeTable("accepts only zero serialization without changing either input", func(kind string) { + desired := serializationObject(kind) + actual := desired.DeepCopy() + serializedValueFields(actual) + beforeDesired, beforeActual := desired.DeepCopy(), actual.DeepCopy() + for _, release := range []string{"26.8.0-rc.1", "26.8.0"} { + Expect(nativeMatches(release, desired, actual)).To(BeTrue(), release) + Expect(nativeMatches(release, actual, desired)).To(BeTrue(), release) + } + Expect(desired).To(Equal(beforeDesired)) + Expect(actual).To(Equal(beforeActual)) + }, Entry("flavor value structs", "DPUFlavor"), Entry("chart source empty path", "DPUService"), Entry("interface template metadata", "DPUServiceInterface"), Entry("chain template metadata", "DPUServiceChain")) + + DescribeTable("retains meaningful and unknown content at normalized paths", func(kind string) { + desired := serializationObject(kind) + actual := desired.DeepCopy() + serializedValueFields(actual) + var path []string + switch kind { + case "DPUFlavor": + path = []string{"spec", "grub", "kernelParameters"} + Expect(unstructured.SetNestedField(actual.Object, []any{"iommu=off"}, path...)).To(Succeed()) + case "DPUService": + path = []string{"spec", "helmChart", "source", "path"} + Expect(unstructured.SetNestedField(actual.Object, "different-chart", path...)).To(Succeed()) + default: + path = []string{"spec", "template", "metadata", "labels", "example.invalid/selected"} + Expect(unstructured.SetNestedField(actual.Object, "foreign", path...)).To(Succeed()) + } + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeFalse()) + actual = desired.DeepCopy() + serializedValueFields(actual) + unknownPath := []string{"spec", "template", "metadata", "unknownEffect"} + switch kind { + case "DPUFlavor": + unknownPath = []string{"spec", "grub", "unknownEffect"} + case "DPUService": + unknownPath = []string{"spec", "helmChart", "source", "unknownEffect"} + } + Expect(unstructured.SetNestedMap(actual.Object, map[string]any{}, unknownPath...)).To(Succeed()) + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeFalse()) + }, Entry("configured flavor", "DPUFlavor"), Entry("configured chart path", "DPUService"), Entry("interface metadata selection", "DPUServiceInterface"), Entry("chain metadata selection", "DPUServiceChain")) + + It("retains presence-gated optional pointer objects and missing required parents", func() { + desired := serializationObject("DPUService") + actual := desired.DeepCopy() + Expect(unstructured.SetNestedMap(actual.Object, map[string]any{}, "spec", "security", "spiffe")).To(Succeed()) + Expect(nativeMatches("26.8.0-rc.1", desired, actual)).To(BeFalse()) + broken := serializationObject("DPUServiceInterface") + unstructured.RemoveNestedField(broken.Object, "spec", "template") + Expect(nativeMatches("26.8.0-rc.1", broken, broken.DeepCopy())).To(BeFalse()) + }) + + It("waits and resumes after controller template metadata serialization while rejecting real drift", func() { + b, g, f, v := lifecycleFixture() + index, files := b.Index(), b.Files() + index.BundleDigest = "" + stage := bundle.Stage{ID: "service-network", Destination: bundle.Management, Namespace: "sentry-system", DependsOn: []string{"provision"}} + for _, kind := range []string{"DPUServiceInterface", "DPUServiceChain"} { + object := serializationObject(kind) + object.SetUID("") + object.SetResourceVersion("") + object.SetGeneration(0) + data, err := yaml.Marshal(object.Object) + Expect(err).NotTo(HaveOccurred()) + path := "objects/" + kind + ".yaml" + files = append(files, bundle.File{Path: path, Data: data}) + index.Files = append(index.Files, bundle.FileRecord{Path: path, Role: bundle.Manifest}) + stage.Files = append(stage.Files, path) + } + index.Stages = append(index.Stages, stage) + var err error + b, err = bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + f.management.onGet = func(reference bundle.ResourceRef) { + if reference.Kind != "DPUServiceInterface" && reference.Kind != "DPUServiceChain" { + return + } + if object := f.management.objects[objectKey(reference)]; object != nil { + serializedValueFields(object) + object.SetGeneration(2) + setReady(object) + } + } + r := dnsTestRunner(f, v) + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + creates := len(f.management.creates) + result, err = r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.management.creates).To(HaveLen(creates)) + object := f.management.objects[objectKey(ref("", "svc.dpu.nvidia.com", "DPUServiceInterface", "sentry-system", "interface"))] + Expect(unstructured.SetNestedField(object.Object, "p1", "spec", "template", "spec", "template", "spec", "physical", "interfaceName")).To(Succeed()) + _, err = r.Run(context.Background(), b, g, Validate) + Expect(err).To(MatchError(ContainSubstring("RESOURCE_OWNERSHIP_CONFLICT"))) + }) +}) diff --git a/pkg/dpf/deployment/preflight.go b/pkg/dpf/deployment/preflight.go new file mode 100644 index 00000000..2fdeade4 --- /dev/null +++ b/pkg/dpf/deployment/preflight.go @@ -0,0 +1,727 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "reflect" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" +) + +func (s *run) management(ctx context.Context) error { + c := s.clients[bundle.Management] + objects, err := c.List(ctx, ref(bundle.Management, "operator.dpu.nvidia.com", "DPFOperatorConfig", s.g.Namespace, ""), "") + if err != nil { + return fail("DPF_CONFIG_UNAVAILABLE", "cannot read effective DPFOperatorConfig") + } + if len(objects) != 1 { + return fail("DPF_CONFIG_AMBIGUOUS", "require exactly one effective DPFOperatorConfig") + } + object := objects[0] + if strings.TrimPrefix(text(object, "status", "version"), "v") != s.g.Release { + return fail("DPF_VERSION_MISMATCH", "effective DPF version differs from bundle release") + } + target := text(object, "status", "targetVersion") + if target != "" && strings.TrimPrefix(target, "v") != s.g.Release { + return fail("DPF_UPGRADE_IN_PROGRESS", "DPF target version differs from active version") + } + _, redfish, _ := unstructured.NestedMap(object.Object, "spec", "provisioningController", "installInterface", "installViaRedfish") + if text(object, "spec", "deploymentMode") != "zero-trust" || !redfish { + return fail("INSTALL_INTERFACE_MISMATCH", "DPF must already use zero-trust installViaRedfish") + } + if !condition(object, "Ready") { + if err := s.check("global-operator-readiness", object.GetName(), "management:DPFOperatorConfig", "diagnostic", "GLOBAL_OPERATOR_NOT_READY", conditionMessage(object)+"; selected controller and tenant checks remain independently required"); err != nil { + return err + } + } + deployments, err := c.List(ctx, ref(bundle.Management, "apps", "Deployment", s.g.Namespace, ""), "") + if err != nil { + return fail("CONTROLLER_READ_FAILED", "cannot inspect DPF management controllers") + } + roles := map[string][]*unstructured.Unstructured{} + for _, deployment := range deployments { + role := deployment.GetLabels()["dpu.nvidia.com/component"] + roles[role] = append(roles[role], deployment) + } + for _, role := range []string{"dpf-operator-controller-manager", "dpf-provisioning-controller-manager", "dpuservice-controller-manager"} { + if len(roles[role]) != 1 { + return fail("MANAGEMENT_CONTROLLER_MISSING", role+" must resolve to exactly one Deployment by component label") + } + ready, _, message := Readiness(roles[role][0]) + if !ready { + return fail("MANAGEMENT_CONTROLLER_NOT_READY", role+": "+message) + } + } + return s.check("existing-management", s.g.Namespace, "management:DPFOperatorConfig", "passed", "MANAGEMENT_READY", "") +} + +func (s *run) cluster(ctx context.Context) error { + binding := s.g.Provisioning.DPUCluster + objects, err := s.clients[bundle.Management].List(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUCluster", "", ""), labels.SelectorFromSet(binding.Selector).String()) + if err != nil { + return fail("DPU_CLUSTER_SELECTION_UNAVAILABLE", "cannot enumerate DPUCluster selector domain across all namespaces") + } + if len(objects) != 1 { + return fail("DPU_CLUSTER_SELECTION_MISMATCH", "DPUCluster selector must match exactly one existing tenant") + } + object := objects[0] + if object.GetName() != binding.Name || object.GetNamespace() != binding.Namespace || string(object.GetUID()) != binding.ResourceUID { + return fail("DPU_CLUSTER_IDENTITY_MISMATCH", "selected DPUCluster identity differs") + } + if object.GetGeneration() < 1 { + return fail("DPU_CLUSTER_GENERATION_MISSING", "selected DPUCluster generation is unavailable") + } + if s.clusterGeneration != 0 && object.GetGeneration() != s.clusterGeneration { + return fail("DPU_CLUSTER_GENERATION_DRIFT", "selected DPUCluster spec changed during this run") + } + ready, _, _ := Readiness(object) + if !ready { + return fail("DPU_CLUSTER_NOT_READY", "existing DPUCluster is not Ready") + } + if s.clusterGeneration == 0 { + s.clusterGeneration = object.GetGeneration() + if err := s.check("dpucluster-freshness", binding.Namespace+"/"+binding.Name, "DPF DPUCluster controller", "diagnostic", "FRESHNESS_NOT_EXPOSED", fmt.Sprintf("rc.1/GA publish no observedGeneration; phase/Ready and unchanged generation=%d are required, with tenant UID/API version/access checked separately", s.clusterGeneration)); err != nil { + return err + } + } + if text(object, "spec", "type") == "kamaji" { + controllers, err := s.clients[bundle.Management].List(ctx, ref(bundle.Management, "apps", "Deployment", s.g.Namespace, ""), "dpu.nvidia.com/component=kamaji-cluster-manager") + if err != nil || len(controllers) != 1 { + return fail("KAMAJI_CONTROLLER_MISSING", "selected Kamaji tenant requires exactly one ready Kamaji cluster manager") + } + ready, _, message := Readiness(controllers[0]) + if !ready { + return fail("KAMAJI_CONTROLLER_NOT_READY", message) + } + } + return s.check("existing-tenant", binding.Namespace+"/"+binding.Name, "management:DPUCluster", "passed", "TENANT_BINDING_MATCH", "") +} + +func (s *run) selection(ctx context.Context) error { + c := s.clients[bundle.Management] + expected := map[string]config.DeviceBinding{} + selectedNodes := map[string]map[string]bool{} + for _, binding := range s.g.Provisioning.Devices { + expected[binding.DPUDevice] = binding + if selectedNodes[binding.DPUNode] == nil { + selectedNodes[binding.DPUNode] = map[string]bool{} + } + selectedNodes[binding.DPUNode][binding.DPUDevice] = true + var inventoryDevice inventory.Device + for _, d := range s.g.Devices { + if d.ID == binding.ID { + inventoryDevice = d + } + } + device, err := c.Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUDevice", s.g.Namespace, binding.DPUDevice)) + if err != nil { + return fail("DEVICE_ENROLLMENT_MISSING", binding.DPUDevice+" is not enrolled") + } + endpoint, _ := url.Parse(inventoryDevice.BMCEndpoint) + if inventoryDevice.SerialNumber == "" || text(device, "spec", "serialNumber") != inventoryDevice.SerialNumber || text(device, "status", "psid") != s.g.DevicePSIDs[binding.ID] || text(device, "spec", "bmcIp") != endpoint.Hostname() { + return fail("DEVICE_IDENTITY_MISMATCH", binding.DPUDevice+" serial/PSID/BMC differs from selected inventory") + } + if device.GetLabels()["provisioning.dpu.nvidia.com/dpunode-name"] != binding.DPUNode { + return fail("DEVICE_NODE_MISMATCH", binding.DPUDevice+" is attached elsewhere") + } + for _, name := range []string{"NodeAttached", "Initialized", "Discovered", "Ready"} { + if !condition(device, name) { + return fail("DEVICE_NOT_READY", binding.DPUDevice+" condition "+name+" is not True") + } + } + for _, name := range []string{"BMCCredentialsReady", "BMCServerCertificateReady", "CATrustBundleReady"} { + if hasCondition(device, name) && !condition(device, name) { + return fail("DEVICE_SECURITY_NOT_READY", binding.DPUDevice+" security condition "+name+" is not True") + } + } + if err = c.VerifyBMC(ctx, s.g, binding, inventoryDevice, s.g.DevicePSIDs[binding.ID]); err != nil { + return fail("BMC_IDENTITY_UNVERIFIED", binding.DPUDevice+": "+err.Error()) + } + } + for name, members := range selectedNodes { + node, err := c.Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUNode", s.g.Namespace, name)) + if err != nil { + return fail("DEVICE_NODE_MISSING", name+" is absent") + } + if text(node, "status", "dpuInstallInterface") != "redfish" { + return fail("DEVICE_NODE_INTERFACE_MISMATCH", name+" did not snapshot redfish") + } + if err := reviewedRebootMethod(node); err != nil { + return err + } + if message := pendingNodeTransition(node); message != "" { + return fail("NODE_TRANSITION_PENDING", name+": "+message) + } + refs, _, _ := unstructured.NestedSlice(node.Object, "spec", "dpus") + if len(refs) != len(members) { + return fail("EFFECT_DOMAIN_MISMATCH", name+" contains an unselected or missing device") + } + for _, item := range refs { + entry, ok := item.(map[string]any) + if !ok || !members[fmt.Sprint(entry["name"])] { + return fail("EFFECT_DOMAIN_MISMATCH", name+" membership differs") + } + } + } + dpus, err := c.List(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPU", s.g.Namespace, ""), "") + if err != nil { + return fail("OWNERSHIP_READ_FAILED", "cannot inspect selected DPU ownership") + } + ownSets := map[string]bool{} + for _, doc := range resourcesInBundle(s.bundle, "DPUSet") { + ownSets[doc.Ref().Name] = true + } + for _, dpu := range dpus { + if _, selected := expected[text(dpu, "spec", "dpuDeviceName")]; !selected { + continue + } + owned := false + for _, owner := range dpu.GetOwnerReferences() { + if owner.Kind == "DPUSet" && ownSets[owner.Name] { + set, err := c.Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUSet", s.g.Namespace, owner.Name)) + if err != nil || set.GetUID() != owner.UID || set.GetAnnotations()[bundleAnnotation] != s.result.BundleDigest { + return fail("STALE_DPU_OWNER", "selected DPU owner UID/bundle is missing or changed; explicit recovery is required") + } + owned = true + } + } + if !owned { + return fail("FOREIGN_DPU_OWNER", "selected DPU remains owned outside this bundle; explicit external transition is required") + } + } + sets, err := c.List(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUSet", s.g.Namespace, ""), "") + if err != nil { + return fail("OWNERSHIP_READ_FAILED", "cannot inspect DPUSet selector overlap") + } + devices, err := c.List(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUDevice", s.g.Namespace, ""), "") + if err != nil { + return fail("SELECTION_READ_FAILED", "cannot inspect exact live DPUDevice membership") + } + for _, set := range sets { + if ownSets[set.GetName()] && set.GetAnnotations()[bundleAnnotation] == s.result.BundleDigest { + continue + } + for _, device := range devices { + if _, selected := expected[device.GetName()]; selected && matchesDeviceSet(set, device) { + return fail("FOREIGN_DPUSET_SELECTOR", "foreign DPUSet still selects a requested device") + } + } + } + return s.check("selected-scope", fmt.Sprint(len(expected)), "management:DPUDevice/DPUNode/DPUSet", "passed", "EXACT_SCOPE_MATCH", "") +} + +func matchesDeviceSet(set, device *unstructured.Unstructured) bool { + raw, exists, _ := unstructured.NestedMap(set.Object, "spec", "dpuDeviceSelector") + if !exists { + raw, _, _ = unstructured.NestedMap(set.Object, "spec", "dpuSelector") + if len(raw) == 0 { + return true + } + for key, value := range raw { + if device.GetLabels()[key] != value { + return false + } + } + return true + } + data, _ := json.Marshal(raw) + var selector metav1.LabelSelector + if json.Unmarshal(data, &selector) != nil { + return true + } + parsed, err := metav1.LabelSelectorAsSelector(&selector) + return err != nil || parsed.Matches(labels.Set(device.GetLabels())) +} + +func (s *run) externalSecrets(ctx context.Context) error { + namespace := s.g.Namespace + if s.g.Sentry.Namespace != nil { + namespace = *s.g.Sentry.Namespace + } + for _, destination := range []bundle.Destination{bundle.Management, bundle.DPU} { + object, err := s.clients[destination].Get(ctx, ref(destination, "", "Namespace", "", namespace)) + if err != nil || text(object, "status", "phase") != "Active" || object.GetUID() == "" { + return fail("EXTERNAL_NAMESPACE_UNAVAILABLE", string(destination)+" service namespace must already exist and be Active") + } + if s.namespaceUIDs[destination] == nil { + s.namespaceUIDs[destination] = map[string]string{} + } + s.namespaceUIDs[destination][namespace] = string(object.GetUID()) + if err = s.check("external-namespace", string(destination)+"/"+namespace, "kubernetes:Namespace", "passed", "NAMESPACE_IDENTITY_OBSERVED", string(object.GetUID())); err != nil { + return err + } + } + for _, destination := range []bundle.Destination{bundle.Management, bundle.DPU} { + c := s.clients[destination] + if err := c.SecretKeys(ctx, namespace, s.g.Sentry.ImagePullSecret, []string{".dockerconfigjson"}); err != nil { + return fail("EXTERNAL_SECRET_MISSING", err.Error()) + } + } + for _, destination := range []bundle.Destination{bundle.Management, bundle.DPU} { + if err := s.clients[destination].SecretKeys(ctx, namespace, s.g.Sentry.MITMSecret, []string{"ca.crt", "tls.crt", "tls.key"}); err != nil { + return fail("EXTERNAL_SECRET_MISSING", err.Error()) + } + } + if s.g.Sentry.LLMInspection != nil && *s.g.Sentry.LLMInspection { + if err := s.clients[bundle.DPU].SecretKeys(ctx, namespace, s.g.Sentry.InferenceSecret, []string{"INFERENCE_API_KEY"}); err != nil { + return fail("EXTERNAL_SECRET_MISSING", err.Error()) + } + } else { + if err := s.check("llm-inspection", namespace, "resolved:Sentry.llmInspection", "disabled", "LLM_INSPECTION_DISABLED", "optional external inference path is disabled; no LLM inspection result is claimed"); err != nil { + return err + } + } + if err := s.serviceOwnership(ctx, namespace); err != nil { + return err + } + return s.check("existing-secrets", namespace, "bound Kubernetes Secret keys", "passed", "SECRET_BINDINGS_PRESENT", "") +} + +func (s *run) serviceOwnership(ctx context.Context, namespace string) error { + expected := map[string]*unstructured.Unstructured{} + for _, doc := range resourcesInBundle(s.bundle, "DPUService") { + expected[doc.Ref().Name] = doc.ObjectCopy() + } + services, err := s.clients[bundle.Management].List(ctx, ref(bundle.Management, "svc.dpu.nvidia.com", "DPUService", namespace, ""), "") + if err != nil { + return fail("SERVICE_OWNERSHIP_UNAVAILABLE", "cannot inspect DPUService ownership in service namespace") + } + instanceExists := false + for _, service := range services { + desired := expected[service.GetName()] + if desired == nil || !owns(service, desired, s.result.BundleDigest, s.g.Release) { + return fail("FOREIGN_SERVICE_OWNER", "service namespace contains a foreign DPUService") + } + instanceExists = true + } + projection := map[bundle.ResourceRef]*unstructured.Unstructured{} + for _, stage := range s.bundle.Index().Stages { + view, _ := s.bundle.Stage(stage.ID) + for _, doc := range view.Projections() { + projection[doc.Ref()] = doc.ObjectCopy() + } + } + for _, kind := range []string{"Deployment", "DaemonSet", "StatefulSet"} { + objects, err := s.clients[bundle.DPU].List(ctx, ref(bundle.DPU, "apps", kind, namespace, ""), "") + if err != nil { + return fail("SERVICE_OWNERSHIP_UNAVAILABLE", "cannot inspect existing tenant service workloads") + } + for _, object := range objects { + reference := ref(bundle.DPU, "apps", kind, namespace, object.GetName()) + if !instanceExists || projection[reference] == nil || !desiredMatches(projection[reference], object) { + return fail("FOREIGN_SERVICE_WORKLOAD", "service namespace contains an unrelated tenant workload") + } + } + } + policies, err := s.clients[bundle.DPU].List(ctx, ref(bundle.DPU, "tower.dpu.nvidia.com", "DocaTowerPolicy", namespace, ""), "") + if err != nil && !meta.IsNoMatchError(err) { + return fail("POLICY_OWNERSHIP_UNAVAILABLE", "cannot inspect existing Sentry policies") + } + if len(policies) > 0 && !instanceExists { + return fail("FOREIGN_POLICY_OWNER", "service namespace contains policies before this bundle's service exists") + } + return nil +} + +// A fresh tenant can have no workers until DPU provisioning completes. DNS is +// therefore a post-provisioning prerequisite for Sentry, not a preflight gate. +func (s *run) baseWorkloads(ctx context.Context) error { + for { + observation, err := s.observeDNS(ctx) + if err != nil { + return err + } + if observation.ready { + return s.check("tenant-dns", "bound tenant", observation.source, "passed", observation.code, observation.message) + } + if s.result.Mode == Validate && !s.runner.Wait { + return fail(observation.code, observation.message) + } + if err := s.check("tenant-dns", "bound tenant", observation.source, "pending", observation.code, observation.message); err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(s.runner.PollInterval): + } + } +} + +func (s *run) provisioningGate(ctx context.Context, set *unstructured.Unstructured) error { + if err := s.management(ctx); err != nil { + return err + } + if err := s.cluster(ctx); err != nil { + return err + } + if err := s.identity(ctx, s.clients[bundle.DPU], s.g.Provisioning.DPUCluster.ClusterUID, "tenant"); err != nil { + return err + } + devices, err := s.clients[bundle.Management].List(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUDevice", s.g.Namespace, ""), "") + if err != nil { + return fail("SELECTION_READ_FAILED", "cannot check generated DPUSet selector") + } + expected := map[string]bool{} + for _, binding := range s.g.Provisioning.Devices { + expected[binding.DPUDevice] = true + } + matches := 0 + for _, device := range devices { + if matchesDeviceSet(set, device) { + if !expected[device.GetName()] { + return fail("DPUSET_SCOPE_MISMATCH", "generated DPUSet selects an unselected device") + } + matches++ + } + } + if matches != 1 { + return fail("DPUSET_SCOPE_MISMATCH", "per-device DPUSet must select exactly one enrolled device") + } + name := text(set, "spec", "dpuTemplate", "spec", "blueFieldSoftware", "name") + software, err := s.clients[bundle.Management].Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "BlueFieldSoftware", s.g.Namespace, name)) + if err != nil { + return fail("SOFTWARE_NOT_READY", "DPUSet software is absent") + } + ready, _, _ := Readiness(software) + if !ready { + return fail("SOFTWARE_NOT_READY", "DPUSet software is not Ready") + } + if err = s.runner.Artifacts.Cached(ctx, s.clients[bundle.Management], s.g, software); err != nil { + return fail("SOFTWARE_INTEGRITY_FAILED", err.Error()) + } + // DPUCluster has no controller observation generation. Recheck the exact + // tenant object generation after potentially long artifact hashing, before + // any DPUSet may trigger physical provisioning. + if err := s.cluster(ctx); err != nil { + return err + } + // Large cached ISO hashes can take time. Reconfirm selected physical identity + // and owner scope after streaming, then re-read the immutable software binding. + if err := s.selection(ctx); err != nil { + return err + } + after, err := s.clients[bundle.Management].Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "BlueFieldSoftware", s.g.Namespace, name)) + if err != nil { + return fail("SOFTWARE_BINDING_DRIFT", "cannot re-read software before provisioning") + } + beforePaths, _, _ := unstructured.NestedMap(software.Object, "status", "downloadedComponents") + afterPaths, _, _ := unstructured.NestedMap(after.Object, "status", "downloadedComponents") + ready, _, _ = Readiness(after) + if !ready || after.GetUID() != software.GetUID() || after.GetGeneration() != software.GetGeneration() || !reflect.DeepEqual(beforePaths, afterPaths) { + return fail("SOFTWARE_BINDING_DRIFT", "software changed after consumed-byte/identity verification") + } + return s.check("software-integrity", name, "management:BlueFieldSoftware cached consumed files", "passed", "CACHED_BYTES_MATCH", "external invariant: cache payloads must not be replaced or removed during provisioning") +} + +func (s *run) selectedReady(ctx context.Context) error { + for { + dpus, err := s.clients[bundle.Management].List(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPU", s.g.Namespace, ""), "") + if err != nil { + return fail("DPU_READINESS_UNAVAILABLE", "cannot list selected DPU children") + } + readyCount := 0 + selected := map[string]map[string]string{} + pending := []string{} + for _, binding := range s.g.Provisioning.Devices { + dpuNode, err := s.clients[bundle.Management].Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUNode", s.g.Namespace, binding.DPUNode)) + if err != nil { + return fail("DPU_NODE_READINESS_UNAVAILABLE", "cannot inspect selected DPUNode transition") + } + if err := reviewedRebootMethod(dpuNode); err != nil { + return err + } + transition := pendingNodeTransition(dpuNode) + if transition != "" { + pending = append(pending, binding.DPUNode+": "+transition) + } + var children []*unstructured.Unstructured + for _, dpu := range dpus { + if text(dpu, "spec", "dpuDeviceName") == binding.DPUDevice { + children = append(children, dpu) + } + } + if len(children) > 1 { + return fail("DPU_CHILD_MEMBERSHIP_MISMATCH", "selected device must have exactly one DPU child") + } + if len(children) == 0 { + pending = append(pending, binding.DPUDevice+": no DPU child") + continue + } + child := children[0] + if text(child, "spec", "cluster", "name") != s.g.Provisioning.DPUCluster.Name || text(child, "spec", "cluster", "namespace") != s.g.Provisioning.DPUCluster.Namespace { + return fail("DPU_TENANT_MISMATCH", "selected DPU is assigned to a different tenant") + } + ownerVerified := false + nodeLabels := map[string]string{} + for _, owner := range child.GetOwnerReferences() { + if owner.Kind != "DPUSet" { + continue + } + for _, doc := range resourcesInBundle(s.bundle, "DPUSet") { + if owner.Name != doc.Ref().Name { + continue + } + set, err := s.clients[bundle.Management].Get(ctx, doc.Ref()) + if err != nil { + return fail("DPU_OWNER_UNAVAILABLE", "cannot recheck selected DPUSet owner") + } + if set.GetUID() != owner.UID || !owns(set, doc.ObjectCopy(), s.result.BundleDigest, s.g.Release) { + return fail("DPU_OWNER_MISMATCH", "DPU child owner identity differs from the retained bundle") + } + if text(child, "spec", "blueFieldSoftware") != text(set, "spec", "dpuTemplate", "spec", "blueFieldSoftware", "name") || text(child, "spec", "dpuFlavor") != text(set, "spec", "dpuTemplate", "spec", "dpuFlavor") { + return fail("DPU_TEMPLATE_MISMATCH", "selected DPU child software/flavor differs from generated intent") + } + nodeLabels, err = expectedDPUNodeLabels(set, binding.DPUNode, s.g.Namespace, s.g.Release) + if err != nil { + return err + } + childLabels, _, err := unstructured.NestedStringMap(child.Object, "spec", "cluster", "nodeLabels") + if err != nil || !jsonEqual(nodeLabels, childLabels) { + return fail("DPU_NODE_SCOPE_MISMATCH", "DPU child node labels differ from its bundle DPUSet") + } + ownerVerified = true + } + } + if !ownerVerified { + return fail("DPU_OWNER_MISMATCH", "selected DPU lacks the expected bundle DPUSet owner") + } + ready, terminal, message := Readiness(children[0]) + if terminal { + return fail("DPU_NOT_READY", message) + } + if !ready { + pending = append(pending, binding.DPUDevice+": DPU phase="+text(child, "status", "phase")+" "+message) + continue + } + node, err := s.clients[bundle.DPU].Get(ctx, ref(bundle.DPU, "", "Node", "", nodeForDPU(children[0]))) + if err != nil { + pending = append(pending, binding.DPUDevice+": tenant Node absent") + continue + } + selected[node.GetName()] = nodeLabels + ready, _, _ = Readiness(node) + labelsMatch := true + for key, value := range nodeLabels { + if node.GetLabels()[key] != value { + labelsMatch = false + } + } + if !labelsMatch { + pending = append(pending, node.GetName()+": tenant Node scope labels do not match") + } + if ready && labelsMatch && transition == "" { + baseReady, err := s.selectedNodeBase(ctx, node.GetName()) + if err != nil { + return err + } + if baseReady { + readyCount++ + } + } + } + if readyCount == len(s.g.Provisioning.Devices) { + if _, err := s.tenantScope(ctx, selected); err != nil { + return err + } + // Re-read after tenant observations: a reset marker or executor + // change that appeared during this sampling round blocks success. + clear, diagnostic, err := s.selectedTransitionsClear(ctx) + if err != nil { + return err + } + if !clear { + if s.result.Mode == Validate && !s.runner.Wait { + return fail("NODE_TRANSITION_PENDING", diagnostic) + } + if err := s.check("selected-dpu-readiness", "DPUNode", "management:DPUNode", "pending", "NODE_TRANSITION_PENDING", diagnostic); err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(s.runner.PollInterval): + continue + } + } + s.selectedNodes = selected + return s.check("selected-dpu-readiness", fmt.Sprint(readyCount), "management:DPU + tenant:Node", "passed", "SELECTED_CHILDREN_READY", "") + } + if s.result.Mode == Validate && !s.runner.Wait { + return fail("SELECTED_DPUS_NOT_READY", "selected DPU children or tenant Nodes are not Ready: "+strings.Join(pending, "; ")) + } + if err := s.check("selected-dpu-readiness", fmt.Sprint(readyCount), "management:DPU + tenant:Node/base Pods", "pending", "SELECTED_CHILDREN_NOT_READY", "waiting for exact selected children, tenant Nodes and selected-node networking components: "+strings.Join(pending, "; ")); err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(s.runner.PollInterval): + } + } +} + +func (s *run) selectedNodeBase(ctx context.Context, node string) (bool, error) { + checks := []struct{ namespace, selector, name string }{ + {s.g.Namespace, "dpu.nvidia.com/component=cni-installer", "cni-installer"}, + {s.g.Namespace, "dpu.nvidia.com/component=multus", "multus"}, + {s.g.Namespace, "app=flannel,tier=node", "flannel"}, + {s.g.Namespace, "dpu.nvidia.com/component=sriov-device-plugin", "sriov-device-plugin"}, + {s.g.Namespace, "dpu.nvidia.com/component=sfc-controller", "sfc-controller"}, + {s.g.Namespace, "dpu.nvidia.com/component=nvidia-k8s-ipam", "nvidia-k8s-ipam"}, + {"kube-system", "k8s-app=kube-proxy", "kube-proxy"}, + } + allReady := true + for _, check := range checks { + pods, err := s.clients[bundle.DPU].List(ctx, ref(bundle.DPU, "", "Pod", check.namespace, ""), check.selector) + if err != nil { + return false, fail("TENANT_BASE_READ_FAILED", "cannot inspect selected-node "+check.name) + } + readyCount := 0 + for _, pod := range pods { + if text(pod, "spec", "nodeName") != node || !pod.GetDeletionTimestamp().IsZero() { + continue + } + ready, _, _ := Readiness(pod) + if ready { + readyCount++ + } + } + if readyCount != 1 { + allReady = false + } + } + if allReady { + if err := s.check("selected-node-base", node, "tenant:Pod networking component selectors", "passed", "SELECTED_NODE_BASE_READY", ""); err != nil { + return false, err + } + } + npd, err := s.clients[bundle.DPU].List(ctx, ref(bundle.DPU, "", "Pod", s.g.Namespace, ""), "dpu.nvidia.com/component=node-problem-detector") + if err != nil { + return false, fail("TENANT_BASE_READ_FAILED", "cannot inspect selected-node problem detector diagnostic") + } + npdReady := false + for _, pod := range npd { + if text(pod, "spec", "nodeName") == node { + ready, _, _ := Readiness(pod) + npdReady = npdReady || ready + } + } + if !npdReady { + if err := s.check("selected-node-problem-detector", node, "tenant:Pod node-problem-detector", "diagnostic", "NODE_PROBLEM_DETECTOR_NOT_READY", "node problem detector is missing/not Ready on the selected node; no healthy detector claim"); err != nil { + return false, err + } + } + return allReady, nil +} + +// A previous operation must finish externally before a new physical operation. +// rebootMethod alone records the most recent reset type and is not an active flag. +func pendingNodeTransition(node *unstructured.Unstructured) string { + active, _, _ := unstructured.NestedBool(node.Object, "status", "rebootInProgress") + if active { + return "rebootInProgress=true (" + text(node, "status", "rebootMethod") + ")" + } + const annotation = "provisioning.dpu.nvidia.com/dpunode-external-reboot-required" + if value, exists := node.GetAnnotations()[annotation]; exists { + return "external reboot annotation remains: " + value + } + for _, name := range []string{"DPUNodeRebootInProgress", "DPUUpdateInProgress", "DPUNodeNodeEffectInProgress"} { + if condition(node, name) { + return name + "=True" + } + } + return "" +} + +// DPUSet readiness may stay False throughout firmware, ISO installation and +// external host reset. Preserve selected-child phase and node reset evidence +// during that wait instead of hiding the physical transition behind a counter. +func (s *run) selectedTransitionDiagnostics(ctx context.Context) (string, error) { + children, err := s.clients[bundle.Management].List(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPU", s.g.Namespace, ""), "") + if err != nil { + return "", fail("DPU_READINESS_UNAVAILABLE", "cannot inspect selected DPU transition diagnostics") + } + out := []string{} + for _, binding := range s.g.Provisioning.Devices { + for _, child := range children { + if text(child, "spec", "dpuDeviceName") == binding.DPUDevice { + out = append(out, binding.DPUDevice+": phase="+text(child, "status", "phase")+" "+conditionMessage(child)) + } + } + node, err := s.clients[bundle.Management].Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUNode", s.g.Namespace, binding.DPUNode)) + if err != nil { + return "", fail("DPU_NODE_READINESS_UNAVAILABLE", "cannot inspect selected DPUNode reset evidence") + } + if transition := pendingNodeTransition(node); transition != "" { + out = append(out, binding.DPUNode+": "+transition) + } + } + return strings.Join(out, "; "), nil +} + +// The first slice permits only a reviewed external executor. Reuse this exact +// contract before writes and at convergence so late changes cannot be ignored. +func reviewedRebootMethod(node *unstructured.Unstructured) error { + if node == nil { + return fail("DPU_NODE_READINESS_UNAVAILABLE", "selected DPUNode is absent") + } + method, exists, err := unstructured.NestedMap(node.Object, "spec", "nodeRebootMethod") + external, ok := method["external"].(map[string]any) + if err != nil || !exists || !ok || len(method) != 1 || len(external) != 0 { + return fail("NODE_REBOOT_METHOD_UNSUPPORTED", node.GetName()+" requires the explicitly reviewed external:{} reboot method") + } + return nil +} +func (s *run) selectedTransitionsClear(ctx context.Context) (bool, string, error) { + pending := []string{} + for _, binding := range s.g.Provisioning.Devices { + node, err := s.clients[bundle.Management].Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUNode", s.g.Namespace, binding.DPUNode)) + if err != nil { + return false, "", fail("DPU_NODE_READINESS_UNAVAILABLE", "cannot recheck selected DPUNode at convergence") + } + if err := reviewedRebootMethod(node); err != nil { + return false, "", err + } + if transition := pendingNodeTransition(node); transition != "" { + pending = append(pending, binding.DPUNode+": "+transition) + } + } + return len(pending) == 0, strings.Join(pending, "; "), nil +} + +// rc.1 dpuset_controller.go buildDPUClusterNodeLabels: the controller appends +// these three reserved labels to frozen user labels. Do not accept arbitrary +// additional labels: they can change service/workload selection. +func expectedDPUNodeLabels(set *unstructured.Unstructured, node, namespace, release string) (map[string]string, error) { + expected, _, err := unstructured.NestedStringMap(set.Object, "spec", "dpuTemplate", "spec", "cluster", "nodeLabels") + if err != nil || expected[sentry.ScopeLabel] == "" { + return nil, fail("DPU_NODE_SCOPE_MISSING", "DPUSet must bind the Sentry node scope label") + } + reserved := map[string]string{ + "provisioning.dpu.nvidia.com/dpunode-name": node, + "provisioning.dpu.nvidia.com/dpunode-namespace": namespace, + "operator.dpu.nvidia.com/dpf-version": "v" + strings.TrimPrefix(release, "v"), + } + for key, value := range reserved { + if supplied, exists := expected[key]; exists && supplied != value { + return nil, fail("DPU_NODE_LABEL_INTENT_CONFLICT", "DPUSet reserved node label conflicts with its selected enrollment/release") + } + expected[key] = value + } + return expected, nil +} diff --git a/pkg/dpf/deployment/private.go b/pkg/dpf/deployment/private.go new file mode 100644 index 00000000..768ef947 --- /dev/null +++ b/pkg/dpf/deployment/private.go @@ -0,0 +1,333 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "reflect" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func (c *nativeClient) VerifyBMC(ctx context.Context, g config.Generation, binding config.DeviceBinding, device inventory.Device, psid string) error { + enrolled, err := c.Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "DPUDevice", g.Namespace, binding.DPUDevice)) + if err != nil { + return fmt.Errorf("cannot read bound BMC credential reference") + } + // Status records the credential currently in use; spec can already point to + // its replacement while the operator is still performing rotation. + secret := text(enrolled, "status", "bmcCredentialSecretName") + if secret == "" { + secret = text(enrolled, "spec", "bmcCredentialSecretName") + } + if secret == "" { + secret = "bmc-shared-password" + } + password, err := c.secretData(ctx, g.Namespace, secret, "password") + if err != nil { + return err + } + certPEM, err := c.secretData(ctx, g.Namespace, "dpf-provisioning-redfish-client-secret-bf4", "tls.crt") + if err != nil { + return err + } + keyPEM, err := c.secretData(ctx, g.Namespace, "dpf-provisioning-redfish-client-secret-bf4", "tls.key") + if err != nil { + return err + } + trust, err := c.Get(ctx, ref(bundle.Management, "", "ConfigMap", g.Namespace, "dpf-ca-trust-bundle")) + if err != nil { + return fmt.Errorf("DPF CA trust bundle is unavailable") + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM([]byte(text(trust, "data", "ca.crt"))) { + return fmt.Errorf("DPF CA trust bundle is invalid") + } + certificate, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return fmt.Errorf("DPF BF4 client keypair is invalid") + } + endpoint, err := url.Parse(device.BMCEndpoint) + if err != nil || endpoint.Scheme != "https" || endpoint.User != nil || endpoint.Host == "" { + return fmt.Errorf("BMC endpoint is not a bound HTTPS origin") + } + tlsConfig := verifiedBMCTLS(pool, certificate, endpoint.Hostname()) + transport := &http.Transport{TLSClientConfig: tlsConfig, TLSHandshakeTimeout: 15 * time.Second, ResponseHeaderTimeout: 15 * time.Second, Proxy: nil} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + get := func(uri string) (map[string]any, error) { + if !strings.HasPrefix(uri, "/redfish/v1/") || path.Clean(uri) != strings.TrimSuffix(uri, "/") || strings.ContainsAny(uri, "?%\\") { + return nil, fmt.Errorf("unsupported BMC identity path") + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.Scheme+"://"+endpoint.Host+uri, nil) + if err != nil { + return nil, fmt.Errorf("cannot create bounded identity request") + } + request.SetBasicAuth("admin", string(password)) + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("verified BMC GET failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("BMC identity GET returned HTTP %d", response.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(response.Body, (4<<20)+1)) + if err != nil || len(data) > 4<<20 { + return nil, fmt.Errorf("BMC identity response is unavailable or oversized") + } + var object map[string]any + if json.Unmarshal(data, &object) != nil { + return nil, fmt.Errorf("BMC identity response is malformed") + } + return object, nil + } + if _, err = get("/redfish/v1/Managers"); err != nil { + return err + } + chassis, err := get("/redfish/v1/Chassis/BlueField_0") + if err != nil { + return err + } + if chassis["SerialNumber"] != device.SerialNumber { + return fmt.Errorf("live BF4 serial differs from selected identity") + } + system, err := get("/redfish/v1/Systems/BlueField_0") + if err != nil { + return err + } + livePSID, _ := system["AssetTag"].(string) + if livePSID == "" || livePSID == "N/A" { + livePSID, _ = chassis["AssetTag"].(string) + } + if !strings.EqualFold(livePSID, psid) { + return fmt.Errorf("live BF4 PSID differs from selected firmware scope") + } + return nil +} + +func verifiedBMCTLS(roots *x509.CertPool, certificate tls.Certificate, host string) *tls.Config { + // BF4 firmware can issue CN-only IP certificates. The custom verification + // below verifies the complete CA chain and identity; it is never a TLS bypass. + return &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{certificate}, InsecureSkipVerify: true, VerifyConnection: func(state tls.ConnectionState) error { + if len(state.PeerCertificates) == 0 || roots == nil || host == "" { + return fmt.Errorf("BMC peer identity is missing") + } + leaf := state.PeerCertificates[0] + intermediates := x509.NewCertPool() + for _, cert := range state.PeerCertificates[1:] { + intermediates.AddCert(cert) + } + if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { + return fmt.Errorf("BMC certificate does not verify against DPF CA") + } + if leaf.VerifyHostname(host) == nil { + return nil + } + if len(leaf.DNSNames) == 0 && len(leaf.IPAddresses) == 0 && leaf.Subject.CommonName == host { + return nil + } + return fmt.Errorf("BMC certificate identity differs from selected endpoint") + }} +} + +// CachedArtifacts hashes the actual DPF registry payload through an authenticated +// Kubernetes Pod proxy, then re-observes the same software identity/status. +// DPF lacks native digest enforcement: cache replacement/deletion during active +// provisioning remains an explicit external invariant. +type CachedArtifacts struct{} + +func (CachedArtifacts) Cached(ctx context.Context, client Client, g config.Generation, software *unstructured.Unstructured) error { + native, ok := client.(*nativeClient) + if !ok { + return fmt.Errorf("native cached-byte verification requires the bound native client") + } + ready, _, _ := Readiness(software) + if !ready || software.GetUID() == "" { + return fmt.Errorf("software snapshot is not Ready and identified") + } + components, _, _ := unstructured.NestedMap(software.Object, "status", "downloadedComponents") + paths := map[string]string{} + iso, _ := components["osIso"].(string) + paths[iso] = g.Provisioning.Software.OSISO.SHA256 + pldm, _ := components["pldmFwBundle"].(map[string]any) + for psid, artifact := range g.Provisioning.Software.PLDM { + var filename string + for candidate, value := range pldm { + if strings.EqualFold(candidate, psid) { + filename, _ = value.(string) + } + } + paths[filename] = artifact.SHA256 + } + if g.Provisioning.Software.NICFirmware != nil { + filename, _ := components["nicFw"].(string) + paths[filename] = g.Provisioning.Software.NICFirmware.SHA256 + } + service, err := native.Get(ctx, ref(bundle.Management, "", "Service", g.Namespace, "bfb-registry")) + if err != nil || service.GetUID() == "" { + return fmt.Errorf("bound DPF bfb-registry Service is unavailable") + } + registry, leader, err := native.registryStorage(ctx, g.Namespace) + if err != nil { + return err + } + prefix := software.GetNamespace() + "-" + software.GetName() + "-" + for filename, expected := range paths { + if filename == "" || path.Clean(filename) != filename || !strings.HasPrefix(filename, "/bfb/") || strings.ContainsAny(filename, "?%\\") || !strings.HasPrefix(path.Base(filename), prefix) { + return fmt.Errorf("software status contains an unsupported cache path") + } + if path.Dir(filename) != "/bfb" && path.Dir(filename) != "/bfb/components" { + return fmt.Errorf("software cache path escapes qualified roots") + } + stream, err := native.core.RESTClient().Get().Namespace(g.Namespace).Resource("pods").Name("https:bfb-registry:8443").SubResource("proxy").Suffix(strings.TrimPrefix(filename, "/")).Stream(ctx) + if err != nil { + return fmt.Errorf("cannot stream bound cached payload through Kubernetes service proxy") + } + hash := sha256.New() + size, copyErr := io.Copy(hash, io.LimitReader(stream, (32<<30)+1)) + closeErr := stream.Close() + if copyErr != nil || closeErr != nil || size == 0 || size > 32<<30 { + return fmt.Errorf("cached artifact stream failed or exceeded bound") + } + if hex.EncodeToString(hash.Sum(nil)) != expected { + return fmt.Errorf("cached artifact SHA256 differs from locked software") + } + } + after, err := native.Get(ctx, ref(bundle.Management, "provisioning.dpu.nvidia.com", "BlueFieldSoftware", software.GetNamespace(), software.GetName())) + if err != nil { + return fmt.Errorf("cannot re-observe software after cache verification") + } + afterComponents, _, _ := unstructured.NestedMap(after.Object, "status", "downloadedComponents") + ready, _, _ = Readiness(after) + if !ready || after.GetUID() != software.GetUID() || after.GetGeneration() != software.GetGeneration() || !reflect.DeepEqual(components, afterComponents) { + return fmt.Errorf("software identity/generation/cache paths changed while verifying consumed bytes") + } + afterService, err := native.Get(ctx, ref(bundle.Management, "", "Service", g.Namespace, "bfb-registry")) + if err != nil || afterService.GetUID() != service.GetUID() || afterService.GetResourceVersion() != service.GetResourceVersion() { + return fmt.Errorf("registry Service changed during cache verification") + } + afterRegistry, afterLeader, err := native.registryStorage(ctx, g.Namespace) + if err != nil || afterRegistry.GetUID() != registry.GetUID() || afterRegistry.GetResourceVersion() != registry.GetResourceVersion() || afterLeader.GetUID() != leader.GetUID() { + return fmt.Errorf("registry/leader storage binding changed while hashing cached files") + } + return nil +} + +func (c *nativeClient) registryStorage(ctx context.Context, namespace string) (*unstructured.Unstructured, *unstructured.Unstructured, error) { + registry, err := c.Get(ctx, ref(bundle.Management, "", "Pod", namespace, "bfb-registry")) + if err != nil { + return nil, nil, fmt.Errorf("cannot inspect bound registry Pod") + } + ready, _, _ := Readiness(registry) + if !ready { + return nil, nil, fmt.Errorf("registry Pod is not Ready") + } + var ownerName string + var ownerUID string + for _, owner := range registry.GetOwnerReferences() { + if owner.Kind == "Pod" { + if ownerName != "" { + return nil, nil, fmt.Errorf("registry has ambiguous provisioning leader ownership") + } + ownerName = owner.Name + ownerUID = string(owner.UID) + } + } + if ownerName == "" { + return nil, nil, fmt.Errorf("registry lacks the qualified provisioning leader Pod owner") + } + leader, err := c.Get(ctx, ref(bundle.Management, "", "Pod", namespace, ownerName)) + if err != nil || string(leader.GetUID()) != ownerUID { + return nil, nil, fmt.Errorf("registry provisioning leader identity is unresolved") + } + lease, err := c.Get(ctx, ref(bundle.Management, "coordination.k8s.io", "Lease", namespace, "provisioning.dpu.nvidia.com")) + if err != nil { + return nil, nil, fmt.Errorf("cannot read qualified provisioning leader Lease") + } + holder := text(lease, "spec", "holderIdentity") + hostname := text(leader, "spec", "hostname") + if hostname == "" { + hostname = ownerName + } + if !strings.HasPrefix(holder, hostname+"_") { + return nil, nil, fmt.Errorf("registry owner is not the current provisioning Lease holder") + } + renewed, err := time.Parse(time.RFC3339Nano, text(lease, "spec", "renewTime")) + duration := number(lease, "spec", "leaseDurationSeconds") + if err != nil || duration <= 0 || time.Now().After(renewed.Add(time.Duration(duration)*time.Second)) { + return nil, nil, fmt.Errorf("provisioning leader Lease is expired or unobserved") + } + registryVolume, ok := bfbStorage(registry) + if !ok { + return nil, nil, fmt.Errorf("registry /bfb storage mount is unsupported") + } + leaderVolume, ok := bfbStorage(leader) + if !ok { + return nil, nil, fmt.Errorf("provisioning leader /bfb storage mount is unsupported") + } + if !reflect.DeepEqual(registryVolume, leaderVolume) { + return nil, nil, fmt.Errorf("registry and provisioning leader do not consume the same cache storage") + } + if _, hostPath := registryVolume["hostPath"]; hostPath && (text(registry, "spec", "nodeName") == "" || text(registry, "spec", "nodeName") != text(leader, "spec", "nodeName")) { + return nil, nil, fmt.Errorf("hostPath registry and provisioning cache are on different nodes") + } + return registry, leader, nil +} + +func bfbStorage(pod *unstructured.Unstructured) (map[string]any, bool) { + containers, _, _ := unstructured.NestedSlice(pod.Object, "spec", "containers") + volumeName := "" + for _, item := range containers { + container, ok := item.(map[string]any) + if !ok { + continue + } + mounts, _ := container["volumeMounts"].([]any) + for _, item := range mounts { + mount, ok := item.(map[string]any) + if ok && mount["mountPath"] == "/bfb" { + if sub, _ := mount["subPath"].(string); sub != "" { + return nil, false + } + name, _ := mount["name"].(string) + if volumeName != "" && volumeName != name { + return nil, false + } + volumeName = name + } + } + } + volumes, _, _ := unstructured.NestedSlice(pod.Object, "spec", "volumes") + for _, item := range volumes { + volume, ok := item.(map[string]any) + if !ok || volume["name"] != volumeName { + continue + } + if host, ok := volume["hostPath"].(map[string]any); ok && host["path"] == "/var/lib/nvidia/dpf/bfb" { + return map[string]any{"hostPath": host["path"]}, true + } + if pvc, ok := volume["persistentVolumeClaim"].(map[string]any); ok { + if claim, ok := pvc["claimName"].(string); ok && claim != "" { + return map[string]any{"persistentVolumeClaim": claim}, true + } + } + } + return nil, false +} diff --git a/pkg/dpf/deployment/private_test.go b/pkg/dpf/deployment/private_test.go new file mode 100644 index 00000000..5521432a --- /dev/null +++ b/pkg/dpf/deployment/private_test.go @@ -0,0 +1,188 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + typedcore "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + clienttesting "k8s.io/client-go/testing" +) + +var _ = Describe("native private boundaries", func() { + It("uses the recorded current BMC credential before a different pending spec reference", func() { + requests := 0 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + user, password, ok := r.BasicAuth() + if r.Method != http.MethodGet || r.TLS == nil || len(r.TLS.PeerCertificates) == 0 || !ok || user != "admin" || password != "recorded-current" { + w.WriteHeader(http.StatusUnauthorized) + return + } + switch r.URL.Path { + case "/redfish/v1/Managers": + _, _ = w.Write([]byte(`{"Members":[]}`)) + case "/redfish/v1/Chassis/BlueField_0": + _, _ = w.Write([]byte(`{"SerialNumber":"SERIAL-A","AssetTag":"TEST-PSID"}`)) + case "/redfish/v1/Systems/BlueField_0": + _, _ = w.Write([]byte(`{"AssetTag":"TEST-PSID"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + server.TLS = &tls.Config{MinVersion: tls.VersionTLS12, ClientAuth: tls.RequireAnyClientCert} + server.StartTLS() + defer server.Close() + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + key, err := x509.MarshalPKCS8PrivateKey(server.TLS.Certificates[0].PrivateKey) + Expect(err).NotTo(HaveOccurred()) + privateKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}) + secret := func(name string, data map[string][]byte) *unstructured.Unstructured { + o := testObject("v1", "Secret", "dpf-system", name, nil) + encoded := map[string]any{} + for key, value := range data { + encoded[key] = base64.StdEncoding.EncodeToString(value) + } + o.Object["data"] = encoded + return o + } + device := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPUDevice", "dpf-system", "device-a", map[string]any{"bmcCredentialSecretName": "future"}) + _ = unstructured.SetNestedField(device.Object, "current", "status", "bmcCredentialSecretName") + trust := testObject("v1", "ConfigMap", "dpf-system", "dpf-ca-trust-bundle", nil) + trust.Object["data"] = map[string]any{"ca.crt": string(certificate)} + dynamic := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), device, trust, secret("current", map[string][]byte{"password": []byte("recorded-current")}), secret("future", map[string][]byte{"password": []byte("not-adopted-yet")}), secret("dpf-provisioning-redfish-client-secret-bf4", map[string][]byte{"tls.crt": certificate, "tls.key": privateKey})) + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{{Version: "v1"}, {Group: "provisioning.dpu.nvidia.com", Version: "v1alpha1"}}) + for _, kind := range []string{"Secret", "ConfigMap"} { + mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: kind}, meta.RESTScopeNamespace) + } + mapper.Add(schema.GroupVersionKind{Group: "provisioning.dpu.nvidia.com", Version: "v1alpha1", Kind: "DPUDevice"}, meta.RESTScopeNamespace) + client := &nativeClient{dynamic: dynamic, mapper: mapper} + Expect(client.VerifyBMC(context.Background(), config.Generation{Namespace: "dpf-system"}, config.DeviceBinding{DPUDevice: "device-a"}, inventory.Device{SerialNumber: "SERIAL-A", BMCEndpoint: server.URL}, "TEST-PSID")).To(Succeed()) + Expect(requests).To(Equal(3)) + }) + It("verifies DPF CA chain and CN-only BMC identity rather than bypassing TLS", func() { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "192.0.2.10"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + Expect(err).NotTo(HaveOccurred()) + cert, err := x509.ParseCertificate(der) + Expect(err).NotTo(HaveOccurred()) + pool := x509.NewCertPool() + pool.AddCert(cert) + state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}} + Expect(verifiedBMCTLS(pool, tls.Certificate{}, "192.0.2.10").VerifyConnection(state)).To(Succeed()) + Expect(verifiedBMCTLS(pool, tls.Certificate{}, "192.0.2.11").VerifyConnection(state)).NotTo(Succeed()) + Expect(verifiedBMCTLS(x509.NewCertPool(), tls.Certificate{}, "192.0.2.10").VerifyConnection(state)).NotTo(Succeed()) + Expect(verifiedBMCTLS(pool, tls.Certificate{}, "192.0.2.10").VerifyConnection(tls.ConnectionState{})).NotTo(Succeed()) + }) + + It("rejects kubeconfigs with insecure TLS, external execution, or ambient file credentials", func() { + base := "apiVersion: v1\nkind: Config\ncurrent-context: explicit\nclusters:\n- name: cluster\n cluster:\n server: https://192.0.2.1\n%scontexts:\n- name: explicit\n context: {cluster: cluster, user: user}\nusers:\n- name: user\n user:\n%s" + for _, input := range []string{fmt.Sprintf(base, " insecure-skip-tls-verify: true\n", " token: unused\n"), fmt.Sprintf(base, "", " exec: {apiVersion: client.authentication.k8s.io/v1, command: forbidden}\n"), fmt.Sprintf(base, "", " tokenFile: /ambient/token\n")} { + _, err := clientFromBytes([]byte(input), "explicit") + Expect(err).To(HaveOccurred()) + } + }) + + DescribeTable("hashes actual registry-consumed bytes and refuses drift", func(tamper, drift, wrongLeader bool) { + iso, fw := []byte("exact ISO fixture bytes"), []byte("exact PLDM fixture bytes") + digest := func(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } + isoPath := "/bfb/dpf-system-software-os.iso" + fwPath := "/bfb/components/dpf-system-software-TEST-PSID-fw.fwpkg" + software := testObject("provisioning.dpu.nvidia.com/v1alpha1", "BlueFieldSoftware", "dpf-system", "software", map[string]any{"osIso": "https://example.invalid/os.iso"}) + setReady(software) + _ = unstructured.SetNestedMap(software.Object, map[string]any{"osIso": isoPath, "pldmFwBundle": map[string]any{"TEST-PSID": fwPath}}, "status", "downloadedComponents") + service := testObject("v1", "Service", "dpf-system", "bfb-registry", map[string]any{"ports": []any{map[string]any{"port": int64(8443)}}}) + cacheSpec := map[string]any{"nodeName": "management-a", "containers": []any{map[string]any{"name": "controller", "volumeMounts": []any{map[string]any{"name": "cache", "mountPath": "/bfb"}}}}, "volumes": []any{map[string]any{"name": "cache", "hostPath": map[string]any{"path": "/var/lib/nvidia/dpf/bfb"}}}} + leader := testObject("v1", "Pod", "dpf-system", "provisioning-leader", cacheSpec) + setReady(leader) + _ = unstructured.SetNestedField(leader.Object, "Running", "status", "phase") + registry := testObject("v1", "Pod", "dpf-system", "bfb-registry", cacheSpec) + registry.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: "v1", Kind: "Pod", Name: leader.GetName(), UID: leader.GetUID()}}) + setReady(registry) + _ = unstructured.SetNestedField(registry.Object, "Running", "status", "phase") + lease := testObject("coordination.k8s.io/v1", "Lease", "dpf-system", "provisioning.dpu.nvidia.com", map[string]any{"holderIdentity": "provisioning-leader_unique-election-id", "renewTime": time.Now().UTC().Format(time.RFC3339Nano), "leaseDurationSeconds": int64(60)}) + if wrongLeader { + _ = unstructured.SetNestedField(lease.Object, "another-leader_unique-election-id", "spec", "holderIdentity") + } + dynamic := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), software, service, leader, registry, lease) + if drift { + dynamic.PrependReactor("get", "bluefieldsoftwares", func(action clienttesting.Action) (bool, runtime.Object, error) { + changed := software.DeepCopy() + changed.SetGeneration(2) + return true, changed, nil + }) + } + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{{Version: "v1"}, {Group: "provisioning.dpu.nvidia.com", Version: "v1alpha1"}, {Group: "coordination.k8s.io", Version: "v1"}}) + mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "Service"}, meta.RESTScopeNamespace) + mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "Pod"}, meta.RESTScopeNamespace) + mapper.Add(schema.GroupVersionKind{Group: "coordination.k8s.io", Version: "v1", Kind: "Lease"}, meta.RESTScopeNamespace) + mapper.Add(schema.GroupVersionKind{Group: "provisioning.dpu.nvidia.com", Version: "v1alpha1", Kind: "BlueFieldSoftware"}, meta.RESTScopeNamespace) + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + switch { + case strings.HasSuffix(r.URL.Path, isoPath): + if tamper { + _, _ = w.Write([]byte("tampered")) + } else { + _, _ = w.Write(iso) + } + case strings.HasSuffix(r.URL.Path, fwPath): + _, _ = w.Write(fw) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + core, err := typedcore.NewForConfig(&rest.Config{Host: server.URL}) + Expect(err).NotTo(HaveOccurred()) + client := &nativeClient{dynamic: dynamic, mapper: mapper, core: core} + g := config.Generation{Namespace: "dpf-system", Provisioning: config.Provisioning{Software: config.Software{OSISO: config.Artifact{SHA256: digest(iso)}, PLDM: map[string]config.Artifact{"TEST-PSID": {SHA256: digest(fw)}}}}} + err = (CachedArtifacts{}).Cached(context.Background(), client, g, software) + if wrongLeader { + Expect(err).To(MatchError(ContainSubstring("Lease holder"))) + Expect(requests).To(BeZero()) + } else if tamper { + Expect(err).To(MatchError(ContainSubstring("SHA256 differs"))) + } else if drift { + Expect(err).To(MatchError(ContainSubstring("changed while verifying"))) + } else { + Expect(err).NotTo(HaveOccurred()) + Expect(requests).To(Equal(2)) + } + }, Entry("matches exact cached ISO and firmware", false, false, false), Entry("rejects cached tampering", true, false, false), Entry("rejects software generation drift", false, true, false), Entry("rejects registry storage from an inactive provisioning leader", false, false, true)) +}) diff --git a/pkg/dpf/deployment/readiness.go b/pkg/dpf/deployment/readiness.go new file mode 100644 index 00000000..a924fd2e --- /dev/null +++ b/pkg/dpf/deployment/readiness.go @@ -0,0 +1,161 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "fmt" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Readiness is a closed kind registry. Structural existence is sufficient only +// for declarative kinds without a reconciliation/readiness API. +func Readiness(o *unstructured.Unstructured) (ready, terminal bool, message string) { + if o == nil { + return false, false, "resource is absent" + } + if !o.GetDeletionTimestamp().IsZero() { + return false, true, "resource is deleting" + } + if requiresObservedGeneration(o.GetKind()) && (o.GetGeneration() < 1 || number(o, "status", "observedGeneration") != o.GetGeneration()) { + return false, false, "current generation is not observed" + } + switch o.GetKind() { + case "ConfigMap", "Secret", "ServiceAccount", "Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding", "NetworkAttachmentDefinition", "DPUFlavor": + return true, false, "resource exists; this kind has no readiness condition" + case "Namespace": + return text(o, "status", "phase") == "Active", false, "namespace must be Active" + case "Node": + return condition(o, "Ready"), false, "Kubernetes Node Ready condition" + case "Pod": + phase := text(o, "status", "phase") + return phase == "Running" && condition(o, "Ready"), phase == "Failed", "Pod must be Running and Ready" + case "Deployment": + want, supplied, err := unstructured.NestedInt64(o.Object, "spec", "replicas") + if !supplied { + want = 1 + } + // During a surge, updated and available replicas can be disjoint: + // the previous ReplicaSet still serves while the new pod is unready. + return err == nil && want > 0 && o.GetGeneration() > 0 && + number(o, "status", "observedGeneration") == o.GetGeneration() && + number(o, "status", "replicas") == want && + number(o, "status", "updatedReplicas") == want && + number(o, "status", "readyReplicas") == want && + number(o, "status", "availableReplicas") == want, false, "Deployment must have a current fully available completed rollout" + case "DaemonSet": + want := number(o, "status", "desiredNumberScheduled") + return want > 0 && number(o, "status", "observedGeneration") >= o.GetGeneration() && number(o, "status", "numberReady") == want && number(o, "status", "updatedNumberScheduled") == want, false, "DaemonSet must have a positive fully Ready rollout" + case "StatefulSet": + want := number(o, "spec", "replicas") + if want == 0 { + want = 1 + } + return number(o, "status", "observedGeneration") >= o.GetGeneration() && number(o, "status", "readyReplicas") == want, false, "StatefulSet replicas must be Ready" + case "Job": + return condition(o, "Complete"), condition(o, "Failed"), "Job completion condition" + case "Service": + return text(o, "spec", "clusterIP") != "", false, "Service clusterIP allocation" + case "CustomResourceDefinition": + return condition(o, "Established") && condition(o, "NamesAccepted"), false, "CRD must be established with accepted names" + case "BlueFieldSoftware": + return text(o, "status", "phase") == "Ready" && condition(o, "Ready"), text(o, "status", "phase") == "Error", "BlueFieldSoftware must be Ready" + case "DPU": + return text(o, "status", "phase") == "Ready" && condition(o, "Ready") && number(o, "status", "observedGeneration") == o.GetGeneration(), text(o, "status", "phase") == "Error", "DPU phase, condition and observed generation must be Ready" + case "DPUCluster": + // rc.1 and GA use cutil.NewCondition without observedGeneration. + phase := text(o, "status", "phase") + return phase == "Ready" && condition(o, "Ready"), phase == "Failed", "DPUCluster phase=" + phase + "; " + conditionMessage(o) + case "DPUServiceNAD", "DPUServiceCredentialRequest", "Certificate", "Issuer", "ClusterIssuer": + return freshCondition(o, "Ready"), condition(o, "Error"), conditionMessage(o) + case "DPUSet", "DPUService", "DPUServiceInterface", "DPUServiceChain": + return condition(o, "Ready"), condition(o, "Error"), conditionMessage(o) + default: + return false, true, fmt.Sprintf("no readiness contract for %s", o.GetKind()) + } +} + +func conditionMessage(o *unstructured.Unstructured) string { + conditions, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + for _, raw := range conditions { + if c, ok := raw.(map[string]any); ok && c["type"] == "Ready" { + reason, _ := c["reason"].(string) + if len(reason) > 256 { + reason = reason[:256] + } + return fmt.Sprintf("%s Ready=%v reason=%s", o.GetKind(), c["status"], reason) + } + } + return o.GetKind() + " has no observed Ready condition" +} + +func condition(o *unstructured.Unstructured, kind string) bool { + if o == nil { + return false + } + conditions, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + for _, raw := range conditions { + if c, ok := raw.(map[string]any); ok && c["type"] == kind && c["status"] == "True" { + return true + } + } + return false +} + +func hasCondition(o *unstructured.Unstructured, kind string) bool { + conditions, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + for _, raw := range conditions { + if c, ok := raw.(map[string]any); ok && c["type"] == kind { + return true + } + } + return false +} + +func nodeForDPU(dpu *unstructured.Unstructured) string { + // Exact rc.1 GenerateNodeName and GetNodeFromDPUCluster use the DPU name. + return dpu.GetName() +} + +func resourcesInBundle(b *bundle.Bundle, kind string) []bundle.Document { + var documents []bundle.Document + for _, stage := range b.Index().Stages { + view, _ := b.Stage(stage.ID) + for _, doc := range view.Documents() { + if doc.Ref().Kind == kind { + documents = append(documents, doc) + } + } + } + return documents +} + +// These exact rc.1 APIs publish status.observedGeneration. NAD and +// CredentialRequest instead use condition.observedGeneration, stamped by +// pkg/conditions.SetSummary. DPUCluster exposes neither observation field. +func requiresObservedGeneration(kind string) bool { + switch kind { + case "BlueFieldSoftware", "DPU", "DPUSet", "DPUService", "DPUServiceInterface", "DPUServiceChain": + return true + default: + return false + } +} + +func freshCondition(o *unstructured.Unstructured, kind string) bool { + if o == nil || o.GetGeneration() < 1 { + return false + } + entries, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + for _, entry := range entries { + c, ok := entry.(map[string]any) + if !ok || c["type"] != kind || c["status"] != "True" { + continue + } + observation, exists, err := unstructured.NestedInt64(c, "observedGeneration") + return err == nil && exists && observation == o.GetGeneration() + } + return false +} diff --git a/pkg/dpf/deployment/service_account_conversion_test.go b/pkg/dpf/deployment/service_account_conversion_test.go new file mode 100644 index 00000000..49d95164 --- /dev/null +++ b/pkg/dpf/deployment/service_account_conversion_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +func accountWorkload(kind string) *unstructured.Unstructured { + return testObject("apps/v1", kind, "service-scope", "example", map[string]any{ + "selector": map[string]any{"matchLabels": map[string]any{"app": "example"}}, + "template": map[string]any{"metadata": map[string]any{"labels": map[string]any{"app": "example"}}, "spec": map[string]any{ + "serviceAccountName": "example-account", "automountServiceAccountToken": false, + "containers": []any{map[string]any{"name": "server", "image": "example.invalid/server:v1"}}, + }}, + }) +} + +var _ = Describe("Kubernetes PodSpec service account conversion", func() { + It("passes owned service preflight with an API alias and refuses a conflicting alias", func() { + b, g, f, _ := lifecycleFixture() + native := serializationObject("DPUService") + workload := accountWorkload("DaemonSet") + workload.SetNamespace("sentry-system") + for _, object := range []*unstructured.Unstructured{native, workload} { + object.SetUID("") + object.SetResourceVersion("") + object.SetGeneration(0) + } + nativeBytes, err := yaml.Marshal(native.Object) + Expect(err).NotTo(HaveOccurred()) + projectionBytes, err := yaml.Marshal(workload.Object) + Expect(err).NotTo(HaveOccurred()) + index, files := b.Index(), b.Files() + index.BundleDigest = "" + index.Files = append(index.Files, bundle.FileRecord{Path: "objects/account-service.yaml", Role: bundle.Manifest}, bundle.FileRecord{Path: "projections/account-workload.yaml", Role: bundle.Projection}) + index.Stages = append(index.Stages, bundle.Stage{ID: "account-service", Destination: bundle.Management, Namespace: "sentry-system", Files: []string{"objects/account-service.yaml"}, DependsOn: []string{"provision"}}, bundle.Stage{ID: "account-workload", Destination: bundle.DPU, Namespace: "sentry-system", ProjectionFiles: []string{"projections/account-workload.yaml"}, DependsOn: []string{"account-service"}}) + files = append(files, bundle.File{Path: "objects/account-service.yaml", Data: nativeBytes}, bundle.File{Path: "projections/account-workload.yaml", Data: projectionBytes}) + b, err = bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + f.management.put(mark(native, b.Index().BundleDigest)) + actual := workload.DeepCopy() + actual.SetUID("captured-workload-uid") + Expect(unstructured.SetNestedField(actual.Object, "example-account", "spec", "template", "spec", "serviceAccount")).To(Succeed()) + f.tenant.put(actual) + state := &run{bundle: b, g: g, result: Result{BundleDigest: b.Index().BundleDigest}, clients: map[bundle.Destination]Client{bundle.Management: f.management, bundle.DPU: f.tenant}} + Expect(state.serviceOwnership(context.Background(), "sentry-system")).To(Succeed()) + Expect(unstructured.SetNestedField(actual.Object, "foreign-account", "spec", "template", "spec", "serviceAccount")).To(Succeed()) + f.tenant.put(actual) + Expect(state.serviceOwnership(context.Background(), "sentry-system")).To(MatchError(ContainSubstring("FOREIGN_SERVICE_WORKLOAD"))) + }) + + DescribeTable("accepts the API-produced matching alias without changing inputs", func(kind string) { + desired := accountWorkload(kind) + actual := desired.DeepCopy() + Expect(unstructured.SetNestedField(actual.Object, "example-account", "spec", "template", "spec", "serviceAccount")).To(Succeed()) + beforeDesired, beforeActual := desired.DeepCopy(), actual.DeepCopy() + Expect(desiredMatches(desired, actual)).To(BeTrue()) + Expect(desiredMatches(actual, desired)).To(BeTrue()) + Expect(desired).To(Equal(beforeDesired)) + Expect(actual).To(Equal(beforeActual)) + // The legacy input alias resolves to the same canonical account. + legacy := desired.DeepCopy() + unstructured.RemoveNestedField(legacy.Object, "spec", "template", "spec", "serviceAccountName") + Expect(unstructured.SetNestedField(legacy.Object, "example-account", "spec", "template", "spec", "serviceAccount")).To(Succeed()) + Expect(desiredMatches(legacy, actual)).To(BeTrue()) + }, Entry("DaemonSet", "DaemonSet"), Entry("Deployment", "Deployment")) + + DescribeTable("does not hide conflicting aliases, changed identity, token mount or unknown content", func(kind, mutation string) { + desired := accountWorkload(kind) + actual := desired.DeepCopy() + Expect(unstructured.SetNestedField(actual.Object, "example-account", "spec", "template", "spec", "serviceAccount")).To(Succeed()) + switch mutation { + case "conflict": + Expect(unstructured.SetNestedField(actual.Object, "other-account", "spec", "template", "spec", "serviceAccount")).To(Succeed()) + // Even self-comparison cannot make conflicting supplied identities valid. + Expect(desiredMatches(actual, actual.DeepCopy())).To(BeFalse()) + case "identity": + Expect(unstructured.SetNestedField(actual.Object, "other-account", "spec", "template", "spec", "serviceAccountName")).To(Succeed()) + Expect(unstructured.SetNestedField(actual.Object, "other-account", "spec", "template", "spec", "serviceAccount")).To(Succeed()) + case "token": + Expect(unstructured.SetNestedField(actual.Object, true, "spec", "template", "spec", "automountServiceAccountToken")).To(Succeed()) + case "unknown": + Expect(unstructured.SetNestedMap(actual.Object, map[string]any{}, "spec", "template", "spec", "unknownIdentityControl")).To(Succeed()) + } + Expect(desiredMatches(desired, actual)).To(BeFalse()) + }, Entry("DaemonSet conflicting alias", "DaemonSet", "conflict"), Entry("Deployment conflicting alias", "Deployment", "conflict"), Entry("DaemonSet different account", "DaemonSet", "identity"), Entry("Deployment different account", "Deployment", "identity"), Entry("DaemonSet token change", "DaemonSet", "token"), Entry("Deployment token change", "Deployment", "token"), Entry("DaemonSet unknown field", "DaemonSet", "unknown"), Entry("Deployment unknown field", "Deployment", "unknown")) +}) diff --git a/pkg/dpf/deployment/service_stage_test.go b/pkg/dpf/deployment/service_stage_test.go new file mode 100644 index 00000000..36f5cb5a --- /dev/null +++ b/pkg/dpf/deployment/service_stage_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +var _ = Describe("mutually dependent native service creation stage", func() { + It("creates Tower, controller and Argus before waiting for Tower's controller-dependent readiness", func() { + b, g, f, v := lifecycleFixture() + index, files := b.Index(), b.Files() + index.BundleDigest = "" + stage := bundle.Stage{ID: "services", Destination: bundle.Management, Namespace: "sentry-system", DependsOn: []string{"provision"}} + for _, name := range []string{"tower", "controller", "argus"} { + service := serializationObject("DPUService") + service.SetName(name) + service.SetUID("") + service.SetResourceVersion("") + service.SetGeneration(0) + data, err := yaml.Marshal(service.Object) + Expect(err).NotTo(HaveOccurred()) + path := "objects/" + name + ".yaml" + files = append(files, bundle.File{Path: path, Data: data}) + index.Files = append(index.Files, bundle.FileRecord{Path: path, Role: bundle.Manifest}) + stage.Files = append(stage.Files, path) + } + index.Stages = append(index.Stages, stage) + var err error + b, err = bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + serviceKey := func(name string) string { + return objectKey(ref("", "svc.dpu.nvidia.com", "DPUService", "sentry-system", name)) + } + towerBlocked := false + f.management.afterCreate = func(created *unstructured.Unstructured) { + if created.GetKind() != "DPUService" || created.GetName() != "tower" { + return + } + tower := f.management.objects[serviceKey("tower")] + Expect(unstructured.SetNestedSlice(tower.Object, []any{map[string]any{"type": "Ready", "status": "False"}}, "status", "conditions")).To(Succeed()) + ready, _, _ := Readiness(tower) + Expect(ready).To(BeFalse()) + Expect(f.management.objects[serviceKey("controller")]).To(BeNil()) + towerBlocked = true + } + readyObservations := 0 + f.management.onGet = func(reference bundle.ResourceRef) { + if reference.Kind != "DPUService" || reference.Name != "tower" { + return + } + if tower := f.management.objects[serviceKey("tower")]; tower != nil { + // A separate Tower wait stage would reach here before the + // controller exists and could never satisfy its own readiness. + Expect(f.management.objects[serviceKey("controller")]).NotTo(BeNil()) + Expect(f.management.objects[serviceKey("argus")]).NotTo(BeNil()) + setReady(tower) + readyObservations++ + } + } + r := dnsTestRunner(f, v) + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(towerBlocked).To(BeTrue()) + Expect(readyObservations).To(BeNumerically(">", 0)) + Expect(f.management.creates).To(ContainElements("DPUService/tower", "DPUService/controller", "DPUService/argus")) + creates := len(f.management.creates) + result, err = r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.management.creates).To(HaveLen(creates)) + }) +}) diff --git a/pkg/dpf/deployment/types.go b/pkg/dpf/deployment/types.go new file mode 100644 index 00000000..aded48c9 --- /dev/null +++ b/pkg/dpf/deployment/types.go @@ -0,0 +1,85 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package deployment implements the existing-infrastructure DPF lifecycle slice. +// It creates native objects only; ownership transitions, registration, Helm, +// firmware actions and host reboot commands are not implicit remediation. +package deployment + +import ( + "context" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +type Mode string + +const ( + Deploy Mode = "deploy" + Validate Mode = "validate" + DryRun Mode = "dry-run" +) + +// Client hides private access credentials and exposes no update/delete operation. +type Client interface { + ServerVersion(context.Context) (string, error) + Get(context.Context, bundle.ResourceRef) (*unstructured.Unstructured, error) + List(context.Context, bundle.ResourceRef, string) ([]*unstructured.Unstructured, error) + Create(context.Context, *unstructured.Unstructured, bool) (*unstructured.Unstructured, error) + SecretKeys(context.Context, string, string, []string) error + VerifyBMC(context.Context, config.Generation, config.DeviceBinding, inventory.Device, string) error +} + +type ClientFactory interface { + Management(context.Context, config.Generation) (Client, error) + Tenant(context.Context, Client, config.ClusterBinding) (Client, error) +} + +// ArtifactVerifier separates source bytes from controller-consumed cached bytes. +// Cached must establish the latter before a DPUSet can cause hardware effects. +type ArtifactVerifier interface { + Cached(context.Context, Client, config.Generation, *unstructured.Unstructured) error +} + +type Check struct { + ID string `json:"id"` + Subject string `json:"subject"` + Source string `json:"source"` + State string `json:"state"` + ReasonCode string `json:"reasonCode"` + Message string `json:"message,omitempty"` +} + +type Event struct { + At time.Time `json:"at"` + Stage string `json:"stage"` + Action string `json:"action"` + Resource bundle.ResourceRef `json:"resource"` + UID string `json:"uid,omitempty"` + ResourceVersion string `json:"resourceVersion,omitempty"` +} + +type Result struct { + Mode Mode `json:"mode"` + BundleDigest string `json:"bundleDigest"` + Outcome string `json:"outcome"` + Checks []Check `json:"checks"` + Events []Event `json:"events"` + ReportPath string `json:"reportPath"` +} + +// Runner requires a private evidence directory outside the immutable bundle. +// Zero client/verifier fields select the native adapters. The cached-artifact +// gate must succeed before any new DPUSet can cause physical-device effects. +type Runner struct { + Clients ClientFactory + Artifacts ArtifactVerifier + EvidenceDir string + PollInterval time.Duration + Timeout time.Duration + Wait bool +} diff --git a/pkg/dpf/generation/generate.go b/pkg/dpf/generation/generate.go new file mode 100644 index 00000000..1ce92884 --- /dev/null +++ b/pkg/dpf/generation/generate.go @@ -0,0 +1,182 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package generation composes pure resolution/rendering and atomic publication. +package generation + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/nvidia/k8s-launch-kit/pkg/configinput" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" + "github.com/nvidia/k8s-launch-kit/pkg/profiles" + "github.com/nvidia/k8s-launch-kit/pkg/target" +) + +type Request struct { + UserConfig string + OutputPath string + Overrides configinput.Values +} + +type Result struct { + Outcome string `json:"outcome"` + BundlePath string `json:"bundlePath"` + BundleDigest string `json:"bundleDigest"` + Unchanged bool `json:"unchanged"` + Checks []Check `json:"checks"` +} + +type Check struct { + ID string `json:"id"` + State string `json:"state"` + ReasonCode string `json:"reasonCode"` + Message string `json:"message"` +} + +// Build returns a complete immutable snapshot without writing output, connecting +// to a cluster or opening any access credential reference. +func Build(ctx context.Context, in config.Input, overrides configinput.Values) (*bundle.Bundle, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + in, err := config.WithOverrides(in, overrides) + if err != nil { + return nil, err + } + g, err := config.ResolveGeneration(in) + if err != nil { + return nil, err + } + archives := map[string][]byte{} + for _, chart := range sentry.Catalog() { + if err = ctx.Err(); err != nil { + return nil, err + } + data, readErr := config.ReadReference(filepath.Join(g.Sentry.ChartDirectory, chart.Filename), 16<<20) + if readErr != nil { + return nil, fmt.Errorf("chart archive %s: %w", chart.Filename, readErr) + } + archives[chart.Name] = data + } + return renderBundle(ctx, g, archives) +} + +// renderBundle is shared by generation and admission of a loaded bundle. It uses +// only resolved intent and retained chart bytes, never credentials or source paths. +func renderBundle(ctx context.Context, g config.Generation, archives map[string][]byte) (*bundle.Bundle, error) { + p, err := profiles.LoadProfileForTarget(target.DPF, g.Profile) + if err != nil { + return nil, err + } + rendered, err := sentry.Render(g, p, archives) + if err != nil { + return nil, err + } + sidecar, err := config.EncodeResolved(g) + if err != nil { + return nil, err + } + rendered.Files = append(rendered.Files, bundle.File{Path: bundle.ResolvedConfigPath, Data: sidecar}) + rendered.Records = append(rendered.Records, bundle.FileRecord{Path: bundle.ResolvedConfigPath, Role: bundle.ResolvedConfig}) + if err = ctx.Err(); err != nil { + return nil, err + } + return bundle.New(bundle.Index{FormatVersion: bundle.FormatVersion, Target: bundle.Target, Phase: bundle.Phase, Profile: g.Profile, Release: g.Release, Renderer: sentry.RendererVersion, SelectedDevices: rendered.SelectedDevices, Files: rendered.Records, Stages: rendered.Stages, References: rendered.References, ExternalPrerequisites: rendered.ExternalPrerequisites}, rendered.Files) +} + +func Run(ctx context.Context, r Request) (Result, error) { + result := Result{Outcome: "failed"} + fail := func(id string, err error) (Result, error) { + result.Checks = append(result.Checks, Check{ID: id, State: "failed", ReasonCode: "GENERATION_FAILED", Message: err.Error()}) + return result, err + } + if r.OutputPath == "" { + return fail("output", fmt.Errorf("--save-deployment-files is required for DPF generation")) + } + in, err := config.Load(r.UserConfig) + if err != nil { + return fail("configuration", err) + } + b, err := Build(ctx, in, r.Overrides) + if err != nil { + return fail("render", err) + } + publication, err := b.Publish(ctx, r.OutputPath) + if err != nil { + return fail("publication", err) + } + result.Outcome = "complete" + result.BundlePath = publication.Path + result.BundleDigest = b.Index().BundleDigest + result.Unchanged = publication.Unchanged + result.Checks = append(result.Checks, Check{ID: "generation", State: "passed", ReasonCode: "VALIDATED_BUNDLE", Message: "Local configuration, release artifacts and rendered resources validated; deployment readiness has not been measured."}) + return result, nil +} + +// Configuration retains the bundle's resolved intent; a supplied source file may +// change access paths, but must not silently diverge from the generated resources. +func Configuration(b *bundle.Bundle, userConfig string) (config.Generation, error) { + if b == nil { + return config.Generation{}, fmt.Errorf("DPF bundle is required") + } + var data []byte + for _, f := range b.Files() { + if f.Path == bundle.ResolvedConfigPath { + data = f.Data + } + } + g, err := config.DecodeResolved(data) + if err != nil { + return g, err + } + idx := b.Index() + if idx.Profile != g.Profile || idx.Release != g.Release || len(idx.SelectedDevices) != len(g.Devices) { + return g, fmt.Errorf("bundle index conflicts with resolved DPF configuration") + } + for i, d := range g.Devices { + if idx.SelectedDevices[i] != d.ID { + return g, fmt.Errorf("bundle selected devices conflict with resolved DPF configuration") + } + } + // A digest proves byte integrity, not that an arbitrary reindexed bundle is + // the supported Sentry plan. Reconstruct the bounded profile contract before + // any client is opened, using retained archives rather than local chart paths. + if idx.Renderer != sentry.RendererVersion { + return g, fmt.Errorf("unsupported DPF bundle renderer %q; regenerate with this binary", idx.Renderer) + } + archives := map[string][]byte{} + for _, chart := range sentry.Catalog() { + for _, f := range b.Files() { + if f.Path == "charts/"+chart.Filename { + archives[chart.Name] = f.Data + } + } + } + expected, err := renderBundle(context.Background(), g, archives) + if err != nil { + return g, fmt.Errorf("validate retained DPF intent: %w", err) + } + if expected.Index().BundleDigest != idx.BundleDigest { + return g, fmt.Errorf("DPF bundle differs from its resolved profile intent; regenerate instead of editing its contents") + } + if userConfig == "" { + return g, nil + } + in, err := config.Load(userConfig) + if err != nil { + return g, err + } + explicit, err := config.ResolveGeneration(in) + if err != nil { + return g, err + } + if !config.SameIntent(g, explicit) { + return g, fmt.Errorf("explicit configuration changes generated DPF intent; run generate into a new bundle directory") + } + return explicit, nil +} diff --git a/pkg/dpf/generation/generation_test.go b/pkg/dpf/generation/generation_test.go new file mode 100644 index 00000000..413fc23a --- /dev/null +++ b/pkg/dpf/generation/generation_test.go @@ -0,0 +1,221 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package generation + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/nvidia/k8s-launch-kit/pkg/configinput" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sentry" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + yaml "gopkg.in/yaml.v3" +) + +func TestGeneration(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "offline DPF generation") } + +var _ = Describe("offline DPF generation", func() { + var root, work string + var in config.Input + var configPath, chartDir string + BeforeEach(func() { + _, file, _, ok := runtime.Caller(0) + Expect(ok).To(BeTrue()) + root = filepath.Clean(filepath.Join(filepath.Dir(file), "../../..")) + old, err := os.Getwd() + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(os.Chdir(old)).To(Succeed()) }) + Expect(os.Chdir(root)).To(Succeed()) + work, err = os.MkdirTemp(root, ".dpf-generation-test-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(os.RemoveAll(work)).To(Succeed()) }) + chartDir = filepath.Join(work, "charts") + Expect(os.MkdirAll(chartDir, 0o700)).To(Succeed()) + for _, c := range sentry.Catalog() { + data, err := os.ReadFile(filepath.Join(root, "pkg/dpf/sentry/testdata/charts", c.Filename)) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(chartDir, c.Filename), data, 0o600)).To(Succeed()) + } + in = generationInput(work, chartDir) + configPath = filepath.Join(work, "site.yaml") + saveInput(in, configPath) + in, err = config.Load(configPath) + Expect(err).NotTo(HaveOccurred()) + }) + + It("builds and reloads a complete bundle without mutating source or reading access references", func() { + before, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + b, err := Build(context.Background(), in, configinput.Values{}) + Expect(err).NotTo(HaveOccurred()) + Expect(b.Index().Profile).To(Equal("sentry-bf4")) + Expect(b.Index().SelectedDevices).To(Equal([]string{"dpu-1"})) + Expect(b.Index().Files).To(ContainElement(HaveField("Path", bundle.ResolvedConfigPath))) + outDir := filepath.Join(work, "bundle") + _, err = b.Publish(context.Background(), outDir) + Expect(err).NotTo(HaveOccurred()) + loaded, err := bundle.Load(outDir) + Expect(err).NotTo(HaveOccurred()) + resolved, err := Configuration(loaded, "") + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Devices).To(HaveLen(1)) + Expect(loaded.Index().SelectedDevices).To(Equal([]string{resolved.Devices[0].ID})) + badIndex := loaded.Index() + badIndex.SelectedDevices = []string{"different-device"} + _, err = bundle.New(badIndex, loaded.Files()) + Expect(err).To(MatchError(ContainSubstring("bundle digest mismatch"))) + after, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + Expect(after).To(Equal(before)) + for _, p := range []string{resolved.KubeconfigRef.File, filepath.Join(work, "missing-password"), filepath.Join(work, "missing-known-hosts")} { + _, err := os.Stat(p) + Expect(os.IsNotExist(err)).To(BeTrue(), "generation must not create access-reference files") + } + }) + + It("rejects a consistently rehashed bundle whose resources diverge from its intent", func() { + b, err := Build(context.Background(), in, configinput.Values{}) + Expect(err).NotTo(HaveOccurred()) + files := b.Files() + for i := range files { + if files[i].Path == bundle.ResolvedConfigPath { + g, err := config.DecodeResolved(files[i].Data) + Expect(err).NotTo(HaveOccurred()) + port := 30081 + g.Sentry.PolicyServerNodePort = &port + files[i].Data, err = config.EncodeResolved(g) + Expect(err).NotTo(HaveOccurred()) + } + } + index := b.Index() + index.BundleDigest = "" + for i := range index.Files { + index.Files[i].SHA256 = "" + } + reindexed, err := bundle.New(index, files) + Expect(err).NotTo(HaveOccurred(), "byte integrity and structure still hold") + _, err = Configuration(reindexed, "") + Expect(err).To(MatchError(ContainSubstring("differs from its resolved profile intent"))) + }) + + It("applies shared CLI overrides and rejects explicit empty values", func() { + b, err := Build(context.Background(), in, configinput.Values{Overrides: []configinput.Override{{Flag: "--dpf-release", Path: "dpf.release", Value: "26.8.0-rc.1"}}}) + Expect(err).NotTo(HaveOccurred()) + resolved, err := config.DecodeResolved(findFile(b, bundle.ResolvedConfigPath)) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Release).To(Equal("26.8.0-rc.1")) + out := filepath.Join(work, "empty-release") + _, err = Run(context.Background(), Request{UserConfig: configPath, OutputPath: out, Overrides: configinput.Values{Overrides: []configinput.Override{{Flag: "--dpf-release", Path: "dpf.release", Value: ""}}}}) + Expect(err).To(HaveOccurred()) + _, err = os.Stat(out) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("does not publish on cancellation or incomplete chart input", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + out := filepath.Join(work, "cancelled") + _, err := Run(ctx, Request{UserConfig: configPath, OutputPath: out}) + Expect(errors.Is(err, context.Canceled)).To(BeTrue()) + _, err = os.Stat(out) + Expect(os.IsNotExist(err)).To(BeTrue()) + + Expect(os.Remove(filepath.Join(chartDir, sentry.Catalog()[0].Filename))).To(Succeed()) + out = filepath.Join(work, "wrong-chart-input") + _, err = Run(context.Background(), Request{UserConfig: configPath, OutputPath: out}) + Expect(err).To(HaveOccurred()) + _, err = os.Stat(out) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + It("keeps an identical publication unchanged and preserves a destination with different intent", func() { + out := filepath.Join(work, "published") + first, err := Run(context.Background(), Request{UserConfig: configPath, OutputPath: out}) + Expect(err).NotTo(HaveOccurred()) + second, err := Run(context.Background(), Request{UserConfig: configPath, OutputPath: out}) + Expect(err).NotTo(HaveOccurred()) + Expect(second.Unchanged).To(BeTrue()) + original, err := os.ReadFile(filepath.Join(out, bundle.IndexPath)) + Expect(err).NotTo(HaveOccurred()) + + changed := in + changed.Site.DPF.Name = stringPtr("other-sentry") + changedPath := filepath.Join(work, "changed.yaml") + saveInput(changed, changedPath) + _, err = Run(context.Background(), Request{UserConfig: changedPath, OutputPath: out}) + Expect(err).To(HaveOccurred()) + current, err := os.ReadFile(filepath.Join(out, bundle.IndexPath)) + Expect(err).NotTo(HaveOccurred()) + Expect(current).To(Equal(original)) + Expect(first.BundleDigest).To(Equal(second.BundleDigest)) + }) + + It("allows moved source paths with stable intent and rejects changed intent", func() { + b, err := Build(context.Background(), in, configinput.Values{}) + Expect(err).NotTo(HaveOccurred()) + moved := in + moved.Site.Management.KubeconfigRef.File = "new-location/management.kubeconfig" + movedPath := filepath.Join(work, "moved", "site.yaml") + saveInput(moved, movedPath) + resolved, err := Configuration(b, movedPath) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.KubeconfigRef.File).To(Equal(filepath.Join(filepath.Dir(movedPath), "new-location/management.kubeconfig"))) + + changed := moved + changed.Site.DPF.Name = stringPtr("different-sentry") + changedPath := filepath.Join(work, "changed-uid.yaml") + saveInput(changed, changedPath) + _, err = Configuration(b, changedPath) + Expect(err).To(MatchError(ContainSubstring("run generate into a new bundle directory"))) + }) +}) + +func generationInput(base, charts string) config.Input { + const psid = "MT_0000000010" + observed := inventory.Observation{State: "observed", Values: []string{psid}} + dpu := inventory.Device{ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "BlueField-4", ObservationState: "observed", BMCEndpoint: "https://192.0.2.21", Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} + host := inventory.Device{ID: "host-1", Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} + site := config.Site{} + site.DPF.Release = stringPtr("26.8.0") + site.Management.KubeconfigRef.File = "missing/management.kubeconfig" + site.Management.Context = stringPtr("management-context") + site.Management.ClusterUID = stringPtr("management-uid-1") + site.Management.KubernetesVersion = stringPtr("v1.35.4") + passwordFile := filepath.Join(base, "private", "password") + site.Credentials = map[string]config.Credential{"remote": {Username: "operator", PasswordRef: config.PasswordRef{File: &passwordFile}}} + site.Discovery.Hosts = []config.Endpoint{{CredentialRef: "remote", SSH: &config.SSH{Address: "192.0.2.12", KnownHostsFile: filepath.Join(base, "private", "known_hosts")}}} + site.Provisioning = &config.Provisioning{Devices: []config.DeviceBinding{{ID: "dpu-1", DPUDevice: "dpu-device-1", DPUNode: "dpu-node-1"}}, DPUCluster: config.ClusterBinding{KubernetesVersion: "v1.35.6", Name: "tenant-one", Namespace: "dpf-operator-system", ClusterUID: "tenant-cluster-uid", ResourceUID: "resource-uid-1", Selector: map[string]string{"site.example.com/tenant": "sentry"}}, Software: config.Software{OSISO: artifact("dpf-os.iso"), PLDM: map[string]config.Artifact{psid: artifact("bf4.pldm")}}} + site.Sentry = &config.Sentry{ChartDirectory: charts, IngressCIDR: "10.70.0.0/24", IngressIP: "10.70.0.20", ImagePullSecret: "sentry-pull", MITMSecret: "sentry-mitm", InferenceSecret: "sentry-inference", PolicyServerAddress: "192.0.2.40"} + site.Inventory = &inventory.Inventory{FormatVersion: inventory.FormatVersion, Management: &inventory.Management{SystemNamespaceUID: "management-uid-1", KubernetesVersion: "v1.35.4"}, DPUs: []inventory.Device{dpu}, Hosts: []inventory.Device{host}, Associations: []inventory.Association{{DPUID: "dpu-1", HostID: "host-1", SerialNumber: "BF4-SERIAL-1"}}} + return config.Input{Site: site, Path: filepath.Join(base, "site.yaml")} +} + +func artifact(name string) config.Artifact { + return config.Artifact{URL: "https://artifacts.example.invalid/" + name, SHA256: strings.Repeat("a", 64)} +} +func stringPtr(s string) *string { return &s } +func saveInput(in config.Input, path string) { + data, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + Expect(os.MkdirAll(filepath.Dir(path), 0o700)).To(Succeed()) + Expect(os.WriteFile(path, data, 0o600)).To(Succeed()) +} +func findFile(b *bundle.Bundle, path string) []byte { + for _, f := range b.Files() { + if f.Path == path { + return f.Data + } + } + Fail("bundle file not found: " + path) + return nil +} diff --git a/pkg/dpf/sentry/catalog.go b/pkg/dpf/sentry/catalog.go new file mode 100644 index 00000000..c7c8f335 --- /dev/null +++ b/pkg/dpf/sentry/catalog.go @@ -0,0 +1,43 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package sentry renders the bounded BlueField-4 Sentry profile offline. +package sentry + +// ChartLock identifies exact local archive bytes and their upstream coordinates. +// Archives are explicit inputs; this package never downloads them. +type ChartLock struct { + Name, Version, Repository, Filename, SHA256 string +} + +// ImageLock identifies immutable image bytes without inspecting a registry. +type ImageLock struct { + Name, Repository, Tag, Digest string +} + +func (i ImageLock) TagDigest() string { return i.Tag + "@" + i.Digest } +func (i ImageLock) Reference() string { return i.Repository + ":" + i.TagDigest() } + +// Catalog returns isolated release locks for the reviewed staging profile. These +// coordinates establish source identity, not release or hardware qualification. +func Catalog() []ChartLock { + return []ChartLock{ + {"doca-tower", "0.2.16", "oci://nvcr.io/nvstaging/doca", "doca-tower-0.2.16.tgz", "f88885808c8b3a84a2c0f1871e5c3341ded3000e77342e34a289717ae45a3628"}, + {"doca-sentry-policy-controller", "0.2.5", "https://helm.ngc.nvidia.com/nvstaging/doca", "doca-sentry-policy-controller-0.2.5.tgz", "553dd95f207fdc1194803d2b041721c93393c8b91a592371c4203d3f4bc4a548"}, + {"doca-argus", "1.5.1-ai-gateway.8", "https://helm.ngc.nvidia.com/nvstaging/doca", "doca-argus-1.5.1-ai-gateway.8.tgz", "de0d228f81167d641bb25f12415544c5eaff91cfd673f47e62ae4fcbbbdfd79f"}, + {"doca-sentry-policy-server", "0.2.5", "https://helm.ngc.nvidia.com/nvstaging/doca", "doca-sentry-policy-server-0.2.5.tgz", "ffad2db31039cc4c4e80c718cac1974a9f88acdd88af7376943aacb6956807ea"}, + } +} + +// Images returns the immutable image identities used by the profile templates. +func Images() []ImageLock { + return []ImageLock{ + {"envoy", "docker.io/envoyproxy/envoy", "v1.39.0", "sha256:d59f7f5fa10cff6d5892b6c5e7df5c9297ddfb2c3683e33fbfb82da24de4fa66"}, + {"openflow", "nvcr.io/nvstaging/doca/doca-tower-openflow", "0.1.10", "sha256:7019071feb4fba5e5123efad56e0cda640d4b7edf4b8129f2238b26dde0f189a"}, + {"plugin", "nvcr.io/nvstaging/doca/doca-tower-policy-plugin", "0.2.4", "sha256:06db03322fde0971bc13b83ff4f43f97212e6fb49963b30d0ef7de8ebd9fa8a2"}, + {"tap", "nvcr.io/nvstaging/doca/doca-tower-tap", "0.2.5", "sha256:59617284dee716701c7d1540bde2ba64a6a5dfdd1fd136aa12339c6a8f7c561a"}, + {"controller", "nvcr.io/nvstaging/doca/doca-tower-policy-sidecar", "0.2.5", "sha256:6c2f18fa661394d01cbbabe6be7d160da573ec526db971f5ff0a271eaac7371c"}, + {"server", "nvcr.io/nvstaging/doca/doca-tower-policy-rest", "0.2.5", "sha256:e1861e1bc153c249c9d07d05bf2d034870dee04afec9498d63b26c24cc9c1843"}, + {"argus", "nvcr.io/nvstaging/doca/doca_argus_ai_gateway", "20261017", "sha256:e216a90398052b6ba177f5400e0ab462a711b7f242f44246404f412f20d25ff2"}, + } +} diff --git a/pkg/dpf/sentry/projection.go b/pkg/dpf/sentry/projection.go new file mode 100644 index 00000000..dea17d20 --- /dev/null +++ b/pkg/dpf/sentry/projection.go @@ -0,0 +1,129 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package sentry + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "path" + "sort" + "strings" + + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/engine" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func loadCharts(archives map[string][]byte) (map[string]*chart.Chart, error) { + if len(archives) != len(Catalog()) { + return nil, fmt.Errorf("sentry requires exactly four pinned local chart archives") + } + out := make(map[string]*chart.Chart, len(archives)) + for _, lock := range Catalog() { + data, ok := archives[lock.Name] + if !ok { + return nil, fmt.Errorf("missing local chart archive %s", lock.Filename) + } + if len(data) > 20<<20 { + return nil, fmt.Errorf("chart %s exceeds archive size limit", lock.Name) + } + h := sha256.Sum256(data) + if hex.EncodeToString(h[:]) != lock.SHA256 { + return nil, fmt.Errorf("chart archive %s SHA-256 mismatch", lock.Filename) + } + c, err := loader.LoadArchive(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("load chart archive %s: %w", lock.Filename, err) + } + if c.Metadata.Name != lock.Name || c.Metadata.Version != lock.Version { + return nil, fmt.Errorf("chart archive %s has unexpected name or version", lock.Filename) + } + if len(c.Metadata.Dependencies) != 0 { + return nil, fmt.Errorf("chart %s requires unsupported unresolved dependencies", lock.Name) + } + out[lock.Name] = c + } + return out, nil +} + +// projectChart is deliberately offline. engine.Render has no Kubernetes client; +// the pinned policy controller's random certificate path is disabled by explicit +// webhook.enabled=false. CRDs and cluster-scoped RBAC remain validation-only. +func projectChart(c *chart.Chart, values map[string]interface{}, release, namespace, kubernetesVersion string) ([]*unstructured.Unstructured, error) { + version, err := chartutil.ParseKubeVersion(kubernetesVersion) + if err != nil { + return nil, fmt.Errorf("invalid captured Kubernetes API version: %w", err) + } + caps := &chartutil.Capabilities{KubeVersion: *version, APIVersions: projectionAPIVersions(), HelmVersion: chartutil.DefaultCapabilities.HelmVersion} + renderValues, err := chartutil.ToRenderValues(c, values, chartutil.ReleaseOptions{Name: release, Namespace: namespace, Revision: 1, IsInstall: true}, caps) + if err != nil { + return nil, err + } + rendered, err := engine.Render(c, renderValues) + if err != nil { + return nil, err + } + keys := make([]string, 0, len(rendered)) + for key := range rendered { + keys = append(keys, key) + } + sort.Strings(keys) + var out []*unstructured.Unstructured + for _, key := range keys { + body := strings.TrimSpace(rendered[key]) + if body == "" || strings.HasSuffix(key, "NOTES.txt") || strings.HasPrefix(path.Base(key), "_") { + continue + } + objects, err := decodeDocuments([]byte(body)) + if err != nil { + return nil, fmt.Errorf("template %s: %w", key, err) + } + for _, obj := range objects { + if obj.GetAnnotations()["helm.sh/hook"] != "" { + return nil, fmt.Errorf("chart %s has unsupported hook %s", c.Name(), obj.GetName()) + } + switch obj.GetKind() { + case "ClusterRole", "ClusterRoleBinding", "CustomResourceDefinition", "MutatingWebhookConfiguration", "ValidatingWebhookConfiguration": + if obj.GetNamespace() != "" { + return nil, fmt.Errorf("cluster-scoped projection %s has namespace", obj.GetName()) + } + default: + if obj.GetNamespace() == "" { + obj.SetNamespace(namespace) + } + if obj.GetNamespace() != namespace { + return nil, fmt.Errorf("projection %s escapes configured namespace", obj.GetName()) + } + } + out = append(out, obj) + } + } + for _, crd := range c.CRDObjects() { + objects, err := decodeDocuments(crd.File.Data) + if err != nil { + return nil, fmt.Errorf("CRD %s: %w", crd.Name, err) + } + for _, obj := range objects { + if obj.GetKind() != "CustomResourceDefinition" || obj.GetNamespace() != "" { + return nil, fmt.Errorf("invalid chart CRD %s", crd.Name) + } + out = append(out, obj) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("chart %s produced no validation projection", c.Name()) + } + return out, nil +} + +// The pinned four charts do not branch on Capabilities.APIVersions. Freeze the +// advertised set rather than inferring or probing an ambient cluster. A future +// chart version using .Has must review this explicit profile contract. +func projectionAPIVersions() chartutil.VersionSet { + return chartutil.VersionSet{"v1", "apps/v1", "rbac.authorization.k8s.io/v1", "apiextensions.k8s.io/v1"} +} diff --git a/pkg/dpf/sentry/render.go b/pkg/dpf/sentry/render.go new file mode 100644 index 00000000..1478df49 --- /dev/null +++ b/pkg/dpf/sentry/render.go @@ -0,0 +1,574 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package sentry + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/netip" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "text/template" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/profiles" + "github.com/nvidia/k8s-launch-kit/pkg/target" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + kjson "k8s.io/apimachinery/pkg/util/json" + kyaml "k8s.io/apimachinery/pkg/util/yaml" + yaml "sigs.k8s.io/yaml" +) + +const ( + RendererVersion = "sentry-bf4/v1alpha1" + ScopeLabel = "nvidia.kubernetes-launch-kit.dpf-instance" + DeviceBMCLabel = "provisioning.dpu.nvidia.com/dpudevice-bmc-ip" + DeviceNodeLabel = "provisioning.dpu.nvidia.com/dpunode-name" +) + +// Rendered is one complete snapshot, ready for the DPF bundle constructor. Chart +// projections participate in validation only; deploy applies the DPUService CRs. +type Rendered struct { + Files []bundle.File + Records []bundle.FileRecord + Stages []bundle.Stage + References []bundle.Reference + ExternalPrerequisites []string + SelectedDevices []string +} + +type names struct { + Flavor, Software, Tower, Controller, Argus, Server, CredentialSecret string + Physical, PF, Patch, Passthrough, IngressNAD, EgressNAD string + IngressInterface, EgressInterface string +} + +type templateData struct { + LLMInspection bool + ProvisioningNamespace string + Names names + Namespace, TenantName, TenantNamespace, ScopeKey, ScopeValue string + PassthroughLabel, TowerLabel, PullSecret, MITMSecret, InferenceSecret string + IngressCIDR, IngressIP string + IngressPrefix, NodePort int + EnvoyTag, OpenFlowTag, PluginTag, TapTag, ControllerTag, ServerTag, ArgusImage string +} + +type templateSpec struct { + filename, stage, workloadStage string + destination bundle.Destination +} + +var templateSpecs = []templateSpec{ + {"10-flavor.yaml", "provisioning-artifacts", "", bundle.Management}, + {"20-passthrough-interfaces.yaml", "passthrough", "", bundle.Management}, + {"21-passthrough-chain.yaml", "passthrough", "", bundle.Management}, + {"30-tower-nads.yaml", "services", "", bundle.Management}, + {"31-tower-interfaces.yaml", "services", "", bundle.Management}, + {"32-tower.yaml", "services", "tower-workloads", bundle.Management}, + {"33-tower-chain.yaml", "services", "", bundle.Management}, + {"40-controller.yaml", "services", "policy-controller-workloads", bundle.Management}, + {"50-argus.yaml", "services", "argus-workloads", bundle.Management}, + {"60-policy-rbac.yaml", "policy-rbac", "", bundle.DPU}, + {"61-policy-credential.yaml", "policy-credentials", "", bundle.Management}, + {"62-policy-server.yaml", "policy-server", "", bundle.Management}, +} + +// Render uses the common profile's local templates and supplied archive bytes. +// It never resolves kubeconfigs, reads Secrets, discovers hardware, or downloads +// charts. All archives are checked before any output is returned. +func Render(g config.Generation, profile *profiles.ProfileMetadata, archives map[string][]byte) (Rendered, error) { + if err := validateInputs(g, profile); err != nil { + return Rendered{}, err + } + charts, err := loadCharts(archives) + if err != nil { + return Rendered{}, err + } + templates, err := readTemplates(profile) + if err != nil { + return Rendered{}, err + } + d := newTemplateData(g) + flavorObjects, err := renderTemplate("10-flavor.yaml", templates["10-flavor.yaml"], d) + if err != nil { + return Rendered{}, err + } + if len(flavorObjects) != 1 || flavorObjects[0].GetKind() != "DPUFlavor" { + return Rendered{}, fmt.Errorf("sentry flavor template must render exactly one DPUFlavor") + } + flavorSpec, err := json.Marshal(flavorObjects[0].Object["spec"]) + if err != nil { + return Rendered{}, err + } + d.Names.Flavor += "-" + shortHash(string(flavorSpec)) + flavorObjects[0].SetName(d.Names.Flavor) + r := Rendered{ExternalPrerequisites: []string{"existing-management", "existing-tenant", "enrolled-devices", "selected-scope", "external-transition-complete", "software-integrity", "existing-secrets"}} + r.Stages = stages(g.Namespace, *g.Sentry.Namespace) + stageByID := map[string]int{} + for i := range r.Stages { + stageByID[r.Stages[i].ID] = i + } + add := func(path string, data []byte, role bundle.Role, stage string) { + r.Files = append(r.Files, bundle.File{Path: path, Data: append([]byte(nil), data...)}) + r.Records = append(r.Records, bundle.FileRecord{Path: path, Role: role}) + if stage != "" { + s := &r.Stages[stageByID[stage]] + if role == bundle.Projection { + s.ProjectionFiles = append(s.ProjectionFiles, path) + } else { + s.Files = append(s.Files, path) + } + } + } + capabilities, err := yaml.Marshal(map[string]interface{}{"managementKubernetesVersion": g.ManagementKubernetesVersion, "dpuKubernetesVersion": g.Provisioning.DPUCluster.KubernetesVersion, "apiVersions": projectionAPIVersions()}) + if err != nil { + return Rendered{}, err + } + add("metadata/helm-capabilities.yaml", capabilities, bundle.Metadata, "") + chartStages := map[string]string{"doca-tower": "services", "doca-sentry-policy-controller": "services", "doca-argus": "services", "doca-sentry-policy-server": "policy-server"} + for _, lock := range Catalog() { + add("charts/"+lock.Filename, archives[lock.Name], bundle.Chart, chartStages[lock.Name]) + } + software := softwareObject(g, d.Names.Software) + b, err := marshalDocuments([]*unstructured.Unstructured{software}) + if err != nil { + return Rendered{}, err + } + add("management/00-software.yaml", b, bundle.Manifest, "provisioning-artifacts") + bindings := append([]config.DeviceBinding(nil), g.Provisioning.Devices...) + sort.Slice(bindings, func(i, j int) bool { return bindings[i].ID < bindings[j].ID }) + for _, binding := range bindings { + var endpoint string + for _, device := range g.Devices { + if device.ID == binding.ID { + endpoint = device.BMCEndpoint + } + } + u, _ := url.Parse(endpoint) + set := deviceSet(g, d, binding, u.Hostname()) + b, err = marshalDocuments([]*unstructured.Unstructured{set}) + if err != nil { + return Rendered{}, err + } + add("management/15-dpuset-"+shortHash(binding.ID)+".yaml", b, bundle.Manifest, "provisioning") + r.SelectedDevices = append(r.SelectedDevices, binding.ID) + for _, to := range []string{d.Names.Flavor, d.Names.Software} { + kind := "DPUFlavor" + if to == d.Names.Software { + kind = "BlueFieldSoftware" + } + r.References = append(r.References, bundle.Reference{From: resource(bundle.Management, set), To: bundle.ResourceRef{Destination: bundle.Management, Group: "provisioning.dpu.nvidia.com", Kind: kind, Namespace: g.Namespace, Name: to}}) + } + } + projectionDependencies := projectionInputs{interfaces: map[string]*unstructured.Unstructured{}, nads: map[string]*unstructured.Unstructured{}} + for _, spec := range templateSpecs { + objects := flavorObjects + if spec.filename != "10-flavor.yaml" { + objects, err = renderTemplate(spec.filename, templates[spec.filename], d) + if err != nil { + return Rendered{}, err + } + } + for _, obj := range objects { + if err = bindObject(g, d, obj); err != nil { + return Rendered{}, fmt.Errorf("bind %s/%s: %w", obj.GetKind(), obj.GetName(), err) + } + if err = projectionDependencies.record(obj); err != nil { + return Rendered{}, err + } + r.References = append(r.References, objectReferences(spec.destination, obj)...) + if obj.GetKind() == "DPUService" { + chartName, _, _ := unstructured.NestedString(obj.Object, "spec", "helmChart", "source", "chart") + version, _, _ := unstructured.NestedString(obj.Object, "spec", "helmChart", "source", "version") + repository, _, _ := unstructured.NestedString(obj.Object, "spec", "helmChart", "source", "repoURL") + chart, ok := charts[chartName] + if !ok || chart.Metadata.Version != version { + return Rendered{}, fmt.Errorf("DPUService %s refers to an unlocked chart %s:%s", obj.GetName(), chartName, version) + } + for _, lock := range Catalog() { + if lock.Name == chartName && repository != lock.Repository { + return Rendered{}, fmt.Errorf("DPUService %s refers to an unlocked chart repository", obj.GetName()) + } + } + values, err := effectiveHelmValues(g.Release, obj, projectionDependencies) + if err != nil { + return Rendered{}, fmt.Errorf("effective Helm inputs for %s: %w", obj.GetName(), err) + } + kubernetesVersion := g.Provisioning.DPUCluster.KubernetesVersion + if obj.GetName() == d.Names.Server { + kubernetesVersion = g.ManagementKubernetesVersion + } + projected, err := projectChart(chart, values, obj.GetName(), *g.Sentry.Namespace, kubernetesVersion) + if err != nil { + return Rendered{}, fmt.Errorf("project DPUService %s: %w", obj.GetName(), err) + } + projection, err := marshalDocuments(projected) + if err != nil { + return Rendered{}, err + } + vb, err := yaml.Marshal(values) + if err != nil { + return Rendered{}, err + } + add("values/"+obj.GetName()+".yaml", vb, bundle.HelmValues, spec.stage) + projectionStage := spec.stage + if spec.workloadStage != "" { + projectionStage = spec.workloadStage + } + add("projections/"+obj.GetName()+".yaml", projection, bundle.Projection, projectionStage) + } + } + b, err = marshalDocuments(objects) + if err != nil { + return Rendered{}, err + } + add(string(spec.destination)+"/"+spec.filename, b, bundle.Manifest, spec.stage) + } + sort.Slice(r.Files, func(i, j int) bool { return r.Files[i].Path < r.Files[j].Path }) + sort.Slice(r.Records, func(i, j int) bool { return r.Records[i].Path < r.Records[j].Path }) + if err := validateRendered(r, g.Release); err != nil { + return Rendered{}, err + } + return r, nil +} + +func renderTemplate(name string, body []byte, data templateData) ([]*unstructured.Unstructured, error) { + t, err := template.New(name).Option("missingkey=error").Parse(string(body)) + if err != nil { + return nil, fmt.Errorf("parse profile template %s: %w", name, err) + } + var out bytes.Buffer + if err = t.Execute(&out, data); err != nil { + return nil, fmt.Errorf("render profile template %s: %w", name, err) + } + if !safeTemplateOutput(out.String()) { + return nil, fmt.Errorf("profile template %s contains unresolved site parameters or forbidden lab credentials", name) + } + objects, err := decodeDocuments(out.Bytes()) + if err != nil { + return nil, fmt.Errorf("profile template %s: %w", name, err) + } + if len(objects) == 0 { + return nil, fmt.Errorf("profile template %s contains no Kubernetes documents", name) + } + return objects, nil +} + +func readTemplates(p *profiles.ProfileMetadata) (map[string][]byte, error) { + paths := map[string]string{} + for _, path := range p.TemplatePaths { + name := filepath.Base(path) + if _, exists := paths[name]; exists { + return nil, fmt.Errorf("duplicate Sentry template %s", name) + } + paths[name] = path + } + out := map[string][]byte{} + for _, spec := range templateSpecs { + path, ok := paths[spec.filename] + if !ok { + return nil, fmt.Errorf("sentry profile requires template %s", spec.filename) + } + body, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read profile template %s: %w", spec.filename, err) + } + if len(body) > 1<<20 { + return nil, fmt.Errorf("sentry template %s exceeds size limit", spec.filename) + } + out[spec.filename] = body + } + return out, nil +} + +func validateInputs(g config.Generation, p *profiles.ProfileMetadata) error { + if p == nil || p.Target != target.DPF || g.Profile != "sentry-bf4" { + return fmt.Errorf("sentry requires the DPF sentry-bf4 profile") + } + if g.Release != "26.8.0" && g.Release != "26.8.0-rc.1" { + return fmt.Errorf("unsupported Sentry DPF release %q", g.Release) + } + if g.Name == "" || g.Namespace == "" || g.Provisioning.DPUCluster.Name == "" || g.Provisioning.DPUCluster.Namespace == "" || len(g.Provisioning.DPUCluster.Selector) == 0 { + return fmt.Errorf("sentry requires explicit instance, namespace and selected tenant identity/selector") + } + if len(g.Provisioning.Devices) == 0 || len(g.Devices) != len(g.Provisioning.Devices) { + return fmt.Errorf("sentry requires exactly the selected resolved devices") + } + if len(p.TemplatePaths) != len(templateSpecs) { + return fmt.Errorf("sentry profile has unexpected template membership") + } + seen := map[string]bool{} + bmcs := map[string]bool{} + for _, b := range g.Provisioning.Devices { + if b.ID == "" || b.DPUDevice == "" || b.DPUNode == "" || seen[b.ID] { + return fmt.Errorf("sentry device bindings require unique IDs and existing enrollment names") + } + seen[b.ID] = true + matches := 0 + for _, device := range g.Devices { + if device.ID == b.ID { + matches++ + u, err := url.Parse(device.BMCEndpoint) + if err != nil || u.Hostname() == "" { + return fmt.Errorf("device %s has no BMC selector identity", b.ID) + } + addr, err := netip.ParseAddr(u.Hostname()) + if err != nil || !addr.Is4() { + return fmt.Errorf("device %s requires an observed BMC IPv4 selector", b.ID) + } + key := b.DPUNode + "/" + addr.String() + if bmcs[key] { + return fmt.Errorf("duplicate node/BMC selector for %s", b.ID) + } + bmcs[key] = true + } + } + if matches != 1 { + return fmt.Errorf("device %s is absent or ambiguous in resolved inventory", b.ID) + } + } + if _, err := netip.ParsePrefix(g.Sentry.IngressCIDR); err != nil { + return fmt.Errorf("invalid Sentry ingress CIDR: %w", err) + } + if g.Sentry.PolicyServerNodePort == nil || g.Sentry.ImagePullSecret == "" || g.Sentry.MITMSecret == "" || g.Sentry.LLMInspection == nil { + return fmt.Errorf("sentry requires resolved NodePort and existing Secret references") + } + if *g.Sentry.LLMInspection && g.Sentry.InferenceSecret == "" { + return fmt.Errorf("sentry LLM inspection requires an existing inference Secret reference") + } + if g.Sentry.Namespace == nil || *g.Sentry.Namespace == "" || *g.Sentry.Namespace == g.Namespace { + return fmt.Errorf("sentry requires a dedicated resolved service namespace") + } + return nil +} + +func newTemplateData(g config.Generation) templateData { + prefix := g.Name + if len(prefix) > 24 { + prefix = prefix[:15] + "-" + shortHash(prefix) + } + n := names{Flavor: prefix + "-flavor", Software: prefix + "-software", Tower: prefix + "-tower", Controller: prefix + "-controller", Argus: prefix + "-argus", Server: prefix + "-policy-server", CredentialSecret: prefix + "-policy-kubeconfig", Physical: prefix + "-p0", PF: prefix + "-pf0hpf", Patch: prefix + "-patch", Passthrough: prefix + "-passthrough", IngressNAD: prefix + "-ingress", EgressNAD: prefix + "-egress", IngressInterface: prefix + "-ingress-interface", EgressInterface: prefix + "-egress-interface"} + // Immutable provisioning resources receive a content-derived suffix, so a + // changed flavor/software does not attempt an immutable spec update. + software, _ := json.Marshal(g.Provisioning.Software) + n.Software += "-" + shortHash(string(software)) + images := map[string]ImageLock{} + for _, i := range Images() { + images[i.Name] = i + } + p, _ := netip.ParsePrefix(g.Sentry.IngressCIDR) + return templateData{LLMInspection: *g.Sentry.LLMInspection, Names: n, ProvisioningNamespace: g.Namespace, Namespace: *g.Sentry.Namespace, TenantName: g.Provisioning.DPUCluster.Name, TenantNamespace: g.Provisioning.DPUCluster.Namespace, ScopeKey: ScopeLabel, ScopeValue: prefix, PassthroughLabel: "sentry.l8k.nvidia.com/" + prefix + "-passthrough", TowerLabel: "sentry.l8k.nvidia.com/" + prefix + "-tower", PullSecret: g.Sentry.ImagePullSecret, MITMSecret: g.Sentry.MITMSecret, InferenceSecret: g.Sentry.InferenceSecret, IngressCIDR: g.Sentry.IngressCIDR, IngressIP: g.Sentry.IngressIP, IngressPrefix: p.Bits(), NodePort: *g.Sentry.PolicyServerNodePort, EnvoyTag: images["envoy"].TagDigest(), OpenFlowTag: images["openflow"].TagDigest(), PluginTag: images["plugin"].TagDigest(), TapTag: images["tap"].TagDigest(), ControllerTag: images["controller"].TagDigest(), ServerTag: images["server"].TagDigest(), ArgusImage: images["argus"].Reference()} +} + +func softwareObject(g config.Generation, name string) *unstructured.Unstructured { + pldm := map[string]interface{}{} + for psid, a := range g.Provisioning.Software.PLDM { + pldm[psid] = a.URL + } + spec := map[string]interface{}{"osIso": g.Provisioning.Software.OSISO.URL, "pldmFwBundle": pldm} + if g.Provisioning.Software.NICFirmware != nil { + spec["nicFw"] = g.Provisioning.Software.NICFirmware.URL + } + return object("provisioning.dpu.nvidia.com/v1alpha1", "BlueFieldSoftware", g.Namespace, name, spec) +} + +func deviceSet(g config.Generation, d templateData, b config.DeviceBinding, bmc string) *unstructured.Unstructured { + spec := map[string]interface{}{ + "strategy": map[string]interface{}{"type": "OnDelete"}, + "dpuDeviceSelector": map[string]interface{}{"matchLabels": map[string]interface{}{DeviceNodeLabel: b.DPUNode, DeviceBMCLabel: bmc}}, + "dpuTemplate": map[string]interface{}{"spec": map[string]interface{}{ + "blueFieldSoftware": map[string]interface{}{"name": d.Names.Software}, "dpuFlavor": d.Names.Flavor, + "nodeEffect": map[string]interface{}{"noEffect": true}, + "cluster": map[string]interface{}{"selector": map[string]interface{}{"matchLabels": stringMap(g.Provisioning.DPUCluster.Selector)}, "nodeLabels": map[string]interface{}{ScopeLabel: d.ScopeValue}}, + }}, + } + return object("provisioning.dpu.nvidia.com/v1alpha1", "DPUSet", g.Namespace, d.ScopeValue+"-"+shortHash(b.ID), spec) +} + +func object(api, kind, namespace, name string, spec map[string]interface{}) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": api, "kind": kind, "metadata": map[string]interface{}{"name": name, "namespace": namespace}, "spec": spec}} +} + +// bindObject completes explicit native tenant and chart inputs once. The DPF +// controller transformation for offline projections remains in effectiveHelmValues. +func bindObject(g config.Generation, d templateData, obj *unstructured.Unstructured) error { + expectedNamespace := *g.Sentry.Namespace + if obj.GetKind() == "DPUFlavor" { + expectedNamespace = g.Namespace + } + if obj.GetNamespace() != expectedNamespace { + return fmt.Errorf("resource requires explicit configured namespace") + } + if obj.GetKind() == "Secret" || obj.GetKind() == "DPUDevice" || obj.GetKind() == "DPUNode" { + return fmt.Errorf("profile cannot create credentials or enrollment objects") + } + kind := obj.GetKind() + if kind == "DPUServiceInterface" || kind == "DPUServiceNAD" || kind == "DPUServiceChain" || (kind == "DPUService" && obj.GetName() != d.Names.Server) { + selector := map[string]interface{}{} + for k, v := range g.Provisioning.DPUCluster.Selector { + selector[k] = v + } + if err := unstructured.SetNestedMap(obj.Object, map[string]interface{}{"matchLabels": selector}, "spec", "dpuClusterSelector"); err != nil { + return err + } + } + if kind != "DPUService" { + return nil + } + if err := unstructured.SetNestedField(obj.Object, obj.GetName(), "spec", "helmChart", "source", "releaseName"); err != nil { + return err + } + values, _, err := unstructured.NestedMap(obj.Object, "spec", "helmChart", "values") + if err != nil { + return err + } + values["fullnameOverride"] = obj.GetName() + if obj.GetName() == d.Names.Tower { + // DPF's ServiceInterfaceSet entry name is namespace_name; the SFC + // controller uses that entry name as dpf-id on both patch peers. + if err := unstructured.SetNestedField(values, obj.GetNamespace()+"_"+d.Names.Patch, "egress", "patchDPFID"); err != nil { + return err + } + } + if obj.GetName() == d.Names.Server { + values["policyNamespace"] = *g.Sentry.Namespace + values["dpuNamespace"] = g.Namespace + values["mitmCA"] = map[string]interface{}{"secretName": g.Sentry.MITMSecret, "namespace": *g.Sentry.Namespace, "key": "ca.crt"} + } + return unstructured.SetNestedMap(obj.Object, values, "spec", "helmChart", "values") +} + +func stages(provisioningNamespace, serviceNamespace string) []bundle.Stage { + // Apply all three services before the first readiness barrier: Tower's + // openflow watches the policy CRD installed by the controller chart, while + // the controller programs Tower through its hostPath socket. + ids := []string{"provisioning-artifacts", "provisioning", "passthrough", "services", "tower-workloads", "policy-controller-workloads", "argus-workloads", "policy-rbac", "policy-credentials", "policy-server"} + out := make([]bundle.Stage, 0, len(ids)) + for i, id := range ids { + s := bundle.Stage{ID: id, Destination: bundle.Management, Namespace: serviceNamespace} + if id == "provisioning-artifacts" || id == "provisioning" { + s.Namespace = provisioningNamespace + } + if id == "policy-rbac" || strings.HasSuffix(id, "-workloads") { + s.Destination = bundle.DPU + } + if i > 0 { + s.DependsOn = []string{ids[i-1]} + } + if id == "provisioning-artifacts" { + s.Requires = []string{"existing-management", "existing-tenant", "enrolled-devices", "selected-scope", "external-transition-complete", "existing-secrets"} + } + if id == "provisioning" { + s.Requires = []string{"software-integrity"} + } + out = append(out, s) + } + return out +} + +func decodeDocuments(data []byte) ([]*unstructured.Unstructured, error) { + decoder := kyaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var out []*unstructured.Unstructured + for { + var raw json.RawMessage + if err := decoder.Decode(&raw); err != nil { + if err == io.EOF { + break + } + return nil, err + } + var obj map[string]interface{} + if len(bytes.TrimSpace(raw)) == 0 { + continue + } + if err := kjson.Unmarshal(raw, &obj); err != nil { + return nil, err + } + if len(obj) == 0 { + continue + } + u := &unstructured.Unstructured{Object: obj} + if u.GetAPIVersion() == "" || u.GetKind() == "" || u.GetName() == "" { + return nil, fmt.Errorf("document requires apiVersion, kind and metadata.name") + } + out = append(out, u) + } + return out, nil +} + +func marshalDocuments(objects []*unstructured.Unstructured) ([]byte, error) { + var out bytes.Buffer + for i, obj := range objects { + if i > 0 { + out.WriteString("---\n") + } + b, err := yaml.Marshal(obj.Object) + if err != nil { + return nil, err + } + out.Write(b) + } + return out.Bytes(), nil +} + +func resource(destination bundle.Destination, obj *unstructured.Unstructured) bundle.ResourceRef { + group := obj.GroupVersionKind().Group + return bundle.ResourceRef{Destination: destination, Group: group, Kind: obj.GetKind(), Namespace: obj.GetNamespace(), Name: obj.GetName()} +} + +func objectReferences(destination bundle.Destination, obj *unstructured.Unstructured) []bundle.Reference { + var out []bundle.Reference + add := func(group, kind, name string) { + if name != "" { + out = append(out, bundle.Reference{From: resource(destination, obj), To: bundle.ResourceRef{Destination: destination, Group: group, Kind: kind, Namespace: obj.GetNamespace(), Name: name}}) + } + } + switch obj.GetKind() { + case "DPUService": + interfaces, _, _ := unstructured.NestedStringSlice(obj.Object, "spec", "interfaces") + for _, name := range interfaces { + add("svc.dpu.nvidia.com", "DPUServiceInterface", name) + } + case "DPUServiceInterface": + service, _, _ := unstructured.NestedString(obj.Object, "spec", "template", "spec", "template", "spec", "service", "serviceID") + network, _, _ := unstructured.NestedString(obj.Object, "spec", "template", "spec", "template", "spec", "service", "network") + add("svc.dpu.nvidia.com", "DPUService", service) + add("svc.dpu.nvidia.com", "DPUServiceNAD", network) + case "RoleBinding": + kind, _, _ := unstructured.NestedString(obj.Object, "roleRef", "kind") + name, _, _ := unstructured.NestedString(obj.Object, "roleRef", "name") + if kind == "Role" { + add("rbac.authorization.k8s.io", kind, name) + } + } + return out +} + +func shortHash(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:4]) } + +func safeTemplateOutput(s string) bool { + for _, forbidden := range []string{"${BFS_", "${DPU_", "${TOWER_", "${BMC_", "{{ .", "usermod --password", "ubuntu_PASSWORD=", "PermitRootLogin yes"} { + if strings.Contains(s, forbidden) { + return false + } + } + return true +} + +func stringMap(in map[string]string) map[string]interface{} { + out := make(map[string]interface{}, len(in)) + for k, v := range in { + out[k] = v + } + return out +} diff --git a/pkg/dpf/sentry/render_test.go b/pkg/dpf/sentry/render_test.go new file mode 100644 index 00000000..427179a7 --- /dev/null +++ b/pkg/dpf/sentry/render_test.go @@ -0,0 +1,501 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package sentry + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/profiles" + "github.com/nvidia/k8s-launch-kit/pkg/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/chart" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + yaml "sigs.k8s.io/yaml" +) + +func TestSentry(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Sentry offline rendering") } + +var _ = Describe("the BF4 Sentry renderer", func() { + var g config.Generation + var profile *profiles.ProfileMetadata + var archives map[string][]byte + BeforeEach(func() { + g = generationFixture() + profile = profileFixture() + archives = archiveFixture() + }) + It("freezes source-pinned charts, images and exact selected scope into a valid bundle", func() { + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + Expect(r.SelectedDevices).To(Equal([]string{"device-a", "device-b"})) + b, err := freezeFixture(r, g) + Expect(err).NotTo(HaveOccurred()) + Expect(b.Index().BundleDigest).NotTo(BeEmpty()) + for _, id := range []string{"tower-workloads", "policy-controller-workloads", "argus-workloads"} { + s, ok := b.Stage(id) + Expect(ok).To(BeTrue()) + Expect(s.Stage().Destination).To(Equal(bundle.DPU)) + Expect(s.Documents()).To(BeEmpty()) + Expect(s.Projections()).NotTo(BeEmpty()) + } + sets := objectsOf(r, bundle.Manifest, "DPUSet") + Expect(sets).To(HaveLen(2)) + for _, set := range sets { + selector, _, err := unstructured.NestedStringMap(set.Object, "spec", "dpuDeviceSelector", "matchLabels") + Expect(err).NotTo(HaveOccurred()) + Expect(selector).To(HaveLen(2)) + Expect(selector).To(HaveKeyWithValue(DeviceNodeLabel, "worker-a")) + Expect(selector[DeviceBMCLabel]).To(BeElementOf("192.0.2.10", "192.0.2.11")) + clusterSelector, found, err := unstructured.NestedStringMap(set.Object, "spec", "dpuTemplate", "spec", "cluster", "selector", "matchLabels") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(clusterSelector).To(Equal(g.Provisioning.DPUCluster.Selector)) + _, hasName, _ := unstructured.NestedFieldNoCopy(set.Object, "spec", "dpuTemplate", "spec", "cluster", "name") + Expect(hasName).To(BeFalse()) + _, hasNamespace, _ := unstructured.NestedFieldNoCopy(set.Object, "spec", "dpuTemplate", "spec", "cluster", "namespace") + Expect(hasNamespace).To(BeFalse()) + } + for _, kind := range []string{"DPUDevice", "DPUNode", "DPFOperatorConfig", "DPUCluster", "Secret"} { + Expect(objectsOf(r, bundle.Manifest, kind)).To(BeEmpty()) + } + Expect(objectsOf(r, bundle.Manifest, "Namespace")).To(BeEmpty()) + for _, kind := range []string{"DPUService", "DPUServiceInterface", "DPUServiceChain", "DPUServiceNAD", "DPUServiceCredentialRequest", "Role", "RoleBinding"} { + for _, obj := range objectsOf(r, bundle.Manifest, kind) { + Expect(obj.GetNamespace()).To(Equal(*g.Sentry.Namespace)) + } + } + for _, kind := range []string{"DPUService", "DPUServiceInterface", "DPUServiceChain", "DPUServiceNAD"} { + for _, obj := range objectsOf(r, bundle.Manifest, kind) { + inCluster, _, _ := unstructured.NestedBool(obj.Object, "spec", "deployInCluster") + if inCluster { + continue + } + selector, found, err := unstructured.NestedStringMap(obj.Object, "spec", "dpuClusterSelector", "matchLabels") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(selector).To(Equal(g.Provisioning.DPUCluster.Selector)) + } + } + flavor := objectsOf(r, bundle.Manifest, "DPUFlavor")[0] + _, hasBFCfg, _ := unstructured.NestedSlice(flavor.Object, "spec", "bfcfgParameters") + Expect(hasBFCfg).To(BeFalse()) + body, _ := json.Marshal(flavor.Object) + Expect(string(body)).NotTo(ContainSubstring("PermitRootLogin")) + Expect(string(body)).NotTo(ContainSubstring("usermod --password")) + Expect(string(body)).NotTo(ContainSubstring("docatower$")) + Expect(string(body)).To(ContainSubstring("br-envoy-vfs-monitor")) + }) + It("projects the actual five-container Tower and the controller CRD without applying them", func() { + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + var tower *unstructured.Unstructured + for _, obj := range objectsOf(r, bundle.Projection, "DaemonSet") { + if obj.GetName() == "site-a-tower" { + tower = obj + } + } + Expect(tower).NotTo(BeNil()) + containers, _, _ := unstructured.NestedSlice(tower.Object, "spec", "template", "spec", "containers") + Expect(containers).To(HaveLen(5)) + for _, raw := range containers { + c := raw.(map[string]interface{}) + Expect(c["image"]).To(ContainSubstring("@sha256:")) + } + selector, _, _ := unstructured.NestedMap(tower.Object, "spec", "template", "spec", "affinity", "nodeAffinity", "requiredDuringSchedulingIgnoredDuringExecution") + Expect(selector).To(HaveKey("nodeSelectorTerms")) + crds := objectsOf(r, bundle.Projection, "CustomResourceDefinition") + Expect(crds).To(HaveLen(1)) + Expect(crds[0].GetName()).To(Equal("docatowerpolicies.tower.dpu.nvidia.com")) + Expect(crds[0].GetNamespace()).To(BeEmpty()) + Expect(objectsOf(r, bundle.Projection, "Secret")).To(BeEmpty()) + for _, svc := range objectsOf(r, bundle.Manifest, "DPUService") { + name, _, _ := unstructured.NestedString(svc.Object, "spec", "helmChart", "source", "releaseName") + Expect(name).To(Equal(svc.GetName())) + fullName, _, _ := unstructured.NestedString(svc.Object, "spec", "helmChart", "values", "fullnameOverride") + Expect(fullName).To(Equal(svc.GetName())) + if svc.GetName() == "site-a-policy-server" { + policyNS, _, _ := unstructured.NestedString(svc.Object, "spec", "helmChart", "values", "policyNamespace") + Expect(policyNS).To(Equal("site-a-services")) + dpuNS, _, _ := unstructured.NestedString(svc.Object, "spec", "helmChart", "values", "dpuNamespace") + Expect(dpuNS).To(Equal(g.Namespace)) + secret, _, _ := unstructured.NestedString(svc.Object, "spec", "helmChart", "values", "mitmCA", "secretName") + Expect(secret).To(Equal("site-ca")) + } + } + }) + It("creates the coupled services before any workload barrier and retains every downstream readiness stage", func() { + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + b, err := freezeFixture(r, g) + Expect(err).NotTo(HaveOccurred()) + stageIDs := []string{} + for _, stage := range b.Index().Stages { + stageIDs = append(stageIDs, stage.ID) + } + Expect(stageIDs).To(Equal([]string{"provisioning-artifacts", "provisioning", "passthrough", "services", "tower-workloads", "policy-controller-workloads", "argus-workloads", "policy-rbac", "policy-credentials", "policy-server"})) + services, ok := b.Stage("services") + Expect(ok).To(BeTrue()) + Expect(services.Stage().Destination).To(Equal(bundle.Management)) + Expect(services.Projections()).To(BeEmpty()) + createdServices := []string{} + createdKinds := []string{} + for _, doc := range services.Documents() { + createdKinds = append(createdKinds, doc.Ref().Kind) + if doc.Ref().Kind == "DPUService" { + createdServices = append(createdServices, doc.Ref().Name) + } + } + Expect(createdServices).To(Equal([]string{"site-a-tower", "site-a-controller", "site-a-argus"})) + Expect(createdKinds).To(ContainElements("DPUServiceNAD", "DPUServiceInterface", "DPUServiceChain")) + for index, stage := range b.Index().Stages { + if index > 0 { + Expect(stage.DependsOn).To(Equal([]string{stageIDs[index-1]})) + } + if strings.HasSuffix(stage.ID, "-workloads") { + view, ok := b.Stage(stage.ID) + Expect(ok).To(BeTrue()) + Expect(index).To(BeNumerically(">", 3)) + Expect(view.Documents()).To(BeEmpty()) + Expect(view.Projections()).NotTo(BeEmpty()) + } + } + controller, ok := b.Stage("policy-controller-workloads") + Expect(ok).To(BeTrue()) + projectionKinds := []string{} + for _, doc := range controller.Projections() { + projectionKinds = append(projectionKinds, doc.Ref().Kind) + } + Expect(projectionKinds).To(ContainElement("CustomResourceDefinition")) + }) + It("binds the egress patch lookup to the actual generated interface for each namespace, instance and release", func() { + for _, release := range []string{"26.8.0-rc.1", "26.8.0"} { + for _, site := range []struct{ name, namespace string }{{"site-a", "site-a-services"}, {"other-site", "isolated-services"}} { + g.Release, g.Name, g.Sentry.Namespace = release, site.name, &site.namespace + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + patchID := "" + for _, iface := range objectsOf(r, bundle.Manifest, "DPUServiceInterface") { + kind, _, _ := unstructured.NestedString(iface.Object, "spec", "template", "spec", "template", "spec", "interfaceType") + if kind == "patch" { + patchID = iface.GetNamespace() + "_" + iface.GetName() + } + } + Expect(patchID).To(Equal(site.namespace + "_" + site.name + "-patch")) + for _, service := range objectsOf(r, bundle.Manifest, "DPUService") { + if service.GetName() == site.name+"-tower" { + actual, found, err := unstructured.NestedString(service.Object, "spec", "helmChart", "values", "egress", "patchDPFID") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(actual).To(Equal(patchID)) + } + } + for _, file := range r.Files { + if file.Path == "values/"+site.name+"-tower.yaml" { + var values map[string]interface{} + Expect(yaml.Unmarshal(file.Data, &values)).To(Succeed()) + actual, _, _ := unstructured.NestedString(values, "egress", "patchDPFID") + Expect(actual).To(Equal(patchID)) + } + } + for _, ds := range objectsOf(r, bundle.Projection, "DaemonSet") { + if ds.GetName() != site.name+"-tower" { + continue + } + containers, _, _ := unstructured.NestedSlice(ds.Object, "spec", "template", "spec", "containers") + for _, raw := range containers { + container := raw.(map[string]interface{}) + if container["name"] == "tower-egress" { + env, _, _ := unstructured.NestedSlice(container, "env") + Expect(env).To(ContainElement(map[string]interface{}{"name": "PATCH_DPF_ID", "value": patchID})) + } + } + } + } + } + }) + It("has deterministic concurrent output and never mutates caller-owned inputs", func() { + before, _ := json.Marshal(g) + one, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + type result struct { + r Rendered + err error + } + results := make(chan result, 3) + for range 3 { + go func() { r, err := Render(g, profile, archives); results <- result{r, err} }() + } + for range 3 { + other := <-results + Expect(other.err).NotTo(HaveOccurred()) + Expect(other.r).To(Equal(one)) + } + after, _ := json.Marshal(g) + Expect(after).To(Equal(before)) + for _, f := range one.Files { + if strings.HasSuffix(f.Path, ".yaml") { + Expect(safeTemplateOutput(string(f.Data))).To(BeTrue()) + } + } + }) + It("keeps a strict subset exact and rejects duplicate physical selectors", func() { + g.Provisioning.Devices = g.Provisioning.Devices[:1] + g.Devices = g.Devices[:1] + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + Expect(r.SelectedDevices).To(Equal([]string{"device-b"})) + Expect(objectsOf(r, bundle.Manifest, "DPUSet")).To(HaveLen(1)) + g = generationFixture() + g.Devices[1].BMCEndpoint = g.Devices[0].BMCEndpoint + r, err = Render(g, profile, archives) + Expect(err).To(MatchError(ContainSubstring("duplicate node/BMC selector"))) + Expect(r.Files).To(BeEmpty()) + }) + It("rejects missing or tampered chart bytes before returning any output", func() { + delete(archives, "doca-argus") + r, err := Render(g, profile, archives) + Expect(err).To(HaveOccurred()) + Expect(r.Files).To(BeEmpty()) + archives = archiveFixture() + archives["doca-tower"][20] ^= 0xff + r, err = Render(g, profile, archives) + Expect(err).To(MatchError(ContainSubstring("SHA-256 mismatch"))) + Expect(r.Files).To(BeEmpty()) + }) + It("rejects unresolved substitutions, lab credentials and dependency identity changes", func() { + for _, injection := range []string{"${TOWER_INGRESS_IP}", "{{ .MissingRequiredValue }}", "usermod --password accidental"} { + p := copyProfile(profile) + body, err := os.ReadFile(p.TemplatePaths[0]) + Expect(err).NotTo(HaveOccurred()) + path := filepath.Join(GinkgoT().TempDir(), "10-flavor.yaml") + Expect(os.WriteFile(path, append(body, []byte("\n# "+injection+"\n")...), 0600)).To(Succeed()) + p.TemplatePaths[0] = path + r, err := Render(g, p, archives) + Expect(err).To(HaveOccurred()) + Expect(r.Files).To(BeEmpty()) + } + p := copyProfile(profile) + for i, path := range p.TemplatePaths { + if filepath.Base(path) == "32-tower.yaml" { + body, err := os.ReadFile(path) + Expect(err).NotTo(HaveOccurred()) + body = bytes.ReplaceAll(body, []byte("version: \"0.2.16\""), []byte("version: \"0.0.0\"")) + path = filepath.Join(GinkgoT().TempDir(), "32-tower.yaml") + Expect(os.WriteFile(path, body, 0600)).To(Succeed()) + p.TemplatePaths[i] = path + } + } + r, err := Render(g, p, archives) + Expect(err).To(MatchError(ContainSubstring("unlocked chart"))) + Expect(r.Files).To(BeEmpty()) + }) + It("does not open deployment credentials or require a live API", func() { + // Deliberately unreadable/nonexistent references cannot affect offline rendering. + g.KubeconfigRef = config.FileRef{File: "/nonexistent/never-read/kubeconfig"} + g.Context = "no-live-context" + g.Sentry.ImagePullSecret = "not-readable-at-generation" + g.Sentry.MITMSecret = "not-readable-at-generation" + g.Sentry.InferenceSecret = "not-readable-at-generation" + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Files).NotTo(BeEmpty()) + }) + It("checks exact release OpenAPI unknown fields and CEL invariants before returning a bundle", func() { + for _, release := range []string{"26.8.0-rc.1", "26.8.0"} { + g.Release = release + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + set := objectsOf(r, bundle.Manifest, "DPUSet")[0] + Expect(unstructured.SetNestedField(set.Object, "unsupported-tenant-name", "spec", "dpuTemplate", "spec", "cluster", "name")).To(Succeed()) + schema, err := releaseSchema(release, "provisioning.dpu.nvidia.com", "DPUSet", "v1alpha1") + Expect(err).NotTo(HaveOccurred()) + Expect(schema.validate(set)).To(MatchError(ContainSubstring("unknown schema fields: spec.dpuTemplate.spec.cluster.name"))) + for _, server := range objectsOf(r, bundle.Manifest, "DPUService") { + if server.GetName() == "site-a-policy-server" { + Expect(unstructured.SetNestedMap(server.Object, map[string]interface{}{"matchLabels": map[string]interface{}{"site.example/tenant": "tenant-a"}}, "spec", "dpuClusterSelector")).To(Succeed()) + schema, err = releaseSchema(release, "svc.dpu.nvidia.com", "DPUService", "v1alpha1") + Expect(err).NotTo(HaveOccurred()) + Expect(schema.validate(server)).To(MatchError(ContainSubstring("CEL validation"))) + } + } + _, err = releaseSchema(release, "svc.dpu.nvidia.com", "DPUService", "v9") + Expect(err).To(MatchError(ContainSubstring("schema identity mismatch"))) + } + }) + It("uses the captured API version and a frozen capability list for offline Helm projection", func() { + c := &chart.Chart{Metadata: &chart.Metadata{Name: "capability-check", Version: "1.0.0"}, Templates: []*chart.File{{Name: "templates/version.yaml", Data: []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: captured-version\ndata:\n version: {{ .Capabilities.KubeVersion.Version | quote }}\n")}}} + for _, version := range []string{g.ManagementKubernetesVersion, g.Provisioning.DPUCluster.KubernetesVersion} { + objects, err := projectChart(c, map[string]interface{}{}, "capability-check", *g.Sentry.Namespace, version) + Expect(err).NotTo(HaveOccurred()) + actual, _, _ := unstructured.NestedString(objects[0].Object, "data", "version") + Expect(actual).To(Equal(version)) + } + _, err := projectChart(c, map[string]interface{}{}, "capability-check", *g.Sentry.Namespace, "") + Expect(err).To(HaveOccurred()) + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + var capabilities []byte + for _, f := range r.Files { + if f.Path == "metadata/helm-capabilities.yaml" { + capabilities = f.Data + } + } + Expect(string(capabilities)).To(ContainSubstring(g.ManagementKubernetesVersion)) + Expect(string(capabilities)).To(ContainSubstring(g.Provisioning.DPUCluster.KubernetesVersion)) + Expect(string(capabilities)).To(ContainSubstring("apiextensions.k8s.io/v1")) + }) + It("renders core policy without inference credentials and keeps ext_authz fail-closed in both modes", func() { + for _, enabled := range []bool{false, true} { + g = generationFixture() + g.Sentry.LLMInspection = &enabled + if !enabled { + g.Sentry.InferenceSecret = "" + } + r, err := Render(g, profile, archives) + Expect(err).NotTo(HaveOccurred()) + for _, tower := range objectsOf(r, bundle.Projection, "DaemonSet") { + if tower.GetName() == "site-a-tower" { + containers, _, _ := unstructured.NestedSlice(tower.Object, "spec", "template", "spec", "containers") + expected := 4 + if enabled { + expected = 5 + } + Expect(containers).To(HaveLen(expected)) + var tap map[string]interface{} + for _, raw := range containers { + container := raw.(map[string]interface{}) + if container["name"] == "tower-tap" { + tap = container + } + } + if enabled { + Expect(tap).NotTo(BeNil()) + env, _, _ := unstructured.NestedSlice(tap, "env") + Expect(env).To(ContainElement(HaveKeyWithValue("name", "INFERENCE_API_KEY"))) + } else { + Expect(tap).To(BeNil()) + } + } + } + for _, tower := range objectsOf(r, bundle.Manifest, "DPUService") { + if tower.GetName() == "site-a-tower" { + tapEnabled, _, _ := unstructured.NestedBool(tower.Object, "spec", "helmChart", "values", "tap", "enabled") + Expect(tapEnabled).To(Equal(enabled)) + _, hasInference, _ := unstructured.NestedFieldNoCopy(tower.Object, "spec", "helmChart", "values", "tap", "inferenceSecret") + Expect(hasInference).To(Equal(enabled)) + } + } + for _, cm := range objectsOf(r, bundle.Projection, "ConfigMap") { + if cm.GetName() == "site-a-tower" { + configBody, _, _ := unstructured.NestedString(cm.Object, "data", "envoy.yaml") + var envoy map[string]interface{} + Expect(yaml.Unmarshal([]byte(configBody), &envoy)).To(Succeed()) + listeners, _, _ := unstructured.NestedSlice(envoy, "static_resources", "listeners") + authzCount, optionalHookCount := 0, 0 + for _, listener := range listeners { + chains, _, _ := unstructured.NestedSlice(listener.(map[string]interface{}), "filter_chains") + for _, chain := range chains { + filters, _, _ := unstructured.NestedSlice(chain.(map[string]interface{}), "filters") + for _, filter := range filters { + httpFilters, _, _ := unstructured.NestedSlice(filter.(map[string]interface{}), "typed_config", "http_filters") + for _, raw := range httpFilters { + f := raw.(map[string]interface{}) + allow, _, _ := unstructured.NestedBool(f, "typed_config", "failure_mode_allow") + if f["name"] == "envoy.filters.http.ext_authz" { + authzCount++ + Expect(allow).To(BeFalse()) + } + if f["name"] == "envoy.filters.http.ext_proc" { + optionalHookCount++ + Expect(allow).To(BeTrue()) + } + } + } + } + } + Expect(authzCount).To(Equal(2)) + Expect(optionalHookCount).To(Equal(2)) + } + } + } + g = generationFixture() + g.Sentry.InferenceSecret = "" + r, err := Render(g, profile, archives) + Expect(err).To(MatchError(ContainSubstring("LLM inspection requires"))) + Expect(r.Files).To(BeEmpty()) + }) +}) + +func generationFixture() config.Generation { + port := 30080 + namespace := "site-a-services" + inspection := true + return config.Generation{Name: "site-a", Profile: "sentry-bf4", Release: "26.8.0-rc.1", Namespace: "sentry-system", ManagementKubernetesVersion: "v1.35.4", ManagementUID: "management-uid", Context: "management", KubeconfigRef: config.FileRef{File: "/never-read/kubeconfig"}, + Provisioning: config.Provisioning{DPUCluster: config.ClusterBinding{Name: "tenant-a", Namespace: "tenant-system", KubernetesVersion: "v1.35.6", ClusterUID: "tenant-uid", Selector: map[string]string{"site.example/tenant": "tenant-a"}}, Devices: []config.DeviceBinding{{ID: "device-b", DPUDevice: "serial-b", DPUNode: "worker-a"}, {ID: "device-a", DPUDevice: "serial-a", DPUNode: "worker-a"}}, Software: config.Software{OSISO: config.Artifact{URL: "https://artifacts.example/os.iso", SHA256: strings.Repeat("1", 64)}, PLDM: map[string]config.Artifact{"MT_0000001774": {URL: "https://artifacts.example/firmware.fwpkg", SHA256: strings.Repeat("2", 64)}}}}, + Sentry: config.Sentry{LLMInspection: &inspection, Namespace: &namespace, IngressCIDR: "192.168.100.0/24", IngressIP: "192.168.100.1", ImagePullSecret: "registry-pull", MITMSecret: "site-ca", InferenceSecret: "inference-api", PolicyServerAddress: "192.0.2.100", PolicyServerNodePort: &port}, + Devices: []inventory.Device{{ID: "device-b", BMCEndpoint: "https://192.0.2.11", SerialNumber: "SERIAL-B"}, {ID: "device-a", BMCEndpoint: "https://192.0.2.10", SerialNumber: "SERIAL-A"}}, + } +} + +func profileFixture() *profiles.ProfileMetadata { + root, err := filepath.Abs("../../../profiles/sentry-bf4") + Expect(err).NotTo(HaveOccurred()) + p := &profiles.ProfileMetadata{Target: target.DPF, Directory: root, Name: "BlueField-4 Sentry"} + for _, s := range templateSpecs { + p.TemplatePaths = append(p.TemplatePaths, filepath.Join(root, s.filename)) + } + return p +} + +func archiveFixture() map[string][]byte { + out := map[string][]byte{} + for _, lock := range Catalog() { + data, err := os.ReadFile(filepath.Join("testdata", "charts", lock.Filename)) + Expect(err).NotTo(HaveOccurred()) + out[lock.Name] = data + } + return out +} + +func objectsOf(r Rendered, role bundle.Role, kind string) []*unstructured.Unstructured { + roles := map[string]bundle.Role{} + for _, rec := range r.Records { + roles[rec.Path] = rec.Role + } + var out []*unstructured.Unstructured + for _, f := range r.Files { + if roles[f.Path] != role { + continue + } + objects, err := decodeDocuments(f.Data) + Expect(err).NotTo(HaveOccurred()) + for _, obj := range objects { + if obj.GetKind() == kind { + out = append(out, obj) + } + } + } + return out +} + +func freezeFixture(r Rendered, g config.Generation) (*bundle.Bundle, error) { + r.Files = append(r.Files, bundle.File{Path: bundle.ResolvedConfigPath, Data: []byte("formatVersion: v1alpha1\ntarget: dpf\nconfig: {}\n")}) + r.Records = append(r.Records, bundle.FileRecord{Path: bundle.ResolvedConfigPath, Role: bundle.ResolvedConfig}) + return bundle.New(bundle.Index{FormatVersion: bundle.FormatVersion, Target: bundle.Target, Phase: bundle.Phase, Profile: g.Profile, Release: g.Release, Renderer: RendererVersion, SelectedDevices: r.SelectedDevices, Files: r.Records, Stages: r.Stages, References: r.References, ExternalPrerequisites: r.ExternalPrerequisites}, r.Files) +} + +func copyProfile(p *profiles.ProfileMetadata) *profiles.ProfileMetadata { + c := *p + c.TemplatePaths = append([]string(nil), p.TemplatePaths...) + return &c +} diff --git a/pkg/dpf/sentry/schema.go b/pkg/dpf/sentry/schema.go new file mode 100644 index 00000000..ac681631 --- /dev/null +++ b/pkg/dpf/sentry/schema.go @@ -0,0 +1,194 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package sentry + +import ( + "context" + "embed" + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + schemacel "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + schemavalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/validation/field" + celconfig "k8s.io/apiserver/pkg/apis/cel" +) + +// The exact upstream CRD schemas and their source hashes are retained together. +// These checks supplement, never replace, the installed cluster's admission. +// +//go:embed schemas/*.json +var schemas embed.FS + +type schemaDocument struct { + Group string `json:"group"` + Kind string `json:"kind"` + Version string `json:"version"` + Schema apiextensionsv1.JSONSchemaProps `json:"schema"` +} + +type compiledSchema struct { + mu sync.Mutex + structural *structuralschema.Structural + openapi schemavalidation.SchemaValidator + cel *schemacel.Validator +} + +var schemaCache sync.Map + +// DefaultedCopy applies only the selected release's pinned CRD defaults to an +// independent copy of a supported DPF object. It never prunes unknown fields or +// models controller/webhook changes, and performs no I/O. +func DefaultedCopy(release string, obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + if obj == nil { + return nil, fmt.Errorf("DPF object is required for schema defaulting") + } + gvk := obj.GroupVersionKind() + if gvk.Group != "provisioning.dpu.nvidia.com" && gvk.Group != "svc.dpu.nvidia.com" { + return nil, fmt.Errorf("unsupported schema-defaulting group %q", gvk.Group) + } + schema, err := releaseSchema(release, gvk.Group, gvk.Kind, gvk.Version) + if err != nil { + return nil, err + } + copy := obj.DeepCopy() + defaulting.Default(copy.Object, schema.structural) + return copy, nil +} + +func releaseSchema(release, group, kind, version string) (*compiledSchema, error) { + key := release + "/" + group + "/" + version + "/" + kind + if cached, ok := schemaCache.Load(key); ok { + return cached.(*compiledSchema), nil + } + data, err := schemas.ReadFile("schemas/" + release + "-" + kind + ".json") + if err != nil { + return nil, fmt.Errorf("no pinned %s schema for %s/%s", release, group, kind) + } + var doc schemaDocument + if err = json.Unmarshal(data, &doc); err != nil { + return nil, err + } + if doc.Group != group || doc.Kind != kind || doc.Version != version { + return nil, fmt.Errorf("schema identity mismatch for %s/%s %s", group, kind, version) + } + compiled, err := compileSchema(doc.Schema) + if err != nil { + return nil, err + } + actual, _ := schemaCache.LoadOrStore(key, compiled) + return actual.(*compiledSchema), nil +} + +func compileSchema(external apiextensionsv1.JSONSchemaProps) (*compiledSchema, error) { + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&external, &internal, nil); err != nil { + return nil, err + } + structural, err := structuralschema.NewStructural(&internal) + if err != nil { + return nil, err + } + openapi, _, err := schemavalidation.NewSchemaValidator(&internal) + if err != nil { + return nil, err + } + return &compiledSchema{structural: structural, openapi: openapi, cel: schemacel.NewValidator(structural, true, celconfig.PerCallLimit)}, nil +} + +func (s *compiledSchema) validate(obj *unstructured.Unstructured) error { + // The upstream OpenAPI validator has internal validation state. Keep one + // compiled schema and serialize its use while callers render concurrently. + s.mu.Lock() + defer s.mu.Unlock() + unknown := pruning.PruneWithOptions(obj.DeepCopy().Object, s.structural, true, structuralschema.UnknownFieldPathOptions{TrackUnknownFieldPaths: true}) + if len(unknown) > 0 { + return fmt.Errorf("unknown schema fields: %s", strings.Join(unknown, ", ")) + } + if errs := schemavalidation.ValidateCustomResource(field.NewPath("resource"), obj.Object, s.openapi); len(errs) > 0 { + return fmt.Errorf("OpenAPI validation: %s", errs.ToAggregate()) + } + if s.cel != nil { + errs, _ := s.cel.Validate(context.Background(), field.NewPath("resource"), s.structural, obj.Object, nil, celconfig.RuntimeCELCostBudget) + if len(errs) > 0 { + return fmt.Errorf("CEL validation: %s", errs.ToAggregate()) + } + } + return nil +} + +func validateRendered(r Rendered, release string) error { + roles := map[string]bundle.Role{} + for _, record := range r.Records { + roles[record.Path] = record.Role + } + type sourced struct { + path string + obj *unstructured.Unstructured + } + var documents []sourced + custom := map[string]*compiledSchema{} + for _, file := range r.Files { + if roles[file.Path] != bundle.Manifest && roles[file.Path] != bundle.Projection { + continue + } + objects, err := decodeDocuments(file.Data) + if err != nil { + return fmt.Errorf("%s: %w", file.Path, err) + } + for _, obj := range objects { + documents = append(documents, sourced{file.Path, obj}) + if obj.GetKind() != "CustomResourceDefinition" { + continue + } + var crd apiextensionsv1.CustomResourceDefinition + data, err := json.Marshal(obj.Object) + if err != nil { + return err + } + if err = json.Unmarshal(data, &crd); err != nil { + return err + } + for _, version := range crd.Spec.Versions { + if !version.Served || version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + continue + } + compiled, err := compileSchema(*version.Schema.OpenAPIV3Schema) + if err != nil { + return fmt.Errorf("chart CRD %s: %w", crd.Name, err) + } + custom[crd.Spec.Group+"/"+version.Name+"/"+crd.Spec.Names.Kind] = compiled + } + } + } + for _, d := range documents { + gvk := d.obj.GroupVersionKind() + var compiled *compiledSchema + var err error + if gvk.Group == "provisioning.dpu.nvidia.com" || gvk.Group == "svc.dpu.nvidia.com" { + compiled, err = releaseSchema(release, gvk.Group, gvk.Kind, gvk.Version) + } else { + compiled = custom[gvk.Group+"/"+gvk.Version+"/"+gvk.Kind] + } + if err != nil { + return fmt.Errorf("%s: %w", d.path, err) + } + if compiled == nil { + continue + } + if err = compiled.validate(d.obj); err != nil { + return fmt.Errorf("%s %s/%s: %w", d.path, gvk.Kind, d.obj.GetName(), err) + } + } + return nil +} diff --git a/pkg/dpf/sentry/schemas/26.8.0-BlueFieldSoftware.json b/pkg/dpf/sentry/schemas/26.8.0-BlueFieldSoftware.json new file mode 100644 index 00000000..edd8c2cc --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-BlueFieldSoftware.json @@ -0,0 +1 @@ +{"group":"provisioning.dpu.nvidia.com","kind":"BlueFieldSoftware","version":"v1alpha1","schema":{"description":"BlueFieldSoftware is the Schema for the bluefieldsoftware API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"BlueFieldSpec defines the desired state of BlueFieldSoftware.","properties":{"nicFw":{"description":"NicFw points to the NIC firmware binary used for E/W NIC firmware updates.\nUse this when a specific NIC firmware binary is required and is not included in the\nplatform PLDM firmware bundle.\nIn production, prefer using PlatformPldmFwBundle.","minLength":1,"type":"string"},"osIso":{"description":"OS ISO points to the OS ISO used by DPU OS installation flow.","minLength":1,"type":"string"},"platformPldmFwBundle":{"description":"PlatformPldmFwBundle points to the platform PLDM firmware bundle used for E/W NIC\nfirmware updates.","minLength":1,"type":"string"},"pldmFwBundle":{"additionalProperties":{"type":"string"},"description":"PldmFwBundle maps each PSID to the PLDM firmware bundle URL used for that DPU model's\nbaseline firmware updates. Each PSID's bundle is a complete PLDM package for that device.\nKeys are matched case-insensitively against the BMC-reported DPUDevice status PSID.","minProperties":1,"type":"object","x-kubernetes-validations":[{"message":"pldmFwBundle entries must have non-empty PSID keys and non-empty bundle URLs","rule":"self.all(k, size(k) > 0 && size(self[k]) > 0)"}]}},"required":["osIso"],"type":"object","x-kubernetes-validations":[{"message":"BlueFieldSpec is immutable","rule":"self == oldSelf"},{"message":"nicFw and platformPldmFwBundle are mutually exclusive; set only one","rule":"!(has(self.nicFw) && has(self.platformPldmFwBundle))"}]},"status":{"default":{"phase":"Initializing"},"description":"BlueFieldSoftwareStatus defines the observed state of BlueFieldSoftware","properties":{"conditions":{"description":"Conditions represent the latest available observations of BlueFieldSoftware state","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"downloadedComponents":{"description":"DownloadedComponents tracks which components have been successfully downloaded","properties":{"nicFw":{"description":"NicFw is the local path of the E/W NIC firmware binary, either downloaded from\nspec.nicFw or unpacked from the platform PLDM firmware bundle.","type":"string"},"osIso":{"description":"OsIso is the local path of the downloaded OS ISO.","type":"string"},"platformPldmFwBundle":{"description":"PlatformPldmFwBundle is the local path of the downloaded platform PLDM firmware bundle.","type":"string"},"pldmFwBundle":{"additionalProperties":{"type":"string"},"description":"PldmFwBundle maps each PSID to the local path of its downloaded PLDM firmware bundle.\nKeys are matched case-insensitively against the BMC-reported DPUDevice PSID.","type":"object"}},"type":"object"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"},"phase":{"default":"Initializing","description":"The current state of BlueFieldSoftware.","enum":["Initializing","Downloading","Extracting","Ready","Deleting","Error"],"type":"string"},"versions":{"description":"Versions tracks the versions of the components","properties":{"bluefieldSoftwareVersions":{"additionalProperties":{"description":"BluefieldDeviceVersions holds firmware versions extracted from a single PSID's PLDM bundle.","properties":{"bfNicFwVersion":{"description":"BFNicFwVersion is the BlueField NIC firmware version shipped in the bundle.","minLength":1,"type":"string"},"bmcErotVersion":{"description":"BMCErotVersion is the BMC ERoT (External Root of Trust) firmware version shipped in the bundle.","minLength":1,"type":"string"},"bmcVersion":{"description":"BMCVersion is the DPU BMC firmware version shipped in the bundle.","minLength":1,"type":"string"},"sbiosVersion":{"description":"SBIOSVersion is the DPU SBIOS firmware version shipped in the bundle.","minLength":1,"type":"string"}},"type":"object"},"description":"BluefieldSoftwareVersions maps each PSID to the firmware versions from that device's\nPLDM bundle. Keys are matched case-insensitively against the BMC-reported DPUDevice PSID.","type":"object"},"doca":{"description":"DOCA is the formatted, user-facing DOCA version derived from the OS ISO.","type":"string"},"ewNicFwVersion":{"description":"EWNicFwVersion is the E/W NIC firmware version taken from the platform PLDM bundle.\nIt is not keyed by PSID because a BlueFieldSoftware carries a single platform bundle.","type":"string"},"osISOVersion":{"description":"OSISOVersion is the raw DOCA version for the OS ISO, taken from the ISO filename","type":"string"}},"type":"object"}},"required":["phase"],"type":"object","x-kubernetes-validations":[{"message":"versions cannot be removed once set","rule":"!has(oldSelf.versions) || has(self.versions)"}]}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 187 chars","rule":"self.metadata.name.size() <= 187"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-DPUFlavor.json b/pkg/dpf/sentry/schemas/26.8.0-DPUFlavor.json new file mode 100644 index 00000000..fa82fe96 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-DPUFlavor.json @@ -0,0 +1 @@ +{"group":"provisioning.dpu.nvidia.com","kind":"DPUFlavor","version":"v1alpha1","schema":{"description":"DPUFlavor is the Schema for the dpuflavors API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUFlavorSpec defines the content of DPUFlavor","properties":{"bfcfgParameters":{"description":"BFCfgParameters are the parameters to be set in the bf.cfg file.","items":{"type":"string"},"type":"array"},"configFiles":{"description":"ConfigFiles are the files to be written on the DPU.","items":{"description":"ConfigFile describes a file materialized from inline raw content or external contentFrom.","properties":{"contentFrom":{"description":"ContentFrom references external content for the file.\nSupported only when type is agent-applied.","properties":{"configMapKeyRef":{"description":"ConfigMapKeyRef selects a key from a ConfigMap in the DPU namespace.","properties":{"key":{"description":"The key to select.","type":"string"},"name":{"default":"","description":"Name of the referent.\nThis field is effectively required, but due to backwards compatibility is\nallowed to be empty. Instances of this type with an empty value here are\nalmost certainly wrong.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names","type":"string"},"optional":{"description":"Specify whether the ConfigMap or its key must be defined","type":"boolean"}},"required":["key"],"type":"object","x-kubernetes-map-type":"atomic"}},"type":"object"},"operation":{"description":"Operation is the operation to be performed on the file.","enum":["override","append"],"type":"string"},"path":{"description":"Path is the path of the file to be written.","type":"string"},"permissions":{"description":"Permissions are the permissions to be set on the file.","type":"string"},"raw":{"description":"Raw is the inline file content.\nSupported only when type is cloud-init. When type is omitted, type defaults\nto cloud-init and raw must be set.","type":"string"},"type":{"default":"cloud-init","description":"Type controls when the file content is materialized.\ncloud-init files use raw inline content and are written during cloud-init.\nagent-applied files use contentFrom and are written later by dpu-agent.\nDefaults to cloud-init when omitted.","enum":["cloud-init","agent-applied"],"type":"string"}},"required":["path"],"type":"object","x-kubernetes-validations":[{"message":"exactly one of raw or contentFrom must be specified","rule":"has(self.raw) != has(self.contentFrom)"},{"message":"contentFrom.configMapKeyRef must be specified","rule":"!has(self.contentFrom) || has(self.contentFrom.configMapKeyRef)"},{"message":"type cloud-init supports raw only","rule":"!has(self.type) || self.type != 'cloud-init' || has(self.raw)"},{"message":"type agent-applied supports contentFrom only","rule":"!has(self.type) || self.type != 'agent-applied' || has(self.contentFrom)"},{"message":"type defaults to cloud-init, which requires raw content","rule":"has(self.type) || has(self.raw)"}]},"type":"array"},"containerdConfig":{"description":"ContainerdConfig contains the configuration for containerd.","properties":{"registryEndpoint":{"description":"RegistryEndpoint is the endpoint of the container registry.","type":"string"}},"type":"object"},"dma":{"description":"DMA configures the DMA SF that e.g. SNAP DOCA service uses to DMA host\nmemory over the second Grace PCI link on BlueField-4 socket-direct\nsystems.","properties":{"enabled":{"description":"Enabled controls whether the dpu-agent creates the DMA SF. Defaults to\nfalse when unset, so the presence of the dma struct alone does not enable\ncreation. Only takes effect on BlueField-4 socket-direct systems. The\ncreated Scalable Function always uses sfnum 8000, the SNAP discovery ABI\nvalue.","type":"boolean"}},"type":"object"},"dpuMode":{"description":"DpuMode is deprecated and no longer used by provisioning workflows.\nDeployment mode is sourced from DPFOperatorConfig and exposed on DPU.status.deploymentMode.","enum":["dpu","zero-trust","nic"],"type":"string"},"dpuResources":{"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","x-kubernetes-int-or-string":true},"description":"DPUResources indicates the minimum amount of resources needed for a BFB with that flavor to be installed on a\nDPU. Using this field, the controller can understand if that flavor can be installed on a particular DPU. It\nshould be set to the total amount of resources the system needs + the resources that should be made available for\nDPUServices to consume.","type":"object"},"ewNicConfigurations":{"description":"EWNicConfigurations lists per-NIC configuration for the E/W NICs.\nOnly the first entry is applied in this release; additional entries are ignored until a future\nrelease adds multi-NIC support. The field is modeled as a list now so the API shape does not\nneed to change when multiple entries are supported.","items":{"description":"NicConfiguration is a set of configurations for the NICs","properties":{"force":{"default":false,"description":"Force passes `--force` to mlxconfig set commands. When set, the daemon\napplies the nv config batch and set_system_conf with --force, letting\nmlxconfig accept a batch it would otherwise refuse due to implicit\nparameter dependencies.","type":"boolean"},"linkType":{"description":"LinkType to be configured, Ethernet|Infiniband. Required unless networkBay is configured;\nfor Network Bay the link type is governed by the system configuration and must not be set.","enum":["Ethernet","Infiniband"],"type":"string"},"networkBay":{"description":"NetworkBay configures a ConnectX-9 Network Bay card (per-ASIC set_system_conf). Allowed only for ConnectX-9 (nicType 1025).","properties":{"conf":{"description":"Conf is the mlxconfig system configuration profile name. The daemon resolves\nthe profile parameters for the device's detected Network Bay ASIC and manages\nthem through the regular mlxconfig validation and apply flow.","type":"string"}},"required":["conf"],"type":"object"},"numVfs":{"description":"Number of VFs to be configured","type":"integer"},"rawNvConfig":{"description":"List of arbitrary nv config parameters","items":{"properties":{"name":{"description":"Name of the arbitrary nvconfig parameter","maxLength":64,"type":"string"},"value":{"description":"Value of the arbitrary nvconfig parameter","maxLength":128,"type":"string"}},"required":["name","value"],"type":"object"},"type":"array"},"spectrumXOptimized":{"description":"Spectrum-X optimization settings. Works only with linkType==Ethernet && numVfs==1. RawNvConfig parameters, if provided, are merged as overrides on top of Spectrum-X calculated params.","properties":{"enabled":{"description":"Optimize Spectrum X","type":"boolean"},"multiplaneMode":{"default":"none","description":"Multiplane mode to be configured\nCan be \"none\", \"swplb\", \"hwplb\", or \"uniplane\"","enum":["none","swplb","hwplb","uniplane"],"type":"string"},"numberOfPlanes":{"default":1,"description":"Number of planes to be configured","enum":[1,2,4],"type":"integer"},"overlay":{"default":"none","description":"Overlay mode to be configured\nCan be \"l3\" or \"none\"","enum":["l3","none"],"type":"string"},"version":{"description":"Version of the Spectrum-X architecture to optimize for. Should match the name of the config map with Spectrum-X profile","type":"string"}},"required":["enabled","version"],"type":"object","x-kubernetes-validations":[{"message":"when MultiplaneMode is none, numberOfPlanes must be 1","rule":"!has(self.multiplaneMode) || !has(self.numberOfPlanes) || self.multiplaneMode != 'none' || self.numberOfPlanes == 1"},{"message":"when MultiplaneMode is not none, numberOfPlanes must not be 1","rule":"!has(self.multiplaneMode) || !has(self.numberOfPlanes) || self.multiplaneMode == 'none' || self.numberOfPlanes != 1"}]}},"required":["numVfs"],"type":"object","x-kubernetes-validations":[{"message":"spectrumXOptimized can be enabled only when linkType=='Ethernet' (or unset for Network Bay) and numVfs==1","rule":"!(has(self.spectrumXOptimized) && self.spectrumXOptimized.enabled) || ((!has(self.linkType) || self.linkType == 'Ethernet') && self.numVfs == 1)"},{"message":"linkType is required unless networkBay is configured","rule":"has(self.networkBay) || has(self.linkType)"},{"message":"linkType must not be set when networkBay is configured (the Network Bay link type is governed by the system configuration)","rule":"!has(self.networkBay) || !has(self.linkType)"},{"message":"networkBay.conf must not be empty","rule":"!has(self.networkBay) || self.networkBay.conf != \"\""}]},"maxItems":16,"type":"array","x-kubernetes-list-type":"atomic"},"grub":{"description":"Grub contains the grub configuration for the DPUFlavor.","properties":{"kernelParameters":{"description":"KernelParameters are the kernel parameters to be set in the grub configuration.","items":{"type":"string"},"type":"array"}},"type":"object"},"hostNetworkInterfaceConfigs":{"description":"HostNetworkInterfaceConfigs contains the configuration for the host-side network interfaces.","items":{"description":"NetworkInterfaceConfig defines the configuration for a network interface","properties":{"dhcp":{"description":"DHCP is the DHCP configuration for the network interface.","type":"boolean"},"mtu":{"description":"MTU is the MTU value to be set on the network interface.","format":"int32","maximum":9216,"minimum":1280,"type":"integer"},"nvconfig":{"description":"NVConfig contains port-specific configuration for this network interface.\nThis configuration is applied in addition to the global NVConfig settings in DPUFlavorSpec.\nBoth global and per-interface NVConfig settings can coexist without collision.","properties":{"device":{"description":"Device is the device to which the configuration applies. If not specified, the configuration applies to all.\nSupported values: \"*\" (wildcard for all devices), \"p0\"/\"P0\" (port 0), \"p1\"/\"P1\" (port 1). Case-insensitive.","enum":["*","p0","p1","P0","P1"],"type":"string"},"force":{"description":"force applies the parameters with `mlxconfig --force` and skips the `mlxconfig q` filter,\nso parameters firmware does not yet expose are applied now instead of deferred to a later\nreboot. Required when a parameter is gated behind another parameter in the same batch.\nRequires DOCA 3.5.0 or later; on an older DOCA version the operation fails.\n`--force` skips validation for the whole batch, so an invalid value is applied silently.\nIgnored under `spec.hostNetworkInterfaceConfigs[].nvconfig`.","type":"boolean"},"parameters":{"description":"Parameters are the parameters to be set for the device.\nDELAY_HOST_OS_INIT=ENABLE_USER (0x3) holds the host at UEFI and is rejected by the DPU agent\noutside zero-trust, where the agent needs the host to reach the kube-apiserver.","items":{"maxLength":200,"pattern":"^[^=\\s]+=[^\\s]*$","type":"string"},"maxItems":64,"minItems":1,"type":"array","x-kubernetes-list-type":"atomic"}},"type":"object"},"portNumber":{"description":"PortNumber identifies which port this configuration applies to.","format":"int32","maximum":1,"minimum":0,"type":"integer"}},"required":["portNumber"],"type":"object"},"type":"array"},"nvconfig":{"description":"NVConfig contains the device-specific configuration (firmware settings, device parameters).\nEach entry specifies a device (wildcard '*', or port identifiers 'p0'/'P0'/'p1'/'P1') and its parameters.\nIf device is '*' or unspecified (defaults to '*'), it applies to all devices and must be the only entry.\nEach device (including unspecified as '*') must be unique across all nvconfig entries (case-insensitive).\nValidation enforces: device enum values, parameter format (KEY=VALUE), case-insensitive uniqueness, and size limits.","items":{"properties":{"device":{"description":"Device is the device to which the configuration applies. If not specified, the configuration applies to all.\nSupported values: \"*\" (wildcard for all devices), \"p0\"/\"P0\" (port 0), \"p1\"/\"P1\" (port 1). Case-insensitive.","enum":["*","p0","p1","P0","P1"],"type":"string"},"force":{"description":"force applies the parameters with `mlxconfig --force` and skips the `mlxconfig q` filter,\nso parameters firmware does not yet expose are applied now instead of deferred to a later\nreboot. Required when a parameter is gated behind another parameter in the same batch.\nRequires DOCA 3.5.0 or later; on an older DOCA version the operation fails.\n`--force` skips validation for the whole batch, so an invalid value is applied silently.\nIgnored under `spec.hostNetworkInterfaceConfigs[].nvconfig`.","type":"boolean"},"parameters":{"description":"Parameters are the parameters to be set for the device.\nDELAY_HOST_OS_INIT=ENABLE_USER (0x3) holds the host at UEFI and is rejected by the DPU agent\noutside zero-trust, where the agent needs the host to reach the kube-apiserver.","items":{"maxLength":200,"pattern":"^[^=\\s]+=[^\\s]*$","type":"string"},"maxItems":64,"minItems":1,"type":"array","x-kubernetes-list-type":"atomic"}},"type":"object"},"maxItems":3,"type":"array","x-kubernetes-list-type":"atomic","x-kubernetes-validations":[{"message":"when device is '*', it must be the only nvconfig entry","rule":"size(self) == 0 || !self.exists(x, has(x.device) && x.device == '*') || size(self) == 1"},{"message":"when device is unspecified (defaults to '*'), it must be the only nvconfig entry","rule":"size(self) == 0 || !self.exists(x, !has(x.device)) || size(self) == 1"},{"message":"each nvconfig.device (including unspecified as '*') must be unique (case-insensitive)","rule":"self.all(p1, self.exists_one(p2, (has(p1.device) ? p1.device.lowerAscii() : '*') == (has(p2.device) ? p2.device.lowerAscii() : '*')))"}]},"ovs":{"description":"OVS contains the OVS configuration for the DPUFlavor.","properties":{"rawConfigScript":{"description":"RawConfigScript is the raw configuration script for OVS.\nThe DPU agent runs this script once per boot. A restart of the agent in the\nsame boot does not re-run it; a reboot or power cycle does.","type":"string"}},"type":"object"},"packages":{"description":"Packages are the packages to reconcile on the node.","items":{"description":"PackageSpec defines a package to reconcile on the node.","properties":{"name":{"description":"Name is the package name.","maxLength":253,"minLength":1,"type":"string"},"repoFileRef":{"description":"RepoFileRef constrains package resolution to a specific repository file available on the node.\nIf empty, any configured repository may satisfy the package.\nIf specified, only the referenced repository file may provide candidates.\nIf that repository file does not provide the package or requested version, the dpu-agent flow does not continue.","type":"string"},"version":{"description":"Version constrains the package version.\nIf empty, any installed version satisfies the spec.","properties":{"matchPolicy":{"default":"AtLeast","description":"MatchPolicy controls how Value is matched.\nIf omitted, AtLeast is used.","enum":["Exact","AtLeast"],"type":"string"},"value":{"description":"Value is the package version to compare against.","minLength":1,"type":"string"}},"required":["value"],"type":"object"}},"required":["name"],"type":"object"},"maxItems":64,"type":"array","x-kubernetes-validations":[{"message":"package names must be unique","rule":"self.all(x, self.exists_one(y, x.name == y.name))"}]},"serviceReadiness":{"description":"serviceReadiness configures the Service Readiness phase.","properties":{"gate":{"description":"gate is the DPU.status.operationalConditions entry that must be True before the DPU leaves\nthe Service Readiness phase. When unset the phase does not wait, and a host hold requested\nvia DELAY_HOST_OS_INIT is released on DPUServiceCriticalPodsReady.","enum":["DPUServiceCriticalPodsReady","OperationalReady"],"type":"string"}},"type":"object"},"sysctl":{"description":"Sysctl contains the sysctl configuration for the DPUFlavor.","properties":{"parameters":{"description":"Parameters are the sysctl parameters to be set.","items":{"type":"string"},"type":"array"}},"type":"object"},"systemReservedResources":{"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","x-kubernetes-int-or-string":true},"description":"SystemReservedResources indicates the resources that are consumed by the system (OS, OVS, DPF system etc) and are\nnot made available for DPUServices to consume. DPUServices can consume the difference between DPUResources and\nSystemReservedResources. This field must not be specified if dpuResources are not specified.","type":"object"},"systemdServices":{"description":"SystemdServices are the systemd services to manage on the node.","items":{"description":"SystemdServiceSpec defines a systemd service to manage on the node.","properties":{"name":{"description":"Name is the systemd service name.","maxLength":253,"minLength":1,"type":"string"},"operation":{"description":"Operation is the systemd operation to perform on the service.","enum":["Start","Enable","EnableAndStart"],"type":"string"}},"required":["name","operation"],"type":"object"},"maxItems":64,"type":"array","x-kubernetes-validations":[{"message":"systemd service names must be unique","rule":"self.all(x, self.exists_one(y, x.name == y.name))"}]}},"type":"object","x-kubernetes-validations":[{"message":"DPUFlavor spec is immutable","rule":"self == oldSelf"}]}},"type":"object"}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-DPUService.json b/pkg/dpf/sentry/schemas/26.8.0-DPUService.json new file mode 100644 index 00000000..9ba317f0 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-DPUService.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUService","version":"v1alpha1","schema":{"description":"DPUService is the Schema for the dpuservices API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceSpec defines the desired state of DPUService","properties":{"configPorts":{"description":"ConfigPorts defines the desired state of port configurations for a DPUService.\nThis struct determines how ports are exposed from the DPU to the host cluster.\nA DPUService can only have a single ServiceType across all ports.","properties":{"ports":{"description":"Ports defines the list of port configurations that will be exposed by the DPUService.\nEach port must specify a name, port number, and protocol.\n\nConstraints:\n- If ServiceType is \"NodePort\", ports may optionally specify a NodePort.\n- If ServiceType is \"None\" or \"ClusterIP\", ports **cannot** specify a NodePort.","items":{"description":"ConfigPort defines the configuration of a single port within a DPUService.\nEach port must have a unique name within the service.","properties":{"name":{"description":"Name is a unique identifier for the port within the DPUService.\nThis name is used for reference inside the service.","minLength":1,"pattern":"^[a-z0-9-]+$","type":"string"},"nodePort":{"description":"NodePort is the external port assigned on each node in the cluster.\nIf not set, Kubernetes will automatically allocate a NodePort.\n\nConstraints:\n- Can only be set when ServiceType is \"NodePort\".\n- Must be within the clusters valid NodePort range (Kubernetes default is 30000-32767).","type":"integer"},"port":{"description":"Port is the port number that will be exposed by the service.\nMust be within the valid range of TCP/UDP ports (1-65535).","type":"integer"},"protocol":{"description":"Protocol specifies the transport protocol used by the port.\nSupported values: TCP, UDP","enum":["TCP","UDP"],"type":"string"}},"required":["name","port","protocol"],"type":"object"},"type":"array"},"serviceType":{"default":"NodePort","description":"ServiceType specifies the type of Kubernetes Service to create.\nAll ports within this ConfigPorts will have the same ServiceType.\nThe value is immutable and cannot be changed after creation.\nSupported values:\n- \"NodePort\": Exposes ports externally on a node.\n- \"ClusterIP\": Exposes ports internally within the cluster.\n- \"None\": Internal-only service with no cluster IP.\nDefault: \"NodePort\"","enum":["NodePort","ClusterIP","None"],"type":"string","x-kubernetes-validations":[{"message":"Value is immutable","rule":"self == oldSelf"}]}},"required":["ports","serviceType"],"type":"object","x-kubernetes-validations":[{"message":"nodePort can only be set when serviceType is NodePort","rule":"!(self.serviceType != 'NodePort' && self.ports.exists(p, has(p.nodePort)))"}]},"deployInCluster":{"description":"DeployInCluster indicates if the DPUService Helm Chart will be deployed on\nthe Host cluster. Default to false.","type":"boolean"},"dpuClusterSelector":{"description":"Select the Clusters with specific labels, Applications will be created only for these Clusters","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"helmChart":{"description":"HelmChart reflects the Helm related configuration","properties":{"source":{"description":"Source specifies information about the Helm chart","properties":{"chart":{"description":"Chart is the name of the helm chart.","type":"string"},"path":{"description":"Path is the location of the chart inside the repo.","type":"string"},"releaseName":{"description":"ReleaseName is the name to give to the release generate from the DPUService.","type":"string"},"repoURL":{"description":"RepoURL specifies the URL to the repository that contains the application Helm chart.\nThe URL must begin with either 'oci://' or 'https://', ensuring it points to a valid\nOCI registry or a web-based repository.","pattern":"^(oci://|https://).+$","type":"string"},"version":{"description":"Version is a semver tag for the Chart's version.","minLength":1,"type":"string"}},"required":["repoURL","version"],"type":"object"},"values":{"description":"Values specifies Helm values to be passed to Helm template, defined as a map. This takes precedence over Values.","type":"object","x-kubernetes-preserve-unknown-fields":true}},"required":["source"],"type":"object"},"interfaces":{"description":"Interfaces specifies the DPUServiceInterface names that the DPUService\nuses in the same namespace.","items":{"type":"string"},"maxItems":50,"minItems":1,"type":"array"},"paused":{"description":"Paused indicates that the DPUService is paused.\nUnderlying resources are also paused when this is set to true.\nNo deletion of resources will occur when this is set to true.","type":"boolean"},"security":{"description":"Security contains security-related settings for the DPUService.","properties":{"privileged":{"description":"Privileged, when set to true, allows workloads governed by this security\nsetting to run containers with `securityContext.privileged: true` in the\nDPUCluster. When set to false, a ValidatingAdmissionPolicy in the\nDPUCluster rejects such workloads. On DPUService objects, this field must\nbe unset when deployInCluster is true. For DPUServices that target a\nDPUCluster, Privileged must be set explicitly. When set on a\nDPUServiceTemplate, the DPUDeployment controller propagates the value to\ngenerated DPUServices that target DPUClusters.\n\nScope of the policy enforcement:\n - Only `securityContext.privileged: true` is gated. Other\n privilege-escalation vectors (hostPID, hostIPC, hostNetwork,\n allowPrivilegeEscalation, capabilities, hostPath volumes) are NOT\n gated by this field.\n - Enforcement matches workloads via the\n `svc.dpu.nvidia.com/service` label. The controller adds this\n label to the resources it manages, and to pod templates of\n workload-controller resources (Deployment, DaemonSet, etc.). If a\n Helm chart strips that label from the pod template, the parent\n resource will be admitted but the child Pods will be denied at\n Pod admission time.","type":"boolean"},"spiffe":{"description":"SPIFFE opts this DPUService into SPIFFE workload identity. When set, one SPIRE\nClusterStaticEntry is registered per targeted DPU, parented to that DPU's SPIRE\nagent, so the service's pods can obtain an SVID. The identity defaults to\n`spiffe:///dpu//service//` and is\nconfigurable through `DPFOperatorConfig.spec.security.spiffe`.\nThe namespace qualifies the service ID, which is only unique within one.\n\nPresence-gated: `spiffe: {}` enables it. It has no sub-fields yet.\n\nRequires `DPFOperatorConfig.spec.security.spiffe`, and only covers DPUs in SPIFFE\nidentity mode (`DPU.status.identityMode: Spiffe`); bootstrap-token DPUs are skipped.\nBoth are reported on the SPIFFEEntriesReady condition rather than rejected at\nadmission. Must not be set when deployInCluster is true, since in-cluster\nDPUServices run on the host and have no per-DPU SPIRE agent to parent to.\n\nWhen set on a DPUServiceTemplate, the DPUDeployment controller propagates it to\ngenerated DPUServices that target DPUClusters.","type":"object"}},"type":"object"},"serviceDaemonSet":{"description":"ServiceDaemonSet specifies the configuration for the ServiceDaemonSet.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations specifies annotations which are added to the ServiceDaemonSet.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels specifies labels which are added to the ServiceDaemonSet.","type":"object"},"nodeSelector":{"description":"NodeSelector specifies which Nodes to deploy the ServiceDaemonSet to.","properties":{"nodeSelectorTerms":{"description":"Required. A list of node selector terms. The terms are ORed.","items":{"description":"A null or empty node selector term matches no objects. The requirements of\nthem are ANDed.\nThe TopologySelectorTerm type implements a subset of the NodeSelectorTerm.","properties":{"matchExpressions":{"description":"A list of node selector requirements by node's labels.","items":{"description":"A node selector requirement is a selector that contains values, a key, and an operator\nthat relates the key and values.","properties":{"key":{"description":"The label key that the selector applies to.","type":"string"},"operator":{"description":"Represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.","type":"string"},"values":{"description":"An array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. If the operator is Gt or Lt, the values\narray must have a single element, which will be interpreted as an integer.\nThis array is replaced during a strategic merge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchFields":{"description":"A list of node selector requirements by node's fields.","items":{"description":"A node selector requirement is a selector that contains values, a key, and an operator\nthat relates the key and values.","properties":{"key":{"description":"The label key that the selector applies to.","type":"string"},"operator":{"description":"Represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.","type":"string"},"values":{"description":"An array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. If the operator is Gt or Lt, the values\narray must have a single element, which will be interpreted as an integer.\nThis array is replaced during a strategic merge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"}},"type":"object","x-kubernetes-map-type":"atomic"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["nodeSelectorTerms"],"type":"object","x-kubernetes-map-type":"atomic"},"resources":{"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","x-kubernetes-int-or-string":true},"description":"Resources specifies resources which are added to the ServiceDaemonSet.","type":"object"},"updateStrategy":{"description":"UpdateStrategy specifies the DeaemonSet update strategy for the ServiceDaemonset.","properties":{"rollingUpdate":{"description":"Rolling update config params. Present only if type = \"RollingUpdate\".","properties":{"maxSurge":{"anyOf":[{"type":"integer"},{"type":"string"}],"description":"The maximum number of nodes with an existing available DaemonSet pod that\ncan have an updated DaemonSet pod during during an update.\nValue can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).\nThis can not be 0 if MaxUnavailable is 0.\nAbsolute number is calculated from percentage by rounding up to a minimum of 1.\nDefault value is 0.\nExample: when this is set to 30%, at most 30% of the total number of nodes\nthat should be running the daemon pod (i.e. status.desiredNumberScheduled)\ncan have their a new pod created before the old pod is marked as deleted.\nThe update starts by launching new pods on 30% of nodes. Once an updated\npod is available (Ready for at least minReadySeconds) the old DaemonSet pod\non that node is marked deleted. If the old pod becomes unavailable for any\nreason (Ready transitions to false, is evicted, or is drained) an updated\npod is immediately created on that node without considering surge limits.\nAllowing surge implies the possibility that the resources consumed by the\ndaemonset on any given node can double if the readiness check fails, and\nso resource intensive daemonsets should take into account that they may\ncause evictions during disruption.","x-kubernetes-int-or-string":true},"maxUnavailable":{"anyOf":[{"type":"integer"},{"type":"string"}],"description":"The maximum number of DaemonSet pods that can be unavailable during the\nupdate. Value can be an absolute number (ex: 5) or a percentage of total\nnumber of DaemonSet pods at the start of the update (ex: 10%). Absolute\nnumber is calculated from percentage by rounding up.\nThis cannot be 0 if MaxSurge is 0\nDefault value is 1.\nExample: when this is set to 30%, at most 30% of the total number of nodes\nthat should be running the daemon pod (i.e. status.desiredNumberScheduled)\ncan have their pods stopped for an update at any given time. The update\nstarts by stopping at most 30% of those DaemonSet pods and then brings\nup new DaemonSet pods in their place. Once the new pods are available,\nit then proceeds onto other DaemonSet pods, thus ensuring that at least\n70% of original number of DaemonSet pods are available at all times during\nthe update.","x-kubernetes-int-or-string":true}},"type":"object"},"type":{"description":"Type of daemon set update. Can be \"RollingUpdate\" or \"OnDelete\". Default is RollingUpdate.","type":"string"}},"type":"object"}},"type":"object"},"serviceID":{"description":"ServiceID is the ID of the service that the DPUService is associated with.","type":"string"}},"required":["helmChart"],"type":"object","x-kubernetes-validations":[{"message":"serviceID must be provided when interfaces are provided","rule":"(has(self.interfaces) && has(self.serviceID)) || (!has(self.interfaces) && !has(self.serviceID)) || has(self.serviceID)"},{"message":"configPorts cannot be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.configPorts))"},{"message":"dpuClusterSelector cannot be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.dpuClusterSelector))"},{"message":"security.privileged must not be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.security) && has(self.security.privileged))"},{"message":"security.spiffe must not be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.security) && has(self.security.spiffe))"},{"message":"security.privileged must be set when deployInCluster is false","optionalOldSelf":true,"rule":"(has(self.deployInCluster) && self.deployInCluster) || (has(self.security) && has(self.security.privileged)) || (oldSelf.hasValue() && !(has(oldSelf.value().deployInCluster) && oldSelf.value().deployInCluster) && !(has(oldSelf.value().security) && has(oldSelf.value().security.privileged)))"}]},"status":{"description":"DPUServiceStatus defines the observed state of DPUService","properties":{"conditions":{"description":"Conditions defines current service state.","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"configPorts":{"additionalProperties":{"items":{"description":"ConfigPort defines the configuration of a single port within a DPUService.\nEach port must have a unique name within the service.","properties":{"name":{"description":"Name is a unique identifier for the port within the DPUService.\nThis name is used for reference inside the service.","minLength":1,"pattern":"^[a-z0-9-]+$","type":"string"},"nodePort":{"description":"NodePort is the external port assigned on each node in the cluster.\nIf not set, Kubernetes will automatically allocate a NodePort.\n\nConstraints:\n- Can only be set when ServiceType is \"NodePort\".\n- Must be within the clusters valid NodePort range (Kubernetes default is 30000-32767).","type":"integer"},"port":{"description":"Port is the port number that will be exposed by the service.\nMust be within the valid range of TCP/UDP ports (1-65535).","type":"integer"},"protocol":{"description":"Protocol specifies the transport protocol used by the port.\nSupported values: TCP, UDP","enum":["TCP","UDP"],"type":"string"}},"required":["name","port","protocol"],"type":"object"},"type":"array"},"description":"ConfigPorts defines the observed state of the config ports.\nIt contains the actual port numbers that are exposed on the DPUService per cluster.","type":"object"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"},"serviceID":{"description":"ServiceID is the ID of the service that the DPUService is associated with.\nThis is set when the DPUService is created.","type":"string"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-DPUServiceChain.json b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceChain.json new file mode 100644 index 00000000..0cef25d2 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceChain.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceChain","version":"v1alpha1","schema":{"description":"DPUServiceChain is the Schema for the DPUServiceChain API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceChainSpec defines the desired state of DPUServiceChainSpec","properties":{"clusterSelector":{"description":"Select the Clusters with specific labels, ServiceChainSet CRs will be created only for these Clusters\n\nDeprecated: This field is deprecated and will be removed with v27.1.0. Use DPUClusterSelector instead.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuClusterSelector":{"description":"DPUClusterSelector determines in which clusters the DPUServiceChain controller should apply the configuration.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"Template describes the ServiceChainSet that will be created for each selected Cluster.","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceChainSetSpec defines the desired state of ServiceChainSet","properties":{"nodeSelector":{"description":"Select the Nodes with specific labels, ServiceChain CRs will be created\nonly for these Nodes","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"ServiceChainSpecTemplate holds the template for the ServiceChainSpec","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceChainSpec is the spec for the ServiceChainSpec","properties":{"node":{"description":"Node where this ServiceChain applies to","type":"string"},"switches":{"description":"The switches of the ServiceChain, order is significant","items":{"description":"Switch defines the switch configuration","properties":{"ports":{"description":"Ports of the switch","items":{"description":"Port defines the port configuration","properties":{"serviceInterface":{"description":"ServiceIfc defines the service interface configuration","properties":{"ipam":{"description":"IPAM defines the IPAM configuration when referencing a serviceInterface of type 'service'","properties":{"defaultGateway":{"description":"DefaultGateway adds gateway as default gateway in the routes list if true.","type":"boolean"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"Labels matching service IPAM","maxProperties":50,"minProperties":1,"type":"object"},"setDefaultRoute":{"description":"SetDefaultRoute adds a default route to the routing table if true.","type":"boolean"}},"required":["matchLabels"],"type":"object"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"Labels matching service interface","maxProperties":50,"minProperties":1,"type":"object"}},"required":["matchLabels"],"type":"object"}},"required":["serviceInterface"],"type":"object"},"maxItems":50,"minItems":1,"type":"array"},"serviceMTU":{"default":1500,"description":"ServiceMTU of the switch\nThe default is 1500.","maximum":9216,"minimum":1280,"type":"integer"}},"required":["ports"],"type":"object"},"maxItems":50,"minItems":1,"type":"array"}},"required":["switches"],"type":"object"}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"},"status":{"description":"DPUServiceChainStatus defines the observed state of DPUServiceChain","properties":{"conditions":{"description":"Conditions reflect the status of the object","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-DPUServiceCredentialRequest.json b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceCredentialRequest.json new file mode 100644 index 00000000..545f0fdf --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceCredentialRequest.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceCredentialRequest","version":"v1alpha1","schema":{"description":"DPUServiceCredentialRequest is the Schema for the dpuserviceCredentialRequests API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceCredentialRequestSpec defines the desired state of DPUServiceCredentialRequest","properties":{"duration":{"description":"Duration is the duration for which the token will be valid.\nValue must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration.\ne.g. \"1h\", \"1m\", \"1s\", \"1ms\", \"1.5h\", \"2h45m\".\nValue duration must not be less than 10 minutes.\n**Note:** The maximum TTL for a token is 24 hours, after which the token\nwill be rotated.","pattern":"^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$","type":"string"},"metadata":{"description":"ObjectMeta defines the metadata of the secret.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"secret":{"description":"Secret defines the needed information to create the secret.\nThe secret will be of the type specified in the `spec.type` field.","properties":{"name":{"description":"Name of the object.","type":"string"},"namespace":{"description":"Namespace of the object, if not provided the object will be looked up in\nthe same namespace as the referring object","type":"string"}},"required":["name"],"type":"object"},"serviceAccount":{"description":"ServiceAccount defines the needed information to create the service account.","properties":{"name":{"description":"Name of the object.","type":"string"},"namespace":{"description":"Namespace of the object, if not provided the object will be looked up in\nthe same namespace as the referring object","type":"string"}},"required":["name"],"type":"object"},"targetCluster":{"description":"TargetCluster defines the target cluster where the service account will\nbe created, and where a token for that service account will be requested.\nIf not provided, the token will be requested for the same cluster where\nthe DPUServiceCredentialRequest object is created.","properties":{"name":{"description":"Name of the object.","type":"string"},"namespace":{"description":"Namespace of the object, if not provided the object will be looked up in\nthe same namespace as the referring object","type":"string"}},"required":["name"],"type":"object"},"type":{"description":"Type is the type of the secret that will be created.\nThe supported types are `kubeconfig` and `tokenFile`.\nIf `kubeconfig` is selected, the secret will contain a kubeconfig file,\nthat can be used to access the cluster.\nIf `tokenFile` is selected, the secret will contain a token file and several\nenvironment variables that can be used to access the cluster. It can be used\nwith https://github.com/kubernetes/client-go/blob/v11.0.0/rest/config.go#L52\nto create a client that will handle file rotation.","enum":["kubeconfig","tokenFile"],"type":"string"}},"required":["secret","serviceAccount","type"],"type":"object"},"status":{"description":"DPUServiceCredentialRequestStatus defines the observed state of DPUServiceCredentialRequest","properties":{"conditions":{"description":"Conditions defines current service state.","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"expirationTimestamp":{"description":"ExpirationTimestamp is the time when the token will expire.","format":"date-time","type":"string"},"issuedAt":{"description":"IssuedAt is the time when the token was issued.","format":"date-time","type":"string"},"secret":{"description":"Sercet is the namespaced name of the Secret resource created by the controller for\nthe DPUServiceCredentialRequest.","type":"string"},"serviceAccount":{"description":"ServiceAccount is the namespaced name of the ServiceAccount resource created by\nthe controller for the DPUServiceCredentialRequest.","type":"string"},"targetCluster":{"description":"TargetCluster is the cluster where the service account was created.\nIt has to be persisted in the status to be able to delete the service account\nwhen the DPUServiceCredentialRequest is updated.","type":"string"},"targetClusterUID":{"description":"TargetClusterUID is the UID of the DPUCluster the credentials were last issued against.\nUsed to detect cluster recreation with the same name and force credential refresh.","type":"string"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-DPUServiceInterface.json b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceInterface.json new file mode 100644 index 00000000..9f11d88c --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceInterface.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceInterface","version":"v1alpha1","schema":{"description":"DPUServiceInterface is the Schema for the DPUServiceInterface API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceInterfaceSpec defines the desired state of DPUServiceInterfaceSpec","properties":{"clusterSelector":{"description":"Select the Clusters with specific labels, ServiceInterfaceSet CRs will be created only for these Clusters\n\nDeprecated: This field is deprecated and will be removed with v27.1.0. Use DPUClusterSelector instead.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuClusterSelector":{"description":"DPUClusterSelector determines in which clusters the DPUServiceInterface controller should apply the configuration.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"Template describes the ServiceInterfaceSet that will be created for each selected Cluster.","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceInterfaceSetSpec defines the desired state of ServiceInterfaceSet","properties":{"nodeSelector":{"description":"Select the Nodes with specific labels, ServiceInterface CRs will be\ncreated only for these Nodes","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"Template holds the template for the serviceInterfaceSpec","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceInterfaceSpec is the spec for the ServiceInterfaceSpec","properties":{"interfaceType":{"description":"The interface type (\"vlan\", \"physical\", \"pf\", \"vf\", \"ovn\", \"patch\", \"service\")","enum":["vlan","physical","pf","vf","ovn","patch","service"],"type":"string"},"node":{"description":"Node where this interface exists","type":"string"},"ovn":{"description":"The OVN definition\nDeprecated: This field is deprecated and will be removed with v26.10.0.\nMigrate to interfaceType=\"patch\" with spec.patch.peerBridge and spec.patch.peerPatchName instead.","properties":{"externalBridge":{"default":"br-ovn","description":"ExternalBridge is the name of the OVN bridge","type":"string"}},"type":"object"},"patch":{"description":"The Patch definition","properties":{"peerBridge":{"description":"PeerBridge is the name of the bridge to which the patch port is connected.\nThis bridge must be created before the ServiceInterface is created.","type":"string"},"peerExternalIDs":{"additionalProperties":{"type":"string"},"description":"PeerExternalIDs are the external IDs used to identify the peer patch port.","type":"object"},"peerPatchName":{"description":"PeerPatchName is the name of the patch port on the peer bridge.\nIf not set, it is auto-generated in the format: `p__to__`\nwhere bridge names have hyphens removed and `` is an 8-character FNV-1a hash\nderived from the ServiceInterface's namespace/name.\nExample: p_brovn_to_brsfc_7aea60f7 (for bridges br-ovn and br-sfc).","type":"string"}},"required":["peerBridge"],"type":"object"},"pf":{"description":"The PF definition","properties":{"nicSelector":{"description":"NICSelector defines the NIC selected for the PF interface","properties":{"controllerNumber":{"description":"ControllerNumber specifies the controller number that will be used to find the matching representor on the DPU.\nA value of 0 targets the local controller, >=1 targets external controllers with the specified number. If unspecified controller number 1 is used.\nIn case of a DPU/NIC with socket direct or MultiHost, the user should specify the controller number explicitly","format":"int32","maximum":255,"minimum":0,"type":"integer"},"pci":{"description":"PCI is the PCI selector. valid only for selector type pci","properties":{"address":{"description":"Address is the PCI address of any of the NIC's Embedded CPU PFs (ECPFs)","pattern":"^[0-9a-f]{4}:[0-9a-f]{2}:[0-1][0-9a-f]\\.[0-7]{1}$","type":"string"}},"required":["address"],"type":"object"},"type":{"description":"Type is the type of selector to be used to identify the NIC.","enum":["dpu","pci"],"type":"string"}},"required":["type"],"type":"object","x-kubernetes-validations":[{"message":"for type=pci, pci must be set","rule":"(self.type == 'dpu') || (self.type == 'pci' && has(self.pci))"}]},"pfID":{"description":"The PF ID","maximum":255,"minimum":0,"type":"integer"},"virtualNetwork":{"description":"VirtualNetwork is the VirtualNetwork name in the same namespace","type":"string"}},"required":["pfID"],"type":"object","x-kubernetes-validations":[{"message":"virtualNetwork is immutable","rule":"(!has(oldSelf.virtualNetwork) && !has(self.virtualNetwork)) || ((has(oldSelf.virtualNetwork) && has(self.virtualNetwork)) && self.virtualNetwork==oldSelf.virtualNetwork)"}]},"physical":{"description":"The physical interface definition","properties":{"interfaceName":{"description":"The interface name","type":"string"}},"required":["interfaceName"],"type":"object"},"service":{"description":"The Service definition","properties":{"interfaceName":{"description":"The interface name","maxLength":15,"minLength":1,"type":"string"},"network":{"description":"Network is the Network Attachment Definition in the form of \"namespace/name\"\nor just \"name\" if the namespace is the same as the ServiceInterface.","type":"string"},"serviceID":{"description":"ServiceID is the DPU Service Identifier","type":"string"},"virtualNetwork":{"description":"VirtualNetwork is the VirtualNetwork name in the same namespace","type":"string"}},"required":["interfaceName","network","serviceID"],"type":"object","x-kubernetes-validations":[{"message":"virtualNetwork is immutable","rule":"(!has(oldSelf.virtualNetwork) && !has(self.virtualNetwork)) || ((has(oldSelf.virtualNetwork) && has(self.virtualNetwork)) && self.virtualNetwork==oldSelf.virtualNetwork)"}]},"vf":{"description":"The VF definition","properties":{"nicSelector":{"description":"NICSelector defines the NIC selected for the VF interface","properties":{"controllerNumber":{"description":"ControllerNumber specifies the controller number that will be used to find the matching representor on the DPU.\nA value of 0 targets the local controller, >=1 targets external controllers with the specified number. If unspecified controller number 1 is used.\nIn case of a DPU/NIC with socket direct or MultiHost, the user should specify the controller number explicitly","format":"int32","maximum":255,"minimum":0,"type":"integer"},"pci":{"description":"PCI is the PCI selector. valid only for selector type pci","properties":{"address":{"description":"Address is the PCI address of any of the NIC's Embedded CPU PFs (ECPFs)","pattern":"^[0-9a-f]{4}:[0-9a-f]{2}:[0-1][0-9a-f]\\.[0-7]{1}$","type":"string"}},"required":["address"],"type":"object"},"type":{"description":"Type is the type of selector to be used to identify the NIC.","enum":["dpu","pci"],"type":"string"}},"required":["type"],"type":"object","x-kubernetes-validations":[{"message":"for type=pci, pci must be set","rule":"(self.type == 'dpu') || (self.type == 'pci' && has(self.pci))"}]},"parentInterfaceRef":{"description":"The parent interface reference","type":"string"},"pfID":{"description":"The PF ID","maximum":255,"minimum":0,"type":"integer"},"vfID":{"description":"The VF ID","maximum":255,"minimum":0,"type":"integer"},"virtualNetwork":{"description":"VirtualNetwork is the VirtualNetwork name in the same namespace","type":"string"}},"required":["pfID","vfID"],"type":"object","x-kubernetes-validations":[{"message":"virtualNetwork is immutable","rule":"(!has(oldSelf.virtualNetwork) && !has(self.virtualNetwork)) || ((has(oldSelf.virtualNetwork) && has(self.virtualNetwork)) && self.virtualNetwork==oldSelf.virtualNetwork)"}]},"vlan":{"description":"The VLAN definition","properties":{"parentInterfaceRef":{"description":"The parent interface reference","type":"string"},"vlanID":{"description":"The VLAN ID","type":"integer"}},"required":["parentInterfaceRef","vlanID"],"type":"object"}},"required":["interfaceType"],"type":"object","x-kubernetes-validations":[{"messageExpression":"'for interfaceType=' + self.interfaceType + ', ' + self.interfaceType + ' must be set'","rule":"(self.interfaceType == 'vlan' && has(self.vlan)) || (self.interfaceType == 'pf' && has(self.pf)) || (self.interfaceType == 'vf' && has(self.vf)) || (self.interfaceType == 'physical' && has(self.physical)) || (self.interfaceType == 'service' && has(self.service)) || (self.interfaceType == 'ovn') || (self.interfaceType == 'patch' && has(self.patch))"}]}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"},"status":{"description":"DPUServiceInterfaceStatus defines the observed state of DPUServiceInterface","properties":{"conditions":{"description":"Conditions defines current service state.","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-DPUServiceNAD.json b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceNAD.json new file mode 100644 index 00000000..cde03602 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-DPUServiceNAD.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceNAD","version":"v1alpha1","schema":{"description":"DPUServiceNAD is the Schema for the dpuservicenads API.","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceNADSpec defines the desired state of DPUServiceNAD.","properties":{"bridge":{"description":"Bridge specifies the name of the OVS bridge to which the network interface will be connected.\nThis bridge name is used in the CNI configuration for the OVS plugin.","type":"string"},"chainedCNIs":{"description":"ChainedCNIs specifies additional CNI plugins to be chained after the base OVS plugin.\nWhen specified, the NAD will use the CNI chaining format with the OVS plugin as the\nfirst plugin, followed by the plugins defined in this list.\nThis allows adding capabilities like RDMA support on top of the base network interface.\nIf empty, the NAD uses a single OVS plugin configuration (backward compatible format).","items":{"description":"CNIPlugin defines a CNI plugin to be used in a chained CNI configuration.\nWhen multiple CNI plugins are specified in ChainedCNIs, they are executed in order\nafter the base OVS CNI plugin to provide additional network functionality.","properties":{"config":{"description":"Config contains optional plugin-specific configuration as raw JSON.\nThe configuration is merged into the CNI plugin configuration.","type":"object","x-kubernetes-preserve-unknown-fields":true},"type":{"description":"Type specifies the CNI plugin type to be used in the chain.\nCurrently only \"rdma\" is supported, which enables RDMA capabilities for the network interface.","enum":["rdma"],"type":"string"}},"required":["type"],"type":"object"},"type":"array"},"dpuClusterSelector":{"description":"DPUClusterSelector determines in which clusters the DPUServiceNAD controller should apply the configuration.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"ipam":{"description":"IPAM enables IP Address Management for the network interfaces attached to this network\nWhen set to true, a DPUServiceChain that references the DPUServiceInterface that has\nrequested this network must be created and include the relevant IPAM information. See\nDPUServiceChain documentation for more.\nWhen set to false, the network interfaces attached to this network will not get an IP","type":"boolean"},"resourceType":{"description":"ResourceType specifies the type of network resource to allocate for pods using this NAD.\n- \"vf\": Virtual Function (SR-IOV VF) from the DPU's physical ports\n- \"sf\": Scalable Function from the DPU (maps to nvidia.com/bf_sf or nvidia.com/bf_sf_trusted)\n- \"veth\": Virtual Ethernet pair (no device plugin resource required)\nThe resource type determines which SR-IOV device plugin resource will be requested.","enum":["vf","sf","veth"],"type":"string"},"serviceMTU":{"default":1500,"description":"ServiceMTU specifies the MTU size in bytes for the network interface.\nThis value is passed to the OVS CNI plugin and determines the maximum packet size.\nIf there is a DPUServiceChain that references an interface that is part of this network,\nthen the MTU that is defined in the DPUServiceChain takes precedence.\nThe default is 1500.","maximum":9216,"minimum":1280,"type":"integer"}},"required":["resourceType"],"type":"object"},"status":{"description":"DPUServiceNADStatus defines the observed state of DPUServiceNAD.","properties":{"conditions":{"description":"Conditions reflect the status of the object","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-DPUSet.json b/pkg/dpf/sentry/schemas/26.8.0-DPUSet.json new file mode 100644 index 00000000..53d97803 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-DPUSet.json @@ -0,0 +1 @@ +{"group":"provisioning.dpu.nvidia.com","kind":"DPUSet","version":"v1alpha1","schema":{"description":"DPUSet is the Schema for the dpusets API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUSetSpec defines the desired state of DPUSet","properties":{"dpuDeviceSelector":{"description":"DPUDeviceSelector defines the selector for DPUDevices that the DPUSet should target and should create a DPU for.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuNodeSelector":{"description":"Select the DPUNodes with specific labels","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuSelector":{"additionalProperties":{"type":"string"},"description":"Select the DPU with specific labels\n\nDeprecated: This field is deprecated and will be removed with v27.1.0. Use DPUDeviceSelector instead.","type":"object"},"dpuTemplate":{"description":"Object that describes the DPU that will be created if insufficient replicas are detected","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations specifies annotations which are added to the DPU.","type":"object"},"spec":{"description":"Spec specifies the DPU specification.","properties":{"astraEnabled":{"description":"AstraEnabled indicates whether E/W NIC configuration (Astra) is enabled","type":"boolean"},"bfb":{"description":"Specifies a BFB CR","properties":{"name":{"description":"Specifies name of the bfb CR to use for this DPU","minLength":1,"type":"string"}},"type":"object"},"blueFieldSoftware":{"description":"Specifies a BlueFieldSoftware CR","properties":{"name":{"description":"Specifies name of the BlueFieldSoftware CR to use for this DPU","minLength":1,"type":"string"}},"type":"object"},"cluster":{"description":"Specifies details on the K8S cluster to join","properties":{"nodeAnnotations":{"additionalProperties":{"type":"string"},"description":"NodeAnnotations specifies the annotations to be added to the node.","type":"object"},"nodeLabels":{"additionalProperties":{"type":"string"},"description":"NodeLabels specifies the labels to be added to the node.","type":"object"},"selector":{"description":"Selector defines the selector of the DPUClusters the produced DPUs should join","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"}},"type":"object"},"dpuFlavor":{"description":"DPUFlavor is the name of the DPUFlavor that will be used to deploy the DPU.\nMutually exclusive with DPUFlavorTemplate.","minLength":1,"type":"string"},"dpuFlavorTemplate":{"description":"DPUFlavorTemplate is the name of a DPUFlavorTemplate that is rendered per-DPU\n(against DPUDevice.spec.values) into a generated DPUFlavor. Mutually exclusive\nwith DPUFlavor.","minLength":1,"type":"string"},"nodeEffect":{"description":"Specifies how changes to the DPU should affect the Node","properties":{"applyOnLabelChange":{"default":false,"description":"Apply node effect when labels change on the DPU object\nWhen set to true, label changes in Ready state will trigger node effect logic","type":"boolean"},"customAction":{"description":"Name of a config map which contains a pod yaml definition to run which will apply the nodeEffect.\nThe pod is expected to exit when node effect is done, if pod terminates with error then DPU would move to an error phase.\nThe DPUNode's name will be exported as an environment variable, named as DPUNODE_NAME, to each container and init container in the pod.\nThe labels and annotations of DPUNode will be exported in `/etc/dpu/dpf-pod-info/labels` and `/etc/dpu/dpf-pod-info/annotations` accordingly; the volume name `dpf-pod-info` is used to mount the labels and annotations.\nIf any name confliction for env or volume, the controller will not export the name or labels/annotations of DPUNode accordingly.","type":"string"},"customLabel":{"additionalProperties":{"type":"string"},"description":"Add specify labels on the DPU node","type":"object"},"drain":{"description":"Drain the K8s host node by NodeMaintenance operator","type":"boolean"},"force":{"default":false,"description":"Force is the flag to indicate if the node effect should be applied immediately.\nIf true, dpfOperatorConfig.multiDPUOperationsSyncWaitTime and dpfOperatorConfig.maxUnavailableDPUNodes will be ignored when applying node effect for DPUNodeMaintenance CR","type":"boolean"},"hold":{"description":"Places annotation `wait-for-external-nodeeffect` and waits for it to be removed\nthis is the default behavior in a non K8S environment","type":"boolean"},"noEffect":{"description":"Do not do any action on the DPU node","type":"boolean"},"nodeMaintenanceAdditionalRequestors":{"description":"Additional requestors to be added to the NvidiaNodeMaintenance CR when Drain is selected","items":{"type":"string"},"type":"array"},"taint":{"description":"Add specify taint on the DPU node","properties":{"effect":{"description":"Required. The effect of the taint on pods\nthat do not tolerate the taint.\nValid effects are NoSchedule, PreferNoSchedule and NoExecute.","type":"string"},"key":{"description":"Required. The taint key to be applied to a node.","type":"string"},"timeAdded":{"description":"TimeAdded represents the time at which the taint was added.","format":"date-time","type":"string"},"value":{"description":"The taint value corresponding to the taint key.","type":"string"}},"required":["effect","key"],"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"exactly one of taint, noEffect=true, drain=true, non-empty customLabel, non-empty customAction, hold=true must be set","rule":"(has(self.taint) ? 1 : 0) + ((has(self.noEffect) && self.noEffect) ? 1 : 0) + ((has(self.customLabel) && size(self.customLabel) > 0) ? 1 : 0) + ((has(self.drain) && self.drain) ? 1 : 0) + ((has(self.customAction) && size(self.customAction) > 0) ? 1 : 0) + ((has(self.hold) && self.hold) ? 1 : 0) == 1"}]},"secureBoot":{"description":"SecureBoot specifies whether UEFI Secure Boot should be enabled.","type":"boolean"}},"required":["nodeEffect"],"type":"object","x-kubernetes-validations":[{"message":"exactly one of bfb or blueFieldSoftware must be set","rule":"(has(self.bfb) && has(self.bfb.name) ? 1 : 0) + (has(self.blueFieldSoftware) && has(self.blueFieldSoftware.name) ? 1 : 0) == 1"},{"message":"exactly one of dpuFlavor or dpuFlavorTemplate must be set","rule":"has(self.dpuFlavor) != has(self.dpuFlavorTemplate)"}]}},"type":"object"},"strategy":{"description":"The rolling update strategy to use to updating existing DPUs with new ones.","properties":{"rollingUpdate":{"description":"Rolling update config params. Present only if StrategyType = RollingUpdate.","properties":{"maxUnavailable":{"anyOf":[{"type":"integer"},{"type":"string"}],"description":"MaxUnavailable is the maximum number of DPUs that can be unavailable during the update.\n\nDeprecated: This field is deprecated and will be removed with v27.1.0.","x-kubernetes-int-or-string":true}},"type":"object"},"type":{"description":"Can be \"OnDelete\" or \"RollingUpdate\".","enum":["OnDelete","RollingUpdate"],"type":"string"}},"required":["type"],"type":"object"}},"required":["dpuTemplate","strategy"],"type":"object","x-kubernetes-validations":[{"message":"only one of dpuSelector or dpuDeviceSelector can be specified","rule":"!(has(self.dpuSelector) && has(self.dpuDeviceSelector))"}]},"status":{"description":"DPUSetStatus defines the observed state of DPUSet","properties":{"conditions":{"description":"Conditions reflect the status of the object","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"dpuStatistics":{"additionalProperties":{"type":"integer"},"description":"DPUStatistics is a map of DPUPhase to the number of DPUs in that phase.","type":"object"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-BlueFieldSoftware.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-BlueFieldSoftware.json new file mode 100644 index 00000000..edd8c2cc --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-BlueFieldSoftware.json @@ -0,0 +1 @@ +{"group":"provisioning.dpu.nvidia.com","kind":"BlueFieldSoftware","version":"v1alpha1","schema":{"description":"BlueFieldSoftware is the Schema for the bluefieldsoftware API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"BlueFieldSpec defines the desired state of BlueFieldSoftware.","properties":{"nicFw":{"description":"NicFw points to the NIC firmware binary used for E/W NIC firmware updates.\nUse this when a specific NIC firmware binary is required and is not included in the\nplatform PLDM firmware bundle.\nIn production, prefer using PlatformPldmFwBundle.","minLength":1,"type":"string"},"osIso":{"description":"OS ISO points to the OS ISO used by DPU OS installation flow.","minLength":1,"type":"string"},"platformPldmFwBundle":{"description":"PlatformPldmFwBundle points to the platform PLDM firmware bundle used for E/W NIC\nfirmware updates.","minLength":1,"type":"string"},"pldmFwBundle":{"additionalProperties":{"type":"string"},"description":"PldmFwBundle maps each PSID to the PLDM firmware bundle URL used for that DPU model's\nbaseline firmware updates. Each PSID's bundle is a complete PLDM package for that device.\nKeys are matched case-insensitively against the BMC-reported DPUDevice status PSID.","minProperties":1,"type":"object","x-kubernetes-validations":[{"message":"pldmFwBundle entries must have non-empty PSID keys and non-empty bundle URLs","rule":"self.all(k, size(k) > 0 && size(self[k]) > 0)"}]}},"required":["osIso"],"type":"object","x-kubernetes-validations":[{"message":"BlueFieldSpec is immutable","rule":"self == oldSelf"},{"message":"nicFw and platformPldmFwBundle are mutually exclusive; set only one","rule":"!(has(self.nicFw) && has(self.platformPldmFwBundle))"}]},"status":{"default":{"phase":"Initializing"},"description":"BlueFieldSoftwareStatus defines the observed state of BlueFieldSoftware","properties":{"conditions":{"description":"Conditions represent the latest available observations of BlueFieldSoftware state","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"downloadedComponents":{"description":"DownloadedComponents tracks which components have been successfully downloaded","properties":{"nicFw":{"description":"NicFw is the local path of the E/W NIC firmware binary, either downloaded from\nspec.nicFw or unpacked from the platform PLDM firmware bundle.","type":"string"},"osIso":{"description":"OsIso is the local path of the downloaded OS ISO.","type":"string"},"platformPldmFwBundle":{"description":"PlatformPldmFwBundle is the local path of the downloaded platform PLDM firmware bundle.","type":"string"},"pldmFwBundle":{"additionalProperties":{"type":"string"},"description":"PldmFwBundle maps each PSID to the local path of its downloaded PLDM firmware bundle.\nKeys are matched case-insensitively against the BMC-reported DPUDevice PSID.","type":"object"}},"type":"object"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"},"phase":{"default":"Initializing","description":"The current state of BlueFieldSoftware.","enum":["Initializing","Downloading","Extracting","Ready","Deleting","Error"],"type":"string"},"versions":{"description":"Versions tracks the versions of the components","properties":{"bluefieldSoftwareVersions":{"additionalProperties":{"description":"BluefieldDeviceVersions holds firmware versions extracted from a single PSID's PLDM bundle.","properties":{"bfNicFwVersion":{"description":"BFNicFwVersion is the BlueField NIC firmware version shipped in the bundle.","minLength":1,"type":"string"},"bmcErotVersion":{"description":"BMCErotVersion is the BMC ERoT (External Root of Trust) firmware version shipped in the bundle.","minLength":1,"type":"string"},"bmcVersion":{"description":"BMCVersion is the DPU BMC firmware version shipped in the bundle.","minLength":1,"type":"string"},"sbiosVersion":{"description":"SBIOSVersion is the DPU SBIOS firmware version shipped in the bundle.","minLength":1,"type":"string"}},"type":"object"},"description":"BluefieldSoftwareVersions maps each PSID to the firmware versions from that device's\nPLDM bundle. Keys are matched case-insensitively against the BMC-reported DPUDevice PSID.","type":"object"},"doca":{"description":"DOCA is the formatted, user-facing DOCA version derived from the OS ISO.","type":"string"},"ewNicFwVersion":{"description":"EWNicFwVersion is the E/W NIC firmware version taken from the platform PLDM bundle.\nIt is not keyed by PSID because a BlueFieldSoftware carries a single platform bundle.","type":"string"},"osISOVersion":{"description":"OSISOVersion is the raw DOCA version for the OS ISO, taken from the ISO filename","type":"string"}},"type":"object"}},"required":["phase"],"type":"object","x-kubernetes-validations":[{"message":"versions cannot be removed once set","rule":"!has(oldSelf.versions) || has(self.versions)"}]}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 187 chars","rule":"self.metadata.name.size() <= 187"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUFlavor.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUFlavor.json new file mode 100644 index 00000000..fa82fe96 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUFlavor.json @@ -0,0 +1 @@ +{"group":"provisioning.dpu.nvidia.com","kind":"DPUFlavor","version":"v1alpha1","schema":{"description":"DPUFlavor is the Schema for the dpuflavors API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUFlavorSpec defines the content of DPUFlavor","properties":{"bfcfgParameters":{"description":"BFCfgParameters are the parameters to be set in the bf.cfg file.","items":{"type":"string"},"type":"array"},"configFiles":{"description":"ConfigFiles are the files to be written on the DPU.","items":{"description":"ConfigFile describes a file materialized from inline raw content or external contentFrom.","properties":{"contentFrom":{"description":"ContentFrom references external content for the file.\nSupported only when type is agent-applied.","properties":{"configMapKeyRef":{"description":"ConfigMapKeyRef selects a key from a ConfigMap in the DPU namespace.","properties":{"key":{"description":"The key to select.","type":"string"},"name":{"default":"","description":"Name of the referent.\nThis field is effectively required, but due to backwards compatibility is\nallowed to be empty. Instances of this type with an empty value here are\nalmost certainly wrong.\nMore info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names","type":"string"},"optional":{"description":"Specify whether the ConfigMap or its key must be defined","type":"boolean"}},"required":["key"],"type":"object","x-kubernetes-map-type":"atomic"}},"type":"object"},"operation":{"description":"Operation is the operation to be performed on the file.","enum":["override","append"],"type":"string"},"path":{"description":"Path is the path of the file to be written.","type":"string"},"permissions":{"description":"Permissions are the permissions to be set on the file.","type":"string"},"raw":{"description":"Raw is the inline file content.\nSupported only when type is cloud-init. When type is omitted, type defaults\nto cloud-init and raw must be set.","type":"string"},"type":{"default":"cloud-init","description":"Type controls when the file content is materialized.\ncloud-init files use raw inline content and are written during cloud-init.\nagent-applied files use contentFrom and are written later by dpu-agent.\nDefaults to cloud-init when omitted.","enum":["cloud-init","agent-applied"],"type":"string"}},"required":["path"],"type":"object","x-kubernetes-validations":[{"message":"exactly one of raw or contentFrom must be specified","rule":"has(self.raw) != has(self.contentFrom)"},{"message":"contentFrom.configMapKeyRef must be specified","rule":"!has(self.contentFrom) || has(self.contentFrom.configMapKeyRef)"},{"message":"type cloud-init supports raw only","rule":"!has(self.type) || self.type != 'cloud-init' || has(self.raw)"},{"message":"type agent-applied supports contentFrom only","rule":"!has(self.type) || self.type != 'agent-applied' || has(self.contentFrom)"},{"message":"type defaults to cloud-init, which requires raw content","rule":"has(self.type) || has(self.raw)"}]},"type":"array"},"containerdConfig":{"description":"ContainerdConfig contains the configuration for containerd.","properties":{"registryEndpoint":{"description":"RegistryEndpoint is the endpoint of the container registry.","type":"string"}},"type":"object"},"dma":{"description":"DMA configures the DMA SF that e.g. SNAP DOCA service uses to DMA host\nmemory over the second Grace PCI link on BlueField-4 socket-direct\nsystems.","properties":{"enabled":{"description":"Enabled controls whether the dpu-agent creates the DMA SF. Defaults to\nfalse when unset, so the presence of the dma struct alone does not enable\ncreation. Only takes effect on BlueField-4 socket-direct systems. The\ncreated Scalable Function always uses sfnum 8000, the SNAP discovery ABI\nvalue.","type":"boolean"}},"type":"object"},"dpuMode":{"description":"DpuMode is deprecated and no longer used by provisioning workflows.\nDeployment mode is sourced from DPFOperatorConfig and exposed on DPU.status.deploymentMode.","enum":["dpu","zero-trust","nic"],"type":"string"},"dpuResources":{"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","x-kubernetes-int-or-string":true},"description":"DPUResources indicates the minimum amount of resources needed for a BFB with that flavor to be installed on a\nDPU. Using this field, the controller can understand if that flavor can be installed on a particular DPU. It\nshould be set to the total amount of resources the system needs + the resources that should be made available for\nDPUServices to consume.","type":"object"},"ewNicConfigurations":{"description":"EWNicConfigurations lists per-NIC configuration for the E/W NICs.\nOnly the first entry is applied in this release; additional entries are ignored until a future\nrelease adds multi-NIC support. The field is modeled as a list now so the API shape does not\nneed to change when multiple entries are supported.","items":{"description":"NicConfiguration is a set of configurations for the NICs","properties":{"force":{"default":false,"description":"Force passes `--force` to mlxconfig set commands. When set, the daemon\napplies the nv config batch and set_system_conf with --force, letting\nmlxconfig accept a batch it would otherwise refuse due to implicit\nparameter dependencies.","type":"boolean"},"linkType":{"description":"LinkType to be configured, Ethernet|Infiniband. Required unless networkBay is configured;\nfor Network Bay the link type is governed by the system configuration and must not be set.","enum":["Ethernet","Infiniband"],"type":"string"},"networkBay":{"description":"NetworkBay configures a ConnectX-9 Network Bay card (per-ASIC set_system_conf). Allowed only for ConnectX-9 (nicType 1025).","properties":{"conf":{"description":"Conf is the mlxconfig system configuration profile name. The daemon resolves\nthe profile parameters for the device's detected Network Bay ASIC and manages\nthem through the regular mlxconfig validation and apply flow.","type":"string"}},"required":["conf"],"type":"object"},"numVfs":{"description":"Number of VFs to be configured","type":"integer"},"rawNvConfig":{"description":"List of arbitrary nv config parameters","items":{"properties":{"name":{"description":"Name of the arbitrary nvconfig parameter","maxLength":64,"type":"string"},"value":{"description":"Value of the arbitrary nvconfig parameter","maxLength":128,"type":"string"}},"required":["name","value"],"type":"object"},"type":"array"},"spectrumXOptimized":{"description":"Spectrum-X optimization settings. Works only with linkType==Ethernet && numVfs==1. RawNvConfig parameters, if provided, are merged as overrides on top of Spectrum-X calculated params.","properties":{"enabled":{"description":"Optimize Spectrum X","type":"boolean"},"multiplaneMode":{"default":"none","description":"Multiplane mode to be configured\nCan be \"none\", \"swplb\", \"hwplb\", or \"uniplane\"","enum":["none","swplb","hwplb","uniplane"],"type":"string"},"numberOfPlanes":{"default":1,"description":"Number of planes to be configured","enum":[1,2,4],"type":"integer"},"overlay":{"default":"none","description":"Overlay mode to be configured\nCan be \"l3\" or \"none\"","enum":["l3","none"],"type":"string"},"version":{"description":"Version of the Spectrum-X architecture to optimize for. Should match the name of the config map with Spectrum-X profile","type":"string"}},"required":["enabled","version"],"type":"object","x-kubernetes-validations":[{"message":"when MultiplaneMode is none, numberOfPlanes must be 1","rule":"!has(self.multiplaneMode) || !has(self.numberOfPlanes) || self.multiplaneMode != 'none' || self.numberOfPlanes == 1"},{"message":"when MultiplaneMode is not none, numberOfPlanes must not be 1","rule":"!has(self.multiplaneMode) || !has(self.numberOfPlanes) || self.multiplaneMode == 'none' || self.numberOfPlanes != 1"}]}},"required":["numVfs"],"type":"object","x-kubernetes-validations":[{"message":"spectrumXOptimized can be enabled only when linkType=='Ethernet' (or unset for Network Bay) and numVfs==1","rule":"!(has(self.spectrumXOptimized) && self.spectrumXOptimized.enabled) || ((!has(self.linkType) || self.linkType == 'Ethernet') && self.numVfs == 1)"},{"message":"linkType is required unless networkBay is configured","rule":"has(self.networkBay) || has(self.linkType)"},{"message":"linkType must not be set when networkBay is configured (the Network Bay link type is governed by the system configuration)","rule":"!has(self.networkBay) || !has(self.linkType)"},{"message":"networkBay.conf must not be empty","rule":"!has(self.networkBay) || self.networkBay.conf != \"\""}]},"maxItems":16,"type":"array","x-kubernetes-list-type":"atomic"},"grub":{"description":"Grub contains the grub configuration for the DPUFlavor.","properties":{"kernelParameters":{"description":"KernelParameters are the kernel parameters to be set in the grub configuration.","items":{"type":"string"},"type":"array"}},"type":"object"},"hostNetworkInterfaceConfigs":{"description":"HostNetworkInterfaceConfigs contains the configuration for the host-side network interfaces.","items":{"description":"NetworkInterfaceConfig defines the configuration for a network interface","properties":{"dhcp":{"description":"DHCP is the DHCP configuration for the network interface.","type":"boolean"},"mtu":{"description":"MTU is the MTU value to be set on the network interface.","format":"int32","maximum":9216,"minimum":1280,"type":"integer"},"nvconfig":{"description":"NVConfig contains port-specific configuration for this network interface.\nThis configuration is applied in addition to the global NVConfig settings in DPUFlavorSpec.\nBoth global and per-interface NVConfig settings can coexist without collision.","properties":{"device":{"description":"Device is the device to which the configuration applies. If not specified, the configuration applies to all.\nSupported values: \"*\" (wildcard for all devices), \"p0\"/\"P0\" (port 0), \"p1\"/\"P1\" (port 1). Case-insensitive.","enum":["*","p0","p1","P0","P1"],"type":"string"},"force":{"description":"force applies the parameters with `mlxconfig --force` and skips the `mlxconfig q` filter,\nso parameters firmware does not yet expose are applied now instead of deferred to a later\nreboot. Required when a parameter is gated behind another parameter in the same batch.\nRequires DOCA 3.5.0 or later; on an older DOCA version the operation fails.\n`--force` skips validation for the whole batch, so an invalid value is applied silently.\nIgnored under `spec.hostNetworkInterfaceConfigs[].nvconfig`.","type":"boolean"},"parameters":{"description":"Parameters are the parameters to be set for the device.\nDELAY_HOST_OS_INIT=ENABLE_USER (0x3) holds the host at UEFI and is rejected by the DPU agent\noutside zero-trust, where the agent needs the host to reach the kube-apiserver.","items":{"maxLength":200,"pattern":"^[^=\\s]+=[^\\s]*$","type":"string"},"maxItems":64,"minItems":1,"type":"array","x-kubernetes-list-type":"atomic"}},"type":"object"},"portNumber":{"description":"PortNumber identifies which port this configuration applies to.","format":"int32","maximum":1,"minimum":0,"type":"integer"}},"required":["portNumber"],"type":"object"},"type":"array"},"nvconfig":{"description":"NVConfig contains the device-specific configuration (firmware settings, device parameters).\nEach entry specifies a device (wildcard '*', or port identifiers 'p0'/'P0'/'p1'/'P1') and its parameters.\nIf device is '*' or unspecified (defaults to '*'), it applies to all devices and must be the only entry.\nEach device (including unspecified as '*') must be unique across all nvconfig entries (case-insensitive).\nValidation enforces: device enum values, parameter format (KEY=VALUE), case-insensitive uniqueness, and size limits.","items":{"properties":{"device":{"description":"Device is the device to which the configuration applies. If not specified, the configuration applies to all.\nSupported values: \"*\" (wildcard for all devices), \"p0\"/\"P0\" (port 0), \"p1\"/\"P1\" (port 1). Case-insensitive.","enum":["*","p0","p1","P0","P1"],"type":"string"},"force":{"description":"force applies the parameters with `mlxconfig --force` and skips the `mlxconfig q` filter,\nso parameters firmware does not yet expose are applied now instead of deferred to a later\nreboot. Required when a parameter is gated behind another parameter in the same batch.\nRequires DOCA 3.5.0 or later; on an older DOCA version the operation fails.\n`--force` skips validation for the whole batch, so an invalid value is applied silently.\nIgnored under `spec.hostNetworkInterfaceConfigs[].nvconfig`.","type":"boolean"},"parameters":{"description":"Parameters are the parameters to be set for the device.\nDELAY_HOST_OS_INIT=ENABLE_USER (0x3) holds the host at UEFI and is rejected by the DPU agent\noutside zero-trust, where the agent needs the host to reach the kube-apiserver.","items":{"maxLength":200,"pattern":"^[^=\\s]+=[^\\s]*$","type":"string"},"maxItems":64,"minItems":1,"type":"array","x-kubernetes-list-type":"atomic"}},"type":"object"},"maxItems":3,"type":"array","x-kubernetes-list-type":"atomic","x-kubernetes-validations":[{"message":"when device is '*', it must be the only nvconfig entry","rule":"size(self) == 0 || !self.exists(x, has(x.device) && x.device == '*') || size(self) == 1"},{"message":"when device is unspecified (defaults to '*'), it must be the only nvconfig entry","rule":"size(self) == 0 || !self.exists(x, !has(x.device)) || size(self) == 1"},{"message":"each nvconfig.device (including unspecified as '*') must be unique (case-insensitive)","rule":"self.all(p1, self.exists_one(p2, (has(p1.device) ? p1.device.lowerAscii() : '*') == (has(p2.device) ? p2.device.lowerAscii() : '*')))"}]},"ovs":{"description":"OVS contains the OVS configuration for the DPUFlavor.","properties":{"rawConfigScript":{"description":"RawConfigScript is the raw configuration script for OVS.\nThe DPU agent runs this script once per boot. A restart of the agent in the\nsame boot does not re-run it; a reboot or power cycle does.","type":"string"}},"type":"object"},"packages":{"description":"Packages are the packages to reconcile on the node.","items":{"description":"PackageSpec defines a package to reconcile on the node.","properties":{"name":{"description":"Name is the package name.","maxLength":253,"minLength":1,"type":"string"},"repoFileRef":{"description":"RepoFileRef constrains package resolution to a specific repository file available on the node.\nIf empty, any configured repository may satisfy the package.\nIf specified, only the referenced repository file may provide candidates.\nIf that repository file does not provide the package or requested version, the dpu-agent flow does not continue.","type":"string"},"version":{"description":"Version constrains the package version.\nIf empty, any installed version satisfies the spec.","properties":{"matchPolicy":{"default":"AtLeast","description":"MatchPolicy controls how Value is matched.\nIf omitted, AtLeast is used.","enum":["Exact","AtLeast"],"type":"string"},"value":{"description":"Value is the package version to compare against.","minLength":1,"type":"string"}},"required":["value"],"type":"object"}},"required":["name"],"type":"object"},"maxItems":64,"type":"array","x-kubernetes-validations":[{"message":"package names must be unique","rule":"self.all(x, self.exists_one(y, x.name == y.name))"}]},"serviceReadiness":{"description":"serviceReadiness configures the Service Readiness phase.","properties":{"gate":{"description":"gate is the DPU.status.operationalConditions entry that must be True before the DPU leaves\nthe Service Readiness phase. When unset the phase does not wait, and a host hold requested\nvia DELAY_HOST_OS_INIT is released on DPUServiceCriticalPodsReady.","enum":["DPUServiceCriticalPodsReady","OperationalReady"],"type":"string"}},"type":"object"},"sysctl":{"description":"Sysctl contains the sysctl configuration for the DPUFlavor.","properties":{"parameters":{"description":"Parameters are the sysctl parameters to be set.","items":{"type":"string"},"type":"array"}},"type":"object"},"systemReservedResources":{"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","x-kubernetes-int-or-string":true},"description":"SystemReservedResources indicates the resources that are consumed by the system (OS, OVS, DPF system etc) and are\nnot made available for DPUServices to consume. DPUServices can consume the difference between DPUResources and\nSystemReservedResources. This field must not be specified if dpuResources are not specified.","type":"object"},"systemdServices":{"description":"SystemdServices are the systemd services to manage on the node.","items":{"description":"SystemdServiceSpec defines a systemd service to manage on the node.","properties":{"name":{"description":"Name is the systemd service name.","maxLength":253,"minLength":1,"type":"string"},"operation":{"description":"Operation is the systemd operation to perform on the service.","enum":["Start","Enable","EnableAndStart"],"type":"string"}},"required":["name","operation"],"type":"object"},"maxItems":64,"type":"array","x-kubernetes-validations":[{"message":"systemd service names must be unique","rule":"self.all(x, self.exists_one(y, x.name == y.name))"}]}},"type":"object","x-kubernetes-validations":[{"message":"DPUFlavor spec is immutable","rule":"self == oldSelf"}]}},"type":"object"}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUService.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUService.json new file mode 100644 index 00000000..9ba317f0 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUService.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUService","version":"v1alpha1","schema":{"description":"DPUService is the Schema for the dpuservices API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceSpec defines the desired state of DPUService","properties":{"configPorts":{"description":"ConfigPorts defines the desired state of port configurations for a DPUService.\nThis struct determines how ports are exposed from the DPU to the host cluster.\nA DPUService can only have a single ServiceType across all ports.","properties":{"ports":{"description":"Ports defines the list of port configurations that will be exposed by the DPUService.\nEach port must specify a name, port number, and protocol.\n\nConstraints:\n- If ServiceType is \"NodePort\", ports may optionally specify a NodePort.\n- If ServiceType is \"None\" or \"ClusterIP\", ports **cannot** specify a NodePort.","items":{"description":"ConfigPort defines the configuration of a single port within a DPUService.\nEach port must have a unique name within the service.","properties":{"name":{"description":"Name is a unique identifier for the port within the DPUService.\nThis name is used for reference inside the service.","minLength":1,"pattern":"^[a-z0-9-]+$","type":"string"},"nodePort":{"description":"NodePort is the external port assigned on each node in the cluster.\nIf not set, Kubernetes will automatically allocate a NodePort.\n\nConstraints:\n- Can only be set when ServiceType is \"NodePort\".\n- Must be within the clusters valid NodePort range (Kubernetes default is 30000-32767).","type":"integer"},"port":{"description":"Port is the port number that will be exposed by the service.\nMust be within the valid range of TCP/UDP ports (1-65535).","type":"integer"},"protocol":{"description":"Protocol specifies the transport protocol used by the port.\nSupported values: TCP, UDP","enum":["TCP","UDP"],"type":"string"}},"required":["name","port","protocol"],"type":"object"},"type":"array"},"serviceType":{"default":"NodePort","description":"ServiceType specifies the type of Kubernetes Service to create.\nAll ports within this ConfigPorts will have the same ServiceType.\nThe value is immutable and cannot be changed after creation.\nSupported values:\n- \"NodePort\": Exposes ports externally on a node.\n- \"ClusterIP\": Exposes ports internally within the cluster.\n- \"None\": Internal-only service with no cluster IP.\nDefault: \"NodePort\"","enum":["NodePort","ClusterIP","None"],"type":"string","x-kubernetes-validations":[{"message":"Value is immutable","rule":"self == oldSelf"}]}},"required":["ports","serviceType"],"type":"object","x-kubernetes-validations":[{"message":"nodePort can only be set when serviceType is NodePort","rule":"!(self.serviceType != 'NodePort' && self.ports.exists(p, has(p.nodePort)))"}]},"deployInCluster":{"description":"DeployInCluster indicates if the DPUService Helm Chart will be deployed on\nthe Host cluster. Default to false.","type":"boolean"},"dpuClusterSelector":{"description":"Select the Clusters with specific labels, Applications will be created only for these Clusters","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"helmChart":{"description":"HelmChart reflects the Helm related configuration","properties":{"source":{"description":"Source specifies information about the Helm chart","properties":{"chart":{"description":"Chart is the name of the helm chart.","type":"string"},"path":{"description":"Path is the location of the chart inside the repo.","type":"string"},"releaseName":{"description":"ReleaseName is the name to give to the release generate from the DPUService.","type":"string"},"repoURL":{"description":"RepoURL specifies the URL to the repository that contains the application Helm chart.\nThe URL must begin with either 'oci://' or 'https://', ensuring it points to a valid\nOCI registry or a web-based repository.","pattern":"^(oci://|https://).+$","type":"string"},"version":{"description":"Version is a semver tag for the Chart's version.","minLength":1,"type":"string"}},"required":["repoURL","version"],"type":"object"},"values":{"description":"Values specifies Helm values to be passed to Helm template, defined as a map. This takes precedence over Values.","type":"object","x-kubernetes-preserve-unknown-fields":true}},"required":["source"],"type":"object"},"interfaces":{"description":"Interfaces specifies the DPUServiceInterface names that the DPUService\nuses in the same namespace.","items":{"type":"string"},"maxItems":50,"minItems":1,"type":"array"},"paused":{"description":"Paused indicates that the DPUService is paused.\nUnderlying resources are also paused when this is set to true.\nNo deletion of resources will occur when this is set to true.","type":"boolean"},"security":{"description":"Security contains security-related settings for the DPUService.","properties":{"privileged":{"description":"Privileged, when set to true, allows workloads governed by this security\nsetting to run containers with `securityContext.privileged: true` in the\nDPUCluster. When set to false, a ValidatingAdmissionPolicy in the\nDPUCluster rejects such workloads. On DPUService objects, this field must\nbe unset when deployInCluster is true. For DPUServices that target a\nDPUCluster, Privileged must be set explicitly. When set on a\nDPUServiceTemplate, the DPUDeployment controller propagates the value to\ngenerated DPUServices that target DPUClusters.\n\nScope of the policy enforcement:\n - Only `securityContext.privileged: true` is gated. Other\n privilege-escalation vectors (hostPID, hostIPC, hostNetwork,\n allowPrivilegeEscalation, capabilities, hostPath volumes) are NOT\n gated by this field.\n - Enforcement matches workloads via the\n `svc.dpu.nvidia.com/service` label. The controller adds this\n label to the resources it manages, and to pod templates of\n workload-controller resources (Deployment, DaemonSet, etc.). If a\n Helm chart strips that label from the pod template, the parent\n resource will be admitted but the child Pods will be denied at\n Pod admission time.","type":"boolean"},"spiffe":{"description":"SPIFFE opts this DPUService into SPIFFE workload identity. When set, one SPIRE\nClusterStaticEntry is registered per targeted DPU, parented to that DPU's SPIRE\nagent, so the service's pods can obtain an SVID. The identity defaults to\n`spiffe:///dpu//service//` and is\nconfigurable through `DPFOperatorConfig.spec.security.spiffe`.\nThe namespace qualifies the service ID, which is only unique within one.\n\nPresence-gated: `spiffe: {}` enables it. It has no sub-fields yet.\n\nRequires `DPFOperatorConfig.spec.security.spiffe`, and only covers DPUs in SPIFFE\nidentity mode (`DPU.status.identityMode: Spiffe`); bootstrap-token DPUs are skipped.\nBoth are reported on the SPIFFEEntriesReady condition rather than rejected at\nadmission. Must not be set when deployInCluster is true, since in-cluster\nDPUServices run on the host and have no per-DPU SPIRE agent to parent to.\n\nWhen set on a DPUServiceTemplate, the DPUDeployment controller propagates it to\ngenerated DPUServices that target DPUClusters.","type":"object"}},"type":"object"},"serviceDaemonSet":{"description":"ServiceDaemonSet specifies the configuration for the ServiceDaemonSet.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations specifies annotations which are added to the ServiceDaemonSet.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels specifies labels which are added to the ServiceDaemonSet.","type":"object"},"nodeSelector":{"description":"NodeSelector specifies which Nodes to deploy the ServiceDaemonSet to.","properties":{"nodeSelectorTerms":{"description":"Required. A list of node selector terms. The terms are ORed.","items":{"description":"A null or empty node selector term matches no objects. The requirements of\nthem are ANDed.\nThe TopologySelectorTerm type implements a subset of the NodeSelectorTerm.","properties":{"matchExpressions":{"description":"A list of node selector requirements by node's labels.","items":{"description":"A node selector requirement is a selector that contains values, a key, and an operator\nthat relates the key and values.","properties":{"key":{"description":"The label key that the selector applies to.","type":"string"},"operator":{"description":"Represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.","type":"string"},"values":{"description":"An array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. If the operator is Gt or Lt, the values\narray must have a single element, which will be interpreted as an integer.\nThis array is replaced during a strategic merge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchFields":{"description":"A list of node selector requirements by node's fields.","items":{"description":"A node selector requirement is a selector that contains values, a key, and an operator\nthat relates the key and values.","properties":{"key":{"description":"The label key that the selector applies to.","type":"string"},"operator":{"description":"Represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.","type":"string"},"values":{"description":"An array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. If the operator is Gt or Lt, the values\narray must have a single element, which will be interpreted as an integer.\nThis array is replaced during a strategic merge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"}},"type":"object","x-kubernetes-map-type":"atomic"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["nodeSelectorTerms"],"type":"object","x-kubernetes-map-type":"atomic"},"resources":{"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","x-kubernetes-int-or-string":true},"description":"Resources specifies resources which are added to the ServiceDaemonSet.","type":"object"},"updateStrategy":{"description":"UpdateStrategy specifies the DeaemonSet update strategy for the ServiceDaemonset.","properties":{"rollingUpdate":{"description":"Rolling update config params. Present only if type = \"RollingUpdate\".","properties":{"maxSurge":{"anyOf":[{"type":"integer"},{"type":"string"}],"description":"The maximum number of nodes with an existing available DaemonSet pod that\ncan have an updated DaemonSet pod during during an update.\nValue can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).\nThis can not be 0 if MaxUnavailable is 0.\nAbsolute number is calculated from percentage by rounding up to a minimum of 1.\nDefault value is 0.\nExample: when this is set to 30%, at most 30% of the total number of nodes\nthat should be running the daemon pod (i.e. status.desiredNumberScheduled)\ncan have their a new pod created before the old pod is marked as deleted.\nThe update starts by launching new pods on 30% of nodes. Once an updated\npod is available (Ready for at least minReadySeconds) the old DaemonSet pod\non that node is marked deleted. If the old pod becomes unavailable for any\nreason (Ready transitions to false, is evicted, or is drained) an updated\npod is immediately created on that node without considering surge limits.\nAllowing surge implies the possibility that the resources consumed by the\ndaemonset on any given node can double if the readiness check fails, and\nso resource intensive daemonsets should take into account that they may\ncause evictions during disruption.","x-kubernetes-int-or-string":true},"maxUnavailable":{"anyOf":[{"type":"integer"},{"type":"string"}],"description":"The maximum number of DaemonSet pods that can be unavailable during the\nupdate. Value can be an absolute number (ex: 5) or a percentage of total\nnumber of DaemonSet pods at the start of the update (ex: 10%). Absolute\nnumber is calculated from percentage by rounding up.\nThis cannot be 0 if MaxSurge is 0\nDefault value is 1.\nExample: when this is set to 30%, at most 30% of the total number of nodes\nthat should be running the daemon pod (i.e. status.desiredNumberScheduled)\ncan have their pods stopped for an update at any given time. The update\nstarts by stopping at most 30% of those DaemonSet pods and then brings\nup new DaemonSet pods in their place. Once the new pods are available,\nit then proceeds onto other DaemonSet pods, thus ensuring that at least\n70% of original number of DaemonSet pods are available at all times during\nthe update.","x-kubernetes-int-or-string":true}},"type":"object"},"type":{"description":"Type of daemon set update. Can be \"RollingUpdate\" or \"OnDelete\". Default is RollingUpdate.","type":"string"}},"type":"object"}},"type":"object"},"serviceID":{"description":"ServiceID is the ID of the service that the DPUService is associated with.","type":"string"}},"required":["helmChart"],"type":"object","x-kubernetes-validations":[{"message":"serviceID must be provided when interfaces are provided","rule":"(has(self.interfaces) && has(self.serviceID)) || (!has(self.interfaces) && !has(self.serviceID)) || has(self.serviceID)"},{"message":"configPorts cannot be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.configPorts))"},{"message":"dpuClusterSelector cannot be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.dpuClusterSelector))"},{"message":"security.privileged must not be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.security) && has(self.security.privileged))"},{"message":"security.spiffe must not be set when deployInCluster is true","rule":"!(has(self.deployInCluster) && self.deployInCluster && has(self.security) && has(self.security.spiffe))"},{"message":"security.privileged must be set when deployInCluster is false","optionalOldSelf":true,"rule":"(has(self.deployInCluster) && self.deployInCluster) || (has(self.security) && has(self.security.privileged)) || (oldSelf.hasValue() && !(has(oldSelf.value().deployInCluster) && oldSelf.value().deployInCluster) && !(has(oldSelf.value().security) && has(oldSelf.value().security.privileged)))"}]},"status":{"description":"DPUServiceStatus defines the observed state of DPUService","properties":{"conditions":{"description":"Conditions defines current service state.","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"configPorts":{"additionalProperties":{"items":{"description":"ConfigPort defines the configuration of a single port within a DPUService.\nEach port must have a unique name within the service.","properties":{"name":{"description":"Name is a unique identifier for the port within the DPUService.\nThis name is used for reference inside the service.","minLength":1,"pattern":"^[a-z0-9-]+$","type":"string"},"nodePort":{"description":"NodePort is the external port assigned on each node in the cluster.\nIf not set, Kubernetes will automatically allocate a NodePort.\n\nConstraints:\n- Can only be set when ServiceType is \"NodePort\".\n- Must be within the clusters valid NodePort range (Kubernetes default is 30000-32767).","type":"integer"},"port":{"description":"Port is the port number that will be exposed by the service.\nMust be within the valid range of TCP/UDP ports (1-65535).","type":"integer"},"protocol":{"description":"Protocol specifies the transport protocol used by the port.\nSupported values: TCP, UDP","enum":["TCP","UDP"],"type":"string"}},"required":["name","port","protocol"],"type":"object"},"type":"array"},"description":"ConfigPorts defines the observed state of the config ports.\nIt contains the actual port numbers that are exposed on the DPUService per cluster.","type":"object"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"},"serviceID":{"description":"ServiceID is the ID of the service that the DPUService is associated with.\nThis is set when the DPUService is created.","type":"string"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceChain.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceChain.json new file mode 100644 index 00000000..7d7470b7 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceChain.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceChain","version":"v1alpha1","schema":{"description":"DPUServiceChain is the Schema for the DPUServiceChain API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceChainSpec defines the desired state of DPUServiceChainSpec","properties":{"clusterSelector":{"description":"Select the Clusters with specific labels, ServiceChainSet CRs will be created only for these Clusters\n\nDeprecated: This field is deprecated and will be removed with v26.7.0. Use DPUClusterSelector instead.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuClusterSelector":{"description":"DPUClusterSelector determines in which clusters the DPUServiceChain controller should apply the configuration.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"Template describes the ServiceChainSet that will be created for each selected Cluster.","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceChainSetSpec defines the desired state of ServiceChainSet","properties":{"nodeSelector":{"description":"Select the Nodes with specific labels, ServiceChain CRs will be created\nonly for these Nodes","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"ServiceChainSpecTemplate holds the template for the ServiceChainSpec","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceChainSpec is the spec for the ServiceChainSpec","properties":{"node":{"description":"Node where this ServiceChain applies to","type":"string"},"switches":{"description":"The switches of the ServiceChain, order is significant","items":{"description":"Switch defines the switch configuration","properties":{"ports":{"description":"Ports of the switch","items":{"description":"Port defines the port configuration","properties":{"serviceInterface":{"description":"ServiceIfc defines the service interface configuration","properties":{"ipam":{"description":"IPAM defines the IPAM configuration when referencing a serviceInterface of type 'service'","properties":{"defaultGateway":{"description":"DefaultGateway adds gateway as default gateway in the routes list if true.","type":"boolean"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"Labels matching service IPAM","maxProperties":50,"minProperties":1,"type":"object"},"setDefaultRoute":{"description":"SetDefaultRoute adds a default route to the routing table if true.","type":"boolean"}},"required":["matchLabels"],"type":"object"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"Labels matching service interface","maxProperties":50,"minProperties":1,"type":"object"}},"required":["matchLabels"],"type":"object"}},"required":["serviceInterface"],"type":"object"},"maxItems":50,"minItems":1,"type":"array"},"serviceMTU":{"default":1500,"description":"ServiceMTU of the switch\nThe default is 1500.","maximum":9216,"minimum":1280,"type":"integer"}},"required":["ports"],"type":"object"},"maxItems":50,"minItems":1,"type":"array"}},"required":["switches"],"type":"object"}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"},"status":{"description":"DPUServiceChainStatus defines the observed state of DPUServiceChain","properties":{"conditions":{"description":"Conditions reflect the status of the object","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceCredentialRequest.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceCredentialRequest.json new file mode 100644 index 00000000..545f0fdf --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceCredentialRequest.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceCredentialRequest","version":"v1alpha1","schema":{"description":"DPUServiceCredentialRequest is the Schema for the dpuserviceCredentialRequests API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceCredentialRequestSpec defines the desired state of DPUServiceCredentialRequest","properties":{"duration":{"description":"Duration is the duration for which the token will be valid.\nValue must be in units accepted by Go time.ParseDuration https://golang.org/pkg/time/#ParseDuration.\ne.g. \"1h\", \"1m\", \"1s\", \"1ms\", \"1.5h\", \"2h45m\".\nValue duration must not be less than 10 minutes.\n**Note:** The maximum TTL for a token is 24 hours, after which the token\nwill be rotated.","pattern":"^([0-9]+(\\.[0-9]+)?(ms|s|m|h))+$","type":"string"},"metadata":{"description":"ObjectMeta defines the metadata of the secret.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"secret":{"description":"Secret defines the needed information to create the secret.\nThe secret will be of the type specified in the `spec.type` field.","properties":{"name":{"description":"Name of the object.","type":"string"},"namespace":{"description":"Namespace of the object, if not provided the object will be looked up in\nthe same namespace as the referring object","type":"string"}},"required":["name"],"type":"object"},"serviceAccount":{"description":"ServiceAccount defines the needed information to create the service account.","properties":{"name":{"description":"Name of the object.","type":"string"},"namespace":{"description":"Namespace of the object, if not provided the object will be looked up in\nthe same namespace as the referring object","type":"string"}},"required":["name"],"type":"object"},"targetCluster":{"description":"TargetCluster defines the target cluster where the service account will\nbe created, and where a token for that service account will be requested.\nIf not provided, the token will be requested for the same cluster where\nthe DPUServiceCredentialRequest object is created.","properties":{"name":{"description":"Name of the object.","type":"string"},"namespace":{"description":"Namespace of the object, if not provided the object will be looked up in\nthe same namespace as the referring object","type":"string"}},"required":["name"],"type":"object"},"type":{"description":"Type is the type of the secret that will be created.\nThe supported types are `kubeconfig` and `tokenFile`.\nIf `kubeconfig` is selected, the secret will contain a kubeconfig file,\nthat can be used to access the cluster.\nIf `tokenFile` is selected, the secret will contain a token file and several\nenvironment variables that can be used to access the cluster. It can be used\nwith https://github.com/kubernetes/client-go/blob/v11.0.0/rest/config.go#L52\nto create a client that will handle file rotation.","enum":["kubeconfig","tokenFile"],"type":"string"}},"required":["secret","serviceAccount","type"],"type":"object"},"status":{"description":"DPUServiceCredentialRequestStatus defines the observed state of DPUServiceCredentialRequest","properties":{"conditions":{"description":"Conditions defines current service state.","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"expirationTimestamp":{"description":"ExpirationTimestamp is the time when the token will expire.","format":"date-time","type":"string"},"issuedAt":{"description":"IssuedAt is the time when the token was issued.","format":"date-time","type":"string"},"secret":{"description":"Sercet is the namespaced name of the Secret resource created by the controller for\nthe DPUServiceCredentialRequest.","type":"string"},"serviceAccount":{"description":"ServiceAccount is the namespaced name of the ServiceAccount resource created by\nthe controller for the DPUServiceCredentialRequest.","type":"string"},"targetCluster":{"description":"TargetCluster is the cluster where the service account was created.\nIt has to be persisted in the status to be able to delete the service account\nwhen the DPUServiceCredentialRequest is updated.","type":"string"},"targetClusterUID":{"description":"TargetClusterUID is the UID of the DPUCluster the credentials were last issued against.\nUsed to detect cluster recreation with the same name and force credential refresh.","type":"string"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceInterface.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceInterface.json new file mode 100644 index 00000000..c5a20aa6 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceInterface.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceInterface","version":"v1alpha1","schema":{"description":"DPUServiceInterface is the Schema for the DPUServiceInterface API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceInterfaceSpec defines the desired state of DPUServiceInterfaceSpec","properties":{"clusterSelector":{"description":"Select the Clusters with specific labels, ServiceInterfaceSet CRs will be created only for these Clusters\n\nDeprecated: This field is deprecated and will be removed with v26.7.0. Use DPUClusterSelector instead.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuClusterSelector":{"description":"DPUClusterSelector determines in which clusters the DPUServiceInterface controller should apply the configuration.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"Template describes the ServiceInterfaceSet that will be created for each selected Cluster.","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceInterfaceSetSpec defines the desired state of ServiceInterfaceSet","properties":{"nodeSelector":{"description":"Select the Nodes with specific labels, ServiceInterface CRs will be\ncreated only for these Nodes","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"template":{"description":"Template holds the template for the serviceInterfaceSpec","properties":{"metadata":{"description":"ObjectMeta holds metadata like labels and annotations.","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations is a map of string keys and values.","type":"object"},"labels":{"additionalProperties":{"type":"string"},"description":"Labels is a map of string keys and values.","type":"object"}},"type":"object"},"spec":{"description":"ServiceInterfaceSpec is the spec for the ServiceInterfaceSpec","properties":{"interfaceType":{"description":"The interface type (\"vlan\", \"physical\", \"pf\", \"vf\", \"ovn\", \"patch\", \"service\")","enum":["vlan","physical","pf","vf","ovn","patch","service"],"type":"string"},"node":{"description":"Node where this interface exists","type":"string"},"ovn":{"description":"The OVN definition\nDeprecated: This field is deprecated and will be removed with v26.10.0.\nMigrate to interfaceType=\"patch\" with spec.patch.peerBridge and spec.patch.peerPatchName instead.","properties":{"externalBridge":{"default":"br-ovn","description":"ExternalBridge is the name of the OVN bridge","type":"string"}},"type":"object"},"patch":{"description":"The Patch definition","properties":{"peerBridge":{"description":"PeerBridge is the name of the bridge to which the patch port is connected.\nThis bridge must be created before the ServiceInterface is created.","type":"string"},"peerExternalIDs":{"additionalProperties":{"type":"string"},"description":"PeerExternalIDs are the external IDs used to identify the peer patch port.","type":"object"},"peerPatchName":{"description":"PeerPatchName is the name of the patch port on the peer bridge.\nIf not set, it is auto-generated in the format: `p__to__`\nwhere bridge names have hyphens removed and `` is an 8-character FNV-1a hash\nderived from the ServiceInterface's namespace/name.\nExample: p_brovn_to_brsfc_7aea60f7 (for bridges br-ovn and br-sfc).","type":"string"}},"required":["peerBridge"],"type":"object"},"pf":{"description":"The PF definition","properties":{"nicSelector":{"description":"NICSelector defines the NIC selected for the PF interface","properties":{"controllerNumber":{"description":"ControllerNumber specifies the controller number that will be used to find the matching representor on the DPU.\nA value of 0 targets the local controller, >=1 targets external controllers with the specified number. If unspecified controller number 1 is used.\nIn case of a DPU/NIC with socket direct or MultiHost, the user should specify the controller number explicitly","format":"int32","maximum":255,"minimum":0,"type":"integer"},"pci":{"description":"PCI is the PCI selector. valid only for selector type pci","properties":{"address":{"description":"Address is the PCI address of any of the NIC's Embedded CPU PFs (ECPFs)","pattern":"^[0-9a-f]{4}:[0-9a-f]{2}:[0-1][0-9a-f]\\.[0-7]{1}$","type":"string"}},"required":["address"],"type":"object"},"type":{"description":"Type is the type of selector to be used to identify the NIC.","enum":["dpu","pci"],"type":"string"}},"required":["type"],"type":"object","x-kubernetes-validations":[{"message":"for type=pci, pci must be set","rule":"(self.type == 'dpu') || (self.type == 'pci' && has(self.pci))"}]},"pfID":{"description":"The PF ID","maximum":255,"minimum":0,"type":"integer"},"virtualNetwork":{"description":"VirtualNetwork is the VirtualNetwork name in the same namespace","type":"string"}},"required":["pfID"],"type":"object","x-kubernetes-validations":[{"message":"virtualNetwork is immutable","rule":"(!has(oldSelf.virtualNetwork) && !has(self.virtualNetwork)) || ((has(oldSelf.virtualNetwork) && has(self.virtualNetwork)) && self.virtualNetwork==oldSelf.virtualNetwork)"}]},"physical":{"description":"The physical interface definition","properties":{"interfaceName":{"description":"The interface name","type":"string"}},"required":["interfaceName"],"type":"object"},"service":{"description":"The Service definition","properties":{"interfaceName":{"description":"The interface name","maxLength":15,"minLength":1,"type":"string"},"network":{"description":"Network is the Network Attachment Definition in the form of \"namespace/name\"\nor just \"name\" if the namespace is the same as the ServiceInterface.","type":"string"},"serviceID":{"description":"ServiceID is the DPU Service Identifier","type":"string"},"virtualNetwork":{"description":"VirtualNetwork is the VirtualNetwork name in the same namespace","type":"string"}},"required":["interfaceName","network","serviceID"],"type":"object","x-kubernetes-validations":[{"message":"virtualNetwork is immutable","rule":"(!has(oldSelf.virtualNetwork) && !has(self.virtualNetwork)) || ((has(oldSelf.virtualNetwork) && has(self.virtualNetwork)) && self.virtualNetwork==oldSelf.virtualNetwork)"}]},"vf":{"description":"The VF definition","properties":{"nicSelector":{"description":"NICSelector defines the NIC selected for the VF interface","properties":{"controllerNumber":{"description":"ControllerNumber specifies the controller number that will be used to find the matching representor on the DPU.\nA value of 0 targets the local controller, >=1 targets external controllers with the specified number. If unspecified controller number 1 is used.\nIn case of a DPU/NIC with socket direct or MultiHost, the user should specify the controller number explicitly","format":"int32","maximum":255,"minimum":0,"type":"integer"},"pci":{"description":"PCI is the PCI selector. valid only for selector type pci","properties":{"address":{"description":"Address is the PCI address of any of the NIC's Embedded CPU PFs (ECPFs)","pattern":"^[0-9a-f]{4}:[0-9a-f]{2}:[0-1][0-9a-f]\\.[0-7]{1}$","type":"string"}},"required":["address"],"type":"object"},"type":{"description":"Type is the type of selector to be used to identify the NIC.","enum":["dpu","pci"],"type":"string"}},"required":["type"],"type":"object","x-kubernetes-validations":[{"message":"for type=pci, pci must be set","rule":"(self.type == 'dpu') || (self.type == 'pci' && has(self.pci))"}]},"parentInterfaceRef":{"description":"The parent interface reference","type":"string"},"pfID":{"description":"The PF ID","maximum":255,"minimum":0,"type":"integer"},"vfID":{"description":"The VF ID","maximum":255,"minimum":0,"type":"integer"},"virtualNetwork":{"description":"VirtualNetwork is the VirtualNetwork name in the same namespace","type":"string"}},"required":["pfID","vfID"],"type":"object","x-kubernetes-validations":[{"message":"virtualNetwork is immutable","rule":"(!has(oldSelf.virtualNetwork) && !has(self.virtualNetwork)) || ((has(oldSelf.virtualNetwork) && has(self.virtualNetwork)) && self.virtualNetwork==oldSelf.virtualNetwork)"}]},"vlan":{"description":"The VLAN definition","properties":{"parentInterfaceRef":{"description":"The parent interface reference","type":"string"},"vlanID":{"description":"The VLAN ID","type":"integer"}},"required":["parentInterfaceRef","vlanID"],"type":"object"}},"required":["interfaceType"],"type":"object","x-kubernetes-validations":[{"messageExpression":"'for interfaceType=' + self.interfaceType + ', ' + self.interfaceType + ' must be set'","rule":"(self.interfaceType == 'vlan' && has(self.vlan)) || (self.interfaceType == 'pf' && has(self.pf)) || (self.interfaceType == 'vf' && has(self.vf)) || (self.interfaceType == 'physical' && has(self.physical)) || (self.interfaceType == 'service' && has(self.service)) || (self.interfaceType == 'ovn') || (self.interfaceType == 'patch' && has(self.patch))"}]}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"}},"required":["spec"],"type":"object"}},"required":["template"],"type":"object"},"status":{"description":"DPUServiceInterfaceStatus defines the observed state of DPUServiceInterface","properties":{"conditions":{"description":"Conditions defines current service state.","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceNAD.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceNAD.json new file mode 100644 index 00000000..cde03602 --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUServiceNAD.json @@ -0,0 +1 @@ +{"group":"svc.dpu.nvidia.com","kind":"DPUServiceNAD","version":"v1alpha1","schema":{"description":"DPUServiceNAD is the Schema for the dpuservicenads API.","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUServiceNADSpec defines the desired state of DPUServiceNAD.","properties":{"bridge":{"description":"Bridge specifies the name of the OVS bridge to which the network interface will be connected.\nThis bridge name is used in the CNI configuration for the OVS plugin.","type":"string"},"chainedCNIs":{"description":"ChainedCNIs specifies additional CNI plugins to be chained after the base OVS plugin.\nWhen specified, the NAD will use the CNI chaining format with the OVS plugin as the\nfirst plugin, followed by the plugins defined in this list.\nThis allows adding capabilities like RDMA support on top of the base network interface.\nIf empty, the NAD uses a single OVS plugin configuration (backward compatible format).","items":{"description":"CNIPlugin defines a CNI plugin to be used in a chained CNI configuration.\nWhen multiple CNI plugins are specified in ChainedCNIs, they are executed in order\nafter the base OVS CNI plugin to provide additional network functionality.","properties":{"config":{"description":"Config contains optional plugin-specific configuration as raw JSON.\nThe configuration is merged into the CNI plugin configuration.","type":"object","x-kubernetes-preserve-unknown-fields":true},"type":{"description":"Type specifies the CNI plugin type to be used in the chain.\nCurrently only \"rdma\" is supported, which enables RDMA capabilities for the network interface.","enum":["rdma"],"type":"string"}},"required":["type"],"type":"object"},"type":"array"},"dpuClusterSelector":{"description":"DPUClusterSelector determines in which clusters the DPUServiceNAD controller should apply the configuration.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"ipam":{"description":"IPAM enables IP Address Management for the network interfaces attached to this network\nWhen set to true, a DPUServiceChain that references the DPUServiceInterface that has\nrequested this network must be created and include the relevant IPAM information. See\nDPUServiceChain documentation for more.\nWhen set to false, the network interfaces attached to this network will not get an IP","type":"boolean"},"resourceType":{"description":"ResourceType specifies the type of network resource to allocate for pods using this NAD.\n- \"vf\": Virtual Function (SR-IOV VF) from the DPU's physical ports\n- \"sf\": Scalable Function from the DPU (maps to nvidia.com/bf_sf or nvidia.com/bf_sf_trusted)\n- \"veth\": Virtual Ethernet pair (no device plugin resource required)\nThe resource type determines which SR-IOV device plugin resource will be requested.","enum":["vf","sf","veth"],"type":"string"},"serviceMTU":{"default":1500,"description":"ServiceMTU specifies the MTU size in bytes for the network interface.\nThis value is passed to the OVS CNI plugin and determines the maximum packet size.\nIf there is a DPUServiceChain that references an interface that is part of this network,\nthen the MTU that is defined in the DPUServiceChain takes precedence.\nThe default is 1500.","maximum":9216,"minimum":1280,"type":"integer"}},"required":["resourceType"],"type":"object"},"status":{"description":"DPUServiceNADStatus defines the observed state of DPUServiceNAD.","properties":{"conditions":{"description":"Conditions reflect the status of the object","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUSet.json b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUSet.json new file mode 100644 index 00000000..c60b3a3f --- /dev/null +++ b/pkg/dpf/sentry/schemas/26.8.0-rc.1-DPUSet.json @@ -0,0 +1 @@ +{"group":"provisioning.dpu.nvidia.com","kind":"DPUSet","version":"v1alpha1","schema":{"description":"DPUSet is the Schema for the dpusets API","properties":{"apiVersion":{"description":"APIVersion defines the versioned schema of this representation of an object.\nServers should convert recognized schemas to the latest internal value, and\nmay reject unrecognized values.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources","type":"string"},"kind":{"description":"Kind is a string value representing the REST resource this object represents.\nServers may infer this from the endpoint the client submits requests to.\nCannot be updated.\nIn CamelCase.\nMore info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds","type":"string"},"metadata":{"type":"object"},"spec":{"description":"DPUSetSpec defines the desired state of DPUSet","properties":{"dpuDeviceSelector":{"description":"DPUDeviceSelector defines the selector for DPUDevices that the DPUSet should target and should create a DPU for.","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuNodeSelector":{"description":"Select the DPUNodes with specific labels","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"},"dpuSelector":{"additionalProperties":{"type":"string"},"description":"Select the DPU with specific labels\n\nDeprecated: This field is deprecated and will be removed with v26.7.0. Use DPUDeviceSelector instead.","type":"object"},"dpuTemplate":{"description":"Object that describes the DPU that will be created if insufficient replicas are detected","properties":{"annotations":{"additionalProperties":{"type":"string"},"description":"Annotations specifies annotations which are added to the DPU.","type":"object"},"spec":{"description":"Spec specifies the DPU specification.","properties":{"astraEnabled":{"description":"AstraEnabled indicates whether E/W NIC configuration (Astra) is enabled","type":"boolean"},"bfb":{"description":"Specifies a BFB CR","properties":{"name":{"description":"Specifies name of the bfb CR to use for this DPU","minLength":1,"type":"string"}},"type":"object"},"blueFieldSoftware":{"description":"Specifies a BlueFieldSoftware CR","properties":{"name":{"description":"Specifies name of the BlueFieldSoftware CR to use for this DPU","minLength":1,"type":"string"}},"type":"object"},"cluster":{"description":"Specifies details on the K8S cluster to join","properties":{"nodeAnnotations":{"additionalProperties":{"type":"string"},"description":"NodeAnnotations specifies the annotations to be added to the node.","type":"object"},"nodeLabels":{"additionalProperties":{"type":"string"},"description":"NodeLabels specifies the labels to be added to the node.","type":"object"},"selector":{"description":"Selector defines the selector of the DPUClusters the produced DPUs should join","properties":{"matchExpressions":{"description":"matchExpressions is a list of label selector requirements. The requirements are ANDed.","items":{"description":"A label selector requirement is a selector that contains values, a key, and an operator that\nrelates the key and values.","properties":{"key":{"description":"key is the label key that the selector applies to.","type":"string"},"operator":{"description":"operator represents a key's relationship to a set of values.\nValid operators are In, NotIn, Exists and DoesNotExist.","type":"string"},"values":{"description":"values is an array of string values. If the operator is In or NotIn,\nthe values array must be non-empty. If the operator is Exists or DoesNotExist,\nthe values array must be empty. This array is replaced during a strategic\nmerge patch.","items":{"type":"string"},"type":"array","x-kubernetes-list-type":"atomic"}},"required":["key","operator"],"type":"object"},"type":"array","x-kubernetes-list-type":"atomic"},"matchLabels":{"additionalProperties":{"type":"string"},"description":"matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels\nmap is equivalent to an element of matchExpressions, whose key field is \"key\", the\noperator is \"In\", and the values array contains only \"value\". The requirements are ANDed.","type":"object"}},"type":"object","x-kubernetes-map-type":"atomic"}},"type":"object"},"dpuFlavor":{"description":"DPUFlavor is the name of the DPUFlavor that will be used to deploy the DPU.\nMutually exclusive with DPUFlavorTemplate.","minLength":1,"type":"string"},"dpuFlavorTemplate":{"description":"DPUFlavorTemplate is the name of a DPUFlavorTemplate that is rendered per-DPU\n(against DPUDevice.spec.values) into a generated DPUFlavor. Mutually exclusive\nwith DPUFlavor.","minLength":1,"type":"string"},"nodeEffect":{"description":"Specifies how changes to the DPU should affect the Node","properties":{"applyOnLabelChange":{"default":false,"description":"Apply node effect when labels change on the DPU object\nWhen set to true, label changes in Ready state will trigger node effect logic","type":"boolean"},"customAction":{"description":"Name of a config map which contains a pod yaml definition to run which will apply the nodeEffect.\nThe pod is expected to exit when node effect is done, if pod terminates with error then DPU would move to an error phase.\nThe DPUNode's name will be exported as an environment variable, named as DPUNODE_NAME, to each container and init container in the pod.\nThe labels and annotations of DPUNode will be exported in `/etc/dpu/dpf-pod-info/labels` and `/etc/dpu/dpf-pod-info/annotations` accordingly; the volume name `dpf-pod-info` is used to mount the labels and annotations.\nIf any name confliction for env or volume, the controller will not export the name or labels/annotations of DPUNode accordingly.","type":"string"},"customLabel":{"additionalProperties":{"type":"string"},"description":"Add specify labels on the DPU node","type":"object"},"drain":{"description":"Drain the K8s host node by NodeMaintenance operator","type":"boolean"},"force":{"default":false,"description":"Force is the flag to indicate if the node effect should be applied immediately.\nIf true, dpfOperatorConfig.multiDPUOperationsSyncWaitTime and dpfOperatorConfig.maxUnavailableDPUNodes will be ignored when applying node effect for DPUNodeMaintenance CR","type":"boolean"},"hold":{"description":"Places annotation `wait-for-external-nodeeffect` and waits for it to be removed\nthis is the default behavior in a non K8S environment","type":"boolean"},"noEffect":{"description":"Do not do any action on the DPU node","type":"boolean"},"nodeMaintenanceAdditionalRequestors":{"description":"Additional requestors to be added to the NvidiaNodeMaintenance CR when Drain is selected","items":{"type":"string"},"type":"array"},"taint":{"description":"Add specify taint on the DPU node","properties":{"effect":{"description":"Required. The effect of the taint on pods\nthat do not tolerate the taint.\nValid effects are NoSchedule, PreferNoSchedule and NoExecute.","type":"string"},"key":{"description":"Required. The taint key to be applied to a node.","type":"string"},"timeAdded":{"description":"TimeAdded represents the time at which the taint was added.","format":"date-time","type":"string"},"value":{"description":"The taint value corresponding to the taint key.","type":"string"}},"required":["effect","key"],"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"exactly one of taint, noEffect=true, drain=true, non-empty customLabel, non-empty customAction, hold=true must be set","rule":"(has(self.taint) ? 1 : 0) + ((has(self.noEffect) && self.noEffect) ? 1 : 0) + ((has(self.customLabel) && size(self.customLabel) > 0) ? 1 : 0) + ((has(self.drain) && self.drain) ? 1 : 0) + ((has(self.customAction) && size(self.customAction) > 0) ? 1 : 0) + ((has(self.hold) && self.hold) ? 1 : 0) == 1"}]},"secureBoot":{"description":"SecureBoot specifies whether UEFI Secure Boot should be enabled.","type":"boolean"}},"required":["nodeEffect"],"type":"object","x-kubernetes-validations":[{"message":"exactly one of bfb or blueFieldSoftware must be set","rule":"(has(self.bfb) && has(self.bfb.name) ? 1 : 0) + (has(self.blueFieldSoftware) && has(self.blueFieldSoftware.name) ? 1 : 0) == 1"},{"message":"exactly one of dpuFlavor or dpuFlavorTemplate must be set","rule":"has(self.dpuFlavor) != has(self.dpuFlavorTemplate)"}]}},"type":"object"},"strategy":{"description":"The rolling update strategy to use to updating existing DPUs with new ones.","properties":{"rollingUpdate":{"description":"Rolling update config params. Present only if StrategyType = RollingUpdate.","properties":{"maxUnavailable":{"anyOf":[{"type":"integer"},{"type":"string"}],"description":"MaxUnavailable is the maximum number of DPUs that can be unavailable during the update.\n\nDeprecated: This field is deprecated and will be removed with v26.7.0.","x-kubernetes-int-or-string":true}},"type":"object"},"type":{"description":"Can be \"OnDelete\" or \"RollingUpdate\".","enum":["OnDelete","RollingUpdate"],"type":"string"}},"required":["type"],"type":"object"}},"required":["dpuTemplate","strategy"],"type":"object","x-kubernetes-validations":[{"message":"only one of dpuSelector or dpuDeviceSelector can be specified","rule":"!(has(self.dpuSelector) && has(self.dpuDeviceSelector))"}]},"status":{"description":"DPUSetStatus defines the observed state of DPUSet","properties":{"conditions":{"description":"Conditions reflect the status of the object","items":{"description":"Condition contains details for one aspect of the current state of this API Resource.","properties":{"lastTransitionTime":{"description":"lastTransitionTime is the last time the condition transitioned from one status to another.\nThis should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.","format":"date-time","type":"string"},"message":{"description":"message is a human readable message indicating details about the transition.\nThis may be an empty string.","maxLength":32768,"type":"string"},"observedGeneration":{"description":"observedGeneration represents the .metadata.generation that the condition was set based upon.\nFor instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date\nwith respect to the current state of the instance.","format":"int64","minimum":0,"type":"integer"},"reason":{"description":"reason contains a programmatic identifier indicating the reason for the condition's last transition.\nProducers of specific condition types may define expected values and meanings for this field,\nand whether the values are considered a guaranteed API.\nThe value should be a CamelCase string.\nThis field may not be empty.","maxLength":1024,"minLength":1,"pattern":"^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$","type":"string"},"status":{"description":"status of the condition, one of True, False, Unknown.","enum":["True","False","Unknown"],"type":"string"},"type":{"description":"type of condition in CamelCase or in foo.example.com/CamelCase.","maxLength":316,"pattern":"^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$","type":"string"}},"required":["lastTransitionTime","message","reason","status","type"],"type":"object"},"type":"array"},"dpuStatistics":{"additionalProperties":{"type":"integer"},"description":"DPUStatistics is a map of DPUPhase to the number of DPUs in that phase.","type":"object"},"observedGeneration":{"description":"ObservedGeneration records the Generation observed on the object the last time it was patched.","format":"int64","type":"integer"}},"type":"object"}},"type":"object","x-kubernetes-validations":[{"message":"name length can't be bigger than 63 chars","rule":"self.metadata.name.size() <= 63"}]}} diff --git a/pkg/dpf/sentry/schemas/README.md b/pkg/dpf/sentry/schemas/README.md new file mode 100644 index 00000000..4215691f --- /dev/null +++ b/pkg/dpf/sentry/schemas/README.md @@ -0,0 +1,14 @@ + + + +These JSON data files preserve the complete v1alpha1 OpenAPI schema extracted +from each emitted DPF Kind's upstream CRD, without alteration. The envelope binds +group, version and Kind. `provenance.json` records each generated data hash, the +source CRD path/hash and exact upstream source commit. Both release candidate +26.8.0-rc.1 and GA26.8.0 are represented independently. + +The offline validator rejects unknown fields that Kubernetes structural pruning +would otherwise remove, then checks OpenAPI and CEL create invariants. It never +defaults/mutates desired objects and does not simulate controller webhooks, +installed CRD drift, updates, physical effects or runtime qualification. The +matching installed API must still validate the complete plan before deployment. diff --git a/pkg/dpf/sentry/schemas/provenance.json b/pkg/dpf/sentry/schemas/provenance.json new file mode 100644 index 00000000..5e67463b --- /dev/null +++ b/pkg/dpf/sentry/schemas/provenance.json @@ -0,0 +1,114 @@ +[ + { + "file": "26.8.0-rc.1-BlueFieldSoftware.json", + "sha256": "c27bc4cc4dd297d9711861ad45e9bc90a3a4ce2b5ae9b628bd654fafd6c7a968", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/provisioning/crd/bases/provisioning.dpu.nvidia.com_bluefieldsoftwares.yaml", + "sourceSHA256": "28273445837914290a7ea9f1f6b98837c3a763e925bd8a318ae7b7ae7d838d5e" + }, + { + "file": "26.8.0-rc.1-DPUFlavor.json", + "sha256": "0b0d46480fbabb0d3b182077cd23f83fc02ffd846a1a484df076ed18246e4ff2", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/provisioning/crd/bases/provisioning.dpu.nvidia.com_dpuflavors.yaml", + "sourceSHA256": "3d0c5c3000121b96e052eed993e773ce5bf42727025419605958bf61123e4fb6" + }, + { + "file": "26.8.0-rc.1-DPUSet.json", + "sha256": "0129a3c4e6010e80d7ad64585cd08045b3d967cb15ca3c6ca37a87541765e38f", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/provisioning/crd/bases/provisioning.dpu.nvidia.com_dpusets.yaml", + "sourceSHA256": "4223ce3d30d513f8ea4007eb092601082355211b7849594f0f9e36b7cbdf9428" + }, + { + "file": "26.8.0-rc.1-DPUService.json", + "sha256": "8cd8be61c807a85935588e3dc1ea73877f42cf5953fb05caceaa0ad84c6c5db8", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservices.yaml", + "sourceSHA256": "8747081355c8aad5f32734d3cf8c6d584b7fa94fb3e452204dc0d81ec69f26e2" + }, + { + "file": "26.8.0-rc.1-DPUServiceNAD.json", + "sha256": "ec9ec0864824dec9c00659b484fca9ac00777eb8b66f6d33ddda91b08b13134b", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservicenads.yaml", + "sourceSHA256": "07f1514c175f1e471f5c1678a6a54dacdc8deac3a4217ad5a830a9b64dfd706c" + }, + { + "file": "26.8.0-rc.1-DPUServiceInterface.json", + "sha256": "15ea9818fc1114305264f8db423325f2a4a7830ad3fd9fc15697071b5178c836", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuserviceinterfaces.yaml", + "sourceSHA256": "944ed0e9a5e1caeef5c5679fafb12c7bf8a016155c42c1bbd687a3d8bacdb123" + }, + { + "file": "26.8.0-rc.1-DPUServiceChain.json", + "sha256": "b4e9c274c67290ef8d9794c20be63ba5af36e842d1d735dcba26cdbd6a4ef3e7", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservicechains.yaml", + "sourceSHA256": "60f850ad7e67d4dc66aac9281451cb157dd44c6a82045e0f61c54378901771e4" + }, + { + "file": "26.8.0-rc.1-DPUServiceCredentialRequest.json", + "sha256": "9d4c953fc166f2e27b5220ace7803a36a53b6c597bb250461a4a1c8a752eb34e", + "sourceCommit": "6bc1bd3f00f5fb4064600748da7164e3610622cd", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservicecredentialrequests.yaml", + "sourceSHA256": "29c4839e629701b08d520e21fb915c2342e8bf66abde01ff5018b428192697b1" + }, + { + "file": "26.8.0-BlueFieldSoftware.json", + "sha256": "c27bc4cc4dd297d9711861ad45e9bc90a3a4ce2b5ae9b628bd654fafd6c7a968", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/provisioning/crd/bases/provisioning.dpu.nvidia.com_bluefieldsoftwares.yaml", + "sourceSHA256": "28273445837914290a7ea9f1f6b98837c3a763e925bd8a318ae7b7ae7d838d5e" + }, + { + "file": "26.8.0-DPUFlavor.json", + "sha256": "0b0d46480fbabb0d3b182077cd23f83fc02ffd846a1a484df076ed18246e4ff2", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/provisioning/crd/bases/provisioning.dpu.nvidia.com_dpuflavors.yaml", + "sourceSHA256": "3d0c5c3000121b96e052eed993e773ce5bf42727025419605958bf61123e4fb6" + }, + { + "file": "26.8.0-DPUSet.json", + "sha256": "3684f374d9c5ad42a091195aba2dfc5b5f5f3a0a2ec607fde49bc9ccef8f08d9", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/provisioning/crd/bases/provisioning.dpu.nvidia.com_dpusets.yaml", + "sourceSHA256": "c025cf3d5a66ca61a270287b688e883a1a1c1ce8855700a553a8b43f41126d53" + }, + { + "file": "26.8.0-DPUService.json", + "sha256": "8cd8be61c807a85935588e3dc1ea73877f42cf5953fb05caceaa0ad84c6c5db8", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservices.yaml", + "sourceSHA256": "8747081355c8aad5f32734d3cf8c6d584b7fa94fb3e452204dc0d81ec69f26e2" + }, + { + "file": "26.8.0-DPUServiceNAD.json", + "sha256": "ec9ec0864824dec9c00659b484fca9ac00777eb8b66f6d33ddda91b08b13134b", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservicenads.yaml", + "sourceSHA256": "07f1514c175f1e471f5c1678a6a54dacdc8deac3a4217ad5a830a9b64dfd706c" + }, + { + "file": "26.8.0-DPUServiceInterface.json", + "sha256": "02b0faaf3384316124de3c9ecb87b8abf43567418f43b84261e75aaf206eedad", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuserviceinterfaces.yaml", + "sourceSHA256": "71b40c578ea40f24b1a219726d94a99de8e7491d3478a5b152c865e70d5d7317" + }, + { + "file": "26.8.0-DPUServiceChain.json", + "sha256": "706755a9aa556e166257427eac66e6a0f994b6b61f5e2d99e226cd472683e0d7", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservicechains.yaml", + "sourceSHA256": "f43da798e37c3cf5322d61248aed81b28323bcbbc9d0fa6fe7722d41e2f69210" + }, + { + "file": "26.8.0-DPUServiceCredentialRequest.json", + "sha256": "9d4c953fc166f2e27b5220ace7803a36a53b6c597bb250461a4a1c8a752eb34e", + "sourceCommit": "fc4974ed3bc026ae27634673c2fdf818a2972235", + "sourcePath": "config/dpuservice/crd/bases/svc.dpu.nvidia.com_dpuservicecredentialrequests.yaml", + "sourceSHA256": "29c4839e629701b08d520e21fb915c2342e8bf66abde01ff5018b428192697b1" + } +] diff --git a/pkg/dpf/sentry/testdata/PROVENANCE.md b/pkg/dpf/sentry/testdata/PROVENANCE.md new file mode 100644 index 00000000..6eecfc6d --- /dev/null +++ b/pkg/dpf/sentry/testdata/PROVENANCE.md @@ -0,0 +1,46 @@ + + + +These small archive fixtures are the authentic NVIDIA-owned Sentry charts used +by the offline projection tests. They were retrieved on 2026-11-06 from the +coordinates and SHA-256 identities in catalog.go. They contain templates and +public image coordinates, with no registry credentials or lab observations. + +The profile templates derive from gzaidman/doca-sentry-deploy commit +0f3c75e54a7f596fc0c469113d55996562975c39. The adaptation removes example OS +password/login configuration and the lab BMC reboot script. It supplies explicit +instance names, selected tenant and device scope, existing Secret names and +immutable artifact/image identities. Host/DPU registration and base DPF +installation remain external prerequisites. This engineering provenance is not +an assertion of release support or runtime qualification. + +Core policy uses ext_authz and Argus with the source chart's `tap.enabled=false`. +It needs no inference Secret and projects four Tower containers. Opting into +LLM inspection enables the fifth `tower-tap` container and requires the real +`INFERENCE_API_KEY` Secret key. The upstream chart archives are unmodified: +both listeners retain an optional ext_proc hook with failure_mode_allow=true, +including when its sidecar is disabled. Core policy therefore makes no claim +of LLM inspection; the independent ext_authz filters remain fail-closed in +both modes. Runtime positive/negative policy tests are still required. + +Offline workload projections use the DPF controller's effective Helm inputs, +separately from the native DPUService chart inputs. The standard-service path +is identical in DPF26.8.0-rc.1 (`6bc1bd3f00f5fb4064600748da7164e3610622cd`) +and26.8.0 (`fc4974ed3bc026ae27634673c2fdf818a2972235`): + +- `internal/dpuservice/controllers/dpuservice_controller.go:458-513,659-716` + resolves service interfaces and NAD resource names, injecting resource counts. +- `internal/dpuservice/controllers/nad.go:33-72,110-143` merges existing + nonempty network fields, adds the invalid-network sentinel and sorts by + network name/interface. Multus v4.0.2 serialization includes `cni-args:null`. +- `internal/dpuservice/controllers/dpuservice_controller.go:1706-1780,1854-1860` + adds the explicit service-ID label and merges generated daemonset values with + raw chart values, using generated values as authoritative. +- `internal/dpuservice/utils/mergemaps.go:26-68` defines that merge precedence. + +`values.go` implements only the Sentry standard-service branch and its declared +NAD dependencies. It rejects unsupported annotation fields, system-component +and config-port branches instead of pretending to project them. It does not +import later-main update-strategy defaults. The sentinel belongs in Helm's +DaemonSet template; the pod-IPAM webhook removes it from Pods later. Chart +projections neither model that webhook nor submit its output as native inputs. diff --git a/pkg/dpf/sentry/testdata/charts/doca-argus-1.5.1-ai-gateway.8.tgz b/pkg/dpf/sentry/testdata/charts/doca-argus-1.5.1-ai-gateway.8.tgz new file mode 100644 index 0000000000000000000000000000000000000000..bf617faf487ee95dda7ca1c095d7d094820e1a8f GIT binary patch literal 9234 zcmV+tB<Dc zVQyr3R8em|NM&qo0PKBha~n6ZXg=##bXoUYNqb3(k{`Q8A8(ab4@b4OtdeYJx3;zz za0VoCGy_Zk3@L7s|9-0*zzk+M)Wdd?-5m2nV$EQ>8;wSzUjR~-us+kX((E74nYM!! z%d>C3>~A<64qrTbM*klUhwc9lpB@gsIe32f;_0))7f%llz8M}oJv?~+4Gh099i6|@ zSf;-jem1Ul=l(-}OiE>$6-t`X7C>5*1Gx~XV1q>E`y?F;u8mM~1Ur9z1y2XVoh?>~ zkKy6saEld1_2=R6@WsLK;Gmx~W4Yc+xk$m3Tf7&ma{|lCt z-24Ix;O_bV;&Aw4ZT>%d@$BpT|22NwTibA~ij@|#xrN6VJ+Q)N zyhl%kv|MvzK}h=BDzJ0DwY^4B4}?szG8J-0pqMc#5zKN2Bg=SVOO_GMT-%kyWqQCg zf}9VL+;s8&DFKbCDo=?2`KemIAp^O6*3)COjL!F_SPs*fbs| zmmj>Qhwlfb;K`tkY=H2u3768}4+yoG!bDk*Ybtn_y5XNrg%oy$ z{HPM6oZWtqY{5jvCK(^W0lnz=L35)@op3{QJ7<#3xF#*Ygyeb)>J|tLAm_Q#s}URw zUkiGW6y*q>Z8;56n`ziXF%^jy=%7#f{TeukoXz-F_4hgcIRd#z^gyV6xiFT^gq)Fe zN&gv>0WxM{>XP1!;3!+NmD%#&#j0r&+BL1_KCLw4 zhR?$L=o70luJ2}2)FkSj;Yo=T>R3t3Z|w$&YqdhDE3{aMjL&#F0;@~z;eAwDnR6(Z zoj2Dhv$ddCPW&(N{c_GV2j)c6Ps2>j{Oq?%^Ar`9ni&#bKT}7rzhHVlQ?p3?x`DF9 ziUl%K^FnER7w{wB^om+`;ur^TV+XE{bH*hyn5S@bdHpSzRX$NRoh``K^@vjT55^fBrB`)ZQK=K6Xc?o%GEH1;POu63FuYj9;hou%4mq|i0 zRawgQ9w{qFf&o=pBhpIN{j9PIS)R;2L-9ugH+Fp)7bH5}1nj#%sQm!jR}`Hnkjlc6 z38ye;nk9T%W?<%}P1RDuq_kkU&V^)_gUEB9qHB_oNpgMu`uyVk{`L9m_or8H16@_4 zaez|Ew8MHXg%vD|RVBFTQMmvsV|flHVG=tZ24MGK`0C{R`sgojPJ3`Te06+v{{H;< z=uHov4qu&KUA?>N!L#A3?~bl6&M#i~;Q8>?`Ni409=sU7IywEzhu1y$b90~5Yd=xolF z;lWHIN7B!vfnVK$OO(PC&Dij*aZ-)5iRXn~dE~6LiU=OLNmkK>&w$kC<$PsC!ZJ83 zW#Sg^-OIDkXKX?M#>KP^v?XpQ ze9Fqq23wJ0c$~}%becowcOxgfw5sn$C=KuTpp+RYfd$uEq$!uk)L2*@_dMFvAq@kF zbiBI`OKHxem~vyCb;6yU#eW<>%i zw+#XSlo>mGUO8afh@7{i;=!wIBIOalBW=F{Ji8TnnfsLljQAI>gHC2pc0USDQH{4O znG4B{8}A4#ilHZG_ph&RM5cQ1_EpYjtOu`O&BToDHKsYmmZPFY zOQbfns!l~%T=#Zp0SA$CY0;GSj^_X_JYI6noq1bIHl6as^7JLzIHlB%(3Fs(oX9fE z1{IC~beyxI5OP*e`$~6{+Ut?ZAs7)hXN98`=ipN3B`)|=&CMKiDa{~)=v8+^&4t0v z*s5Bl{P(mPab`*!YBsgpJARrkgenaviH1gAWZd!w9vq(^N}*9{LMerX-Q4s5!S!I? zgLLmD4hApChGud`oH-cMC>=hV$FMvtV~UC{7z#q(Ko#@_jXrV6^U{FkWCW!HI9kVs zxDH&jJI0r@+cAO)BVtCrRVPv>XfONpuspYV`|_J>#dzTSFF{|n3@=eYyL3MdDBAJ{rL9C?0-BuF?$ci<{ypgxLzCU zHQEK#4Gxx>8}w>Q4!cnYm1P;I$^TH;-+O52Q0@&V`e_vmNo=KZ5+CDND)Ux>pRF}^ zx1fw#`i{aYWD@uVCqyN13e^7Cz~xY!orJ`M5Dm^2NUNF?yZx?K4ik!XexoR%o>-PDD7Nn@h_gj^|wc0#7r_=r3R+s%J(Hi!o-YYxFk7f12{#$n0VnWa&J4ksWjnX ziSzLGDM;$_W)!KS1Xct0RFxWXZW$@&jO^200xr-JPr2^F4PU(?mtwCeL-F_#ChFv? z|8`iZ!+`n|%1J40IXXNX44?h^9uTRRabp7z_gByJd4$WxRNPjfnEQYQ_oIS}Gu&9c zsvVO@W5{zZ?W_NuYR>yprE`>j-UBg{WMNKL&TA8L*8KJjXA2I@IcLu8bj!;o7Le{S zH=+PLXBK5wi%DrYdg*B82Q)8zsEg*Z_ZVr-4Ih{W9V=C$HW|Q?g^V+tP?8i=*I@QG z5V!uk#p)_vgs2+$-=pms1a6Y4x&zSy&f8ytQLz~fCrfHe*Dg>V)U(i%k60MZoR3$f@(kWA=_9sk3|IaXC zdJh>*QAZlQnfxAO<_X{ zZjz2D<&8w6e^;so&RiHG2Kg$i;n@^T*@lAIjIzpm(_r!+(2K?QM5>eHT_#uE2sNj5 zY;OSPQxduur(9JNkF~@wj)kJZNX?8RAyG2X-14l2lMpf+2!9D;4JFNvhl#kej+^)# zj^=oNFdY7X60;OpnlPPK0;<*0XFzvx0=w`tg_rl5Ag;MvPtg z$w`<&|Lw#3v;LoYaCLV4?BGvBIKKW!EjGw|$T{%N<gZ0=T;|AN95i58EsQq=V0%NHw{AH`5(lvp z{GJ4eib-7FB#x`6eClH`%*J_E|A8|mYD=BG?aiLH8Nlg+>y^upP~rxcO|P-gqxP>* z7&BS`RB7Rcb1xG;xkjMp%nbu-PUkIuLt zfw4;;(=$uq)>z874O)^%g-~9N)hjS%?`Z66Z-dmW@ZKnst0i02!0Jx7?RMtKaVsa0 z;G~p6Fl2*;Xe?*xzy)P|WrlyJIi>Q9GxG>k0}Gc{Xw-VqctOtf3sZG0GH6gZKzJ>ou_8x0I7YQDd1=I6Y07Q@qGz$_9IU$E%JVAGKf; zwct&)e4#9Na%PsSXt>gaNMX1~b$c##;k!K6h=hw0% zQ5wNzt427|F)j_MNYC7H-T8m~GaiL{{5N``C&sVXNKr`(X$ zqR4z89qB>saCafLOt{Nq17$Wf-{#_Y}o|;Lty)NK|O=B?6l@W zp`f=dOkqjwA%R@X6UTu&j^gGhIpL&s&G`eBx8BMN_*Vz(aU}fZ>GO6{_L|p5epQY`l2jGmITq}i zJ*WSG-?F@*XcW5V)M{1bm5|MD4e_ZBOWDFY85rp$|IMgnK=|YxIU!ZT6t&t7U-{xn z3cKOj!E{)7T%DAO)3gX7HnN!J=yzu^LLUlr_MFmieiCe#k7xUrXK|G>Nbdb(&61lQ zwDO8BJpvy;_^AE&nJE$h{lx-)f|=$8eEjhJ-k^b7*O;3JeDVD4 zn~T3)4^M_?!zQ^WZ;v45L=b69S_JT)yHWg7H@dXffRN^F>;R12-j_VPeLjw{oca+~ zQ(&H#O(%jTfKQKe1BPrn$F{VQf*-7T3}1d+7t_Pvazr0LpdZf;Tl8LfZ;>qcW+>j% z{_Z;bi$c)5l1#3!=DXo^^VUz#JrGkj^{86jHsRvKpuj@vA03MZUF(VeeVZ5ii|OHh zy8`Tv{&dtrbd^Ei%GN4`XaH@OI8>zgI>Rn9-meOh%Xilqt67!Gdp>M-r=5=pP|qj_ z(u!2Y=@!)}!>PomGgQ!mzMg4mES-S_TFV>CuGOR&5DI%M;=oNMncXp(b;ouaPPi+) zm zm#f-;wO~@1d2`u0l9k1>RArVT0}+6b&fQtLdF#CB7#W+W(o&H?$nN@-xtHr6YFJlZ z;U)vWMZS3&muT`MHK0n}j^otyd1PbPSs`cRDHBxj(KO)VBw0`_-p2JQdKaPJiz6uh zgJuBbf-j@Vl?s#KLeA6fv?N)uodqa0J#&V=#-R+=qbX{0;UaGfVFZpSldD`c{pT7@ z{pFh2oUSEj1HY-XSt!}75TmI2WS`|e<`)S=P)Ln_xL_OlymA#<1Z4hCH&;?W@2FgY zgqZ>%u6)Qik0gwVDbX=BUDW`!j}a9Y#9Z+nK}Q{?BlxY5l)Xl6Vvwx|4BDfPE7nBw<6$y${y{d*y6zhg%lelQ&0d!*8mf`!mAL-mV%a3n)`fSd$D91{ znz`o_F=^mh_~P?u4SIjQSgw06HEDCE`^NevA3W(0!lu|%6RVWo8F|WaSyOA)d!?c(H3PmLi}L8zQoXK-eN|QkzFm^lJ!i00KO*7YmE!j_dr%Nt zE7u0)-VUqcm~;rp(Hgi~NHS#}Dv)h#^lb;aw)_zCunVa}D%)`BZ{OYv>6(mQtO;5q zrp{XmdyJf%+N9_~nHEkS2a!a2nH|@lHVEY=;X~WF!vgu;$>e$;H;CmrNAEB{j%#S2 zY=LwjywezoaFWYV8aCs=ZxhZ@J^KKgAi0_2!lAM>0$`IOMup!QBNlv&z&g;q4=`Mw zF6@z$K#pg_bsWIAA#-f&uj-;ldlDiYd_NX5qYZt1v(?y-<#@+?R2eE9a4 zqFrDW`%d zg&pdcj-Q|g29%4s~eQjz$>9((|U_S>wD$d0QWMrVd%gTAK0K+dfeiFqJhrRm(7;!2D*jhX@+h^FLw zVm#%aA};Z|ednOp+b_W<3px|xRSTamAJhAB^?t)zyeYjNDoQ?o@vUJ#W>@M)X05HU zho&`})|cb+elT4coitFA>EX7jQ8<8=I=q;{UkTO?F`0af9ET z;^hcx5kV4M7p{5PatkFL2Vmw*`)yq*6ryMr-{FJw7sGvYHvVvN{x{E;2I5kR+fYyO ziRc~w3&YD)^}T(O!S3bz>pgJine3a4EjScRG-FMLHTNQY)7o?33W(cfG?(wMJ+4SV zckP@T4jG>bD{{A)oYs%R=C^xP91s66W}fySQ!^p|*{01w9EV_tKkte(vauug?LVWr z{QB|(SXHP@%~l;zky%lW!|M(6w*euly!*mlU4`P8nZpP91868^Sf(m!H1Bh&SZuio zl)Xx03@?UphUJp$K5{M;PWx4)-mK}5TT8WAa1B=xzu0|clKafV-PK$CJHgvx9BZ*T< z*gD>WA2ub(L)aidNKmr_RWaeNRBI&oM}dmH&lfF3{tHQ!=@{~AmuU!uw`Gj6&xj6Z=?aI=aLmSv0H-R9-Br-1>Ja?tX(@ZVD$ULr4 zx=z)z=eM5H6=9z`rPuQ=RXA;DZNDIyxpU46s(Vdtdfjq1Vnq> z`0p#eaRibtTg4kyIO6WiiuQ4JR%YGr~AI_utwq+)t2H%TFQ5LtoDz_(N-0Nc=NY7~c3G@0}tTVX!C6xOEIYW1k|46dTE| z8#mFBXC6o8o_DLbM(OI8Oewex8KFd7IO~Wu(J@`^(r_E^WEu|Qsa}+A&A|j zyq%$@N0Pt-r~QoRLffno-hj2He20?AbA%pat+w|e`W_vCwd8gEa|*29%||YP%kdXX zvz${6kqZFP-ew?AH#9ANqZc6zX7oyg-^rU9K!pNx}YUXgSFwdel+XO{E-Kn40GCO^ebiTe6e!_ zWhh92vws`I!c&NPzk7*w4em&jxKa7^6p2T0M+ML|h0^B2Sfy(R(1qjtBD`gvmW{%>OwGpboA!nVIx5na8D?r0YKq-K0}nVMbr1U0NY`6} z-GPpNL@NU%V9V5uHU|1*LflRq>-X2;5T<-mx+ZAAxcOgq3VkNcSsovJRbi;Z2n)er z$MTG$E4vDNul%W}VYef$FDR~*$(r*#WN7}9mHdo0WT(R1#B7WX>R9LH475L@*lBl4 zgv4K0DWUr8mABpB(-PqmDR07|w)8fb*^ELpW#O7qap+voj+72mVPC2+ej|Q30nxx* zEg`RKTMdulI=pVf?F}y)oyplPtwUx^$Btods=l1iaQYHMzo*b00thl; z6sz1$Q_T~l)Aer9QC}{feev$%bc;?5zvs8L?F+R}-W?x}kFH*SxDNNfc>1;<1f_M1 zG0ACtP^~|9Gdix;ol#M8*bS$nkOjFEhVKpFUE#06*@iwm@yg{1^s7U5{JFk_#+{2< zxufpLbNsM6n#+I48lOeS+RLT-mmjJ`$hNvgew}%{&>!_f@?Fd#h17W=$`w( zp1wGI*1G@a`HREDulN7_8o$l=|2+SC|IdH+{Xbhj{IEu;mlz*YC)48n_z*FA&V&quWr5E#LW_r|=W5t}=laPwAf` zzb>a!aSKoSPr$=Mv+U=eJ#EM2#R97^OiOe&ekfVy13ra28+5Ss9e1#)y+sVz1N3nd zhRZDNdHriSK_*&m)=qjQOE;XSB@FC(KA&7%V_VwwU+`pauysBK%`?u7XPDm=WBj#s z?ikwIN+Ns19g0`_6J|HE6J2>ke^21a_{rm^kIHi;{dG*6=yPu!fpeba12f+z znn&>Ck6!<^J2P-!{CHN3s%&IQZxtm?>KK45aa65z7IpikypqpwTjb$?7O`78= z+D6uub2Pq=hh^>>zM?M{ad{RBd$_>X5z)%#n|#$W$;kN*E6w*lX+ z{||=GpSS)07l#K2U-kd5@%wuJ@9$dw$BLO1h1svPb9FuG1GRH>Bm2X2GrH3VGFa69 zo>69#xxb^+=Mrd*y3f=238YuqFA%un9Bc*SD{3z%y>kF13PG6@)}EN zxLEllT;KUlyFk!pQbHX0y>pQPWs)a$-6bnk(8 zh0?*Nf{w;qnW)a8|H72d+k`{8EZyBJ0LJB%>a+ zZ_Im^zLVLiRi?QnlJ{u+b(HNE)r7iFPiXFM*o=`^4`X3mzo*cl8EjB5;Po6gkNc{F z8C))62!S`(r&k}(k59+P?=H^HUysku-<+=DiM06LV~szmMjXMDXm+m^IKj_9KUu@l oRHF@G_gzxI4(oqubn)x&>+kFD-}(3d0{{U3|LzAIvjCI;0JF|AmjD0& literal 0 HcmV?d00001 diff --git a/pkg/dpf/sentry/testdata/charts/doca-sentry-policy-controller-0.2.5.tgz b/pkg/dpf/sentry/testdata/charts/doca-sentry-policy-controller-0.2.5.tgz new file mode 100644 index 0000000000000000000000000000000000000000..b484af6eb5c5f18e62670c1f119ed7f101f862b1 GIT binary patch literal 5316 zcmV;#6g%r5iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PK8wa~n63c>d<6=%TX~DL>7S)XNTkp+q~r&who+yoV1-wzGRH{9#jgbT_&ijqU~pArFWrktwn85+2cD;RT#2 z!J`O8?|4Rp@)snIzIj0D_xt_B{eAO)zu&L_-`{(B_|4#8_i2Cc>Hc8%={Nns)4|~2 z8|Xg->82%>iimIe55`rW+&@VnjB!O299m?7nPX>O zrSp&G{vTgfum3rTQj`xh0BhF&;BbFXU;lgiPaoI+*C9R!kK2EUM){6^t8N~J<^Hu8<_e2waci)%DTGJTbC_^Yfn|A!_Es zOC)oI35g`a%?z2byc(6@snT=1czyI1G!W@`N+5@JOs8XJ{O--mA#}Bg9`<^@IT1aP zvL;7*_CH^n@otAQsR#>j#G_~kyR!2HPN*^6`gbeEIW)hu=ltT2_B{cHHc{{G(ohsbb> zPbQGE2&LgC9F5LEuNV}1q~E#0@r?6ptun~SIKnVv6u^YgC>59q{*=(jY5EW}3eC$1 zMYfLKUw!EF-NS=}s{P*`91M0J?f=&(A3u5! z;)F7U?n+(owYBVm_vur|jUj2Iz_%g7Yi&yXOY1J^p4wUo8Y27-N~A1+!#(qp#uw>i zLT{n#b-}?x7V4j!&1spWQB^3DG z5dRHENF>^^8J(5NI}(=&%G3n9f0EvxWVZ&(BHdgf?LY4zC$rNQLaGy2=9~TP;t2R% z@3s*M5W4@b3*BG4Uq+9tzhMcSVHEpv)-yB@;p0c3EQr!@(@FZ4j7>*L=S#hGB={vQ zam+Z%<0{=olElAG$0!&ol-l{ogPD&XTZa4&e^0r>8grUZV#3H2LvOsWfHIBA{H=R4 zw?0F!6}^q)Hs_mN_I~?ZGIydnlk(!a`!w8k?%2_)%&topwlYIx z|DlNmB;F?*w0i#^^bhu)*7pD1{@&nm|Nk1LoXR0dB75^e=bEx`Sfp$^F)9)gMNAlY z$K_hmbgjX8h}QuCFy?MEI8kmUe+NcwJhyKlS~w<3$add)0PBWl{p|?F|4QO$IYhVU zs-f+M_)0_Wg;WQ*8rOPE)hOG7szu(N^~fvp%?!647!wuDUY4YGC%HAZxdQj0MVA>F zhpwb-xupA{T!K+gI40M%)z;ARBW&}Uj@ep|dpLBLa{UO#kM=b|b6 z@N)7Y?^8)1D3M}2r8zJih;Lz>sKs)Ql~YsOC7w+tlu@-P7^XWOHEIC_{yn7v!;@4{ zHoXXD7^V?r)3Yh#dG(813{sQPE{&{{)e95;2uk`FP;D zL{BQB=OsW{(y9u;4#@mSt@)Kf<$P7!K%4N%9K!NQSLM5uP}|@cl1R*6I`s5bg_{i<;R3gv}VJnyME>0Lj| zy<^Yj$a-dNu&L=N8^l&LQ5R(Mx1&{0N6ktLz3g6(Q z+2Y!4noyfbvtqSXY{fk~Klw7Kv)#{xIV(|LjE;`KFn*bkpiD%PrRkegS=Ft<&JD@P zdLIxxv_Ha5&%oimf z2^rIfDvGksi%{^Sxq>`6dik>E7}`UNHC{n}7*x!(s z88m=?_qGcydrpO>yKh-` z)t|FyQSoM%Fwin@xm(rXS$6lK%IV}G477J5@zFHzFIpwt*4;JQ zsHfm`%+VqYmydtNLmWbXzu&(@7*;xVD|(gB;_l~B|Iz84wPn-%pBR(izV|@Zo&W9k z>*s&{rw99w`MSD7@4l!Zc!~%t>zjRwHmyoQc*V5)e+~{X#(B(pbK4_LGKz;%hVLNGYAVy zHu$imE$wZ+wi^&X5URVqtX)NNeTQpYbhn$6e-0~gR@kzvZ-&3Zu~nTlEv=WM0-{+u zMRt4yTV?qXE4*#kJ3Q#;-AQEFYt%x!Ay&Ypf&QN|sejAlcCE#i^I&zWAJ}cnk9a_$ za^IX_wCXZNc0s2M!(&~s8V{pUiF7j|J@FRNj1Tvb1-i;NaI3<|TT!TLyXAkS{TEy` zF7vcH?^W*4fLa#SQc86{RHEiE~h>?yF-K-ZL|W1Xtm+H1>zx{Fp1 zTb51T8jjCD=j>Fqlpb%-M<<)5pA`{PR5n*#15isyR}bnx)5&;iR7Jv=O8rydlu9Mq zJ>g3d(U53M@t>CNW6maYn%eH4(JS_2Nan-55o%*dccs&`m8 zd9Z93;HDv?kyfthPC}NRQV*V#F6DQR-@G__`$7k)??xy3SJ82!_}efz;M!E?9#Hs& zFZwMhtNnj}hEYt-ErJ4+PS9MXtGjlMyNZQEc~fNO%KkW|QHY}Q6~nym z51#saxpUZ5u~v60bHArHk-5YrGhT`%$6ar(etK?NbO8EGTkA_Itt*v8sUT4cZPufk zabaV^EDQ)$;(R?eO(P=eBQ@TpCtwoOGGm|zBn%CkNi@FSc@=E>+?7GIbcO)5dzbP0f%1eL+Bj*d4QY*PC^PmswOW>@rnCURB*SE|?u zUJwx}>4c(afI7~bHUgO6Zwdq&6@5!%l#0Yj1?tfhE8xQHgxtje&Ox-I4Q3eU*Om3u zk49&gdl!{zfa;S7wRaE3l!X!qgYyL0h31^Q5}?&3jdf6>IBlau`%e;1MUZU)GuXevdPsV*wWWSfOAH6yS#n8<^|F~LUVn1Xy?A6tT z0XNxxzXtlk+^&#%*`!gZ&BPpq0)YXaGWsEhZQq2LA(&LO0<#1diR^v19Uv^M-{~L= z5U6oN%1ZFudj_@Mued;2>=z>g;={TlJ=9i8@@jBRJ`+4;k z-FFIMy~7!UV-jO@OeB6~S!tEkhYqtr83cFi0@3pPgJY7yk?_Rry7BrnFqj#Rmx56)Z4$wX~ zU{uE$HLXNk)J1E=n@VMK8O2ppo1)1!HIN+KqYCEv12Biy_A(E#S)e>G*2EB#+n30u zdM`NKYqnCBwbpN{E>!_tXmYu~hJ9zF`F_SGp4iPCE>FQ-+lDRsxIvdD-R&JnC5Axi zSIu-GFrh*!FCoD-Dwt8$Fsmy`jY+VEQoX4-LN8BYlCr=U@>fU4J{bLkYg|a9eUwe5 z-hi}hTWB?=s6__%qCN?`~w64oXKZA4t?bOL-^BUv5fS*vdyumEq+zE}*MO(rxl`xt-EwOyumRhzHl_jn8mw zp$g;Hk!$!|?m&bfiE*Vi@^?O}GeJ4DKcn++EK! zj)`bl15j7T-Tf74ma;-1k(^OB{SUp1S1&-8%Sv{y7Nsp?ElkR$%X_S6i7l*k4=YO9 z^tbb*jBSOIm`;sb=}VC<6@lxy$}rrOs+mAJ<59Stn=FmTt!orY-@`2fm?klS;5?WbJ&70gPfHT>sJA`nif6mdd zldp-}G%Q@=J6Cc!u+nA)oY~Z1i#4l|_T6#O6D)a0<`K?t2hNx|pS89AcMU*G?P=8k z%@7oi9EGR|jZCm-;!>hRVD}CmLXZlP9Z}bdV~@)z9gWT)OFFkjjq7ZBM5KBv2$M8R z`7JXd>qFjU5Hm6B7Erpv%rR$*%6sc2E_iG>VpU47X~MXgwXN!`yw#j=sj{(qs86FX z&UUn*Y+Aw=%m_0cJDg4IldH~zTr+;dcJx37sm!uNhTa^aX1Dn!Gy`K2OhJH&BIBr4 z0=i7Nh>5bvQcpv?uNNMpliZ6vd|(S4D3qsX|jrir(8hxUHy)^jM|2T%FJLy3mEL~z}QJCKP1Qv-22-rzhx>^ z!Vfz&I(6`x8jBQ`KZ2_llU)*M&6i6?J8H_N3iElaTO*XIrT$(_8I9Ap{tatUtY%QA zaJs}>vfzn{EMaCL zX@~+tATUjdAWR{K+FGtM4-Q7ja+4yvy~a{F^I~nWo~p~68!NMcd0FA*Z{XpV{JA6V zzlitYYRmoTdq4cTGran~^yk&~k4@*zmbHeg&95p-MO)o|nRyVIHtd+qluzKTNO1>F zNhEOx-Z8V=En^h2wB?}$TRiA$P~GLt2FLPthxRMe#7#@iImc*Dnk>CfmM%VT*g WkL91P{BHmN0RR7u_$mVcY5)Kw`*A7& literal 0 HcmV?d00001 diff --git a/pkg/dpf/sentry/testdata/charts/doca-sentry-policy-server-0.2.5.tgz b/pkg/dpf/sentry/testdata/charts/doca-sentry-policy-server-0.2.5.tgz new file mode 100644 index 0000000000000000000000000000000000000000..65857f6995d1481373c04321517b914a8fbe71a8 GIT binary patch literal 2292 zcmVDc zVQyr3R8em|NM&qo0PI@*Z`-)}@6Y-x9;n~LeTSW7*-5=l!wtA4U03whGm@^rP!tD7 zdTjGTkt#{4=WG4H4-_StlI6q~2TeAZzgXtW^Ty|e{79)tNT85wxe9W@XtD|v$`a-1 zY)PaJS0rOk4s1~rMbp#M>Nkp_{%>?LJ$W*IHkqDIqiHlfeG-kQlapspAUc%sKCMui z$S2Xky!y)hmo&n;&_q+gRqO$zc@grGrj&$<$VQcSlqeMyJce)o`aMj-KY#0yoW5&1 z%ZW$wy!{)6lkl0BqDmyqb=CC`p>&Y2LTQw6@#69=EQrQWWCeOjG<=d&qk?mhkhkI! z%30-waCV_UaL`MH^Vz>VfSQ8GNro}(^jlA_a*ToGIipFXm{)d%duWqCs3H8TlO zS(}BqkUI40f5uQY?==D$b%oS17)d>-_|2-0lgRWK*-3^fCkcX>o82%=VPu{_T>XNH zZWC~&#rf>-jV7U&=0&?ahBTiCB1cKIkbzn$joB9c-Rb=`$;dBsL)-4*E#$qYFe|PY zO&~$3>D=Uctc+v5>*b5LFe=BRgoN;RiI7WLni8ms@nTE zoAP2^%1j6nVr+P`NMMM^kdQEu+A}74-3EZ6pBX=z5xtCIR1!Io1s|n((QN*12|9T> z*hE#k;0~~qoa$9I=skrWk_2;& zsqt#!M!xZ2%8hTBom2Q1&`YWyF-JvM5@lvrc_BltzpJcF;GAG4_$6-Kwe5*(&+JGC zz{nLcRsV5MD7L4_7m}oSsU^|4Sk=8ztd6Rw)#Jw^y>d4nog$2`u{cB=g$G8IHw%J-{}jX$$UNp z>$O+UCk4|G+AE^TR}j9l$3-*qrxK+!74VBW6z~M5C)Fd(E{pk`{tABJLrn{LY977) zJ{r>uIWLy$8lUNa_ZMhG82rrN!QD}+O z^K(hLo_A(`qx_oxzCzPEB!X)~xvEzV^6GZ}PKz{xexoV@yR`Nf*Sf&)Ivr+4Hu<5Cb-+PwM&IOQpF4X2ixrBR_1P5JF#FzV&*d@cS> zGPavuox@a)NsT`+nh<4^Ud_fu2JG4<65vKPca1s1SzS6Ton++aU9Eg$4vQ{emXzsA+T(rfJP`BscHI7xRJ^0K?V zJHlc-*(TqK4byTV^_I8T@K|b{cLtnjnz=Bi+j1$iNCb=F?b*y3lSop^QK^{}SJ-7} z7`?Q(RELyB?aa#6Z~d`~|CWT5Q%wmwM@Ckc#v4-=!%1gKj*^PB9UQv~Fk0fBWLzR) zdU=SDc^t=(XA_vS67i{uSDO*eJ)4B~*Xd*qZa@UakjE3m9D{lwYI zzW&%!Zaair0J2YRPjd_yq^xnyt-7+)1~%Pv4d=qoJB_kCPf?AT>U(8iLGfG2z3%59 z+pJ3u$bXY<4?TGdPF>e{k`&W-UDoAxi1Iq#_gf+G{?4o z+vol-uSjypcL2BC|4*jlNgw|oN8{uD|7*0)r##Ko|FU!8dYsq)AP73&AyrOF3cVDP z{#K>0@cMZLTefP|@6j&=!^f$UESN3g0g#-&l%mL04DbD1ijo>NDPM%$Pqh3G6=&z= z=X*cR3sakv{~efTRnI9=URC1@)Mm(By3Fq846^D(aTv7We4o$<=-YeY9X9zM<7N0`id(O;Un9nb0?8gu zRd1-WXAJLeyu)v93I{MUSPcGn}|E$6>+ zG#wwm|9GHw)A{dl*7D}ffc@Ul$ySYRBI{8$vDXk4Okl;s5dDu)==x<){CLtqIM?pIz)?woA_K>Br0cu^rp79ozp}`%eG> O0RR6gtMKsvH~;|K>4@k6 literal 0 HcmV?d00001 diff --git a/pkg/dpf/sentry/testdata/charts/doca-tower-0.2.16.tgz b/pkg/dpf/sentry/testdata/charts/doca-tower-0.2.16.tgz new file mode 100644 index 0000000000000000000000000000000000000000..a89385f9da6def0d2bc79111f34c615ad5bf8779 GIT binary patch literal 29710 zcmV)EK)}BriwG0|00000|11DQExp+Wo&H#?7e$?Il+`suLrn0wdry?c!vYRS(f5)-CiBG(?kL68I zpUqZu5}b z3T9*y|L)^{+U<7x#opfU+U7I=-}dg_&hI+=J1^UNFZVh-`@d^>y&%^@BSf4!WB`BC$j4SL}?ZzQx-82gnZJ{+%cs>GM;qd`CkrTC-~EI zkEHBJ_i(!7ku=SJc7okM1#K^)G88ODxLvruyt??`zrjf|DVMt<0Jr8m285BXL zG-N0w4EJjEH{dAFq@v;#6H)<@HxiDxpvULE0FLR9WU+!wQVD8IQLO0BJ(n7}cp8K} z83taI$V)Asx)pF9n~)Jjv;e*{5)&X}N)mVZ3)BRU~T1ij-Pgj6W(AyE{h9RLYi-*c%BVZsu%9e4nW zjJhzj(w8Mnoz|f_CSt%vFrjfw5`L#;d>HdA^0^4dRN`1M-TbsgBoz@QNvjYMK2K>f zjQQ>APH9{k?VuBU-7w_3%;LC5LqV17!rOPA$P&dS^e84$o@Qb@*3g>^SDPm#)PV8Io8NDQ{ZRFyxc;Qt%;*HD>@Awe%)IR$>W_ zJjG)w-1K<+&hs|lY|kk`J^6SXQ=!LyAbia_0h}t3V#>qD`GO_s3}3v)Nmsg5?BIqCWkWc z+(=`0XhH^YiPjtN*D+17-7@A(Ld2vl*#to+d`flGQjwUto+3!8G0ha)7Jk#S9 z(MSuM$lCxqLR=u^{Dx8ajUI9|-(JQ1wmkOul_eSSDMz?&xJ4FI8F&Uvg_YcB)lerT zDjXkQzB)bbLX-|s=m?^?@a0S@I_bNjzAoqwhI=@h_yDyQ1wlhDBAFYH*tbwCClGX) zNeToeJfbm5`;;e=2H4SL$VEsYrV9Ij1FlF4B#}7f=$-1Pf`?6DNf>7lOGZFoue}Sm zI1vaMt0}UpBr^$;BScN=h*JsJbT}r-h?Y!W+u(#I3TG^fDCAaf;6+4{G$x7WW5l-W z`saBp$*IW_Ma7VWvD@LLd*B-pwn&#?bgm77EP;|#f}=hi%(Qv4HMg+QLc+&c>q`xLJ0G(Um%(N_On8)Ac8_d zr&PeLU`nAB&=TO_-Y0Nm+x} z`P_oX$Krm7>miNZB~jxpw2G%_F1$iL1S*z6C{Q8a&{^^5hR#rFMbZ$p3s@J220_8M zT{sk@OahB&qL`{%ur(p!c7VJM-8URdkc!1IYF=rk1;Pz%uL7-~bPG%x;>L8SG;szj zAz}tHB_R#Gd=}Kv-wE~_9^^giLW`%W1Q&(B?pqgHdAE@E zS`T-jH6@}YGK-2U5ulnzy90-z$HSX0wA3VR>EHdIGb(1Th{5|JiDx;5HsHjHGD^a+ zE%96$M9}zwm`N4bL7rq%fuJfAM$gha8fJRJYR&PD21pX1lT^+0BZz3itUZE8E0qe> zvbA5o#a3pgmUT)%E&pB%a0IakDN1BwcNs0qB^GO_| zw8SwTX9FM#P-x8n7rQEeW3IJQ%5mN*)Ko+P(NxpWv>^+mn4{VV_K|hqmS=GU0~+!P z+L#m~yP`rf!?cnm1D+)j{P?Ow3dM!7^J|QjT;q1F5Oo?C)j4HTDgwxLVt@(}!ZGK# zgmZcy8#kQqZNQeGDVj49A}ZLFM&AHa_!E0gK+=@psuXS#KG+W6e=;gjKuFM?8VWu! z)91%m-iFrfDDsqf@y{$8QQxp`@K4E;04@a&(N02eMn?#mL=^Rp;>FN-8*oEsph&9g zX+@P1DQN_xY;{=R(6aNxRS4iRCfcy4-Uh@h$?jT2OkV6+F@wYG7{6s6HGvC|s5(({ z0>^cMp#=7djA(>ak}MPS0PRT%DXBo%9bcPy0Wd{4IbpbbA0p?Xm%nSt=m8 zHffBBgVqV~#4H2Rm!WMY4JgFbH5)Q2Kv04v@fhVmGKDD-x|l-=c{}G=rhz$@!(mN^ zRL~@(&S22Qz(pXB(IpY`B$7d{cl7L(Xh)l=JRx`e36+wJXdf5-Xx(88V?in414N9; zz!CV`l<7jF)0U>CLv;VZfXFd~M3I<}0{H8gp)x^2!yxA24R(YlZr+w=?U)L^bj12s zxFG4~GmGA(R45RS!hX?Qe-bY2x z5(>cR0v$^{sKy5NeC}?^;b6OWy2x6gNEM# z3yUTZe)TkhM}nfdCWWn$#qkV&&Pa>`F2W5UEelp-JIvy^iZL7J2)dU!lD5dw_v>^b zmN}eFtPI_Q^{HelvLPC=im*glAi))lDUsCFwZ*CrpNL#4SfYl|cqaX4vN6}dX0k5C zBwGG;*`R1HD@J4 z&h>E-igA`i{s4Dl?u$`&OP%#-G~XS+wk_nDr9{J*mJW+GoddgyW0}7JMgUK+%oFEPsGR=@VA1bDHT4tD!#F+UA=@&(c@FV9>!!O zS4*9I;ew`P!RVWlvM^}abxb1{1#{~iSB(wzx?wA!usJEJZRbzyDyXrYw=^UyhAlU- z=HO}q6+DyM4cMF%(2)4<`{rcfu^E&)qff6#1*K)fG$e5>1G6M58GmTY z5wx9S>LuA^Kn3ujGc}tavv7<{rcV12(oe|UYh!v^z$Nl4B)=o}0s_B);ytB3cCH2+ z!k9=WQYNwin<@vVY1~)<9yLoD1SeHehR# z#jsTtsTvFT1#klrb{E;h@Ik(g=LrnB8UwlzXo9xo{DOYofxqCm4?M@1gKRj&y(>E{ zwHIH{3hk!AWdPNFS5(srY?z|Ukw-J_YV2#5n_B~!^-h0s6=i>R5wFk7axJ_?Fr%LZVMq+RIX z9v0Ls^_HrDnyDGgTPDjevHpe{MB2fQ%-PUFzzLEGs(aDb98!WX5^31C@eXc3Sd+ew zZ@O^rwiWtjj5P)ODES{O28tj(`#jBqsPJ#d#VrvLOGf?N8IZpO$Pyg$s0(i!OSnce zX5*dprWhXY6Qu+jWD3`?ZyWA$5Ms|FstAjvtzs1-wvYXGEnAmue@sb4g(YY2>H##2fziWCKVqU$-W15~Q6FE+hAvrH zw2Sa_=N9~uo1mzg%guR(LPwte5=gl+kn%tDuh#!(jFbsUKTa^{YX4umv%lAwi~oMn z+1vl(|N9*Nze*@~$Gc%k)Gh5;Ir9eXhm|XO=N+?FTpr4)6)TF7x=PO5f+90vNrX`x zd#K!9lf}+(7k=^X??I(8Yy}w1AcOCyf`)VC=LH+v8eAC~fC+E$F)lK|peqfl228bF zV*!o}0`pB6+T#G1R9J*;cY#Y?t$|i5v~{)8tORk0GxLeZtr za50yJET-$gD;Hjb=$tkdOdUU!?5}wqKo%6Zx`Ry&V`#!=U$6fyGBF?I*x+8^SJouG zzsGGSTL~T>Y~!1yPMBAVty#V4CfdL0r&j()I`CUk^*QAK?u)(M-39sovhzj$e{T6t z(p0u`C6Gt||5{4GRl<_}|DRUJlVt&FZ0rHQOsyd9OxgO2e+V zPD|DE6Yx-Trowx?@k)Q4>%MEj?^Y`*Aa3YqbJh0%1Wlt}ItkdE4*{hWqdAzmVCa zys5K5FATte^^{;Oz$df${e2yaeLbh^5Xb7Ie_x*Z+VE5X;Aui&#k5#}T*BSWP4~(< z-%a@A($>pIqqYn1cz7|vFi88`gf<>7Jk6sgqhlQ++zZDv%3_v`PDcsPAD`&hOxy%4 z)tv9>Iw+Qi(b*}`T^Pp&vd|>zwkIOzwDGt(U=aCJ3TIWvg;^`XNT;zz`656 zmmT@~D2fZjE4hgBrzQ7LqkIak$)>9s+S7XZ?8^|zk8fjTpNKo_E^s?EA>Im(dXiC?hthGH-8-ZS7OXEU?jy< z-ZB-AS3q|gTy>=v%$;zB72yt-u0`YzQo40~{|L|L{9agFb#Z}f&xN(-LFo~asg+65 zig`$4Jv-*oV>+etZ&)(q3x#ocoC!RBP{sy} zH#%i{l#)fv4?V%t1C=)=Rza26g+KkNy)#!L60WZCm@nt__|UgrGn`-j(OQB&)%3#>8HeGr@ac5%G-Q> z_1Ho*&;L-GPYFe4G^206_7hcJEu=gytHv^`Re~lmYaP&e3|9?k z4a%KseiTxEHhcvzvspa%v{gNHydY(k@skor);9rKeqzAR4_;CaciI-!z z_Fz@uJq<@6@=z>4Vq)&yvY@Sb5a!6cZ}j3i7?;n;ucGP-@}YWJPCPsdu}=S&Hd31< zpr$=r&Np+>cd01WY1;=M1-;Lrbx4LqPmjw^&ca*SSd)h3G#5446qz}Sk6+#BYUv|a zWpKH^u0D3TJ(gisH0PTvA0OVNO&#)BizZpgVv7}6dCrk#&{ptM#d%*0U=hdo@UV0V zw_>rjq|KG9!(U^S{fO8unUBkhd@}oSO{wg%SihY)t;keIrC~Yk?$K%=s{T@t>qb25#?lTdgA7<+EuE2{!xJPmLzn!#f8%+W|h@+Fr`;%l9@{xJkcIzlQoo ze0((WKW_bg_*PIE8roo(Ds%;?#kWgrjK__Mxk#Q;L5GjgOQzhs%Ky(ZC*02I{PSv9 zV-fzij__Y|^VE|1F{8kGRS&O^zV9DjzUrSIuM4?IJX;U_KU8JZ4w)4vv`(JX?p^Cx3$Ws7!b|K0psgKMkp|NQOvVp<3m@xRDeYd~2H{5@@x4*x$NB27Wdo90% zewC4+DfmCz#KJYru@wA0@HwXG79hv`79u*uEVnx#L!MF?#e6{G9M<$_cn+NxfBGrd z+28vqc)k>-;29=THxSKUYYw?U?D^l_%jawCldak&+kcWMeAE$8TVTZs>EB<*F2EI> zs}+E4lFBt8KQ6uTOwpsF+vF8o0ii}*uNL%6Ds^j%JWae8^_EHY#Y6Nt#{ks~iEFNT77{_$>8bj)k8!vDKxa$VKlv%0 zRah?3pyuBVovp^Oc7T3G0r$SHOIIJAX6riqkeZLV`pR&5wo0HaWFq}!PQ8UOGvien zs-?pex+B3|ua?o)4)?ZU=SI6?s)a7JU+(WOU{E)n`W(hFbNA0YPN3QfpEXYZF%9h4 zs7ls!J#~yZWV-go2QFp{P1dOCmF>;z>}Bw4=Kh>ioFiNfIqFpXO5{EkW~=uMYA{{4 zFR=>S`}=hpTp~W^I(Q*W3e5NF=!#KTw%PIIzKuVyKacHwtU_pEBjm%0w`{Lt`96nH z$yM~qS2&cYhLwZj-7=C@W#nGF{RI3UGi(=>-I@cwj^15mmSr#VlI*-&=Gh zHr{!@?|%RVaTAXvqkk(x@|fCXrz7NwUg7OS3vBV>eURDUpENX!Oy9Q9s~kl2-e-A| z@BJ?39@dZN&bzxw^dJ1GmH#$s|KntTc~t)I?6hB^{NLZ-f3fpo$K?Ot`*Q#5XSx3s zxp_0x2AazP-aRlU#OKl`-85lyTF#kVzPvTSO<$WSLMxQbsrxYp{Xn~L+{=NW>rc*6_cP6NZ3q!O-hgWuc|$CS(RJf=jb0VVo+m>8q=B2IvGvT<fFCGLu@$fpH|B~TOgoJ|ka}}}1Fz)49QM9G30&Ozm?f#;BS9t_g2M*y zqDiws%;fl-D@_U;r$c4#;MB+M6B1vWYb>1)UH8jePP6K@S%SF+y~^=8JEyj4IaE*M z?LZ%ybE8^!j;`)>+``c^MZ}1zxf6CJ9w~iw;^@ogGk;d;e_|}D4>kiH(f{q8-Q79; z|8i&N%lhxLtpDazXRRguE9p~*;+M}S{1gMx3h=@krc+M2w;$*8*!*w5*wgcWuk&)R z^8)if?(XgHeVPBCWBv!ny}mw&=LI9CTKfMD4|rb%@;3Fp`adsV5hX1zn97hxwDlzs z_-`l?cxnDG3*mD`KiT}>?X<1`cYklct^L34mtXY%XVL!~zb`^M@t#;e$DE`hDw7AB|4;v-;h00i>=T0n z!n8vauyGo}7QJH%Ht$WvhwTRZq2ZNdtg%S>clZC5f8hmN_E2~9(D05f zE`B&YsR_&`=s1iS zO;nJw$h<#0IqW%L7=jhE0hUUOqT2Z{Jr6g$`*>7jYa8y-r)bXJ)SmSK--mFFA*?1+ z6&joW)$krX4+E&DQU_4`_E-gU_5sOx$E1>w)A*s`c{&umj~*3yu(fUFF#4Rp7uuO) za$38 zqjc!A=%C7J4-IeI(|x?(D53WDzurAGz^6Y$2j0E21jU995aUw0_Pd9M_Y@lCkm@Pr zd72|X#^6Qiu+O4{#wHST1v%N!G0{)f^){laOvSv38gjzUUs@5JwvsH4k$J2Ga=TGR z;NE;|wSH=}9vUUSvM(Mq%Y22Wop!JvfB1tzD$gb@14hn5YuGKwyX zGAM2|kSxR^ffkl-`F-+J}We9`x6otv%-M4V>KaRX+~H%tMPqkz#sl# z`;(WoJ(5)Q^JiAw>!a;pvJff+U^h1K>%nHngYrAuKXclMKQ__aNh@-D1JCd64}Ezg zY%w^1&Nsl|&j)b+>KkBx{G-9geQ**W{H&;R8=R^__mLXGY=e< zj+0zeEz&aLx5;B=+$P==L1!siaLz95GuanelCWe{;SjVewZ`+cKC2j^xpp_S0pyz8 zB482xxu!-=_LN0P?*D_ozis<}diTf8MOd~~bMDyGXoZs~W(lonSqXUlQ~NpK-=CM& z`o-RliU;COeun2;fB&Vm?PvhOSEx_6;Frzq=SUoFEB8l4(OWXJWT}#(u(#)lfq|`h z2Iv-Be}DV7tD|kY@7{f7u2lss*o1^u&fT^tIDk!eP+De#t=8Lbn(u$OE3KG*jIQj_=<>3CLm#W%77v9lU)DzXyMa zq$?Jkj&Hr(utl{uS7f`PIcf7AF_q;izhCbVZDIK~=KE+)&w5ZyOm~EKUdKO29sj(H z#xF}-A7n`s(@M*J1yTjhU*?oF>~8_BvH-KR1n6%Hs$~O__RUSm2ko*M zs!gJNabjzvR@W{$E4+Y4_1v(R>)p*g`2GJu>+i+s;TG783b48NjaHNJpzR>T8UDXY zeN3xMl!^gSwmA^dA^aX{v`AhVKuj#xE$RY{TX`cXmW7B(a}hM-$#aZZAxT6;gkKcZ zQTyipbS;%ZhJvLkfa@{E7@X(QLd%6Yjs#2S1h-xV)q#}|W>WD<36_B(&4Du-!BB7= z*J6Q%n6J9HITC|hplHG01HZGt8OCFP{DT%xA1(aNEvc96yX|eeN}e0-4;t?$j<_{` z3|OlQ5RHnm+b)*FWyY{?tK3lo%Uk1c`a;)SZ)|_guh1Cl6n)r5e#O@sxgoR4xMcFO zbkPhhqm1f72j_W(w;f$O-V@#K*xoV8ulH#QkdEjjS&YE+kwB2z}{ z+NvXz=Mj`BE0kH`a9mTNl`1T%oZV$Ar%6PL|LgPgU`KzEDLI($gnpq)FIf7RvRaV$ zd551a^PNxl%jf?;pPK!jJpZ^W^Se*F|Fg5#-hFBJf7|<=HpYMNz36<||M@)oKX83{ zb@9J{GXbX|O_hlWMp@At(>S(iIknWyk9)#aIVbRT;B9yt`sN_q=yAtDXaq#T*-MQ7 z?S9?J&^zvJH;ua806~2&*vP_F+(z!|G}RH#S z_dmV@5(^CT^)?`45gecQn%bn&QPC-np37Xh8I^|w8B-XNfnZ@1!;6o&pa_^r$Pz(G zI7Z!r$mMyIThROQ^cX^O&_)8fx)I$>(&!uLU7lW@Af*Y!L{R}!(bNnaW4HTx(4h*MpnXKDuPb;Pa6@To;AyP!m^T~h-(~tR2@@uz4l^&hE)|(c4b~51 zE@>2aI0ERMW>FUkd5wNYGt`x*o(|K-;?~)PU%|H7chBHHe+`)UtsS(Dm%VSZV`02D zpwnxXkO>R>l?yt;NcQcOt(AjveUh^N)P7WAF~tc zMjW+li%%t|7JtV7WO_)@VVp<%7f9IUuWiU-qq`T_SAi=VI0@E|@{~wO(5Nyv_^4>wcDSNgp5iF7VEq9z8!m{RqW#TN~K5no7Z#bfL>|$&Gmwd%;Hj} zL^QMD)U<>oz)bUp`qdeqPsn2v zFQ9)$s%SC4P0UCAAtpze%ZPyg@cu0Oa!w+Y|5$=<=sx;6l2I2LZ*j2R!L#1=;nj8j z`tr6%A$S!Lv7)_TBdv zz3XQ`CFML=TBFyFQ@x~dpD7b6BXJ)$QDnFB$}tK_W|ijv5ufNe2yYu-RdQhEn9PB~ ziK$gS$}q>zjaCC5=8DVf8pbr|r%n zt&$}OKgbp7Cz>nujp1}Txl@M=4L`2X+-K2{s7n!#I-e5Jw@gLCF)b;P;)7;v-==%* z4|p`2?={~3epi&3DLi}jVt;@4#X>=C)$~&$@P@Gdgi1+9^{t{@2ygmCI)F80$_jiD zsWtVuEuH`6JlENm>+y*up!gh2Vp>8DxzgV}ePZzXtXC|#FbRNLhjsV%cEO%*98*D! z+ZJ#BU_%xXvp6toAr0vG&DH7ociOnjmqlH(VtFo03nO zCMk^#x*+mK=0uOoiX|nwI$n_-15-eRU{flClI4WBeH4=k7l6N6lj#vORV@1~!9%}_ zi%*GXr6FX}>sX#v8m=|&3aKXNt{E(MYrd2WR9wu@`u;q%N40HWM~?YzVwUmOv9zfl zORZpB4Nv;m|FAAEq}l&4?~BRTchjg^*mXOfZgX#!b@k(Q7zIR*oD6ufjm#WgruVquDz@t!>XblAIL9Tw@F6(9WYO99!nl>*N~R`t!9Ta~pIY z7e6${6^x`35yu5}HqnY~LFrd24YT;MI#xtEbbib0##~i?SulQr1tU`T*H|#hgdHHH zHz&R8ljDBxGqXQFZF`h?xRJF#IarYZsi+!gPC5>mI399Lu*cK>(#5I zO6p~=j$P{7jY>6}U%vi(642oh^>a4qyF5I76DOOCkz`gM}X?r3Y-FZHCf zMtLFmHgMO|aWx2nhYLWBNur-4v;YnrgX7LQhVWtjAhhCwFeX_3zh=!_f)enXQ+Pyi``yT%ZQvfRwM)3T11 z^tWT8=#&Z^%&MqIT9m*Q+2U}I9Z1eXnw2$cBGv)CrbK2!N0R`-s8`r&@4WEafAV*B zN@HHfu`c~Ur`>*LJ)JsG1D6*~*lD+c3^9O$siMnh2w7D6H*_Y;SB9+iG!-0B?}yUk zi#7#8qZyZ4*ugHes0v$B#w{Fe*$RnY8g?0kLg9XQq1T+{Q&MLb#eI5Iuc&t9Z${s& zqY8GG>jFG(*e6o5QGx-|u4cVmXt z|I5F7{+B#a3+;4jj_|z5kMPu;U|f>=WbxcdN2hi0o&dx1HsFd*_>|h<vB!iyg%tLx)6>M(1&C77x3$o9<}J z`Nh@i!?T9>MgIQ=l4K7Y|qziM+PH|nX*tzvj6hUb?f-#?Bx1H zqN~7veRzbS^BD?#se}4a^5=JrRnIdZu2ksotAxY#(WPtP@bWb9Ji9mCch|#oy<#su zqKVx+F^j13bFXUV!`pY>uRH%0Nn7as63R!bz}FY&_??{qlkd1{A94P7+AsEARL_6y zwA;I1?*I5a_kWGHeyK=xG?uH10N{D5_f!^fJLHo!!Z{i=m-zk&lZtI{~JX7Ho?{W)37|YwSeHB z6h)L!G+}&m6&DmgrIw{Wh3C>hnRs|h=Jy>e0tTo(UT0zZ7Pd{fBC~LeZXlDkiVHw# z0Df3TZ6+0+==3+bib>b=G?We%9`BRJrR;jiQ!0q{K%+J>575o++cu4`%FGRaO&DnV zya_QI2>oTiIr`B}+*lKCs7Pq6`CvlAF-xd_%OYx511VACz#$k4rl?>f?*`IVWSX{z zIv>I!dKW$4!~jJFF&zESQQOoRz?vp$S`%8C6isNwY=AE`rmddtR})>@`kTWYB6pD@ zzBB!38TJMxDAj$!Wz5eRo01sWsnSD4H~_q_QG!m+#fGfdGnF(Y0)vV}K`~5%k+=yY ziwbQ_=`x#DnpiFR2Yzi>+K@j6GQ>kCW$4-=nzbw z>BtSx(7T(Ox^Y~guQ|e)3XT~mnz$`0e5bCP#^c|FLB`^!35rfqZiar%t{_%Y_b}nr zVP!JM5?)h51SilL!VBSCsk} zZaVQq&oYy;$j&v(4i&~nIcHq{PZkBQvE~d&&SzXeq;LMI&Mh4Vf(!PEp#+-CNQfp# z#nV)O!>UY%Xh2~%SXq4G3CICvA+ZLMa}hroVH+8M+eHL&sCK`P3y|4hf)VNPb4F2& zqculrS}ox=!VqW{=7n6)z|#;z>lcWPIKcipaMs)I2&k`u!uKRGq5rsio>C)=*AhS1 z<3dDQ9zt)OL?#}vB$5?0O!Jz<^`PNfchS-9m`f}DB1kiVR{$X=>y0kg1a5dCyDDy> z)=k<9PDX{xX09S-Gv_9%(KPo;N|1c0ZgG0+i7_EF+k`os9Kc`ku^Q)Q0gZ|(w=9k! z;=q%Tg3Y0Wc;dETxdRJx$PEmJQsGAPEkVc5=xM`!l5bX9g5J%Vpt8j2moo6CjZec- z$J?+(LKT(s?FVq-985o6o;zK~6VQH~T$DVSMd&I_%Qzs~*7z`I0ku4AS`)>Mo+d3C zCpZNM6!KH(#+KrllBiN~&`>tUZXN`1m_V=hy=%fXx+T$`xZ1`JUSGXAsocti$HJ|$ zVG?cyQ^*`dYOxz57G2!dw)beE7Is3P_9fD{t;#(*4s_8+iEG? z*$+vqgA!VHx&#t$7MJ6HG~pj4RX!%3v#%hp@DJUwh4)&d$gyvdL;VBH<$JhnH(ai3 zS;FH}4b!13ni#2mnY9>KXFp501P;xf0f)M!~vsxAU z0Y@~lot8zf1Nd*ww4@<=mZq6jhsXe6DAm-e5^Jzzz_TQBGla!Vp{qG^)XLd5DlpZv zBzH9POc@XbUP!A#Tj$m0Wt^R4nmgZfmC$qA!oqSf7AU?oL?=%)1$mO&d%Dp~PZ6B& zT;wpUd5o5u>q1EnqwX?l@M(yiphl&|3dSyLF#HB@_OoLh&z9?E%V#BA1Wr6j-v=FM zq0Hlhi%%tiC26KK8<@<7U^k!5z*<_Dyk=|3`@?EmgJEmPt6X>DUVOqn%t4S>LETp`+KDxpOR z=LFU$VI!%^7$$|n?9w9^eZbQCFw$@)#R_{4bc)HYj7}X?7Rxb9vO8n-8KdEYkZ5Y< zJGCw%JaYnPx&h~ZbSgY=kW~h#geFSKTTH|>W~sHz-h1!E%REf-PYA3*y~%BSIqSMp+m`k+E?Fgnm&*&>Asl&LW8lmpCk?l>mN%(Ezqwbe_ z>~uGyR(7s`TUi#~qQfr}m}A=jiOTjCQNiQWI{n0S0vsnRVAWo-L}hN=pr#re!|3DSQ_; zMa27}UUq}@G$>7BQhsg;>D&8<4f#vh&}7O4PwZmiz0t904xkKfN(8eGs+5LW{>LUn zKA-fmX%Lc4ZOIvSMlq(fiovH$JH2b}_5}1>BL&}^f?7r4Y=<)Fx$%y;qRF(0J{S>d z{3M&Om=%1Y7}Glv(U47Ww}1Tn9Q^j zz+`kZ*ifTzs@8i6<%qry%E#28Y)~~V=y|WGNcF@Sknl#vM2=gSeVc@+;i(Ksihl8* zGn#}I@;T=AanD4#h2ztZLX*qE!7;<_qY2dw-=##yRihaga$B#o7{<-K!iS&%pqsIT z;OVO#=)46VVE6R2KFJb1>xTw~)`U(g>P;nF%9$0QY0q{Bz27*mL8dWN=u;|T=K3JB zfmBSrhf@U!7x6eI!Z(?0#uPK@py!^1H;RQf+R>gjK+7AV&t<+y)$3bh!j|xJmfTbE z`~TSc_U<;0HPti;Jgmy9mfRg->oXv(NDVw_z$xBjp_Bg&&1JeM;986E92c)oB zefN8Q)m1$`0Q3MTC*FIcoUANzrl-2Qy1E{}s_N3WZWQm-LRc?F=21(0PQ*{hPgbw| zE~0H%qk4B>kIl*Qo3%7dm&-&Vw?RE!Pdu(CqU0_wvJqS(+Y{rW;5iS5RVeHbK65lz z17S2i!gwaofpcqpb=AC_;Nd9_gj>U8Br|FgX16h&7wx+)K5p>`!dkJ*l~u<|FW_Kt z0LOY)7^sB~gmgrtz~SE*AR^q{>Vm(s&6+BDf5mq5+g{cAEGPF;WX5q-Qo5VNR86)_ z*Y6oRL6vl*tHQQK9gA7ZHZ3kpAI3A%lq#mz7tg@yzyp~LcR)m#FQcKTx0-iZZbs7O z97>5<_Cw;MM4hx4Juk_3$+T}q9v$IVx}K2qqHdEZ z7Qavbo8>i*um40?)zwAPMXvn-(#0w>VTDnWazCJ<-Ilv@XO?si2`qUHT%VGJD~Vz|n;gFZ4h z@(>;h1N|N`4Db8o--z_550Wy{S%(NaXNMkeg^DhCIx=tl$X*wXq*0;E$(%wNK#L{A1b5j?Ix|k0aiTbm?)tA(rC{YJhRT%ll`48s+ z&Heeu=Rz1wrFVLK6%qfx2kqpaN*ss-@V`?8EG}-mSgSi*-_UgL8((gR3}Q`qnejyN zL;c7_6i5&r!P$fh%+~PAL}Z6ZdN;bcG0`67RjM!^G63K(I67xHX6(KWfqe9AK<)g3 z)(K0CgoZx;=Tdx>u?>LdC55;}0&OkgfrTUg@Vv?(9}_ML!Ui#W(7kWL2kxmkMe7`v z_0{NFk&07{-LkRFsAIC1lodc#wsXin*U-63p9R8;7uId{B>h979{4+?6?^MyQ{>?l zzvbAe$3z^SC+hO))j30C=qt5bl(GF$ng$3bcmaW96%4?}PAb^veyl{5BmBQ+$RQ02 zH>1D{pi<3C)=+<^h>PDPBqcmB($p_-Gm0)p8h+M0V=6hO!w#0g`MS%Dg_dt|#nm-r zETI?wVbGIdBGN~i#a^}fg--_$Xk*&Vj$um%n`}O_o#tlp-G-dN< zCSO%}JB6GgaH62<;5IES&vw<-9jmpodC}x4tq4p=IBv*<*jOGPibh|y19?>d{1BU! z6l?cT;*xxBo=)#~SaLB-nntuBTXDc8tx2>Xa7R*L5TOgwz_hwE!>I+rw9>9IEjke# zGSy9ggp_pRs*nK~k?P>(n4=QWuXcDJ^WHCHBKoCEl*w{F?~&7CRNGShYKO?6z(Nm| zDI31qb<-chb&2JrJapAk$NU#jj`CJBG}@6V!x>aJ^Xfy)f@xx1apC2D9byy$tut~V!1L1&lj_sRMcn^JjWX0dAKCArLmIDGr*`~+K= zE+Xg`m{M^2RJFz693rFho^*ng{wKEXV9z>q(Xo~BXk}E`6F5zyS>Xo2FQ@QKa4eA% zTeqJbC6^!(7zz=WS^S&Hs5DkWxq`G8=LgS2pkE5NbtvgG9K93VjEMK=PvXS)lrwyq zfeQjr!n!_|u|X~tuE!gT3h})Mbep6{ZAotI z&u_UjQIp!vE0;w0I)A!UolmAL;RONDGce4XN|zHebHBEQ=|d5XAb1h|{1~PvW{@f8 zv+b-=ev#}GQ}>BxIaN*#&BB&hXNE#hOg_oN{*s(x+TxWL`z)Yxwp{s$Y97*bQ8&vU zij`Hd4CNWVAt5jfzyvdBE%-q!)q_f>>ozYlnwiL>`%b83Y`H|eTU`>V_&Z4AqI5vu zUHI;_b#(3jYKQBrgO2psmRN@705jj1dJF}n6R4zm$S#3T0ti8f6?Atr`D8k@xLJhh zgI4R3$l?mHGd4KldBj_C@1>3h6QBXHQP3se06+j4X@|OK^GNx_5n;}Z<~0Ifm35F% z_g>pDAj8%jX9_m+4m@z2*XB+aMG|yWn2->YgxjGETgd`#9eOdOv>(J%##vKzt4DL2 z=1w&E!cJ79XBDNAZZ@+c>g^4Ft>^GV5YrS*VBL9z;D-h5K*(tkKAr#b2gyY$fO}|YZ-W0Tb&_k z3)ScyAPwVj&gi(tR<19mgI0kUTtV1;k64Y!_1TH2B!Q`A;JuBP&vPt{s$j&5ixp}* z--Hw|#3s=Rr4pTC0V#6$9=?OK=o1oPd&`)VB}8*F**b^81@_9}Vg`d@S;Gnh*MKhx zfNO4O3R_M>P4`fAU^p*s((oSnbVcurT?%#QWAox>%he7<1Ih<#O~;LZY6@XG?ISZ3 zGx94XUOXlU=SxsPUc>-T5V1&>yBh&Mh-5c`rp<^yyEQdMlu>~IB^i{jW03eE?SjRF z<2n+i2E3QL_F<(WJ*6XC?s9+yy~`q^+#|-lLIz7P-`wwbF)xD>9c6(haq_EL`<wcSzW@a5T}X+kl1me^4hDi+GGjBi!&Y-?a%Yq?; z9#@OwTvs_?hnukjTzN`IHO4~3%MVYK%qrc`UIY~R zSUo?ahj3Fhk}Rd4n5=#7jl_i~f~Wt!YGgYi7sTX&iPVtjOp0IH*tC6|T>a>3wv==d zX^SDu+p#LPit9;2R?ZRTw*{4P09u=xQo-QQ%^)UX#WrdE2(I!*imltMw~j*7MJCdF za80)#r9IqiYR~aX&~s=wTX${;(ZbqQw{;v#k*`Ojg-SZn)fj9uS?~qr zCf`%L){JAKk!-D=o;*E^KI0a%Asn`FFWh_pJ)QyrN~_Me(f1;4W+q;u+HA|d9knGI zij-nHh>ZOUrcQ|O(`joJb;91pf`rH9dQCjBnc0-gpm`;-IIIwOR%@$2MVw*lC2o$$ zl6=As2m0xbXzTEw*s5@y`_gxuCy!NbP{4xG!cNb^RUXkg^S&xVLEaD3Fw^py}Lf2;oOq;tIOdjr3!s=ODu&B&TWvIb<;rn4lE>QLl zdH-fKKS@GU+;URg>UptU`Wte|BtgbI?ngJo*y8Cm6#v70$-5>WPdguvS6{LUv-zx2 zZFF-p!cf(tCaFThG}CUl8-d4}#)7qV6(Sl&MP3c9u3;!_8q~#-(3l>!hcENdZbzll zJQ^7g6*>?~9OOPZqikqS&AdeV)Rk1sqGtt7rM1=UTH(&o0R#~_P zEqb=24Qt{s2u+yBB?nAOwnc4^>A|w&DgdkZBNP9}C+)eQMrt3;# zM%344GMA+4dzeNEWem{!-S7qIsGubT%wqe!hz!PrRU6vyhrYnYYE+@kF%7z92Svr; zms_?NV4iID<>#Gu*V16isFa|;h9e3D5@udCRDEevMmhHHv16C6`xa;xeTTFGc&J#C zZT6Kze$faPRg$>dbQKxf1OT5b1-x@Qo^!X12h|=K>0&hB#&|g77wM8*9hY`uf}6>X zNh6BLpqm1U1c)3>&FIF>XX@6}Tb|h;@h12%%O`n*&6W=&iaP}_i#6C1N&D@YF7vTL zeKN^1%<@SWAWj#6A+I=5zMWFhrrY@O(v|VBChb{f#!gs6!toV@VT@=4d5zT2j3MYr zYH>PUZBP_@wU{mf3cir}{X7aKj{O4>R69OVQB4;PfifaBA|rCk+|%#6T&^gIixnLlK0^VXjz18$ z3-C+u6zZm_^C5r=5pChOQ4Np~o9RSx?{)D7<(5+eNPH5-? zIx$bz=m{7f_3;) zR~P*}xtz+qwtG!=FD>0dJDq859se_BI3{#q5D}5adJxbI_uxdX^wlw*?dZm%h}7Sx zy9=^7^O;GJlrpV@C*_Tc#01#@9Cv${Zx7B)0Pzv+@(ra zzSu`BUhr5%5c{x#v080LkywS4HEJXQXYKS2oX>hwIcc87inj77xa{X z3=&`(S*Y@GM4e0uI91#u_dWS8ER!H#Wn`bKL8PCY$60Z|(=Hc*svA=eO>Lb*=__I) zG;rTOkO9M`4QE6(^3-3}eQDL*+}66=mI%(q1A{B)V9?+^VEKlmFsz5RkqRt?%n z@kv6>Nl$C6oM-0N6gc)Dj9^X?Zy^GyB>yMs`@BiyW{ZAoYjt?iKRjArCo`f}c;ti7 z3K0b9ki$53Zg~kgZOSD}l}5!;>5{c1E+NoxkWKR&@FCDT>%YZRJA)AawAP^%S2T@? zV405^)q1pf!uB(%ZKUjYjD&>9lM9y;dSxw^$}xsdwHHoj1;cZ61Gj*P?gsJgxEZ;Z z7_o7kpoE)qBg*#6>>l;ECo77Lc>@KMNo0rk7f?S_LypenXXm4lF}bIwx^{{AX`Vw1-g&er^V=--Pr zel@Iu-$mY$?uXg^jt~;cNFqqIHjg-1wG>MJfN8`Xj~SC@T)6Xu*YH(G_o4i!FGLg~ zjiF8w7aEeZ&AvR~^BmopRCEICBHJa#0^W0eZ-P6*+W~@DZ(J$8%&8Y>lrGVFj0dHc z(|huAZs!g~(dH#OS#>1jof)!ez9rU*(xTZ8frZW1T}rNq2w%07WO8SQs1}V74Z^80 zzXfk&M5aIBT_mJrDc?JbR~Sr|pPEy`Im^vO5REkQUx>3%q*%&$pBy}2BJAS3gTvSd zO}U${FMWnrro}#G)Rn)d>$Ws^Ocme=?YPNGg>F(AxZ1fFrgRoMc5OLHi~P28Yp>8A z^hM49`T!mimWMcFNS*^GtV)9Ma&Y)8X|27;m?Gfvh?$Daz!hR!nG{@znW+l9*qrU< z)C##jKJOzgIFC~Yh*dDW>vJ(DZU<#*Qir8TWdBz??61M~@E}w*zUB}qicd7yx;v9`7@#e1(}ubT4Jl(OjiRX26?R!;7~}34;pKifl6D4 z`D~e*)TS=tpqG3_-R2`uG`97~r14Ef?b}G_{hO(6Soc6gSAqzg(G1!PAx=@{#oCy9 z374%;caRn0do6LmgE zqrF;Vu`8HK#-0b7u;g!{ozU3p{JlX6km-^Op3}J4kvOIQ&0itL^ zK-90rLd`X8GQDjst)QIZnBaNA;d+Z3OzK1OCpa0D|ehwj!R@s z@Jb-fkmJMoWP1~atT13>9Ve)<>Bag*jNa*fJ5K8s8v4M&n7Uul-#o~y>*pQW`o5T6 zf`I|rl;fibcTbM_LP!lCiwIzraMO8%3Y~bXHbZMSWyvWZIE3V$>H@AUR2EfsJVx0J z2b>j$nJ{($_D6(>!gC)fze%MTfgj$QHsBF_FdKfnwpEKQOIYuu`Q|m3dLL49bi-~A zT-u@Sgoj;6v|)+3;>`AEj)LzC6^+%g54l_uP*@J~}Px-J$e{bIRmh`ZDG zNz514We6cB*PBk%pyu8>aP!jV2-p+Tz+2pz-Tl{1ui{y|3wYn|-E=^_MJJt3I7wuy z>iKOYUoN0uE)Pq4$v@)S_^U7wd6t`;Z0rLK3BXjWu4+3YxKMZnWy40cL~RRO5-tUj zYKn`C^8?92=v;rS^J0z)_L9w^{^_4Ei@*OpUdOyL@Ax^AzYuN;b@dr8 zXvgFn74btZWt8uAbal5u{u~>OFwL|7SIQ|5h7}0_@g9O zZM;ku{9KXg8^Z?HieHHDqtn=O_HcH?6Z&!)61K3Z9y!rBc1 zGXC(5kvDfYC6p;#j|B&vpr+BF@t5d*vczOLgM{1o+m!8}kAZ-MZQls~7JYgc3nF8; zJzC-#*_!#D%Cw5i3}S+0ZAAMz+wvY4=%r~|CB*4m9iKnuy$Et@lUip|(x92yj=&?s1m1%d zpR8XM8n}s9%`)VLD@psE57n$Tr<7A+mKiOK4A+OLht>rVr_5+-iQS~L5lLTX8uc%) zRxwN~_1|R}(ttRMhqLE4Mh#s}^KLmkm09Xs!&SEnR8Q7La6-4f<`xLh0SF|az~QYf z=By8NBf$h2KraHYnv`S-)(2kboG)ss!V%c_sPjUlm$4$UGU?E0QpAZMR8xY^(uAT} zZO(SQ4zX$P^A^3Qp1mV9hjD#@HelT3 z><*s^5*B6^^^X}zngu=s0cRx8iT$%PoHbx7h%NDGD{U1q{nofYRGn zex=$>R1jnZ*o3RBZnkSUum&e3buCz^BCwylv8A@)_Pl2`q7s1iu6r$1D%rLKk$Vfx z*j`;Y%P?W5It{)PA>A#LZm^iHOZp0-Z{nv=RY;0t=nO@qTm$>J4M#_ou$aJWC&AKK zCK*Z+gHf|gN9CeP|T?^G(`xKHW4Rz->GneNVM><-EEE9T1jtQ z)+2DPnAh0H%WZskreUwzGJ(7#gg20d$Qx&hao}sJA=u*2kSn=bCU6PB)kU!V<|i=R zt-*B~F4q*7D`p1N_YW~u{)Z9N6X$S2rcc6`5kRHAnc1^%OoP* z�*hNh$MYV%N~A-$W^0UQB4LeD`m4-3Q=u<g{hRH6@1^R%c!}b6u=v9%DjyMl5EywL&7W|2v)O3aEHbXoTPB+-RlE%zJXxGQn#Y zYvr=US2Y#q2=<|L(1AOS=8IV&^S<72nzmj$({(%qErH4Q3PM~p!X<$na??Tch8FUn zoR1(&TP4MPD;+q)`jp(v$}xQ{n%B=-a(TnNJB+1c)l{E|*%6x?cYz4VCS(CuESs2Zo-34J0%Lp#W%-sRrWr6`&?p7w+Dft?kQW_ao zco(@P6o0_IuFKJsD>_)NPb&p*f~~b~6>2}#*yj}U4X*_cXY)yEYj&#g^mUeP z-6%7RK*V(h?;MWCE_2a)?cc!)SLiH0!?ca?FETgROhW2+h1na`p7k1{VqJfETe{yN zZ)s^8&fI|C!G4?y=u<^(DRrG&IBiJPJw3bn;pFstj!}qgB%K)jrtsWdZOe&DQw7l3LW&ULH(Gsv zd<8pDe6uJWL2m3cRnD%A7)T>E7jo?mG>p)ykOZnHxi;EaN?Pu`5G1$!eUXo*dyv6M z(7!eHP&fGu4V$RQQ>OF;sG$ay7778u>Ld3yg4(7=M537*n7nl$^NyNe(2w2J&Wr4c z8uT7NQb%TFNXk+&;-L5Vfw~|=EDg zmsijL)ZFdyYJT&GrsSr!ZcJwtUtBz6Mxs6scd%Ui#paB0D}1+?giRmx2K(utH~3oF zQbr3(Ov_=Ra;#VLnmem9GqZenhNjFXCHLS>Q=U^yAeC9&^ZW&3;c75OI_Kt3uy;rQHP35~r%5fWh02>&*O=;Cv)5B6Xt_x`+o1ROp^RutCBo+6 zo>PUHf%iaC#9#p)1w*@(>MW7!tPKY6vdGRVgSsp-mVo1Mw7`QZ z(~ECA>y_mW0jfEQtmtylOsSp&zb?ZZ9Q}B5d3Nzvb$0sfuhg`JYe>&}>Y!bf6d+#M zXydulI5qHF5C$8uj(jG=QSV&lfSDz9fzRp1>S4j?SU`Bf3-9|XyBQML;pWN!s54LE zCByS&LmU@k=Bk<3rIfqFpqlxJA_Tx67?t)+~G1x!8!2c+sa}gwHG(KT-@n}N>ieHj$6^? z2sp0Lh5HA`y)d(IN9H&aG`Fe1X9#w$8VfO@8&rgw+AL-$84rXOpnM@o38+Volw=?G zga-)}!*wx1VM*GR_>v`FFqsU<+d*z#&L4oFf*1lPg@_nT))aj&CR)O$LVyeN1i$P*36BI%ST^h9nKbH0` z9MDLPMf{(06Z8_&^Xlx`5iKO=`>4!}^33!m7Pd9{*Q;;t6v$9Hlh_(l1oz08 zJ7E#$L9g&5mKl_RW{N0nqK0Oq=PnYbYN|6n!$VZSZ)_0`9xWz9;Sg)!0l$Q$y){LH zWl(gmJ&|}$E$m$ViVBHSO|$^By|Tkyot{<>TZbJ4SyvYahsSa>0vAD;;cFy%feN>? zu*o#zaE8$Cr%+Dk8JKtP&alpeRZL`iSc=k)p^D2K4%pi;Vn<=kEZ!0o6{|kYM^iP@ zHFfvJQe&^H+S)OBQGQ8wR+uKh-y&vUEf`$DVih8*SoGA1XtAs@qOj#c4UK0`kRsDv zknKcr;!w}Ibko|CEXR-u*hBN)sV>IYM_c1OHkM2wbZ+8%4qv6|Q#_wGscDrnzq7l< zvLeJ?utHlent27=l=j{4Hba__0&5F*t($9@@lApW`JZ`GN4A9HjS)cFsY!m@g881F zTwI2gW9fi*uk9G_(kvd1e}q@BemJ?5w|eRbYD#Yg-5`eJA&g6%gb@F9X!Yt16>wKxn(Ol;b5a zKc7xR_eA;Ebh|KLzy2~f`hMYjXg-VYvx!H+)@YBgSN$B4B1JQ34bVU0Zm_M{A(?zALY!*!4eLH$bq)F)KoJf?NX$hq@z+qdd_8SV zx0{QfhNP$ylfp|?fv{gND9o~0D06fae(|F zbFHwG34@Kzti>QC8~J|+5BDFf@&7*h z=>Pp4_rN$_YrQ7@tYr1{Ipi#KhWD(hh$k%Q_>n3j9oTUR=U zy9Bo89D!F?SzB<6N<_+Ju*0*{rzhXPxHz~vIXi{>5SE??KcXtH)L)`+zHGnu)nob| zwV@84EPj|<^+>SI+tqEJP39w-qOI-q)OYZJHTArij^~9+QyC8m4td9dmk*cQuXflY z=d&LkQW+2O$MH`hHh8omv=$YokX~U_iX;YUd4EgxFGXX?X3F$^-*_ zU7JF~`NXac3`c-@oCB5nUdsw}LamKmDj#it|37VjkNVmD^t1nc{cmfv?>+Uu2M_lj zebv?f_P_dS@KOK!-_-w}`2w`wr%2{O+Muk7zt3ZDF~b4w%GQrX_a}M$!ngb0RmyLjz_gn92VJ!jWqhvg~_PoP23Q<9)oWXYpkND6aOl0p2Q(6~3$!Gml-K?cz%$x!nZ{fdd?Zc(i(%%~k5AK4GAlRsDZ{ z$;TL@qqXjtLAX-IJmyUO3~TDTDTU2nU}Zt z?IfT^g$`h0Ix7+rI_!HL`ZLe-ia z;1m^SwJg&6^AZYiN!4}orl(FhW;m)XHS&WX6j`AHPgAHsj`|&)6+{et)Fgxx5aINz z-`y12ixE~1ReMyVvvpRzb;-5seTT@49wH|e?aIMhsHuz9@2S0`AYqCHbmfuJ)Y<~J zROu@$ap8W9wim5BivbMFAf7uN@1Qz5K74j^igSvkjhO+;=bEj$F$=_e5N9BJJ#(G6 zu*n^w5{OPb&vZi=_0%y!>EKlagurYZ(l&_u+j;#&P4i4pO;o7Da7jE@q5Lp>xm4{y znruV>4}wvwto+`=2rm^ToAAfjJ)EyjpwY-XfG$oLf<1zZWjNIPy}^JP`SAd=dYO$? zI(^xCPCARtCB4p{9h_b&-NN=^F*;Np<`8a>cG01x4u9bKa$YGQzk2E*<8#fsLo%Ji zl#*!SgUwS@R3yu=zBoiw2?(#{P7{c|1;Jg>h{%k)#yE^&t82lLV5l|H)1K=C;L$M%) z-6ByFs|z*Doi4c4175|(!Cq#U3_igLu7V9=!Tn`Q8*R%D41V>r!zqI^fL)79 z8R=(B{z{Ml`DEIN*w7H#MS-jy{LTYryWTPLN{<@$86g^4BhwE5 zpvl)93igcOo(-JX_Pgxe1c69e4!#QP4=vBiZXDINs!SHNpZLO|91{%7yj0q7^kY_v z)gU?$9^$FhOD`Fy$=WsU;g=r1MaPmh0=TB`HzATB^l@gqv>UR*pin0WIxk$FuR3_J z?~b3IT^t8!9bwdXi%rI&g0&phX$w}!Fa+dcxOZBag`w!a2QO1nGqd$7OH&z8Jo@bH zd-e3>+3{t^?C^sV2%@%)XMGq&?LqJ;znAq)rS-V09YTwY;0e6K`@*_`#r=Y!>}lwU*!S4R7lDzsN>8ju+>k}XU6NlIoFK? zjqRE1npBAx+^&MM7A)bgZhLqSd0g>rFfrvEC3@y1x`3jX)FQHxgM}5DYYw)iFCc?E zTybTluz!f7>IUqb-D3p;JnU&pI<2%F{ z$Is8N{wizW!9%v}uniUo92iV)Oryb4*>QXCqfQk-N~B3ZJ0KYaqzigQWVgA4(G;jB zDHyce!Qg?QzGGY(c75d60zdWfCt(N7*&(2kDX%#;bzYIu?HB6k{KcMQcqXxalg~tS z*f9|6ri4j~gEP}NM%_(qVUR)woFWdl&eZAImHP9GqwjGGp6eRRJv4>A>+k|8g(iEV zIJLohYXYtzT1um9cHvObBpd-k{-t)D!VJUh%XJ`$1mgpq_W7OSfEx*CKIfvwsQWDy-i zx~6d*xz^r>NcF8wJdE?s&K07@Qy3#l^+(X%x?Kw)Hv@C(6=vH=pF=q_%gh7cvrp-fRQe!)mSWhh=6i} zv!hT%nU;*I3UT|0rAvT_;3dT^Ko{B9UxKk+qZeG;qQsR{J(Q(eA(n;o=w zYU>#n-3PrT*uYcK69tbuFouF-(INq`xDXeeux=@6Ja@;}LdGPlSDg|2YH z*c^J3&kadHNyS0vhl62IXo=@!RJhIW^n$}$(wuTd0+(}0TktIPu8AWGrC-W8We%38_rk69kM6x^Oc-UKGdxKb5@uZ)z?;e$>9XN zg%XPGZoUm2gL1>GMU$#+BgeDy9k(&gV#Dq-~nU+ zMP&9uLx7a*2ne8gypGJVz`SWmqq=a(GLVexbGB6!KvM8btT<$R@rN{R-LBL5jAD-n zu_jPqX$n+=Q{1_Hq~Y>ig^TsneZfdqg76BNv(%?lWp;+lq1>r|AKALHx9ca!L-g!Hi4g5&ug|uBA4`jw=V4 zSNvveMtMa|36ZFrWk`qo)28%)voZN}(~!qbmu`l&4D|q`a?5dDKr5}Fe1*UdOye^^ z7wLrRZXDCrp-!0-2TEckZ|vW z({_boVElJ$t$5dq$`B7v)YRS?=)<`rXJXIHO!6l&9hn1uqxmq&MtW#v?;mwn04h!b z6m+RwkyaNkPANuvesOm6;t;T9b$NX8WjyF@?Ptg$diFPxGoX? zE8>sNuCOM_7rKxwH=CFrxQ)d*wDB%&E}zC0$ReymR8dN-`xyt?I+6-pBmp)ACSnaN z=%hGbf7*RdTqS0?dtH&^@A?f7t)J0P998mOSus4N_^+hNt_kq8VL!wcV$3{d_dVo#MNuD&6dSsveQ;MgY)Zuj(bw{b`bkX z&D^RguMAqX?d%*r{r-=;pZ)j#lXRKX*SkAA`B?q@v-*Qd$7=Vp!>8Zxs$YJAfosYg zMSW1zZutHaSqVG{KnP+=-zrxagVuWL^W*Z?qBMGQjipoED()JR_nOeDUqDX6edAwJ zc{$F@yjkpwb0Xyr;Sx`8RYfi5AF6-rXXdtFqJ`{(fBUEXH&Z7su(6m4u$Jg;>3kav z-<_QP@w3lmpme16a+Hy86xJ(o3->LmoR`h#_;(NeAU}ue<#V@iJzY<3f8PJ)i!XW; z(|o?`tHSSofxB?GZ>y#s37vXm&(2<4?S2t6%PY0}*>@+WZ)X-fLF!xE$o|7I%S>0Kail0& z+tgb9tNPcsE!_@uHVyOc-HS&9Uh8_|{2KoJ%M-fZ{o?B{cG~sYS(!BvOjGBhO5CSD z`)?Zc=YRg?_3jQm!z(CH@#&N2{{5GK*=_&(_3rDPkN5v?bN^SheYHrzSszFTc*p(! z=*zF;{Ex3c8SH=g=+VdekN+d}A3yy=f9`710Ze(TSPwW1<{|EQFPKWxlB)b@Py3%dNgq6l9y>fax`-dbzq^AD`t-sNS6AoP=VupJcsl&!N&3xze|g#Y?Ir*E z(W8g*^0Sl6tK(C7fd7dBq9MA^$iJVY-#mKs~m-SOYN(#248%uGP|QxUZLgM4Mi^0?q@Rblc&4ODhXQP4r`-t zb`;-L`#YH}&CYubv9t53I+X@c8M^(9nj#diX_E$|FnHTH>RVw%?}eu-i94TC(yTf> zIKMtUzPdg*dVX>WM>zuJ`1{VF5?{n}%x`ce?D&tX^iGwg>E+e!qh8CAe;lk#{O^A! zU&Ffh!1p~}6ur8-9_K}4>Mh>Se1gio#vmH(yfN2=^D0gA$Nl#hWVbs?%lRiyU&r+9 zzHzK}`%Sa7`|}FF)LGW{;Ec}J$@%Mk`!9bSU+<#fKzEoO3<6NHiAIs%L9n_v8 zru@5_)TUDDf2rN^-AvavYVgV*o7ZT_a5JwIADP#?>OXfB7D@&{t!;j3_cOe*E3fQ+ zMz8F)^wGzB=uUnaeS7c!($*;6?!EX+G(CLEKQidi|MdqnwZbk9iY1|+IF$l~OrW>) zP@P{KU!1+Tg5xiBE35Pem1glWo%^|_m1;(nO4E#NfZfl6INMd}uX6a()Rt-dfR{zP zDot6r=q%BLh+Y13hn91d;Dg@}!N2{AUcQsU%eRyGKfk8;CRr?ppY6>rDmwX3U;pu4 zav*+_Ki-V0KHrw0>vj5XPxzmnh}Y2*mrIWbp|8CkxBVVA^7U?zQ>%AaINfVG Multus sentinel/sort -> NAD resources +// -> service-ID label -> authoritative generated values merged over raw values. +// Native DPUService inputs are never mutated. System components/config ports +// belong to a different controller branch and are outside this profile. +func effectiveHelmValues(release string, service *unstructured.Unstructured, inputs projectionInputs) (map[string]interface{}, error) { + if release != "26.8.0-rc.1" && release != "26.8.0" { + return nil, fmt.Errorf("no effective Helm-input contract for DPF %q", release) + } + if service == nil || service.GetKind() != "DPUService" { + return nil, fmt.Errorf("effective Helm inputs require a DPUService") + } + if _, system := service.GetLabels()["dpu.nvidia.com/component"]; system { + return nil, fmt.Errorf("system-component Helm inputs are outside the Sentry profile") + } + if _, found, _ := unstructured.NestedFieldNoCopy(service.Object, "spec", "configPorts"); found { + return nil, fmt.Errorf("config-port Helm inputs are outside the Sentry profile") + } + serviceID, _, err := unstructured.NestedString(service.Object, "spec", "serviceID") + if err != nil || serviceID == "" { + return nil, fmt.Errorf("sentry DPUService requires an explicit serviceID") + } + rawDaemonSet, _, err := unstructured.NestedMap(service.Object, "spec", "serviceDaemonSet") + if err != nil { + return nil, err + } + var daemonSet daemonSetInput + if rawDaemonSet != nil { + data, err := json.Marshal(rawDaemonSet) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err = decoder.Decode(&daemonSet); err != nil { + return nil, fmt.Errorf("unsupported serviceDaemonSet input: %w", err) + } + } + interfaces, hasInterfaces, err := unstructured.NestedStringSlice(service.Object, "spec", "interfaces") + if err != nil { + return nil, err + } + if hasInterfaces { + generated := map[string]networkSelection{"invalid-interface": {Name: "invalid-network", Namespace: "invalid-namespace", Interface: "invalid-interface"}} + resources := map[string]int{} + for _, name := range interfaces { + iface := inputs.interfaces[service.GetNamespace()+"/"+name] + if iface == nil { + return nil, fmt.Errorf("DPUService %s has undeclared interface %s", service.GetName(), name) + } + definition, found, err := unstructured.NestedMap(iface.Object, "spec", "template", "spec", "template", "spec", "service") + if err != nil || !found { + return nil, fmt.Errorf("interface %s has no service definition", name) + } + kind, _, _ := unstructured.NestedString(iface.Object, "spec", "template", "spec", "template", "spec", "interfaceType") + if kind != "service" || definition["serviceID"] != serviceID { + return nil, fmt.Errorf("interface %s has a conflicting service identity", name) + } + network, _ := definition["network"].(string) + networkName, networkNamespace, err := splitNetwork(network) + if err != nil { + return nil, err + } + interfaceName, _ := definition["interfaceName"].(string) + if interfaceName == "" { + return nil, fmt.Errorf("interface %s has no explicit interfaceName", name) + } + generated[interfaceName] = networkSelection{Name: networkName, Namespace: networkNamespace, Interface: interfaceName} + if networkNamespace == "" { + networkNamespace = iface.GetNamespace() + } + nad := inputs.nads[networkNamespace+"/"+networkName] + if nad == nil { + return nil, fmt.Errorf("interface %s has undeclared NAD %s/%s", name, networkNamespace, networkName) + } + resource, err := nadResource(nad) + if err != nil { + return nil, err + } + if resource != "" { + resources[resource]++ + } + } + if daemonSet.Annotations == nil { + daemonSet.Annotations = map[string]string{} + } + annotation, err := mergeNetworks(generated, daemonSet.Annotations[networksAnnotation]) + if err != nil { + return nil, err + } + daemonSet.Annotations[networksAnnotation] = annotation + if len(resources) > 0 { + if daemonSet.Resources == nil { + daemonSet.Resources = corev1.ResourceList{} + } + for name, count := range resources { + daemonSet.Resources[corev1.ResourceName(name)] = *kresource.NewQuantity(int64(count), kresource.DecimalSI) + } + } + } + if daemonSet.Labels == nil { + daemonSet.Labels = map[string]string{} + } + daemonSet.Labels[serviceIDLabel] = serviceID + data, err := json.Marshal(daemonSet) + if err != nil { + return nil, err + } + var daemonSetValues map[string]interface{} + if err = kjson.Unmarshal(data, &daemonSetValues); err != nil { + return nil, err + } + values, _, err := unstructured.NestedMap(service.Object, "spec", "helmChart", "values") + if err != nil { + return nil, err + } + generatedValues := map[string]interface{}{"serviceDaemonSet": daemonSetValues} + mergeMissingValues(generatedValues, values) + return generatedValues, nil +} + +func splitNetwork(network string) (name, namespace string, err error) { + parts := strings.Split(network, "/") + switch len(parts) { + case 1: + name = parts[0] + case 2: + namespace, name = parts[0], parts[1] + default: + err = fmt.Errorf("invalid declared network %q", network) + } + if name == "" { + err = fmt.Errorf("declared network name is empty") + } + return name, namespace, err +} + +func nadResource(nad *unstructured.Unstructured) (string, error) { + kind, _, _ := unstructured.NestedString(nad.Object, "spec", "resourceType") + switch kind { + case "sf": + if _, trusted := nad.GetAnnotations()[trustedSFAnnotation]; trusted { + return "nvidia.com/bf_sf_trusted", nil + } + return "nvidia.com/bf_sf", nil + case "vf": + return "nvidia.com/bf_vf", nil + case "veth": + return "", nil + default: + return "", fmt.Errorf("NAD %s has unsupported resourceType %q", nad.GetName(), kind) + } +} + +func mergeNetworks(generated map[string]networkSelection, raw string) (string, error) { + var existing []networkSelection + if raw != "" { + decoder := json.NewDecoder(bytes.NewBufferString(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&existing); err != nil { + return "", fmt.Errorf("invalid Sentry network annotation: %w", err) + } + if err := decoder.Decode(new(interface{})); err != io.EOF { + return "", fmt.Errorf("invalid Sentry network annotation: trailing JSON data") + } + } + merged := make([]networkSelection, 0, len(existing)+len(generated)) + for _, e := range existing { + if n, ok := generated[e.Interface]; ok { + // Exact upstream mergo.WithOverride behavior for the bounded fields: + // nonempty existing fields override the generated interface selection. + if e.Name != "" { + n.Name = e.Name + } + if e.Namespace != "" { + n.Namespace = e.Namespace + } + if e.Interface != "" { + n.Interface = e.Interface + } + if e.CNIArgs != nil { + n.CNIArgs = e.CNIArgs + } + merged = append(merged, n) + } else { + merged = append(merged, e) + } + delete(generated, e.Interface) + } + for _, n := range generated { + merged = append(merged, n) + } + sort.Slice(merged, func(i, j int) bool { + if merged[i].Name == merged[j].Name { + return merged[i].Interface < merged[j].Interface + } + return merged[i].Name < merged[j].Name + }) + data, err := json.Marshal(merged) + if err != nil { + return "", err + } + return string(data), nil +} + +// DPF MergeMaps gives generated destination values precedence recursively. Copy +// fallback leaves as well, so no retained projection aliases native chart input. +func mergeMissingValues(destination, fallback map[string]interface{}) { + for key, value := range fallback { + current, exists := destination[key] + if !exists { + destination[key] = runtime.DeepCopyJSONValue(value) + continue + } + dst, dstMap := current.(map[string]interface{}) + src, srcMap := value.(map[string]interface{}) + if dstMap && srcMap { + mergeMissingValues(dst, src) + } + } +} diff --git a/pkg/dpf/sentry/values_test.go b/pkg/dpf/sentry/values_test.go new file mode 100644 index 00000000..b1fe3c6e --- /dev/null +++ b/pkg/dpf/sentry/values_test.go @@ -0,0 +1,181 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package sentry + +import ( + "encoding/json" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + yaml "sigs.k8s.io/yaml" +) + +var _ = Describe("exact-release DPF effective Helm inputs", func() { + It("projects the real Tower chart with the controller sentinel, JSON shape, ordering and resources for both releases", func() { + for _, release := range []string{"26.8.0-rc.1", "26.8.0"} { + g := generationFixture() + g.Release = release + r, err := Render(g, profileFixture(), archiveFixture()) + Expect(err).NotTo(HaveOccurred()) + expected := `[{"name":"invalid-network","namespace":"invalid-namespace","interface":"invalid-interface","cni-args":null},{"name":"site-a-egress","namespace":"site-a-services","interface":"net2","cni-args":null},{"name":"site-a-ingress","namespace":"site-a-services","interface":"net1","cni-args":null}]` + for _, tower := range objectsOf(r, bundle.Projection, "DaemonSet") { + if tower.GetName() == "site-a-tower" { + Expect(tower.GetAnnotations()).To(HaveKeyWithValue(networksAnnotation, expected)) + annotations, _, _ := unstructured.NestedStringMap(tower.Object, "spec", "template", "metadata", "annotations") + Expect(annotations).To(HaveKeyWithValue(networksAnnotation, expected)) + labels, _, _ := unstructured.NestedStringMap(tower.Object, "spec", "template", "metadata", "labels") + Expect(labels).To(HaveKeyWithValue(serviceIDLabel, "site-a-tower")) + } + } + for _, svc := range objectsOf(r, bundle.Manifest, "DPUService") { + _, hasDuplicate, _ := unstructured.NestedFieldNoCopy(svc.Object, "spec", "helmChart", "values", "serviceDaemonSet") + Expect(hasDuplicate).To(BeFalse()) + if svc.GetName() == "site-a-tower" { + raw, _, _ := unstructured.NestedString(svc.Object, "spec", "serviceDaemonSet", "annotations", networksAnnotation) + Expect(raw).NotTo(ContainSubstring("invalid-network")) + Expect(strings.Index(raw, "site-a-ingress")).To(BeNumerically("<", strings.Index(raw, "site-a-egress"))) + } + } + var values map[string]interface{} + for _, f := range r.Files { + if f.Path == "values/site-a-tower.yaml" { + Expect(yaml.Unmarshal(f.Data, &values)).To(Succeed()) + } + } + annotation, _, _ := unstructured.NestedString(values, "serviceDaemonSet", "annotations", networksAnnotation) + Expect(annotation).To(Equal(expected)) + trusted, _, _ := unstructured.NestedString(values, "serviceDaemonSet", "resources", "nvidia.com/bf_sf_trusted") + Expect(trusted).To(Equal("2")) + cpu, _, _ := unstructured.NestedString(values, "serviceDaemonSet", "resources", "cpu") + Expect(cpu).To(Equal("1")) + for _, server := range objectsOf(r, bundle.Projection, "Deployment") { + if server.GetName() == "site-a-policy-server" { + labels, _, _ := unstructured.NestedStringMap(server.Object, "spec", "template", "metadata", "labels") + Expect(labels).To(HaveKeyWithValue(serviceIDLabel, "site-a-policy-server")) + } + } + for _, controller := range objectsOf(r, bundle.Projection, "DaemonSet") { + if controller.GetName() == "site-a-controller" { + annotations, _, _ := unstructured.NestedStringMap(controller.Object, "spec", "template", "metadata", "annotations") + Expect(annotations).NotTo(HaveKey(networksAnnotation)) + } + } + } + }) + It("derives resource counts from NADs, gives generated values precedence, and leaves all inputs independent", func() { + g := generationFixture() + r, err := Render(g, profileFixture(), archiveFixture()) + Expect(err).NotTo(HaveOccurred()) + inputs, tower := projectionFixture(r) + Expect(unstructured.SetNestedField(tower.Object, "99", "spec", "serviceDaemonSet", "resources", "nvidia.com/bf_sf_trusted")).To(Succeed()) + lower := map[string]interface{}{"serviceDaemonSet": map[string]interface{}{"labels": map[string]interface{}{serviceIDLabel: "incorrect-lower-priority-id", "extra.example/label": "retained"}, "resources": map[string]interface{}{"nvidia.com/bf_sf_trusted": "77"}, "annotations": map[string]interface{}{networksAnnotation: "incorrect-lower-priority-networks"}}, "customValue": map[string]interface{}{"keep": "unchanged"}} + Expect(unstructured.SetNestedMap(tower.Object, lower, "spec", "helmChart", "values")).To(Succeed()) + before, _ := json.Marshal(tower.Object) + values, err := effectiveHelmValues("26.8.0-rc.1", tower, inputs) + Expect(err).NotTo(HaveOccurred()) + trusted, _, _ := unstructured.NestedString(values, "serviceDaemonSet", "resources", "nvidia.com/bf_sf_trusted") + Expect(trusted).To(Equal("2")) + label, _, _ := unstructured.NestedString(values, "serviceDaemonSet", "labels", serviceIDLabel) + Expect(label).To(Equal("site-a-tower")) + kept, _, _ := unstructured.NestedString(values, "serviceDaemonSet", "labels", "extra.example/label") + Expect(kept).To(Equal("retained")) + after, _ := json.Marshal(tower.Object) + Expect(after).To(Equal(before)) + Expect(unstructured.SetNestedField(values, "changed-result", "customValue", "keep")).To(Succeed()) + after, _ = json.Marshal(tower.Object) + Expect(after).To(Equal(before)) + // Changing one declared NAD changes the controller-injected resource, + // rather than trusting the now-stale request in the native DPUService. + for key, nad := range inputs.nads { + if nad.GetName() == "site-a-egress" { + copy := nad.DeepCopy() + Expect(unstructured.SetNestedField(copy.Object, "vf", "spec", "resourceType")).To(Succeed()) + inputs.nads[key] = copy + } + } + values, err = effectiveHelmValues("26.8.0", tower, inputs) + Expect(err).NotTo(HaveOccurred()) + trusted, _, _ = unstructured.NestedString(values, "serviceDaemonSet", "resources", "nvidia.com/bf_sf_trusted") + Expect(trusted).To(Equal("1")) + vf, _, _ := unstructured.NestedString(values, "serviceDaemonSet", "resources", "nvidia.com/bf_vf") + Expect(vf).To(Equal("1")) + }) + It("rejects missing dependencies and unsupported transformation branches instead of inventing values", func() { + g := generationFixture() + r, err := Render(g, profileFixture(), archiveFixture()) + Expect(err).NotTo(HaveOccurred()) + inputs, tower := projectionFixture(r) + delete(inputs.nads, *g.Sentry.Namespace+"/site-a-egress") + _, err = effectiveHelmValues(g.Release, tower, inputs) + Expect(err).To(MatchError(ContainSubstring("undeclared NAD"))) + inputs, tower = projectionFixture(r) + Expect(unstructured.SetNestedStringSlice(tower.Object, []string{"missing-interface"}, "spec", "interfaces")).To(Succeed()) + _, err = effectiveHelmValues(g.Release, tower, inputs) + Expect(err).To(MatchError(ContainSubstring("undeclared interface"))) + _, err = effectiveHelmValues("26.11.0", tower, inputs) + Expect(err).To(MatchError(ContainSubstring("no effective Helm-input contract"))) + inputs, tower = projectionFixture(r) + tower.SetLabels(map[string]string{"dpu.nvidia.com/component": "system-component"}) + _, err = effectiveHelmValues(g.Release, tower, inputs) + Expect(err).To(MatchError(ContainSubstring("system-component"))) + inputs, tower = projectionFixture(r) + Expect(unstructured.SetNestedMap(tower.Object, map[string]interface{}{}, "spec", "configPorts")).To(Succeed()) + _, err = effectiveHelmValues(g.Release, tower, inputs) + Expect(err).To(MatchError(ContainSubstring("config-port"))) + }) + It("merges explicit existing network fields then sorts exactly, while refusing unsupported annotation data", func() { + generated := func() map[string]networkSelection { + return map[string]networkSelection{"net1": {Name: "generated", Interface: "net1"}, "invalid-interface": {Name: "invalid-network", Namespace: "invalid-namespace", Interface: "invalid-interface"}} + } + actual, err := mergeNetworks(generated(), `[{"name":"custom","namespace":"explicit","interface":"net1","cni-args":{"mtu":"1500"}}]`) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(`[{"name":"custom","namespace":"explicit","interface":"net1","cni-args":{"mtu":"1500"}},{"name":"invalid-network","namespace":"invalid-namespace","interface":"invalid-interface","cni-args":null}]`)) + for _, raw := range []string{`[{"name":"custom","interface":"net1","ips":["192.0.2.1"]}]`, `[] []`, `not-json`} { + _, err = mergeNetworks(generated(), raw) + Expect(err).To(HaveOccurred()) + } + }) +}) + +var _ = Describe("pinned DPF schema default copies", func() { + It("defaults an independent copy for both releases and never prunes additional fields", func() { + for _, release := range []string{"26.8.0-rc.1", "26.8.0"} { + input := object("svc.dpu.nvidia.com/v1alpha1", "DPUServiceNAD", "services", "network", map[string]interface{}{"resourceType": "sf", "additionalDesiredField": "must-remain-visible"}) + copy, err := DefaultedCopy(release, input) + Expect(err).NotTo(HaveOccurred()) + mtu, _, _ := unstructured.NestedFieldNoCopy(copy.Object, "spec", "serviceMTU") + Expect(mtu).To(BeNumerically("==", 1500)) + _, originalHasMTU, _ := unstructured.NestedFieldNoCopy(input.Object, "spec", "serviceMTU") + Expect(originalHasMTU).To(BeFalse()) + extra, _, _ := unstructured.NestedString(copy.Object, "spec", "additionalDesiredField") + Expect(extra).To(Equal("must-remain-visible")) + Expect(unstructured.SetNestedField(copy.Object, "changed", "spec", "additionalDesiredField")).To(Succeed()) + extra, _, _ = unstructured.NestedString(input.Object, "spec", "additionalDesiredField") + Expect(extra).To(Equal("must-remain-visible")) + } + _, err := DefaultedCopy("26.8.0", nil) + Expect(err).To(HaveOccurred()) + _, err = DefaultedCopy("26.8.0", object("v1", "ConfigMap", "services", "unsupported", nil)) + Expect(err).To(HaveOccurred()) + }) +}) + +func projectionFixture(r Rendered) (projectionInputs, *unstructured.Unstructured) { + inputs := projectionInputs{interfaces: map[string]*unstructured.Unstructured{}, nads: map[string]*unstructured.Unstructured{}} + for _, kind := range []string{"DPUServiceInterface", "DPUServiceNAD"} { + for _, obj := range objectsOf(r, bundle.Manifest, kind) { + Expect(inputs.record(obj)).To(Succeed()) + } + } + for _, obj := range objectsOf(r, bundle.Manifest, "DPUService") { + if obj.GetName() == "site-a-tower" { + return inputs, obj + } + } + Fail("Tower fixture is absent") + return inputs, nil +} diff --git a/pkg/networkoperatorplugin/maintenance_render_test.go b/pkg/networkoperatorplugin/maintenance_render_test.go index 24512a70..cccbda85 100644 --- a/pkg/networkoperatorplugin/maintenance_render_test.go +++ b/pkg/networkoperatorplugin/maintenance_render_test.go @@ -389,6 +389,17 @@ func TestMaintenanceScalarTypesAcrossProfiles(t *testing.T) { profilesOnDisk, err := filepath.Glob(filepath.Join("..", "..", "profiles", "*", "profile.yaml")) require.NoError(t, err) for _, profilePath := range profilesOnDisk { + // Maintenance fields belong to Host profiles. Inspect only the target + // header before decoding a target-specific profile payload. + data, err := os.ReadFile(profilePath) + require.NoError(t, err) + var header struct { + Target string `yaml:"target"` + } + require.NoError(t, configyaml.Unmarshal(data, &header)) + if header.Target != "" && header.Target != "host" { + continue + } dir := filepath.Base(filepath.Dir(profilePath)) profile := loadProfileFromDir(t, dir) for _, tc := range []struct { diff --git a/pkg/options/options.go b/pkg/options/options.go index a2adc473..2d76aeef 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -25,6 +25,10 @@ import ( // Options holds all the configuration parameters for the application type Options struct { Flavor string `flag:"flavor" config:"flavor" scopes:"root,generate,discover" usage:"Cluster flavor: k8s or ocp (overrides config)"` // Empty means use the config file. + // DPF profile and release are target-specific generation inputs. Empty + // explicit values remain present in ConfigInputs for validation. + DPFProfile string `flag:"dpf-profile" config:"dpf.profile" target:"dpf" scopes:"generate" usage:"DPF generation profile name"` + DPFRelease string `flag:"dpf-release" config:"dpf.release" target:"dpf" scopes:"generate" usage:"Installed DPF release selected for generation"` // LaunchKitVersion is the version reported by `l8k version`. The CLI // passes it through so generated resources can record their producer. LaunchKitVersion string diff --git a/pkg/profiles/profiles.go b/pkg/profiles/profiles.go index 32278eff..635c3aba 100644 --- a/pkg/profiles/profiles.go +++ b/pkg/profiles/profiles.go @@ -26,6 +26,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/nvidia/k8s-launch-kit/pkg/config" apperrors "github.com/nvidia/k8s-launch-kit/pkg/errors" + launchtarget "github.com/nvidia/k8s-launch-kit/pkg/target" "gopkg.in/yaml.v2" "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -71,6 +72,29 @@ type Profile struct { Templates []string } +// ProfileMetadata contains fields shared by all profile targets. Manifest is +// the original YAML so a target can decode its own payload without using the +// Host profile schema. TemplatePaths are resolved, existing files contained +// within the profiles root. +type ProfileMetadata struct { + Target launchtarget.Name + Directory string + Name string + Plugin string + Description string + DeploymentGuide string + TemplatePaths []string + Manifest []byte +} + +type profileMetadataYAML struct { + Name string `yaml:"name"` + Plugin string `yaml:"plugin"` + Description string `yaml:"description"` + DeploymentGuide string `yaml:"deploymentGuide"` + Templates []string `yaml:"templates"` +} + // getprofilesDir resolves the profiles directory using a lookup chain: // 1. ./profiles (CWD — container/repo root) // 2. /usr/local/share/l8k/profiles (default install) @@ -94,6 +118,182 @@ func getprofilesDir() (string, error) { "Run 'scripts/install.sh' to install l8k, or run from the repository root") } +// LoadProfileForTarget loads target-independent metadata and template paths for +// the exact profile directory name. The returned manifest is raw YAML for the +// target implementation to decode using its own schema. +func LoadProfileForTarget(targetName launchtarget.Name, profileName string) (*ProfileMetadata, error) { + profilesDir, err := getprofilesDir() + if err != nil { + return nil, err + } + return loadProfileForTarget(profilesDir, targetName, profileName) +} + +func loadProfileForTarget(profilesDir string, targetName launchtarget.Name, profileName string) (*ProfileMetadata, error) { + if !knownProfileTarget(targetName) { + return nil, fmt.Errorf("unknown profile target %q", targetName) + } + root, directory, manifest, err := readProfileManifest(profilesDir, profileName) + if err != nil { + return nil, err + } + declaredTarget, err := decodeProfileTarget(manifest) + if err != nil { + return nil, fmt.Errorf("profile %q: %w", profileName, err) + } + if declaredTarget != targetName { + return nil, fmt.Errorf("profile %q targets %q, not %q", profileName, declaredTarget, targetName) + } + return profileMetadataFromManifest(root, directory, profileName, declaredTarget, manifest) +} + +func readProfileManifest(profilesDir, profileName string) (string, string, []byte, error) { + if profileName == "" || profileName == "." || filepath.Base(profileName) != profileName || strings.ContainsAny(profileName, `/\\`) { + return "", "", nil, fmt.Errorf("invalid profile name %q", profileName) + } + + root, err := filepath.EvalSymlinks(profilesDir) + if err != nil { + return "", "", nil, fmt.Errorf("resolve profiles directory: %w", err) + } + root, err = filepath.Abs(root) + if err != nil { + return "", "", nil, fmt.Errorf("resolve profiles directory: %w", err) + } + profilePath := filepath.Join(root, profileName) + info, err := os.Stat(profilePath) + if err != nil { + return "", "", nil, fmt.Errorf("profile %q not found: %w", profileName, err) + } + if !info.IsDir() { + return "", "", nil, fmt.Errorf("profile %q is not a directory", profileName) + } + directory, err := resolveContainedPath(root, root, profileName) + if err != nil { + return "", "", nil, fmt.Errorf("resolve profile %q: %w", profileName, err) + } + + manifestPath, err := resolveContainedPath(root, directory, "profile.yaml") + if err != nil { + return "", "", nil, fmt.Errorf("resolve profile %q manifest: %w", profileName, err) + } + manifest, err := os.ReadFile(manifestPath) + if err != nil { + return "", "", nil, fmt.Errorf("read profile %q manifest: %w", profileName, err) + } + return root, directory, manifest, nil +} + +func profileMetadataFromManifest(root, directory, profileName string, declaredTarget launchtarget.Name, manifest []byte) (*ProfileMetadata, error) { + var metadata profileMetadataYAML + if err := yaml.Unmarshal(manifest, &metadata); err != nil { + return nil, fmt.Errorf("decode profile %q metadata: %w", profileName, err) + } + + templatePaths := make([]string, 0, len(metadata.Templates)) + for _, template := range metadata.Templates { + if template == "" { + return nil, fmt.Errorf("profile %q contains an empty template path", profileName) + } + path, err := resolveContainedPath(root, directory, filepath.FromSlash(template)) + if err != nil { + return nil, fmt.Errorf("profile %q template %q: %w", profileName, template, err) + } + if info, err := os.Stat(path); err != nil { + return nil, fmt.Errorf("profile %q template %q: %w", profileName, template, err) + } else if !info.Mode().IsRegular() { + return nil, fmt.Errorf("profile %q template %q is not a regular file", profileName, template) + } + templatePaths = append(templatePaths, path) + } + + return &ProfileMetadata{ + Target: declaredTarget, + Directory: directory, + Name: metadata.Name, + Plugin: metadata.Plugin, + Description: metadata.Description, + DeploymentGuide: metadata.DeploymentGuide, + TemplatePaths: templatePaths, + Manifest: manifest, + }, nil +} + +func decodeProfileTarget(manifest []byte) (launchtarget.Name, error) { + var header yaml.MapSlice + if err := yaml.Unmarshal(manifest, &header); err != nil { + return "", fmt.Errorf("decode target header: %w", err) + } + var declared string + found := false + for _, item := range header { + key, ok := item.Key.(string) + if !ok || key != "target" { + continue + } + if found { + return "", fmt.Errorf("duplicate target header") + } + found = true + value, ok := item.Value.(string) + if !ok { + return "", fmt.Errorf("invalid target header: expected a string") + } + declared = value + } + if !found { + return launchtarget.Host, nil + } + targetName := launchtarget.Name(declared) + if !knownProfileTarget(targetName) { + return "", fmt.Errorf("unknown or invalid target %q", declared) + } + return targetName, nil +} + +func knownProfileTarget(targetName launchtarget.Name) bool { + return targetName == launchtarget.Host || targetName == launchtarget.DPF +} + +// resolveContainedPath follows symlinks but rejects any result outside root. +// The profile root itself may be a symlink in an installation layout. +func resolveContainedPath(root, base, relative string) (string, error) { + if filepath.IsAbs(relative) { + return "", fmt.Errorf("absolute path is not allowed") + } + candidate := filepath.Join(base, relative) + lexical, err := filepath.Abs(candidate) + if err != nil { + return "", err + } + if err := ensureContained(root, lexical); err != nil { + return "", err + } + resolved, err := filepath.EvalSymlinks(candidate) + if err != nil { + return "", err + } + resolved, err = filepath.Abs(resolved) + if err != nil { + return "", err + } + if err := ensureContained(root, resolved); err != nil { + return "", err + } + return resolved, nil +} + +func ensureContained(root, path string) error { + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("path escapes profiles directory") + } + return nil +} + // FindApplicableProfile selects a profile from disk that matches the user's // requirements, cluster capabilities, and selected Network Operator release. // selectedRelease is the catalog key (MAJOR.MINOR, e.g. "26.4") chosen via @@ -109,7 +309,10 @@ func FindApplicableProfile(requirements *config.Profile, capabilities *config.Cl if err != nil { return nil, err } + return findApplicableProfile(profilesDir, requirements, capabilities, pluginName, selectedRelease, flavor) +} +func findApplicableProfile(profilesDir string, requirements *config.Profile, capabilities *config.ClusterCapabilities, pluginName, selectedRelease, flavor string) (*Profile, error) { entries, err := os.ReadDir(profilesDir) if err != nil { return nil, err @@ -121,16 +324,26 @@ func FindApplicableProfile(requirements *config.Profile, capabilities *config.Cl for _, entry := range entries { if entry.IsDir() { - profileManifest := filepath.Join(profilesDir, entry.Name(), "profile.yaml") - profileData, err := os.ReadFile(profileManifest) + root, directory, manifest, err := readProfileManifest(profilesDir, entry.Name()) + if err != nil { + log.Log.Error(err, "failed to read profile manifest", "profile", entry.Name()) + return nil, err + } + profileTarget, err := decodeProfileTarget(manifest) + if err != nil { + return nil, fmt.Errorf("profile %q: %w", entry.Name(), err) + } + if profileTarget != launchtarget.Host { + continue + } + metadata, err := profileMetadataFromManifest(root, directory, entry.Name(), profileTarget, manifest) if err != nil { - log.Log.Error(err, "failed to read profile manifest", "profileManifest", profileManifest) return nil, err } profile := &Profile{} - err = yaml.Unmarshal(profileData, profile) + err = yaml.Unmarshal(metadata.Manifest, profile) if err != nil { - log.Log.Error(err, "failed to unmarshal profile manifest", "profileManifest", profileManifest) + log.Log.Error(err, "failed to unmarshal Host profile manifest", "profile", entry.Name()) return nil, err } if profile.Plugin != pluginName { @@ -146,7 +359,8 @@ func FindApplicableProfile(requirements *config.Profile, capabilities *config.Cl valid, reason := profile.Validate(requirements, capabilities, selectedRelease) if valid { log.Log.V(1).Info("Found applicable profile", "profile", profile) - profile.UpdateManifestsPaths(filepath.Join(profilesDir, entry.Name())) + profile.Templates = metadata.TemplatePaths + profile.DeploymentGuide = filepath.Join(metadata.Directory, profile.DeploymentGuide) return profile, nil } else { errorMessages = append(errorMessages, fmt.Sprintf("profile %s is not applicable: %s", entry.Name(), reason)) diff --git a/pkg/profiles/target_profile_test.go b/pkg/profiles/target_profile_test.go new file mode 100644 index 00000000..bf7191b5 --- /dev/null +++ b/pkg/profiles/target_profile_test.go @@ -0,0 +1,154 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package profiles + +import ( + "os" + "path/filepath" + "testing" + + "github.com/nvidia/k8s-launch-kit/pkg/config" + "github.com/nvidia/k8s-launch-kit/pkg/target" + "github.com/stretchr/testify/require" +) + +func TestFindApplicableProfileSkipsDPFPayloadBeforeHostDecode(t *testing.T) { + root := t.TempDir() + writeTestProfile(t, root, "host-profile", `name: Host profile +plugin: test-plugin +profileRequirements: + fabric: ethernet + deployment: sriov +nodeCapabilities: + rdma: true +templates: + - host.yaml +`, map[string]string{"host.yaml": "kind: Host\n"}) + writeTestProfile(t, root, "dpf-profile", `target: dpf +name: DPF profile +plugin: test-plugin +profileRequirements: this is deliberately not a Host requirements mapping +templates: + - ../outside.yaml +`, nil) + + profile, err := findApplicableProfile(root, + &config.Profile{Fabric: "ethernet", Deployment: "sriov"}, + &config.ClusterCapabilities{Nodes: &config.NodesCapabilities{Rdma: true}}, + "test-plugin", "", config.FlavorK8s) + require.NoError(t, err) + require.Equal(t, "Host profile", profile.Name) + resolvedRoot, err := filepath.EvalSymlinks(root) + require.NoError(t, err) + require.Equal(t, []string{filepath.Join(resolvedRoot, "host-profile", "host.yaml")}, profile.Templates) +} + +func TestLoadProfileForTargetDefaultsMissingTargetToHost(t *testing.T) { + root := t.TempDir() + writeTestProfile(t, root, "host-profile", `name: Host profile +plugin: test-plugin +templates: + - host.yaml +`, map[string]string{"host.yaml": "kind: Host\n"}) + + metadata, err := loadProfileForTarget(root, target.Host, "host-profile") + require.NoError(t, err) + require.Equal(t, target.Host, metadata.Target) + require.Equal(t, "Host profile", metadata.Name) + require.Equal(t, "test-plugin", metadata.Plugin) + resolvedRoot, err := filepath.EvalSymlinks(root) + require.NoError(t, err) + require.Equal(t, []string{filepath.Join(resolvedRoot, "host-profile", "host.yaml")}, metadata.TemplatePaths) + require.Contains(t, string(metadata.Manifest), "name: Host profile") +} + +func TestDecodeProfileTargetRejectsUnknownInvalidAndMalformedHeaders(t *testing.T) { + for _, tc := range []struct { + name string + manifest string + wantErr string + }{ + {name: "unknown", manifest: "target: future\n", wantErr: `unknown or invalid target "future"`}, + {name: "invalid type", manifest: "target: [host]\n", wantErr: "invalid target header"}, + {name: "null is not omission", manifest: "target: null\n", wantErr: "invalid target header"}, + {name: "duplicate", manifest: "target: host\ntarget: dpf\n", wantErr: "duplicate target header"}, + {name: "malformed yaml", manifest: "target: dpf\nname: [\n", wantErr: "decode target header"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := decodeProfileTarget([]byte(tc.manifest)) + require.ErrorContains(t, err, tc.wantErr) + }) + } +} + +func TestLoadProfileForTargetRequiresExactProfileAndMatchingTarget(t *testing.T) { + root := t.TempDir() + writeTestProfile(t, root, "dpf-profile", `target: dpf +name: DPF profile +plugin: dpf +templates: + - dpf.yaml +`, map[string]string{"dpf.yaml": "kind: DPF\n"}) + + _, err := loadProfileForTarget(root, target.DPF, "missing") + require.ErrorContains(t, err, `profile "missing" not found`) + _, err = loadProfileForTarget(root, target.Host, "dpf-profile") + require.ErrorContains(t, err, `targets "dpf", not "host"`) + _, err = loadProfileForTarget(root, target.DPF, "../dpf-profile") + require.ErrorContains(t, err, "invalid profile name") + _, err = loadProfileForTarget(root, target.Name("future"), "dpf-profile") + require.ErrorContains(t, err, `unknown profile target "future"`) +} + +func TestLoadProfileForTargetRejectsTemplateTraversalAndSymlinkEscape(t *testing.T) { + t.Run("traversal", func(t *testing.T) { + root := t.TempDir() + writeTestProfile(t, root, "dpf-profile", `target: dpf +name: DPF profile +templates: + - ../../outside.yaml +`, nil) + _, err := loadProfileForTarget(root, target.DPF, "dpf-profile") + require.ErrorContains(t, err, "path escapes profiles directory") + }) + + t.Run("symlink", func(t *testing.T) { + root := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.yaml") + require.NoError(t, os.WriteFile(outside, []byte("kind: Outside\n"), 0o600)) + profileDir := filepath.Join(root, "dpf-profile") + require.NoError(t, os.MkdirAll(profileDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`target: dpf +name: DPF profile +templates: + - outside.yaml +`), 0o600)) + require.NoError(t, os.Symlink(outside, filepath.Join(profileDir, "outside.yaml"))) + _, err := loadProfileForTarget(root, target.DPF, "dpf-profile") + require.ErrorContains(t, err, "path escapes profiles directory") + }) +} + +func writeTestProfile(t *testing.T, root, name, manifest string, files map[string]string) { + t.Helper() + directory := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(directory, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "profile.yaml"), []byte(manifest), 0o600)) + for path, contents := range files { + require.NoError(t, os.WriteFile(filepath.Join(directory, path), []byte(contents), 0o600)) + } +} diff --git a/pkg/target/dpf/driver.go b/pkg/target/dpf/driver.go index 41c31d3b..5b492f07 100644 --- a/pkg/target/dpf/driver.go +++ b/pkg/target/dpf/driver.go @@ -26,14 +26,18 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/target" ) -// Descriptor exposes the inventory preview, keeping provisioning unavailable. +// Descriptor exposes the independent preview workflows. Pipeline bootstrap is +// unavailable; deployment consumes explicitly bound existing infrastructure. func Descriptor() target.Descriptor { phases := map[target.Phase]target.Capability{} for _, p := range append(target.PublicPhases(), target.Pipeline) { phases[p] = target.Capability{Available: false, Reason: "DPF provisioning is not implemented in this build"} } phases[target.Discover] = target.Capability{Available: true, Reason: "experimental inventory discovery; management probes require qualified release assets; hardware catalog qualification pending"} - return target.Descriptor{Name: target.DPF, Description: "Independent DPF inventory discovery preview", Phases: phases} + for _, p := range []target.Phase{target.Generate, target.Deploy, target.Validate} { + phases[p] = target.Capability{Available: true, Reason: "experimental Sentry BF4 workflow on existing DPF management, tenant and enrolled devices"} + } + return target.Descriptor{Name: target.DPF, Description: "Independent DPF discovery and Sentry BF4 workflow", Phases: phases} } type Request struct { diff --git a/pkg/target/dpf/driver_test.go b/pkg/target/dpf/driver_test.go index 2f8857bd..f2f570a7 100644 --- a/pkg/target/dpf/driver_test.go +++ b/pkg/target/dpf/driver_test.go @@ -30,11 +30,23 @@ var _ = Describe("target boundary", func() { Expect(result["target"]).To(Equal("dpf")) Expect(result["execution"]).To(Equal("failed")) }) - It("advertises only discovery and rejects other phases before binding", func() { - for _, p := range []target.Phase{target.Generate, target.Deploy, target.Validate, target.Pipeline} { - _, e := NewDiscoverAdapter(Request{}).Bind(target.Invocation{Target: target.DPF, Phase: p}) - Expect(e).To(HaveOccurred()) - Expect(Descriptor().Capability(p).Available).To(BeFalse()) + It("advertises implemented public workflows and keeps the composite pipeline unavailable", func() { + for _, p := range []target.Phase{target.Discover, target.Generate, target.Deploy, target.Validate} { + Expect(Descriptor().Capability(p).Available).To(BeTrue()) } + Expect(Descriptor().Capability(target.Pipeline).Available).To(BeFalse()) + }) + + It("binds the workflow adapter only to its configured phase", func() { + adapter := NewWorkflowAdapter(WorkflowRequest{Phase: target.Generate}) + _, e := adapter.Bind(target.Invocation{Target: target.DPF, Phase: target.Deploy}) + Expect(e).To(MatchError(ContainSubstring("does not bind phase"))) + _, e = adapter.Bind(target.Invocation{Target: target.DPF, Phase: target.Generate}) + Expect(e).NotTo(HaveOccurred()) + }) + + It("retains the discovery adapter phase boundary", func() { + _, e := NewDiscoverAdapter(Request{}).Bind(target.Invocation{Target: target.DPF, Phase: target.Generate}) + Expect(e).To(HaveOccurred()) }) }) diff --git a/pkg/target/dpf/workflow.go b/pkg/target/dpf/workflow.go new file mode 100644 index 00000000..b08a6eb0 --- /dev/null +++ b/pkg/target/dpf/workflow.go @@ -0,0 +1,166 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package dpf + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/configinput" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/deployment" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/generation" + apperrors "github.com/nvidia/k8s-launch-kit/pkg/errors" + "github.com/nvidia/k8s-launch-kit/pkg/target" +) + +type WorkflowRequest struct { + Phase target.Phase + UserConfig string + OutputPath string + DeploymentFiles string + Overrides configinput.Values + Deploy bool + Wait time.Duration + Stdout io.Writer + Stderr io.Writer +} + +type workflowDriver struct{ request WorkflowRequest } + +func NewWorkflowAdapter(r WorkflowRequest) target.Driver { + r.Overrides = r.Overrides.Clone() + return workflowDriver{request: r} +} + +func (workflowDriver) Descriptor() target.Descriptor { return Descriptor() } + +type workflowResult struct { + ExitCode int `json:"exitCode"` + Target string `json:"target"` + Phase target.Phase `json:"phase"` + Outcome string `json:"outcome"` + Generation *generation.Result `json:"generation,omitempty"` + Deployment *deployment.Result `json:"deployment,omitempty"` + Errors []string `json:"errors,omitempty"` +} + +func (d workflowDriver) Bind(inv target.Invocation) (target.Operation, error) { + r := d.request + if inv.Phase != r.Phase || (inv.Phase != target.Generate && inv.Phase != target.Deploy && inv.Phase != target.Validate) { + return nil, fmt.Errorf("DPF workflow adapter does not bind phase %q", inv.Phase) + } + if r.Wait < 0 { + return nil, fmt.Errorf("validation wait must not be negative") + } + if inv.Phase == target.Generate && inv.Execution.DryRun && !r.Deploy { + return nil, fmt.Errorf("DPF generate --dry-run requires --deploy; generation alone is local and does not mutate a cluster") + } + if r.Stdout == nil { + r.Stdout = os.Stdout + } + if r.Stderr == nil { + r.Stderr = os.Stderr + } + return target.OperationFunc(func(parent context.Context) error { + ctx, stop := signal.NotifyContext(parent, terminationSignals()...) + defer stop() + timeout := inv.Execution.Timeout + if inv.Phase == target.Validate { + timeout = r.Wait + } + if timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + result := workflowResult{Target: "dpf", Phase: inv.Phase, Outcome: "failed"} + var runErr error + exitCode := 1 + path := r.DeploymentFiles + if inv.Phase == target.Generate { + generated, err := generation.Run(ctx, generation.Request{UserConfig: r.UserConfig, OutputPath: r.OutputPath, Overrides: r.Overrides}) + result.Generation = &generated + runErr = err + if err != nil && (len(generated.Checks) == 0 || generated.Checks[len(generated.Checks)-1].ID != "publication") { + exitCode = 2 + } + path = generated.BundlePath + } + if runErr == nil && (inv.Phase != target.Generate || r.Deploy) { + var b *bundle.Bundle + b, runErr = bundle.Load(path) + if runErr != nil { + exitCode = 2 + } + if runErr == nil { + // Generate already captured overrides in the resolved snapshot; source + // must not be reapplied over that snapshot during --deploy. + source := r.UserConfig + if inv.Phase == target.Generate { + source = "" + } + g, err := generation.Configuration(b, source) + runErr = err + if err != nil { + exitCode = 2 + } + if runErr == nil { + mode := deployment.Deploy + if inv.Phase == target.Validate { + mode = deployment.Validate + } else if inv.Execution.DryRun { + mode = deployment.DryRun + } + absolute, err := filepath.Abs(path) + runErr = err + if runErr == nil { + deployed, err := (deployment.Runner{EvidenceDir: absolute + ".runs", Timeout: timeout, Wait: r.Wait > 0}).Run(ctx, b, g, mode) + result.Deployment = &deployed + runErr = err + } + } + } + } + if runErr != nil { + result.ExitCode = exitCode + result.Errors = append(result.Errors, runErr.Error()) + } else { + result.Outcome = "complete" + } + if inv.Output.Format == "json" { + if err := json.NewEncoder(r.Stdout).Encode(result); err != nil { + return apperrors.MarkReported(apperrors.NewGeneralError("cannot encode DPF workflow result", err)) + } + } else { + fmt.Fprintf(r.Stdout, "DPF %s: %s\n", inv.Phase, result.Outcome) + if result.Generation != nil && result.Generation.BundlePath != "" { + fmt.Fprintf(r.Stdout, "Bundle: %s\n", result.Generation.BundlePath) + } + if result.Deployment != nil { + for _, check := range result.Deployment.Checks { + if check.State != "passed" { + fmt.Fprintf(r.Stdout, "%s %s: %s (%s): %s\n", check.ID, check.Subject, check.State, check.ReasonCode, check.Message) + } + } + if result.Deployment.ReportPath != "" { + fmt.Fprintf(r.Stdout, "Report: %s\n", result.Deployment.ReportPath) + } + } + for _, message := range result.Errors { + fmt.Fprintln(r.Stderr, message) + } + } + if runErr != nil { + return apperrors.NewExitStatus(exitCode) + } + return nil + }), nil +} diff --git a/profiles/sentry-bf4/10-flavor.yaml b/profiles/sentry-bf4/10-flavor.yaml new file mode 100644 index 00000000..97d841ba --- /dev/null +++ b/profiles/sentry-bf4/10-flavor.yaml @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: provisioning.dpu.nvidia.com/v1alpha1 +kind: DPUFlavor +metadata: + name: {{ .Names.Flavor }} + namespace: {{ .ProvisioningNamespace }} + annotations: + provisioning.dpu.nvidia.com/num-of-trusted-sfs: "2" +spec: + sysctl: + parameters: + - net.ipv6.conf.all.disable_ipv6=1 + - net.ipv6.conf.default.disable_ipv6=1 + - net.ipv6.conf.lo.disable_ipv6=1 + - net.ipv4.conf.all.arp_ignore=1 + - net.ipv4.conf.all.arp_announce=2 + dpuMode: zero-trust + grub: + kernelParameters: + - console=hvc0 + - console=ttyAMA0 + - fixrttc + - net.ifnames=0 + - biosdevname=0 + - iommu.passthrough=1 + - cgroup_no_v1=net_prio,net_cls + - hugepagesz=2048kB + - hugepages=500 + nvconfig: + - device: "*" + parameters: + - PF_BAR2_ENABLE=0 + - PER_PF_NUM_SF=1 + - PF_TOTAL_SF=20 + - PF_SF_BAR_SIZE=14 + - NUM_PF_MSIX_VALID=0 + - PF_NUM_PF_MSIX_VALID=1 + - PF_NUM_PF_MSIX=228 + - INTERNAL_CPU_MODEL=1 + - INTERNAL_CPU_OFFLOAD_ENGINE=0 + - SRIOV_EN=1 + - NUM_OF_VFS=46 + - LAG_RESOURCE_ALLOCATION=1 + - LINK_TYPE_P1=ETH + - LINK_TYPE_P2=ETH + ovs: + rawConfigScript: | + _ovs-vsctl() { + ovs-vsctl --timeout 15 "$@" + } + + + for i in $(seq 0 99); do + _ovs-vsctl --if-exists del-br "ovsbr${i}" + done + ovs-appctl --timeout 15 dpctl/del-dp system@ovs-system &>/dev/null || true + + _ovs-vsctl set Open_vSwitch . other_config:doca-init=true + _ovs-vsctl set Open_vSwitch . other_config:dpdk-max-memzones=50000 + _ovs-vsctl set Open_vSwitch . other_config:hw-offload=true + _ovs-vsctl set Open_vSwitch . other_config:pmd-quiet-idle=false + _ovs-vsctl set Open_vSwitch . other_config:max-idle=20000 + _ovs-vsctl set Open_vSwitch . other_config:max-revalidator=5000 + _ovs-vsctl set Open_vSwitch . other_config:doca-congestion-threshold=60 + _ovs-vsctl set Open_vSwitch . other_config:flow-limit=500000 + _ovs-vsctl set Open_vSwitch . other_config:hw-offload-ct-unidir-udp-enabled=true + _ovs-vsctl set Open_vSwitch . other_config:hw-offload-ct-size=250000 + _ovs-vsctl remove Open_vSwitch . other_config default-datapath-type || true + + if systemctl list-unit-files openvswitch-switch.service &>/dev/null; then + systemctl restart openvswitch-switch + elif systemctl list-unit-files openvswitch.service &>/dev/null; then + systemctl restart openvswitch + fi + + _ovs-vsctl --may-exist add-br br-sfc + _ovs-vsctl set bridge br-sfc datapath_type=netdev + _ovs-vsctl set bridge br-sfc fail_mode=secure + _ovs-vsctl --may-exist add-port br-sfc p0 + _ovs-vsctl set Interface p0 type=dpdk + _ovs-vsctl set Interface p0 mtu_request=9216 + _ovs-vsctl set Port p0 external_ids:dpf-type=physical + + _ovs-vsctl --may-exist add-br br-envoy + _ovs-vsctl set bridge br-envoy datapath_type=netdev + + configFiles: + - path: /etc/mellanox/mlnx-bf.conf + operation: override + raw: | + ALLOW_SHARED_RQ="no" + IPSEC_FULL_OFFLOAD="no" + ENABLE_ESWITCH_MULTIPORT="yes" + SNAP_DMA_SF="no" + permissions: "0644" + - path: /etc/mellanox/mlnx-ovs.conf + operation: override + raw: | + CREATE_OVS_BRIDGES="no" + OVS_DOCA="yes" + permissions: "0644" + - path: /etc/mellanox/mlnx-sf.conf + operation: override + raw: "" + permissions: "0644" + - path: /usr/local/sbin/br-envoy-vfs + operation: override + permissions: "0755" + raw: | + #!/bin/bash + set -euo pipefail + + _ovs-vsctl() { + ovs-vsctl --timeout 15 "$@" + } + + discover_vfs() { + devlink -j port show | python3 -c ' + import json + import sys + + ports = json.load(sys.stdin).get("port", {}) + matches = sorted( + (port.get("vfnum"), port.get("netdev") or port.get("ifname")) + for port in ports.values() + if port.get("flavour") == "pcivf" + and port.get("controller") == 1 + and port.get("pfnum") == 0 + and isinstance(port.get("vfnum"), int) + and (port.get("netdev") or port.get("ifname")) + ) + for _, netdev in matches: + print(netdev) + ' + } + + is_current_vf() { + local port=$1 + local vf + for vf in "${vf_representors[@]}"; do + [[ "$vf" == "$port" ]] && return 0 + done + return 1 + } + + for _ in $(seq 1 30); do + _ovs-vsctl br-exists br-envoy && break + sleep 2 + done + _ovs-vsctl br-exists br-envoy + + vf_representors=() + mapfile -t vf_representors < <(discover_vfs) + + mapfile -t bridge_ports < <(_ovs-vsctl list-ports br-envoy) + for port in "${bridge_ports[@]}"; do + [[ -n "$port" ]] || continue + tag="$(_ovs-vsctl --if-exists get Port "$port" external_ids:br-envoy-vf || true)" + tag="${tag//\"/}" + if [[ "$tag" == "1" ]] && ! is_current_vf "$port"; then + echo "[br-envoy-vfs] removing stale VF port ${port}" >&2 + _ovs-vsctl --if-exists del-port br-envoy "$port" + fi + done + + for vf_representor in "${vf_representors[@]}"; do + [[ -n "$vf_representor" ]] || continue + _ovs-vsctl --may-exist add-port br-envoy "${vf_representor}" -- \ + set Interface "${vf_representor}" type=dpdk mtu_request=9216 -- \ + set Port "${vf_representor}" external_ids:br-envoy-vf=1 + done + + echo "[br-envoy-vfs] attached ${#vf_representors[@]} PF0 VF representor(s)" + - path: /usr/local/sbin/br-envoy-vfs-monitor + operation: override + permissions: "0755" + raw: | + #!/bin/bash + set -uo pipefail + + reconcile() { + until /usr/local/sbin/br-envoy-vfs; do + sleep 5 + done + } + + { + trap 'kill $(jobs -p) 2>/dev/null' EXIT + devlink monitor port & + ovsdb-client monitor Open_vSwitch Port name,external_ids & + echo start + wait -n + } | while read -r _; do + while read -r -t 2 _; do :; done + reconcile + done + - path: /etc/netplan/99-p1.yaml + operation: override + permissions: "0600" + raw: | + network: + version: 2 + ethernets: + p1: + renderer: networkd + dhcp4: false + dhcp6: false + accept-ra: false + link-local: [] + - path: /etc/netplan/99-disable-mlx-ipv6.yaml + operation: override + permissions: "0600" + raw: | + network: + version: 2 + ethernets: + enP2p1s0f0S0: + dhcp4: true + dhcp6: false + accept-ra: false + link-local: [] + enP6p1s0f0S0: + dhcp4: true + dhcp6: false + accept-ra: false + link-local: [] + - path: /etc/systemd/system/br-envoy-vfs.service + operation: override + permissions: "0644" + raw: | + [Unit] + Description=Attach PF0 host VF representors to br-envoy + After=openvswitch-switch.service + + [Service] + Type=simple + ExecStart=/usr/local/sbin/br-envoy-vfs-monitor + Restart=always + RestartSec=5 + + [Install] + WantedBy=multi-user.target + systemdServices: + - name: br-envoy-vfs.service + operation: EnableAndStart diff --git a/profiles/sentry-bf4/20-passthrough-interfaces.yaml b/profiles/sentry-bf4/20-passthrough-interfaces.yaml new file mode 100644 index 00000000..fd9f1a33 --- /dev/null +++ b/profiles/sentry-bf4/20-passthrough-interfaces.yaml @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: {{ .Names.Physical }} + namespace: {{ .Namespace }} +spec: + template: + spec: + nodeSelector: + matchLabels: + {{ .ScopeKey }}: "{{ .ScopeValue }}" + template: + metadata: + labels: + {{ .PassthroughLabel }}: p0 + spec: + interfaceType: physical + physical: + interfaceName: p0 +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: {{ .Names.PF }} + namespace: {{ .Namespace }} +spec: + template: + spec: + nodeSelector: + matchLabels: + {{ .ScopeKey }}: "{{ .ScopeValue }}" + template: + metadata: + labels: + {{ .PassthroughLabel }}: pf0hpf + spec: + interfaceType: pf + pf: + pfID: 0 +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: {{ .Names.Patch }} + namespace: {{ .Namespace }} +spec: + template: + spec: + nodeSelector: + matchLabels: + {{ .ScopeKey }}: "{{ .ScopeValue }}" + template: + metadata: + labels: + {{ .PassthroughLabel }}: br-envoy-patch + spec: + interfaceType: patch + patch: + peerBridge: br-envoy diff --git a/profiles/sentry-bf4/21-passthrough-chain.yaml b/profiles/sentry-bf4/21-passthrough-chain.yaml new file mode 100644 index 00000000..6f8e9851 --- /dev/null +++ b/profiles/sentry-bf4/21-passthrough-chain.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceChain +metadata: + name: {{ .Names.Passthrough }} + namespace: {{ .Namespace }} +spec: + template: + spec: + nodeSelector: + matchLabels: + {{ .ScopeKey }}: "{{ .ScopeValue }}" + template: + spec: + switches: + - serviceMTU: 1500 + ports: + - serviceInterface: + matchLabels: + {{ .PassthroughLabel }}: p0 + - serviceInterface: + matchLabels: + {{ .PassthroughLabel }}: pf0hpf + - serviceInterface: + matchLabels: + {{ .PassthroughLabel }}: br-envoy-patch diff --git a/profiles/sentry-bf4/30-tower-nads.yaml b/profiles/sentry-bf4/30-tower-nads.yaml new file mode 100644 index 00000000..8ca6a85a --- /dev/null +++ b/profiles/sentry-bf4/30-tower-nads.yaml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceNAD +metadata: + name: {{ .Names.IngressNAD }} + namespace: {{ .Namespace }} + annotations: + dpuservicenad.svc.dpu.nvidia.com/use-trusted-sfs: "" +spec: + resourceType: sf + bridge: br-envoy + serviceMTU: 1500 + ipam: false +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceNAD +metadata: + name: {{ .Names.EgressNAD }} + namespace: {{ .Namespace }} + annotations: + dpuservicenad.svc.dpu.nvidia.com/use-trusted-sfs: "" +spec: + resourceType: sf + bridge: br-envoy + serviceMTU: 1500 + ipam: false diff --git a/profiles/sentry-bf4/31-tower-interfaces.yaml b/profiles/sentry-bf4/31-tower-interfaces.yaml new file mode 100644 index 00000000..62f54336 --- /dev/null +++ b/profiles/sentry-bf4/31-tower-interfaces.yaml @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: {{ .Names.IngressInterface }} + namespace: {{ .Namespace }} +spec: + template: + spec: + nodeSelector: + matchLabels: + {{ .ScopeKey }}: "{{ .ScopeValue }}" + template: + metadata: + labels: + {{ .TowerLabel }}: tower-ingress + spec: + interfaceType: service + service: + serviceID: {{ .Names.Tower }} + network: {{ .Names.IngressNAD }} + interfaceName: net1 +--- +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceInterface +metadata: + name: {{ .Names.EgressInterface }} + namespace: {{ .Namespace }} +spec: + template: + spec: + nodeSelector: + matchLabels: + {{ .ScopeKey }}: "{{ .ScopeValue }}" + template: + metadata: + labels: + {{ .TowerLabel }}: tower-egress + spec: + interfaceType: service + service: + serviceID: {{ .Names.Tower }} + network: {{ .Names.EgressNAD }} + interfaceName: net2 diff --git a/profiles/sentry-bf4/32-tower.yaml b/profiles/sentry-bf4/32-tower.yaml new file mode 100644 index 00000000..1537d077 --- /dev/null +++ b/profiles/sentry-bf4/32-tower.yaml @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUService +metadata: + name: {{ .Names.Tower }} + namespace: {{ .Namespace }} +spec: + serviceID: {{ .Names.Tower }} + security: + privileged: true + interfaces: + - {{ .Names.IngressInterface }} + - {{ .Names.EgressInterface }} + helmChart: + source: + repoURL: oci://nvcr.io/nvstaging/doca + chart: doca-tower + version: "0.2.16" + values: + dnsPolicy: Default + # enforce: ext_proc observability_mode false and tower-tap replies, + # so a pack with judgeEnforce can turn a blocked tool call into a 403. + dataplane: + mode: enforce + image: + repository: docker.io/envoyproxy/envoy + tag: {{ .EnvoyTag }} + pullPolicy: IfNotPresent + imagePullSecrets: + - name: {{ .PullSecret }} + initImage: + repository: nvcr.io/nvstaging/doca/doca-tower-openflow + tag: {{ .OpenFlowTag }} + pullPolicy: Always + ingress: + interface: net1 + cidr: {{ .IngressCIDR }} + ip: {{ .IngressIP }} + prefix: {{ .IngressPrefix }} + mitm: + enabled: true + secretName: {{ .MITMSecret }} + tproxy: + httpsListen: 8445 + openflow: + enabled: true + normalActionWhenDisabled: false + image: + repository: nvcr.io/nvstaging/doca/doca-tower-openflow + tag: {{ .OpenFlowTag }} + pullPolicy: Always + egress: + enabled: true + extauthzd: + enabled: true + policyOnly: true + nodeID: doca-tower + image: + repository: nvcr.io/nvstaging/doca/doca-tower-policy-plugin + tag: {{ .PluginTag }} + pullPolicy: IfNotPresent + binary: /extauthzd + # The chart mounts this path into extauthzd. Argus creates + # argus_query.sock in this directory. + hostPath: /etc/doca_argus_config + argusSock: /etc/doca_argus_config/argus_query.sock + policySocket: /var/run/doca-tower-policy/policy.sock + policySocketHostPath: /var/run/doca-tower-policy + policyStore: "" + tap: + enabled: {{ .LLMInspection }} + judgeURL: https://inference-api.nvidia.com/openjev/v1/systemone + judgeModel: openjev-4b +{{- if .LLMInspection }} + inferenceSecret: {{ .InferenceSecret }} +{{- end }} + image: + repository: nvcr.io/nvstaging/doca/doca-tower-tap + tag: {{ .TapTag }} + pullPolicy: IfNotPresent + hostPath: /opt/doca-tower + binary: /usr/local/bin/tower-tap + serviceDaemonSet: + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .ScopeKey }} + operator: In + values: + - "{{ .ScopeValue }}" + labels: + dpuservice.dpu.nvidia.com/name: {{ .Names.Tower }} + svc.dpu.nvidia.com/service: {{ .Names.Tower }} + annotations: + dpuservice.dpu.nvidia.com/name: {{ .Names.Tower }} + k8s.v1.cni.cncf.io/networks: '[{"name":"{{ .Names.IngressNAD }}","namespace":"{{ .Namespace }}","interface":"net1"},{"name":"{{ .Names.EgressNAD }}","namespace":"{{ .Namespace }}","interface":"net2"}]' + resources: + cpu: "1" + memory: 1Gi + nvidia.com/bf_sf_trusted: "2" diff --git a/profiles/sentry-bf4/33-tower-chain.yaml b/profiles/sentry-bf4/33-tower-chain.yaml new file mode 100644 index 00000000..c89ca240 --- /dev/null +++ b/profiles/sentry-bf4/33-tower-chain.yaml @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceChain +metadata: + name: {{ .Names.Tower }} + namespace: {{ .Namespace }} +spec: + template: + spec: + nodeSelector: + matchLabels: + {{ .ScopeKey }}: "{{ .ScopeValue }}" + template: + spec: + switches: + - serviceMTU: 1500 + ports: + - serviceInterface: + matchLabels: + {{ .TowerLabel }}: tower-ingress + - serviceMTU: 1500 + ports: + - serviceInterface: + matchLabels: + {{ .TowerLabel }}: tower-egress diff --git a/profiles/sentry-bf4/40-controller.yaml b/profiles/sentry-bf4/40-controller.yaml new file mode 100644 index 00000000..5593769b --- /dev/null +++ b/profiles/sentry-bf4/40-controller.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUService +metadata: + name: {{ .Names.Controller }} + namespace: {{ .Namespace }} +spec: + serviceID: {{ .Names.Controller }} + security: + privileged: false + helmChart: + source: + repoURL: https://helm.ngc.nvidia.com/nvstaging/doca + chart: doca-sentry-policy-controller + version: "0.2.5" + values: + image: + repository: nvcr.io/nvstaging/doca/doca-tower-policy-sidecar + tag: {{ .ControllerTag }} + pullPolicy: Always + imagePullSecrets: + - name: {{ .PullSecret }} + hostNetwork: true + webhook: + enabled: false + serviceDaemonSet: + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .ScopeKey }} + operator: In + values: + - "{{ .ScopeValue }}" + labels: + dpuservice.dpu.nvidia.com/name: {{ .Names.Controller }} + svc.dpu.nvidia.com/service: {{ .Names.Controller }} + annotations: + dpuservice.dpu.nvidia.com/name: {{ .Names.Controller }} + resources: + cpu: "200m" + memory: 256Mi diff --git a/profiles/sentry-bf4/50-argus.yaml b/profiles/sentry-bf4/50-argus.yaml new file mode 100644 index 00000000..8cc00ee1 --- /dev/null +++ b/profiles/sentry-bf4/50-argus.yaml @@ -0,0 +1,120 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUService +metadata: + name: {{ .Names.Argus }} + namespace: {{ .Namespace }} +spec: + serviceID: {{ .Names.Argus }} + security: + privileged: true + helmChart: + source: + repoURL: https://helm.ngc.nvidia.com/nvstaging/doca + chart: doca-argus + version: "1.5.1-ai-gateway.8" + values: + containerImage: {{ .ArgusImage }} + imagePullSecrets: + - name: {{ .PullSecret }} + config: + isLocalPath: false + hostConfigDir: /etc/doca_argus_config + content: + service: + query_socket_path: /etc/doca_argus_config/argus_query.sock + immediate_shutdown: true + service_log_level: 50 + system_scanner_sleep_time: 10s + pool_num_workers: 7 + apsh_config: + auto_scan: true + only_scan_pf: true + default: + os_symbol_path: auto + mem_regions_path: auto + os_type: linux + service_log_level: 50 + sdk_log_level: 50 + limits: + string: 2100000 + process: 10000 + file_descriptor: 10000 + thread: 10000 + process_memory: 10000 + pool_size: 100000000 + events: + container_filter: false + sbom: + containers: false + none_containers: false + collection: + events: + process: + disable_scan: false + scans_frequency: false + process_created: true + process_terminated: true + process_zombie: true + process_hidden: true + thread: + disable_scan: false + scans_frequency: 1s + thread_created: true + thread_terminated: true + file_descriptor: + disable_scan: false + scans_frequency: false + file_descriptor_open: true + file_descriptor_close: true + file_descriptor_file_content_change: true + network_connection: + disable_scan: false + scans_frequency: false + network_connection_created: true + network_connection_terminated: true + process_memory: + disable_scan: false + scans_frequency: 10s + new_executable_anonymous_memory_mapped: true + file_unmapped: true + executable_permissions_added: true + executable_permissions_removed: true + process_attestation: + disable_scan: false + scans_frequency: 10s + new_file_mapped: true + foreign_binary_executed: true + foreign_library_loaded: true + shell_command: + disable_scan: false + scans_frequency: 1s + shell_command_executed: true + shell_history_cleared: true + shell_history_disabled: true + output: + log_stdout: false + log_folder_path: /var/log/doca_argus_activity_report + log_threshold_size: 100M + log_max_files_count: 10 + telemetry_address: false + telemetry_tag: fluentbit_forward_input + telemetry_format: json + telemetry_user_data: NONE + serviceDaemonSet: + nodeSelector: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .ScopeKey }} + operator: In + values: + - "{{ .ScopeValue }}" + labels: + dpuservice.dpu.nvidia.com/name: {{ .Names.Argus }} + svc.dpu.nvidia.com/service: {{ .Names.Argus }} + annotations: + dpuservice.dpu.nvidia.com/name: {{ .Names.Argus }} + resources: + cpu: "4" + memory: 10Gi diff --git a/profiles/sentry-bf4/60-policy-rbac.yaml b/profiles/sentry-bf4/60-policy-rbac.yaml new file mode 100644 index 00000000..5d1fcd14 --- /dev/null +++ b/profiles/sentry-bf4/60-policy-rbac.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Names.Server }} + namespace: {{ .Namespace }} +rules: + - apiGroups: ["tower.dpu.nvidia.com"] + resources: ["docatowerpolicies"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["tower.dpu.nvidia.com"] + resources: ["docatowerpolicies/status"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Names.Server }} + namespace: {{ .Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Names.Server }} +subjects: + - kind: ServiceAccount + name: {{ .Names.Server }} + namespace: {{ .Namespace }} diff --git a/profiles/sentry-bf4/61-policy-credential.yaml b/profiles/sentry-bf4/61-policy-credential.yaml new file mode 100644 index 00000000..d259ca13 --- /dev/null +++ b/profiles/sentry-bf4/61-policy-credential.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUServiceCredentialRequest +metadata: + name: {{ .Names.Server }} + namespace: {{ .Namespace }} +spec: + duration: 24h + serviceAccount: + name: {{ .Names.Server }} + namespace: {{ .Namespace }} + targetCluster: + name: {{ .TenantName }} + namespace: {{ .TenantNamespace }} + type: kubeconfig + secret: + name: {{ .Names.CredentialSecret }} + namespace: {{ .Namespace }} diff --git a/profiles/sentry-bf4/62-policy-server.yaml b/profiles/sentry-bf4/62-policy-server.yaml new file mode 100644 index 00000000..a9266033 --- /dev/null +++ b/profiles/sentry-bf4/62-policy-server.yaml @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +apiVersion: svc.dpu.nvidia.com/v1alpha1 +kind: DPUService +metadata: + name: {{ .Names.Server }} + namespace: {{ .Namespace }} +spec: + serviceID: {{ .Names.Server }} + deployInCluster: true + helmChart: + source: + repoURL: https://helm.ngc.nvidia.com/nvstaging/doca + chart: doca-sentry-policy-server + version: "0.2.5" + values: + service: + type: NodePort + port: 8080 + nodePort: {{ .NodePort }} + image: + repository: nvcr.io/nvstaging/doca/doca-tower-policy-rest + tag: {{ .ServerTag }} + pullPolicy: Always + imagePullSecrets: + - name: {{ .PullSecret }} + dpuKubeconfig: + secretName: {{ .Names.CredentialSecret }} + secretKey: kubeconfig + mountPath: /var/run/dpu-cluster + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule diff --git a/profiles/sentry-bf4/profile.yaml b/profiles/sentry-bf4/profile.yaml new file mode 100644 index 00000000..98d1c369 --- /dev/null +++ b/profiles/sentry-bf4/profile.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +name: BlueField-4 Sentry +plugin: dpf +target: dpf +description: Zero Trust Sentry on explicitly selected enrolled BlueField-4 devices. +templates: + - 10-flavor.yaml + - 20-passthrough-interfaces.yaml + - 21-passthrough-chain.yaml + - 30-tower-nads.yaml + - 31-tower-interfaces.yaml + - 32-tower.yaml + - 33-tower-chain.yaml + - 40-controller.yaml + - 50-argus.yaml + - 60-policy-rbac.yaml + - 61-policy-credential.yaml + - 62-policy-server.yaml diff --git a/skills/k8s-launch-kit-deploy/SKILL.md b/skills/k8s-launch-kit-deploy/SKILL.md index 3ad6db12..6c8c1a5b 100644 --- a/skills/k8s-launch-kit-deploy/SKILL.md +++ b/skills/k8s-launch-kit-deploy/SKILL.md @@ -11,6 +11,22 @@ metadata: > **PREREQUISITE:** Read `../k8s-launch-kit-shared/SKILL.md` for install paths, global flags, and output modes. +## DPF Sentry deployment + +For an explicit DPF request, use `l8k deploy --target dpf +--deployment-files ` and follow `docs/user/dpf-sentry.md`. The bundle +binds management/tenant access and selected enrolled BF4 devices; do not add +Host `--kubeconfig` or infer an ambient context. Existing namespaces, registry +and MITM Secrets, operator/tenant health and exact foreign-owner transition +are external prerequisites. Preview native API admission with `--dry-run`, +review unresolved checks, then use an explicit `--deploy-timeout`. Real deploy +can reimage devices and require host power handling. It never implicitly +adopts/removes another owner or applies chart projections twice. Reports go to +`.runs/`; preserve the bundle and failed-run evidence before retrying. +The Host procedure below applies to `host` only. + +## Host workflow + This workflow uses the default `host` target; `--target host` is equivalent. The CLI snapshots deployment arguments and runs the standalone Host deploy service through the target registry; command syntax and apply ordering are diff --git a/skills/k8s-launch-kit-generate/SKILL.md b/skills/k8s-launch-kit-generate/SKILL.md index b0b2db80..e60b5977 100644 --- a/skills/k8s-launch-kit-generate/SKILL.md +++ b/skills/k8s-launch-kit-generate/SKILL.md @@ -11,6 +11,20 @@ metadata: > **PREREQUISITE:** Read `../k8s-launch-kit-shared/SKILL.md` for install paths, global flags, and output modes. +## DPF Sentry generation + +For an explicit DPF request, use `--target dpf` and follow +`docs/user/dpf-sentry.md`. Generate from a retained discovery inventory plus +explicit selected device/tenant bindings, site networks, SHA-bound BF4 software +and four catalogued local chart archives. `l8k schema` exposes `dpf.profiles`, +`dpf.generationReleases` and `dpf.charts`. Generation reads no credentials and +contacts no cluster. Core policy defaults to `llmInspection: false`; enabling +LLM inspection requires a real inference Secret. The resolved bundle is +immutable; use a new directory for changed desired intent. Do not apply chart +projections separately. The Host procedure below applies to `host` only. + +## Host workflow + This workflow uses the default `host` target; `--target host` is equivalent. The CLI snapshots generation arguments and runs the concrete Host generation operation through the target registry; command syntax and artifacts are diff --git a/skills/k8s-launch-kit-shared/SKILL.md b/skills/k8s-launch-kit-shared/SKILL.md index 6b03c252..b2771257 100644 --- a/skills/k8s-launch-kit-shared/SKILL.md +++ b/skills/k8s-launch-kit-shared/SKILL.md @@ -68,8 +68,12 @@ the `host` target. Existing invocations must omit `--target` unless the user explicitly requests it; `--target host` is an equivalent explicit form. The `dpf` target exposes experimental inventory discovery through explicit -SSH/Redfish sources. Its provisioning phases remain unavailable. Do not attempt DPF provisioning. Use `l8k schema` and inspect -`targets[].phases` before selecting any non-host target. Host-only flags such as +SSH/Redfish sources and Sentry BF4 generate/deploy/validate on existing DPF +management, tenant and enrolled devices. Follow `docs/user/dpf-sentry.md`; the +root DPF pipeline remains unavailable. Use `l8k schema` and inspect +`targets[].phases`, `dpf.profiles`, `dpf.generationReleases` and `dpf.charts` +before selecting this workflow. DPF uses the explicit management file/context +in its site/resolved input, plus the selected tenant binding. Host-only flags such as `--fabric`, `--kubeconfig`, and the Network Operator/Spectrum-X flags are rejected when explicitly combined with another target. @@ -92,7 +96,7 @@ root-only, `--deploy-timeout` belongs to root/standalone deploy, and | Flag | Description | |------|-------------| -| `--target ` | Lifecycle target: `host` (default); `dpf` offers experimental inventory discovery | +| `--target ` | Lifecycle target: `host` (default); `dpf` offers experimental inventory and Sentry BF4 generation/deployment/resource validation | | `--kubeconfig ` | Path to kubeconfig file (optional — falls back to `$KUBECONFIG` env var) | | `--user-config ` | Path to user-supplied l8k-config.yaml | | `--output ` | Output format: `text` (default), `json` | diff --git a/skills/k8s-launch-kit-validate/SKILL.md b/skills/k8s-launch-kit-validate/SKILL.md index 70909f5b..b8e33acf 100644 --- a/skills/k8s-launch-kit-validate/SKILL.md +++ b/skills/k8s-launch-kit-validate/SKILL.md @@ -11,6 +11,20 @@ metadata: > **PREREQUISITE:** Read `../k8s-launch-kit-shared/SKILL.md` for install paths, global flags, and exit codes. +## DPF Sentry resource validation + +For an explicit DPF request, use `l8k validate --target dpf +--deployment-files --wait 10m` and follow `docs/user/dpf-sentry.md`. +Access comes from the bundle's bound management/tenant inputs. Validation +verifies current resource state, identities and selected coverage without +cluster writes, and writes evidence to `.runs/`. It does not run a +Sentry allow/deny or LLM test. Perform those functional checks separately with +the actual selected VF/MAC and public MITM CA. Do not reuse Host connectivity, +HTML/JUnit report or Network Operator checks for this target. The Host +procedure below applies to `host` only. + +## Host workflow + This workflow uses the default `host` target; `--target host` is equivalent. The CLI snapshots validation arguments and runs the standalone Host validation and report service through the target registry; checks and report semantics From 3b3736ef1eda19ea43a652f9d998d6c70c00cc61 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 6 Nov 2026 20:31:50 +0100 Subject: [PATCH 2/9] fix: serialize DPF device provisioning across host transitions Wait for each DPUSet and selected host transitions before submitting the next physical operation. Preserve global identity, ownership, BMC and artifact gates, keep service creation grouped, and retain projection validation. Cover separate and shared host domains, cancellation, late drift, and projected resource drift. Signed-off-by: Alexander Maslennikov --- docs/user/dpf-sentry.md | 5 +- docs/user/dpf-workflows.md | 2 +- openspec/specs/workflow-execution/spec.md | 6 + openspec/workflows/dpf-discovery.json | 2 +- pkg/dpf/deployment/engine.go | 19 ++ .../deployment/multidevice_regression_test.go | 262 ++++++++++++++++++ 6 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 pkg/dpf/deployment/multidevice_regression_test.go diff --git a/docs/user/dpf-sentry.md b/docs/user/dpf-sentry.md index 94254b92..e2aa5951 100644 --- a/docs/user/dpf-sentry.md +++ b/docs/user/dpf-sentry.md @@ -240,7 +240,10 @@ The CLI checks existing operator/tenant health, selected identities and ownershi service namespaces/Secret keys, then creates the bundle's resources in dependency order. It verifies the downloaded provisioning bytes before each DPUSet creates physical effects. DPF performs reimaging, host power coordination and service -reconciliation. External firmware caches must remain unchanged during provisioning. +reconciliation. Launch Kit provisions devices sequentially, waiting for each +DPUSet to become Ready and host transition markers to clear before submitting +the next. Include all devices and required host power cycles in the deployment +timeout. External firmware caches must remain unchanged during provisioning. A host cold boot also affects co-resident devices and workloads; include that host-wide effect in the maintenance plan. diff --git a/docs/user/dpf-workflows.md b/docs/user/dpf-workflows.md index fda97bbb..3b636433 100644 --- a/docs/user/dpf-workflows.md +++ b/docs/user/dpf-workflows.md @@ -477,7 +477,7 @@ l8k deploy --target dpf --deployment-files ./sentry-bundle \ --deploy-timeout 90m --output json > deploy.json 2> deploy.log ``` -The CLI submits native resources in order. DPF downloads the desired ISO/firmware, and consumed artifact hashes are checked before DPUSet effects. DPF then provisions the selected DPUs. After the selected nodes and DNS are ready, the CLI submits Tower, the policy controller and Argus together, then waits for all three. The controller installs the policy API Tower needs, while Tower provides the controller's policy socket. The policy REST server starts afterward on management. Chart projections are observed, never applied again. Follow the existing DPF host-power procedure when a cold boot is required. Keep firmware caches unchanged during provisioning. +The CLI submits native resources in order. DPF downloads the desired ISO/firmware, and consumed artifact hashes are checked before DPUSet effects. DPF then provisions the selected DPUs. Launch Kit waits for each DPUSet to become Ready and host transition markers to clear before submitting the next; allow time for all devices and required host power cycles. After the selected nodes and DNS are ready, the CLI submits Tower, the policy controller and Argus together, then waits for all three. The controller installs the policy API Tower needs, while Tower provides the controller's policy socket. The policy REST server starts afterward on management. Chart projections are observed, never applied again. Follow the existing DPF host-power procedure when a cold boot is required. Keep firmware caches unchanged during provisioning. Next: [Run completed or stopped](#deploy-sentry-inspect). diff --git a/openspec/specs/workflow-execution/spec.md b/openspec/specs/workflow-execution/spec.md index abc4544e..b7470c94 100644 --- a/openspec/specs/workflow-execution/spec.md +++ b/openspec/specs/workflow-execution/spec.md @@ -95,6 +95,12 @@ Selected nodes SHALL use the external reboot method without pending power action - **WHEN** the selected DPUNode uses a reboot script or has a pending power request - **THEN** preflight blocks provisioning and identifies the unresolved external transition. +#### Scenario: Multiple selected devices + +- **WHEN** deployment submits a DPUSet for one selected device and DPF starts its host transition +- **THEN** the workflow waits for that DPUSet to become Ready and selected host transition markers to clear before submitting the next DPUSet, retaining the full identity, ownership, BMC and artifact checks before each physical create. +- **AND** timeout or cancellation retains the completed and pending observations and does not submit later DPUSets. + ### Requirement: Honest admission and selected readiness Unresolved required admission checks SHALL yield an incomplete result and nonzero diff --git a/openspec/workflows/dpf-discovery.json b/openspec/workflows/dpf-discovery.json index 7aa8cf97..b172719b 100644 --- a/openspec/workflows/dpf-discovery.json +++ b/openspec/workflows/dpf-discovery.json @@ -343,7 +343,7 @@ { "id": "deploy", "title": "CLI and management controllers: create and reconcile in dependency order", - "body": "Use an explicit budget that includes downloads, firmware and OS installation, required host power cycles, and service startup:\n\n```bash\nl8k deploy --target dpf --deployment-files ./sentry-bundle \\\n --deploy-timeout 90m --output json > deploy.json 2> deploy.log\n```\n\nThe CLI submits native resources in order. DPF downloads the desired ISO/firmware, and consumed artifact hashes are checked before DPUSet effects. DPF then provisions the selected DPUs. After the selected nodes and DNS are ready, the CLI submits Tower, the policy controller and Argus together, then waits for all three. The controller installs the policy API Tower needs, while Tower provides the controller's policy socket. The policy REST server starts afterward on management. Chart projections are observed, never applied again. Follow the existing DPF host-power procedure when a cold boot is required. Keep firmware caches unchanged during provisioning.", + "body": "Use an explicit budget that includes downloads, firmware and OS installation, required host power cycles, and service startup:\n\n```bash\nl8k deploy --target dpf --deployment-files ./sentry-bundle \\\n --deploy-timeout 90m --output json > deploy.json 2> deploy.log\n```\n\nThe CLI submits native resources in order. DPF downloads the desired ISO/firmware, and consumed artifact hashes are checked before DPUSet effects. DPF then provisions the selected DPUs. Launch Kit waits for each DPUSet to become Ready and host transition markers to clear before submitting the next; allow time for all devices and required host power cycles. After the selected nodes and DNS are ready, the CLI submits Tower, the policy controller and Argus together, then waits for all three. The controller installs the policy API Tower needs, while Tower provides the controller's policy socket. The policy REST server starts afterward on management. Chart projections are observed, never applied again. Follow the existing DPF host-power procedure when a cold boot is required. Keep firmware caches unchanged during provisioning.", "transitions": [ { "to": "inspect", diff --git a/pkg/dpf/deployment/engine.go b/pkg/dpf/deployment/engine.go index 978b319e..e1bdfdf5 100644 --- a/pkg/dpf/deployment/engine.go +++ b/pkg/dpf/deployment/engine.go @@ -197,9 +197,20 @@ func (s *run) execute(ctx context.Context) error { if err := s.document(ctx, stage, doc); err != nil { return err } + // A DPUSet starts a physical operation. Finish it before the next + // document rechecks the complete selected-device/BMC domain; those + // prerequisites may be unavailable during the operation just started. + if s.result.Mode == Deploy && doc.Ref().Kind == "DPUSet" { + if err := s.wait(ctx, stage.ID, doc); err != nil { + return err + } + } } if s.result.Mode != DryRun { for _, doc := range append(view.Documents(), view.Projections()...) { + if s.result.Mode == Deploy && doc.Ref().Kind == "DPUSet" && !doc.IsProjection() { + continue // Already waited before processing the next document. + } if err := s.wait(ctx, stage.ID, doc); err != nil { return err } @@ -382,6 +393,14 @@ func (s *run) wait(ctx context.Context, stage string, doc bundle.Document) error } } ready, terminal, message := Readiness(object) + if ready && doc.Ref().Kind == "DPUSet" { + // Native Ready can precede removal of the host reset marker. + // Do not start another operation until the selected domain is idle. + ready, message, err = s.selectedTransitionsClear(ctx) + if err != nil { + return err + } + } if ready && doc.IsProjection() && doc.Ref().Destination == bundle.DPU && doc.Ref().Kind == "DaemonSet" { ready, message, err = s.daemonSetCoverage(ctx, object) if err != nil { diff --git a/pkg/dpf/deployment/multidevice_regression_test.go b/pkg/dpf/deployment/multidevice_regression_test.go new file mode 100644 index 00000000..177b149f --- /dev/null +++ b/pkg/dpf/deployment/multidevice_regression_test.go @@ -0,0 +1,262 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package deployment + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/bundle" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +// Extend the existing lifecycle fixture with a second physical device, retaining +// distinct per-device selectors even when both cards share a host reset domain. +func multiDeviceFixture(sharedNode bool) (*bundle.Bundle, config.Generation, *fakeFactory, *fakeArtifacts) { + b, g, f, v := lifecycleFixture() + nodeName := "node-b" + if sharedNode { + nodeName = "node-a" + } + g.Provisioning.Devices = append(g.Provisioning.Devices, config.DeviceBinding{ID: "physical-b", DPUDevice: "device-b", DPUNode: nodeName}) + g.Devices = append(g.Devices, inventory.Device{ID: "physical-b", SerialNumber: "SERIAL-B", BMCEndpoint: "https://192.0.2.11"}) + g.DevicePSIDs["physical-b"] = "TEST-PSID" + device := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUDevice", g.Namespace, "device-a"))].DeepCopy() + device.SetName("device-b") + device.SetUID("device-b-uid") + device.SetLabels(map[string]string{"provisioning.dpu.nvidia.com/dpunode-name": nodeName, "provisioning.dpu.nvidia.com/dpudevice-bmc-ip": "192.0.2.11"}) + Expect(unstructured.SetNestedField(device.Object, "SERIAL-B", "spec", "serialNumber")).To(Succeed()) + Expect(unstructured.SetNestedField(device.Object, "192.0.2.11", "spec", "bmcIp")).To(Succeed()) + f.management.put(device) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", g.Namespace, "node-a"))].DeepCopy() + if sharedNode { + Expect(unstructured.SetNestedSlice(node.Object, []any{map[string]any{"name": "device-a"}, map[string]any{"name": "device-b"}}, "spec", "dpus")).To(Succeed()) + } else { + node.SetName(nodeName) + node.SetUID("node-b-uid") + Expect(unstructured.SetNestedSlice(node.Object, []any{map[string]any{"name": "device-b"}}, "spec", "dpus")).To(Succeed()) + } + f.management.put(node) + set := resourcesInBundle(b, "DPUSet")[0].ObjectCopy() + set.SetName("selected-set-b") + Expect(unstructured.SetNestedStringMap(set.Object, map[string]string{"provisioning.dpu.nvidia.com/dpunode-name": nodeName, "provisioning.dpu.nvidia.com/dpudevice-bmc-ip": "192.0.2.11"}, "spec", "dpuDeviceSelector", "matchLabels")).To(Succeed()) + data, err := yaml.Marshal(set.Object) + Expect(err).NotTo(HaveOccurred()) + i := b.Index() + i.BundleDigest = "" + i.SelectedDevices = append(i.SelectedDevices, "physical-b") + i.Files = append(i.Files, bundle.FileRecord{Path: "objects/2.yaml", Role: bundle.Manifest}) + i.Stages[1].Files = append(i.Stages[1].Files, "objects/2.yaml") + b, err = bundle.New(i, append(b.Files(), bundle.File{Path: "objects/2.yaml", Data: data})) + Expect(err).NotTo(HaveOccurred()) + pods, err := f.tenant.List(context.Background(), ref("", "", "Pod", "", ""), "") + Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + pod.SetName(pod.GetName() + "-b") + pod.SetUID(pod.GetUID() + "-b") + Expect(unstructured.SetNestedField(pod.Object, "dpu-child-b", "spec", "nodeName")).To(Succeed()) + f.tenant.put(pod) + } + return b, g, f, v +} + +func setDeviceReady(device *unstructured.Unstructured, ready bool) { + conditions, _, err := unstructured.NestedSlice(device.Object, "status", "conditions") + Expect(err).NotTo(HaveOccurred()) + for _, item := range conditions { + condition := item.(map[string]any) + if condition["type"] == "Ready" { + condition["status"] = "False" + if ready { + condition["status"] = "True" + } + } + } + Expect(unstructured.SetNestedSlice(device.Object, conditions, "status", "conditions")).To(Succeed()) +} + +var _ = Describe("serialized multi-device provisioning", func() { + newRunner := func(f *fakeFactory, v *fakeArtifacts) Runner { + dir, err := os.MkdirTemp("", "dpf-multidevice-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + dir, err = filepath.EvalSymlinks(dir) + Expect(err).NotTo(HaveOccurred()) + return Runner{Clients: f, Artifacts: v, EvidenceDir: dir, PollInterval: time.Millisecond, Timeout: 10 * time.Second} + } + // Progress each real persisted DPUSet through NotReady, native Ready with + // a still-pending host reset, then a clear host and Ready device. No timer or + // invocation count alone can create the second set before these live states. + progress := func(g config.Generation, f *fakeFactory, finished func(string)) *[]string { + f.management.noChildren = true + completed := []string{} + activeSet, activeNode, activeDevice := "", "", "" + setReads, nodeReads := 0, 0 + f.management.afterCreate = func(set *unstructured.Unstructured) { + if set.GetKind() != "DPUSet" { + return + } + Expect(activeSet).To(BeEmpty(), "another physical provisioning operation is still active") + binding := g.Provisioning.Devices[0] + childName := "dpu-child" + if set.GetName() == "selected-set-b" { + binding = g.Provisioning.Devices[1] + childName = "dpu-child-b" + Expect(completed).To(Equal([]string{"selected-set"})) + } + activeSet, activeNode, activeDevice = set.GetName(), binding.DPUNode, binding.DPUDevice + setReads, nodeReads = 0, 0 + current := f.management.objects[keyFor(set)] + Expect(unstructured.SetNestedField(current.Object, int64(0), "status", "availableReplicas")).To(Succeed()) + conditions, _, _ := unstructured.NestedSlice(current.Object, "status", "conditions") + conditions[0].(map[string]any)["status"] = "False" + Expect(unstructured.SetNestedSlice(current.Object, conditions, "status", "conditions")).To(Succeed()) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", g.Namespace, activeNode))] + node.SetAnnotations(map[string]string{"provisioning.dpu.nvidia.com/dpunode-external-reboot-required": "SystemLevelReset"}) + setDeviceReady(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUDevice", g.Namespace, activeDevice))], false) + child := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPU", g.Namespace, childName, map[string]any{"dpuDeviceName": binding.DPUDevice, "blueFieldSoftware": "software", "cluster": map[string]any{"name": g.Provisioning.DPUCluster.Name, "namespace": g.Provisioning.DPUCluster.Namespace}}) + scope, err := expectedDPUNodeLabels(set, binding.DPUNode, g.Namespace, g.Release) + Expect(err).NotTo(HaveOccurred()) + Expect(unstructured.SetNestedStringMap(child.Object, scope, "spec", "cluster", "nodeLabels")).To(Succeed()) + child.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: set.GetAPIVersion(), Kind: "DPUSet", Name: set.GetName(), UID: set.GetUID()}}) + setReady(child) + f.management.put(child) + tenantNode := testObject("v1", "Node", "", childName, nil) + tenantNode.SetLabels(scope) + setReady(tenantNode) + f.tenant.put(tenantNode) + } + f.management.onGet = func(reference bundle.ResourceRef) { + if activeSet == "" { + return + } + if reference.Kind == "DPUSet" && reference.Name == activeSet { + setReads++ + if setReads >= 2 { + setReady(f.management.objects[objectKey(reference)]) + } + } + if reference.Kind == "DPUNode" && reference.Name == activeNode && setReads >= 2 { + nodeReads++ + if nodeReads >= 2 { + node := f.management.objects[objectKey(reference)] + node.SetAnnotations(nil) + setDeviceReady(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUDevice", g.Namespace, activeDevice))], true) + name := activeSet + completed = append(completed, name) + activeSet = "" + if finished != nil { + finished(name) + } + } + } + } + return &completed + } + DescribeTable("waits for each physical operation and host reset before creating the next set", func(sharedNode bool) { + b, g, f, v := multiDeviceFixture(sharedNode) + completed := progress(g, f, nil) + r := newRunner(f, v) + result, err := r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(*completed).To(Equal([]string{"selected-set", "selected-set-b"})) + Expect(v.calls).To(Equal(2)) + before := len(f.management.creates) + result, err = r.Run(context.Background(), b, g, Deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.management.creates).To(HaveLen(before)) + result, err = r.Run(context.Background(), b, g, Validate) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Outcome).To(Equal("complete")) + Expect(f.management.creates).To(HaveLen(before)) + }, Entry("distinct hosts", false), Entry("two cards sharing one host reset domain", true)) + It("continues validating DPUSet projections that were never submitted for creation", func() { + b, g, f, v := lifecycleFixture() + projection := resourcesInBundle(b, "DPUSet")[0].ObjectCopy() + projection.SetName("projected-set") + Expect(unstructured.SetNestedStringMap(projection.Object, map[string]string{"provisioning.dpu.nvidia.com/dpunode-name": "external-node"}, "spec", "dpuDeviceSelector", "matchLabels")).To(Succeed()) + data, err := yaml.Marshal(projection.Object) + Expect(err).NotTo(HaveOccurred()) + i := b.Index() + i.BundleDigest = "" + i.Files = append(i.Files, bundle.FileRecord{Path: "projections/set.yaml", Role: bundle.Projection}) + i.Stages[1].ProjectionFiles = append(i.Stages[1].ProjectionFiles, "projections/set.yaml") + b, err = bundle.New(i, append(b.Files(), bundle.File{Path: "projections/set.yaml", Data: data})) + Expect(err).NotTo(HaveOccurred()) + actual := projection.DeepCopy() + Expect(unstructured.SetNestedField(actual.Object, "foreign-software", "spec", "dpuTemplate", "spec", "blueFieldSoftware", "name")).To(Succeed()) + f.management.put(actual) + _, err = newRunner(f, v).Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("PROJECTION_DRIFT"))) + Expect(f.management.creates).NotTo(ContainElement("DPUSet/projected-set")) + }) + It("still rejects a pending prior transition on the second selected host before any writes", func() { + b, g, f, v := multiDeviceFixture(false) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", g.Namespace, "node-b"))] + node.SetAnnotations(map[string]string{"provisioning.dpu.nvidia.com/dpunode-external-reboot-required": "SystemLevelReset"}) + _, err := newRunner(f, v).Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring("NODE_TRANSITION_PENDING"))) + Expect(f.management.creates).To(BeEmpty()) + }) + DescribeTable("retains full selected-scope checks before the next physical create", func(change, code string) { + b, g, f, v := multiDeviceFixture(false) + progress(g, f, func(name string) { + if name == "selected-set" { + switch change { + case "owner": + f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", g.Namespace, name))].SetUID("replacement-set-uid") + case "device": + device := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUDevice", g.Namespace, "device-b"))] + Expect(unstructured.SetNestedField(device.Object, "FOREIGN-SERIAL", "spec", "serialNumber")).To(Succeed()) + case "foreign": + child := testObject("provisioning.dpu.nvidia.com/v1alpha1", "DPU", g.Namespace, "foreign-child", map[string]any{"dpuDeviceName": "device-b"}) + child.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: "provisioning.dpu.nvidia.com/v1alpha1", Kind: "DPUSet", Name: "foreign-set", UID: "foreign-set-uid"}}) + f.management.put(child) + } + } + }) + _, err := newRunner(f, v).Run(context.Background(), b, g, Deploy) + Expect(err).To(MatchError(ContainSubstring(code))) + Expect(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", g.Namespace, "selected-set-b"))]).To(BeNil()) + }, Entry("replaced owner UID", "owner", "STALE_DPU_OWNER"), Entry("second device identity drift", "device", "DEVICE_IDENTITY_MISMATCH"), Entry("foreign ownership of the second device", "foreign", "FOREIGN_DPU_OWNER")) + It("stops on cancellation with a pending first host reset without creating the second set", func() { + b, g, f, v := multiDeviceFixture(false) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started := false + nodeReads := 0 + f.management.afterCreate = func(object *unstructured.Unstructured) { + if object.GetKind() == "DPUSet" { + Expect(object.GetName()).To(Equal("selected-set")) + node := f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUNode", g.Namespace, "node-a"))] + node.SetAnnotations(map[string]string{"provisioning.dpu.nvidia.com/dpunode-external-reboot-required": "SystemLevelReset"}) + started = true + } + } + f.management.onGet = func(reference bundle.ResourceRef) { + if started && reference.Kind == "DPUNode" && reference.Name == "node-a" { + nodeReads++ + if nodeReads >= 3 { + cancel() + } + } + } + result, err := newRunner(f, v).Run(ctx, b, g, Deploy) + Expect(err).To(MatchError(context.Canceled)) + Expect(result.Outcome).To(Equal("cancelled")) + Expect(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", g.Namespace, "selected-set"))]).NotTo(BeNil()) + Expect(f.management.objects[objectKey(ref("", "provisioning.dpu.nvidia.com", "DPUSet", g.Namespace, "selected-set-b"))]).To(BeNil()) + Expect(v.calls).To(Equal(1)) + }) +}) From 4ac3a8c9b52477d4c8fb73f94efdc5ad5284441c Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 13 Nov 2026 10:19:17 +0100 Subject: [PATCH 3/9] Require administrator discovery evidence for Sentry handoff Signed-off-by: Alexander Maslennikov --- docs/user/dpf-sentry.md | 20 ++++++++ openspec/specs/artifact-handoff/spec.md | 5 ++ openspec/specs/profile-planning/spec.md | 6 +++ pkg/dpf/config/generation.go | 26 +++++++---- pkg/dpf/config/generation_test.go | 62 ++++++++++++++++++++++++- pkg/dpf/config/resolved.go | 5 +- pkg/dpf/config/resolved_test.go | 16 +++++++ pkg/dpf/generation/generation_test.go | 35 +++++++++++++- skills/k8s-launch-kit-deploy/SKILL.md | 5 ++ skills/k8s-launch-kit-generate/SKILL.md | 8 +++- 10 files changed, 173 insertions(+), 15 deletions(-) diff --git a/docs/user/dpf-sentry.md b/docs/user/dpf-sentry.md index e2aa5951..3bab7c4d 100644 --- a/docs/user/dpf-sentry.md +++ b/docs/user/dpf-sentry.md @@ -78,6 +78,15 @@ device administration process: - Existing DPF BMC credentials, controller client certificates/trust and the necessary API permissions. Artifact serving, DPU management connectivity, DNS, time, image pulls and host power handling must work for the chosen cohort. +- BMC firmware exposing the Redfish schema supported by the selected DPF + operator and this deployment preflight, including + `/redfish/v1/Systems/BlueField_0` and a serial-bearing + `/redfish/v1/Chassis/BlueField_0`. Discovery can identify older B4240 firmware + through its legacy component/Card graph; that does not make it provisionable. + Arrange a supported firmware update through the device administrator before + enrollment when that schema is missing, then rediscover and enroll through the + existing DPF process. Generation does not update firmware, and deployment + does not bypass a failed BMC preflight or substitute SSH access. For a functional host VF test, also prepare the host's VF allocation, device resource and network attachment separately. This DPF workflow does not install @@ -97,6 +106,17 @@ Copy a successful [discovered configuration](dpf-discovery.md) to below; replace every angle-bracket placeholder before generating. Do not edit the inventory to turn an unknown or conflicting observation into a fact. +Every selected device must retain successful discovery evidence under +`observations.bmc-administrator`: `state: observed` and +`reasonCode: BMC_ADMIN_VERIFIED`, with an explicit BMC endpoint and no ARM SSH +endpoint. Generation accepts the exact observed model `BlueField-4`, or `B4240` +from the reviewed `bluefield-card-v1` reader, and preserves that measured model. +Unselected records do not need to satisfy this selection gate. If older inventory +or a retained bundle lacks this proof, rerun discovery with DPU BMC administrator +credentials, review its completed report, and generate into a new bundle +directory. Do not add the proof by editing YAML. Saved privileges are an +observation; deployment still verifies live DPF access and its own prerequisites. + | Input class | What belongs here | | --- | --- | | Required site intent | Explicit installed DPF release; selected inventory device IDs; existing DPUDevice/DPUNode names; selected tenant name, namespace, resource UID, tenant cluster UID and exact label selector; tenant API server version; ISO and PSID-specific firmware URLs with SHA-256; local chart directory; ingress subnet/address; existing Secret names; reachable management policy-server address. | diff --git a/openspec/specs/artifact-handoff/spec.md b/openspec/specs/artifact-handoff/spec.md index a76d190d..c67834c4 100644 --- a/openspec/specs/artifact-handoff/spec.md +++ b/openspec/specs/artifact-handoff/spec.md @@ -100,6 +100,11 @@ explicit access-path relocation SHALL not replace captured deployment intent. - **WHEN** a bundle has consistent hashes but changed resources or stages inconsistent with its resolved profile - **THEN** deploy and validate reject it before cluster access or writes. +#### Scenario: Retained DPF bundle predates administrator discovery evidence + +- **WHEN** selected devices in retained resolved metadata lack successful BMC administrator discovery proof +- **THEN** source generation, resolved-metadata encoding and bundle consumption reject the missing proof with rediscovery/regeneration guidance before publication or cluster access. The inventory/sidecar format version alone does not satisfy the requirement. + #### Scenario: Relocated discovery output - **WHEN** discovery saves a source config in another directory diff --git a/openspec/specs/profile-planning/spec.md b/openspec/specs/profile-planning/spec.md index 0c8800f0..6130f031 100644 --- a/openspec/specs/profile-planning/spec.md +++ b/openspec/specs/profile-planning/spec.md @@ -34,6 +34,12 @@ A profile extension MUST define eligibility and preserve exact selected-group sc - **WHEN** the requested profile combination is outside declared eligibility - **THEN** selection rejects it or follows an explicitly documented selection policy. +#### Scenario: DPF Sentry selects discovered BF4 devices + +- **WHEN** Sentry generation selects inventory device IDs +- **THEN** every selected device has an observed `BlueField-4` identity, or exact `B4240` identity from `bluefield-card-v1`, an explicit BMC endpoint without an ARM SSH substitute, and successful `bmc-administrator` evidence (`observed`, `BMC_ADMIN_VERIFIED`). Measured model values are preserved; unselected records do not broaden or block the selection. +- **AND** missing or failed administrator proof requires rediscovery and regeneration; saved evidence neither qualifies legacy firmware for provisioning nor replaces live DPF authentication and preflight. + ### Requirement: Multiplicity and unique identities A template extension MUST declare render multiplicity, API scope and reference namespaces separately. New or changed identity rules SHALL detect collisions before output-map assignment can overwrite an artifact. Stable inputs SHALL produce stable identities. diff --git a/pkg/dpf/config/generation.go b/pkg/dpf/config/generation.go index 55520c47..7f875069 100644 --- a/pkg/dpf/config/generation.go +++ b/pkg/dpf/config/generation.go @@ -261,8 +261,8 @@ func resolveProvisioning(g *Generation, inv *inventory.Inventory) error { return fmt.Errorf("device %s does not resolve to exactly one observed serial", binding.ID) } d := matches[0] - if d.ObservationState != "observed" || d.Model != "BlueField-4" { - return fmt.Errorf("device %s requires observed BlueField-4 identity", binding.ID) + if err := validateProvisioningDevice(d); err != nil { + return err } associations := []inventory.Association{} for _, a := range inv.Associations { @@ -308,12 +308,6 @@ func resolveProvisioning(g *Generation, inv *inventory.Inventory) error { } g.DevicePSIDs[d.ID] = psid } - if d.BMCEndpoint == "" { - return fmt.Errorf("device %s has no observed BMC endpoint", binding.ID) - } - if _, err := Origin(d.BMCEndpoint); err != nil { - return fmt.Errorf("device %s has invalid BMC endpoint", binding.ID) - } b, err := yaml.Marshal(d) if err != nil { return err @@ -346,6 +340,22 @@ func resolveProvisioning(g *Generation, inv *inventory.Inventory) error { return nil } +// Validate source inventory and retained bundle devices by the same contract. +// Discovery evidence is a prerequisite; deploy still verifies live DPF access. +func validateProvisioningDevice(d inventory.Device) error { + if d.ObservationState != "observed" || (d.Model != "BlueField-4" && (d.Model != "B4240" || d.ReaderAdapter != "bluefield-card-v1")) { + return fmt.Errorf("device %s requires observed BlueField-4 identity; B4240 requires the bluefield-card-v1 reader", d.ID) + } + admin := d.Observations["bmc-administrator"] + if admin.State != "observed" || admin.ReasonCode != "BMC_ADMIN_VERIFIED" || d.BMCEndpoint == "" || d.SSHEndpoint != "" { + return fmt.Errorf("device %s requires verified BMC administrator discovery evidence; rerun discovery with an explicit DPU BMC endpoint and administrator credentials, then regenerate the bundle", d.ID) + } + if _, err := Origin(d.BMCEndpoint); err != nil { + return fmt.Errorf("device %s has invalid BMC endpoint", d.ID) + } + return nil +} + func validateArtifact(a Artifact) error { u, err := url.Parse(a.URL) if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { diff --git a/pkg/dpf/config/generation_test.go b/pkg/dpf/config/generation_test.go index 35f4b111..88c9b41d 100644 --- a/pkg/dpf/config/generation_test.go +++ b/pkg/dpf/config/generation_test.go @@ -153,6 +153,65 @@ var _ = Describe("DPF generation configuration", func() { } }) + It("retains the exact B4240 model only from the reviewed BlueField reader", func() { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.DPUs[0].Model = "B4240" + in.Site.Inventory.DPUs[0].ReaderAdapter = "bluefield-card-v1" + resolved, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.Devices[0].Model).To(Equal("B4240")) + Expect(in.Site.Inventory.DPUs[0].Model).To(Equal("B4240")) + encoded, err := EncodeResolved(resolved) + Expect(err).NotTo(HaveOccurred()) + decoded, err := DecodeResolved(encoded) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Devices[0].Model).To(Equal("B4240")) + + for _, reader := range []string{"", "host-card-v1", "ssh"} { + By(reader) + in.Site.Inventory.DPUs[0].ReaderAdapter = reader + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("B4240 requires the bluefield-card-v1 reader"))) + } + in.Site.Inventory.DPUs[0].ReaderAdapter = "bluefield-card-v1" + in.Site.Inventory.DPUs[0].Model = "B4240X" + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("requires observed BlueField-4 identity"))) + }) + + It("requires successful BMC administrator discovery evidence for each selected device", func() { + for _, tc := range []struct { + name string + edit func(*inventory.Device) + }{ + {"legacy inventory", func(d *inventory.Device) { d.Observations = nil }}, + {"missing observation", func(d *inventory.Device) { delete(d.Observations, "bmc-administrator") }}, + {"failed observation", func(d *inventory.Device) { + d.Observations["bmc-administrator"] = inventory.Observation{State: "failed", ReasonCode: "BMC_ADMIN_REQUIRED"} + }}, + {"conflicting state", func(d *inventory.Device) { + d.Observations["bmc-administrator"] = inventory.Observation{State: "failed", ReasonCode: "BMC_ADMIN_VERIFIED"} + }}, + {"unverified reason", func(d *inventory.Device) { + d.Observations["bmc-administrator"] = inventory.Observation{State: "observed", ReasonCode: "BMC_ADMIN_UNVERIFIABLE"} + }}, + {"missing BMC", func(d *inventory.Device) { d.BMCEndpoint = "" }}, + {"ARM SSH substitute", func(d *inventory.Device) { d.SSHEndpoint = "192.0.2.21:22" }}, + } { + By(tc.name) + in := validGenerationInput(GinkgoT().TempDir()) + tc.edit(&in.Site.Inventory.DPUs[0]) + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("rerun discovery with an explicit DPU BMC endpoint and administrator credentials"))) + } + + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.DPUs = append(in.Site.Inventory.DPUs, inventory.Device{ID: "dpu-2", SerialNumber: "BF4-SERIAL-2", Model: "BlueField-4", ObservationState: "observed", BMCEndpoint: "https://192.0.2.22"}) + in.Site.Provisioning.Devices = append(in.Site.Provisioning.Devices, DeviceBinding{ID: "dpu-2", DPUDevice: "dpu-device-2", DPUNode: "dpu-node-2"}) + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("device dpu-2 requires verified BMC administrator discovery evidence"))) + }) + It("requires explicit management context and either explicit or observed management UID", func() { in := validGenerationInput(GinkgoT().TempDir()) in.Site.Management.Context = nil @@ -315,7 +374,8 @@ func validGenerationInput(base string) Input { psid := inventory.Observation{State: "observed", Values: []string{generationPSID}} dpu := inventory.Device{ ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "BlueField-4", ObservationState: "observed", - BMCEndpoint: "https://192.0.2.21", + BMCEndpoint: "https://192.0.2.21", + Observations: map[string]inventory.Observation{"bmc-administrator": {State: "observed", ReasonCode: "BMC_ADMIN_VERIFIED"}}, Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{ SerialNumber: "BF4-SERIAL-1", PSID: psid, }}}}, diff --git a/pkg/dpf/config/resolved.go b/pkg/dpf/config/resolved.go index 891d31d1..f4661e8d 100644 --- a/pkg/dpf/config/resolved.go +++ b/pkg/dpf/config/resolved.go @@ -111,10 +111,7 @@ func ValidateGeneration(g Generation) error { return fmt.Errorf("resolved device identities or bindings conflict") } ids[d.ID], serials[d.SerialNumber], names[binding.DPUDevice] = true, true, true - if d.Model != "BlueField-4" || d.ObservationState != "observed" { - return fmt.Errorf("resolved device %s has no observed BF4 identity", d.ID) - } - if _, err := Origin(d.BMCEndpoint); err != nil { + if err := validateProvisioningDevice(d); err != nil { return err } psid := g.DevicePSIDs[d.ID] diff --git a/pkg/dpf/config/resolved_test.go b/pkg/dpf/config/resolved_test.go index 2fdf8012..b71d6c6a 100644 --- a/pkg/dpf/config/resolved_test.go +++ b/pkg/dpf/config/resolved_test.go @@ -9,9 +9,25 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + yaml "gopkg.in/yaml.v3" ) var _ = Describe("resolved DPF generation", func() { + It("rejects legacy or failed administrator proof in retained deployment metadata", func() { + for _, admin := range []inventory.Observation{{}, {State: "failed", ReasonCode: "BMC_ADMIN_REQUIRED"}, {State: "failed", ReasonCode: "BMC_ADMIN_VERIFIED"}, {State: "observed", ReasonCode: "BMC_ADMIN_UNVERIFIABLE"}} { + generation := resolvedGeneration() + generation.Devices[0].Observations = map[string]inventory.Observation{"bmc-administrator": admin} + Expect(ValidateGeneration(generation)).To(MatchError(ContainSubstring("rerun discovery"))) + _, err := EncodeResolved(generation) + Expect(err).To(MatchError(ContainSubstring("rerun discovery"))) + // Marshal directly to represent metadata produced before this requirement. + legacy, err := yaml.Marshal(Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: generation}) + Expect(err).NotTo(HaveOccurred()) + _, err = DecodeResolved(legacy) + Expect(err).To(MatchError(ContainSubstring("rerun discovery"))) + } + }) + It("round-trips a validated deployment config and returns independent values", func() { generation := resolvedGeneration() encoded, err := EncodeResolved(generation) diff --git a/pkg/dpf/generation/generation_test.go b/pkg/dpf/generation/generation_test.go index 413fc23a..8b8d2f38 100644 --- a/pkg/dpf/generation/generation_test.go +++ b/pkg/dpf/generation/generation_test.go @@ -69,6 +69,7 @@ var _ = Describe("offline DPF generation", func() { resolved, err := Configuration(loaded, "") Expect(err).NotTo(HaveOccurred()) Expect(resolved.Devices).To(HaveLen(1)) + Expect(resolved.Devices[0].Model).To(Equal("B4240")) Expect(loaded.Index().SelectedDevices).To(Equal([]string{resolved.Devices[0].ID})) badIndex := loaded.Index() badIndex.SelectedDevices = []string{"different-device"} @@ -83,6 +84,38 @@ var _ = Describe("offline DPF generation", func() { } }) + It("rejects an old inventory before publication and an old bundle before cluster access", func() { + b, err := Build(context.Background(), in, configinput.Values{}) + Expect(err).NotTo(HaveOccurred()) + files := b.Files() + for i := range files { + if files[i].Path == bundle.ResolvedConfigPath { + g, err := config.DecodeResolved(files[i].Data) + Expect(err).NotTo(HaveOccurred()) + g.Devices[0].Observations = nil + files[i].Data, err = yaml.Marshal(config.Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: g}) + Expect(err).NotTo(HaveOccurred()) + } + } + index := b.Index() + index.BundleDigest = "" + for i := range index.Files { + index.Files[i].SHA256 = "" + } + legacy, err := bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + _, err = Configuration(legacy, "") + Expect(err).To(MatchError(ContainSubstring("rerun discovery"))) + + in.Site.Inventory.DPUs[0].Observations = nil + saveInput(in, configPath) + out := filepath.Join(work, "legacy-inventory") + _, err = Run(context.Background(), Request{UserConfig: configPath, OutputPath: out}) + Expect(err).To(MatchError(ContainSubstring("rerun discovery"))) + _, err = os.Stat(out) + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + It("rejects a consistently rehashed bundle whose resources diverge from its intent", func() { b, err := Build(context.Background(), in, configinput.Values{}) Expect(err).NotTo(HaveOccurred()) @@ -183,7 +216,7 @@ var _ = Describe("offline DPF generation", func() { func generationInput(base, charts string) config.Input { const psid = "MT_0000000010" observed := inventory.Observation{State: "observed", Values: []string{psid}} - dpu := inventory.Device{ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "BlueField-4", ObservationState: "observed", BMCEndpoint: "https://192.0.2.21", Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} + dpu := inventory.Device{ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "B4240", ReaderAdapter: "bluefield-card-v1", ObservationState: "observed", BMCEndpoint: "https://192.0.2.21", Observations: map[string]inventory.Observation{"bmc-administrator": {State: "observed", ReasonCode: "BMC_ADMIN_VERIFIED"}}, Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} host := inventory.Device{ID: "host-1", Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} site := config.Site{} site.DPF.Release = stringPtr("26.8.0") diff --git a/skills/k8s-launch-kit-deploy/SKILL.md b/skills/k8s-launch-kit-deploy/SKILL.md index 6c8c1a5b..107417ef 100644 --- a/skills/k8s-launch-kit-deploy/SKILL.md +++ b/skills/k8s-launch-kit-deploy/SKILL.md @@ -23,6 +23,11 @@ review unresolved checks, then use an explicit `--deploy-timeout`. Real deploy can reimage devices and require host power handling. It never implicitly adopts/removes another owner or applies chart projections twice. Reports go to `.runs/`; preserve the bundle and failed-run evidence before retrying. +Bundles missing selected-device BMC administrator discovery proof require +rediscovery and regeneration. Saved proof does not replace live DPF access. +Legacy B4240 inventory can identify a card whose BMC schema is unsupported by +the DPF operator/preflight; arrange supported firmware before enrollment and +rediscover. Keep failed preflight closed; do not substitute SSH or edit identity. The Host procedure below applies to `host` only. ## Host workflow diff --git a/skills/k8s-launch-kit-generate/SKILL.md b/skills/k8s-launch-kit-generate/SKILL.md index e60b5977..d10794eb 100644 --- a/skills/k8s-launch-kit-generate/SKILL.md +++ b/skills/k8s-launch-kit-generate/SKILL.md @@ -18,7 +18,13 @@ For an explicit DPF request, use `--target dpf` and follow explicit selected device/tenant bindings, site networks, SHA-bound BF4 software and four catalogued local chart archives. `l8k schema` exposes `dpf.profiles`, `dpf.generationReleases` and `dpf.charts`. Generation reads no credentials and -contacts no cluster. Core policy defaults to `llmInspection: false`; enabling +contacts no cluster. Each selected device requires observed BMC administrator +proof (`bmc-administrator`, `observed`, `BMC_ADMIN_VERIFIED`) from explicit BMC +discovery; ARM SSH cannot substitute. Rediscover older inventories without this +proof and regenerate old bundles into a new directory. The exact model `B4240` +is accepted only from `bluefield-card-v1` and stays unchanged in metadata. +Saved proof does not replace live deployment authentication or qualify legacy +BMC firmware for provisioning. Core policy defaults to `llmInspection: false`; enabling LLM inspection requires a real inference Secret. The resolved bundle is immutable; use a new directory for changed desired intent. Do not apply chart projections separately. The Host procedure below applies to `host` only. From 95d97418c1722d9e87bdb6e518bd7260220ab565 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 13 Nov 2026 10:22:07 +0100 Subject: [PATCH 4/9] Align discovery handoff plan with Sentry implementation Signed-off-by: Alexander Maslennikov --- engineering/architecture/dpf-integration-plan.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/engineering/architecture/dpf-integration-plan.md b/engineering/architecture/dpf-integration-plan.md index bba26b7a..22b1b9f5 100644 --- a/engineering/architecture/dpf-integration-plan.md +++ b/engineering/architecture/dpf-integration-plan.md @@ -322,9 +322,11 @@ inventory requires read-only verification of the authenticated DPU BMC account a its effective Administrator privileges. Host/ARM-only inventory is partial and cannot bypass this gate. Preserve physical identity and authorization failure evidence independently; never rotate passwords or probe privilege by mutation. -The current DPF target still rejects generation, deployment and validation before -consuming inventory. A future generator must require fresh complete discovery and -revalidate its own authorization needs; saved facts do not authorize provisioning. +Sentry generation and retained bundle validation require successful administrator +evidence for each selected DPU. Older inventories and bundles must be regenerated +from completed discovery. Deployment independently verifies live DPF authentication +and provisioning prerequisites; saved facts do not authorize provisioning. The +root DPF pipeline remains unavailable. ### Stage 4: DPF generation for Zero Trust passthrough From ad5f9e977d769718cf8148572101fd73efad42a5 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 13 Nov 2026 10:56:44 +0100 Subject: [PATCH 5/9] Use consistent measured PSIDs for DPF firmware selection Signed-off-by: Alexander Maslennikov --- docs/user/dpf-sentry.md | 7 +++ docs/user/dpf-workflows.md | 2 + openspec/specs/profile-planning/spec.md | 14 +++++ openspec/workflows/dpf-discovery.json | 2 +- pkg/dpf/config/generation.go | 73 +++++++++++++++++++------ pkg/dpf/config/generation_test.go | 56 +++++++++++++++++++ pkg/dpf/config/resolved.go | 9 +++ pkg/dpf/config/resolved_test.go | 37 +++++++++++++ 8 files changed, 183 insertions(+), 17 deletions(-) diff --git a/docs/user/dpf-sentry.md b/docs/user/dpf-sentry.md index 3bab7c4d..c975af6d 100644 --- a/docs/user/dpf-sentry.md +++ b/docs/user/dpf-sentry.md @@ -117,6 +117,13 @@ credentials, review its completed report, and generate into a new bundle directory. Do not add the proof by editing YAML. Saved privileges are an observation; deployment still verifies live DPF access and its own prerequisites. +Firmware selection uses the PSID measured by discovery. Supported BF4 BMCs can +report it through Redfish; review `observations.psid` in the inventory. Existing +host SSH measurements can also provide the PSID. All available measurements +must agree. A malformed, conflicting, changing or incompletely verified Redfish PSID blocks generation +even when another source reports a value. Resolve the recorded cause and +rediscover; do not copy a PSID from a different card or infer it from the model. + | Input class | What belongs here | | --- | --- | | Required site intent | Explicit installed DPF release; selected inventory device IDs; existing DPUDevice/DPUNode names; selected tenant name, namespace, resource UID, tenant cluster UID and exact label selector; tenant API server version; ISO and PSID-specific firmware URLs with SHA-256; local chart directory; ingress subnet/address; existing Secret names; reachable management policy-server address. | diff --git a/docs/user/dpf-workflows.md b/docs/user/dpf-workflows.md index 1ef3f5a8..d9ac1ffa 100644 --- a/docs/user/dpf-workflows.md +++ b/docs/user/dpf-workflows.md @@ -392,6 +392,8 @@ Core policy defaults to `llmInspection: false` and makes no LLM inspection claim `ingressCIDR` and `ingressIP` configure only the `net1` ingress network. The current profile obtains `net2` egress IPv4 and its router through DHCP on the untagged `p0` uplink; these inputs do not configure egress, switch VLANs, a DHCP server/relay or upstream routing. +Use the PSID actually measured by discovery: supported BF4 BMCs report it through Redfish, and existing host SSH inventory can also supply it. Missing BMC metadata is distinct from contradictory, invalid or changing PSID evidence. All available measurements must agree; resolve recorded conflicts and rediscover before generation. A model name or another card's PSID is not a substitute. + Next: [Bindings and intent reviewed](#generate-sentry-stage-charts). ### 2. CLI machine: obtain the pinned local charts {#generate-sentry-stage-charts} diff --git a/openspec/specs/profile-planning/spec.md b/openspec/specs/profile-planning/spec.md index 6130f031..438a9382 100644 --- a/openspec/specs/profile-planning/spec.md +++ b/openspec/specs/profile-planning/spec.md @@ -205,6 +205,20 @@ established unnamed sole source SHALL retain its empty ID through explicit - **WHEN** selected sources sharing a GPU label have different rail-count buckets - **THEN** each resource retains only its intended source IDs, enabling deployment to reject a shared selector that includes another bucket's workers. +### Requirement: Measured DPF firmware selection + +DPF generation SHALL accept a supported BF4 Redfish PSID observation without +requiring an SSH runtime record. It SHALL compare that observation with eligible +serial-bound host/DPU SSH PSIDs and require one consistent identifier. Explicitly +invalid, conflicting, changed or incompletely verified Redfish PSID evidence SHALL remain a blocker even +when another source supplies a value. Missing Redfish metadata alone does not +invalidate an independent eligible measurement. + +#### Scenario: Retained firmware selection contradicts the BMC observation + +- **WHEN** a retained DPF bundle selects a PSID different from any retained serial-bound Redfish or SSH PSID +- **THEN** bundle validation rejects the contradiction before cluster access. + ## Baseline and limits Current profile lookup is first-match; this spec does not introduce universal ambiguity rejection. Existing render maps do not universally detect pre-overwrite collisions: the collision requirement is an extension obligation, with legacy coverage to be assessed when touched. `ScopeClusterWide` may render a namespaced ConfigMap, while SCCs may render per workload namespace. [Resource kinds](../resource-kinds/spec.md) owns API/readiness integration; [artifacts](../artifact-handoff/spec.md) owns post-render structural validation. diff --git a/openspec/workflows/dpf-discovery.json b/openspec/workflows/dpf-discovery.json index be25b3a2..709633d5 100644 --- a/openspec/workflows/dpf-discovery.json +++ b/openspec/workflows/dpf-discovery.json @@ -253,7 +253,7 @@ { "id": "bind-inputs", "title": "CLI operator: select devices and desired site inputs", - "body": "Copy the discovered YAML to `deployment-site.yaml`, retaining its inventory. Choose the exact inventory device IDs and explicitly name their enrolled DPUDevice/DPUNode objects. Bind the existing tenant by name, namespace, selector, resource UID, tenant cluster UID and API server version. Management UID/version may default from saved observations; management kubeconfig/context must be explicit. Supply PSID-matched ISO/PLDM URLs and SHA-256, service networks and existing Secret names. Follow the [configuration fragment and input table](dpf-sentry.md#prepare-the-deployment-configuration). Do not invent missing observations or infer enrollment names.\n\nCore policy defaults to `llmInspection: false` and makes no LLM inspection claim. Enable it only with the real inference Secret and intended external inference access.\n\n`ingressCIDR` and `ingressIP` configure only the `net1` ingress network. The current profile obtains `net2` egress IPv4 and its router through DHCP on the untagged `p0` uplink; these inputs do not configure egress, switch VLANs, a DHCP server/relay or upstream routing.", + "body": "Copy the discovered YAML to `deployment-site.yaml`, retaining its inventory. Choose the exact inventory device IDs and explicitly name their enrolled DPUDevice/DPUNode objects. Bind the existing tenant by name, namespace, selector, resource UID, tenant cluster UID and API server version. Management UID/version may default from saved observations; management kubeconfig/context must be explicit. Supply PSID-matched ISO/PLDM URLs and SHA-256, service networks and existing Secret names. Follow the [configuration fragment and input table](dpf-sentry.md#prepare-the-deployment-configuration). Do not invent missing observations or infer enrollment names.\n\nCore policy defaults to `llmInspection: false` and makes no LLM inspection claim. Enable it only with the real inference Secret and intended external inference access.\n\n`ingressCIDR` and `ingressIP` configure only the `net1` ingress network. The current profile obtains `net2` egress IPv4 and its router through DHCP on the untagged `p0` uplink; these inputs do not configure egress, switch VLANs, a DHCP server/relay or upstream routing.\n\nUse the PSID actually measured by discovery: supported BF4 BMCs report it through Redfish, and existing host SSH inventory can also supply it. Missing BMC metadata is distinct from contradictory, invalid or changing PSID evidence. All available measurements must agree; resolve recorded conflicts and rediscover before generation. A model name or another card's PSID is not a substitute.", "transitions": [ { "to": "stage-charts", diff --git a/pkg/dpf/config/generation.go b/pkg/dpf/config/generation.go index 7f875069..f82b7dc7 100644 --- a/pkg/dpf/config/generation.go +++ b/pkg/dpf/config/generation.go @@ -273,27 +273,15 @@ func resolveProvisioning(g *Generation, inv *inventory.Inventory) error { if len(associations) != 1 || associations[0].SerialNumber != d.SerialNumber { return fmt.Errorf("device %s requires one unambiguous host association", binding.ID) } - psids := map[string]bool{} - collectPSIDs := func(device inventory.Device) { - if device.Runtime == nil || device.Runtime.Hardware == nil { - return - } - for _, card := range device.Runtime.Hardware.Cards { - if card.SerialNumber == d.SerialNumber && card.PSID.State == "observed" { - for _, v := range card.PSID.Values { - if v != "" { - psids[v] = true - } - } - } - } + psids, err := measuredDevicePSIDs(d) + if err != nil { + return err } - collectPSIDs(d) hosts := 0 for _, h := range inv.Hosts { if h.ID == associations[0].HostID { hosts++ - collectPSIDs(h) + collectRuntimePSIDs(psids, d.SerialNumber, h) } } if hosts != 1 { @@ -356,6 +344,59 @@ func validateProvisioningDevice(d inventory.Device) error { return nil } +// measuredDevicePSIDs shares source and retained-device consistency checks. +// Host runtime evidence is added by resolution using the verified association. +func measuredDevicePSIDs(d inventory.Device) (map[string]bool, error) { + psids := map[string]bool{} + psid, err := observedRedfishPSID(d) + if err != nil { + return nil, err + } + if psid != "" { + psids[psid] = true + } + collectRuntimePSIDs(psids, d.SerialNumber, d) + return psids, nil +} + +func collectRuntimePSIDs(psids map[string]bool, serial string, d inventory.Device) { + if d.Runtime == nil || d.Runtime.Hardware == nil { + return + } + for _, card := range d.Runtime.Hardware.Cards { + if card.SerialNumber == serial && card.PSID.State == "observed" { + for _, value := range card.PSID.Values { + if value != "" { + psids[value] = true + } + } + } + } +} + +// Redfish and SSH measurements may complement each other, but an explicitly +// contradictory or changed Redfish identity cannot be hidden by another source. +func observedRedfishPSID(d inventory.Device) (string, error) { + observation := d.Observations["psid"] + switch observation.ReasonCode { + case "PSID_CONFLICT", "PSID_INVALID", "PSID_CHANGED", "PSID_RECHECK_FAILED", "PSID_NOT_VERIFIED": + return "", fmt.Errorf("device %s has rejected Redfish PSID evidence (%s); resolve the reported cause and rediscover", d.ID, observation.ReasonCode) + } + if observation.State != "observed" && observation.ReasonCode != "PSID_OBSERVED" { + return "", nil + } + if observation.State != "observed" || observation.ReasonCode != "PSID_OBSERVED" || observation.Tool != "redfish" || d.Model != "B4240" || d.ReaderAdapter != "bluefield-card-v1" || len(observation.Values) != 1 { + return "", fmt.Errorf("device %s has inconsistent Redfish PSID evidence; rediscover", d.ID) + } + psid := observation.Values[0] + if !strings.HasPrefix(psid, "MT_") || len(psid) > 32 || strings.IndexFunc(psid, func(r rune) bool { + return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '.' || r == '-') + }) >= 0 { + return "", fmt.Errorf("device %s Redfish PSID is not supported for PLDM selection", d.ID) + } + return psid, nil +} + func validateArtifact(a Artifact) error { u, err := url.Parse(a.URL) if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { diff --git a/pkg/dpf/config/generation_test.go b/pkg/dpf/config/generation_test.go index 88c9b41d..a5e73949 100644 --- a/pkg/dpf/config/generation_test.go +++ b/pkg/dpf/config/generation_test.go @@ -291,6 +291,62 @@ var _ = Describe("DPF generation configuration", func() { } }) + It("uses a Redfish PSID without requiring an SSH runtime observation", func() { + in := validGenerationInput(GinkgoT().TempDir()) + d := &in.Site.Inventory.DPUs[0] + d.Model, d.ReaderAdapter = "B4240", "bluefield-card-v1" + d.Runtime = nil + d.Observations["psid"] = inventory.Observation{State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{generationPSID}} + in.Site.Inventory.Hosts[0].Runtime = nil + resolved, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved.DevicePSIDs).To(Equal(map[string]string{"dpu-1": generationPSID})) + Expect(resolved.Devices[0].Runtime).To(BeNil()) + encoded, err := EncodeResolved(resolved) + Expect(err).NotTo(HaveOccurred()) + decoded, err := DecodeResolved(encoded) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Devices[0].Observations["psid"]).To(Equal(d.Observations["psid"])) + }) + + It("rejects contradictory Redfish PSID evidence even when SSH reports a value", func() { + for _, reason := range []string{"PSID_CONFLICT", "PSID_INVALID", "PSID_CHANGED", "PSID_RECHECK_FAILED", "PSID_NOT_VERIFIED"} { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.DPUs[0].Observations["psid"] = inventory.Observation{State: "unavailable", ReasonCode: reason, Tool: "redfish"} + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("rejected Redfish PSID evidence (" + reason + ")"))) + } + in := validGenerationInput(GinkgoT().TempDir()) + d := &in.Site.Inventory.DPUs[0] + d.Model, d.ReaderAdapter = "B4240", "bluefield-card-v1" + d.Observations["psid"] = inventory.Observation{State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{"MT_0000000011"}} + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("one observed, nonconflicting PSID"))) + + d.Observations["psid"] = inventory.Observation{State: "unknown", ReasonCode: "PSID_NOT_EXPOSED", Tool: "redfish"} + _, err = ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + }) + + It("does not accept malformed or unsupported Redfish PSID observations", func() { + for _, observation := range []inventory.Observation{ + {State: "unknown", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{generationPSID}}, + {State: "observed", ReasonCode: "PSID_NOT_EXPOSED", Tool: "redfish", Values: []string{generationPSID}}, + {State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "ssh", Values: []string{generationPSID}}, + {State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: nil}, + {State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{generationPSID, generationPSID}}, + {State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{"MT_invalid value"}}, + {State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{"unknown"}}, + } { + in := validGenerationInput(GinkgoT().TempDir()) + d := &in.Site.Inventory.DPUs[0] + d.Model, d.ReaderAdapter = "B4240", "bluefield-card-v1" + d.Observations["psid"] = observation + _, err := ResolveGeneration(in) + Expect(err).To(HaveOccurred()) + } + }) + It("rejects absent or conflicting observed PSID mappings", func() { in := validGenerationInput(GinkgoT().TempDir()) in.Site.Inventory.DPUs[0].Runtime.Hardware.Cards[0].PSID = inventory.Observation{State: "unavailable"} diff --git a/pkg/dpf/config/resolved.go b/pkg/dpf/config/resolved.go index f4661e8d..b53b58ac 100644 --- a/pkg/dpf/config/resolved.go +++ b/pkg/dpf/config/resolved.go @@ -118,6 +118,15 @@ func ValidateGeneration(g Generation) error { if psid == "" { return fmt.Errorf("resolved device %s has no PSID", d.ID) } + measuredPSIDs, err := measuredDevicePSIDs(d) + if err != nil { + return err + } + for measured := range measuredPSIDs { + if measured != psid { + return fmt.Errorf("resolved device %s PSID differs from its retained measurements", d.ID) + } + } a, ok := g.Provisioning.Software.PLDM[psid] if !ok { return fmt.Errorf("resolved device %s has no matching PLDM artifact", d.ID) diff --git a/pkg/dpf/config/resolved_test.go b/pkg/dpf/config/resolved_test.go index b71d6c6a..83367ebc 100644 --- a/pkg/dpf/config/resolved_test.go +++ b/pkg/dpf/config/resolved_test.go @@ -13,6 +13,43 @@ import ( ) var _ = Describe("resolved DPF generation", func() { + It("rejects retained PSID selection that contradicts the Redfish observation", func() { + generation := resolvedGeneration() + d := &generation.Devices[0] + d.Model, d.ReaderAdapter = "B4240", "bluefield-card-v1" + d.Observations["psid"] = inventory.Observation{State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{"MT_0000000011"}} + Expect(ValidateGeneration(generation)).To(MatchError(ContainSubstring("differs from its retained measurements"))) + d.Observations["psid"] = inventory.Observation{State: "unavailable", ReasonCode: "PSID_CHANGED", Tool: "redfish"} + Expect(ValidateGeneration(generation)).To(MatchError(ContainSubstring("rejected Redfish PSID evidence"))) + }) + + It("compares all retained serial-bound measurements before accepting firmware selection", func() { + generation := resolvedGeneration() + d := &generation.Devices[0] + d.Model, d.ReaderAdapter = "B4240", "bluefield-card-v1" + d.Observations["psid"] = inventory.Observation{State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{generationPSID}} + d.Runtime.Hardware.Cards[0].PSID.Values = []string{"MT_0000000011"} + Expect(ValidateGeneration(generation)).To(MatchError(ContainSubstring("differs from its retained measurements"))) + _, err := EncodeResolved(generation) + Expect(err).To(HaveOccurred()) + encoded, err := yaml.Marshal(Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: generation}) + Expect(err).NotTo(HaveOccurred()) + _, err = DecodeResolved(encoded) + Expect(err).To(HaveOccurred()) + + // An unrelated card's measurement must not participate in this binding. + d.Runtime.Hardware.Cards[0].SerialNumber = "OTHER-CARD" + Expect(ValidateGeneration(generation)).To(Succeed()) + }) + + It("rejects incomplete Redfish verification even when retained SSH has a PSID", func() { + for _, reason := range []string{"PSID_RECHECK_FAILED", "PSID_NOT_VERIFIED"} { + generation := resolvedGeneration() + generation.Devices[0].Observations["psid"] = inventory.Observation{State: "unavailable", ReasonCode: reason, Tool: "redfish"} + Expect(ValidateGeneration(generation)).To(MatchError(ContainSubstring("rejected Redfish PSID evidence"))) + } + }) + It("rejects legacy or failed administrator proof in retained deployment metadata", func() { for _, admin := range []inventory.Observation{{}, {State: "failed", ReasonCode: "BMC_ADMIN_REQUIRED"}, {State: "failed", ReasonCode: "BMC_ADMIN_VERIFIED"}, {State: "observed", ReasonCode: "BMC_ADMIN_UNVERIFIABLE"}} { generation := resolvedGeneration() From e08384e4774ab72512597a067873e71c0fd7f332 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 13 Nov 2026 11:08:34 +0100 Subject: [PATCH 6/9] Clarify the supported PSID character check Signed-off-by: Alexander Maslennikov --- pkg/dpf/config/generation.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/dpf/config/generation.go b/pkg/dpf/config/generation.go index f82b7dc7..3a40e6bd 100644 --- a/pkg/dpf/config/generation.go +++ b/pkg/dpf/config/generation.go @@ -390,7 +390,12 @@ func observedRedfishPSID(d inventory.Device) (string, error) { } psid := observation.Values[0] if !strings.HasPrefix(psid, "MT_") || len(psid) > 32 || strings.IndexFunc(psid, func(r rune) bool { - return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '.' || r == '-') + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '_', r == '.', r == '-': + return false + default: + return true + } }) >= 0 { return "", fmt.Errorf("device %s Redfish PSID is not supported for PLDM selection", d.ID) } From a91ebdcc2283374927a913673afbd9de59ff3ea9 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 13 Nov 2026 19:29:22 +0100 Subject: [PATCH 7/9] Retain typed discovery facts in Sentry resolved configuration Signed-off-by: Alexander Maslennikov --- docs/user/dpf-sentry.md | 19 ++++--- pkg/dpf/bundle/bundle_test.go | 14 +++++ pkg/dpf/bundle/validate.go | 3 +- pkg/dpf/config/generation.go | 4 +- pkg/dpf/config/generation_test.go | 48 +++++++++++++++-- pkg/dpf/config/resolved.go | 39 +++++++++++--- pkg/dpf/config/resolved_output.go | 77 +++++++++++++++++++++++++++ pkg/dpf/config/resolved_test.go | 60 +++++++++++++++++++-- pkg/dpf/generation/generate.go | 14 +++-- pkg/dpf/generation/generation_test.go | 28 +++++++++- 10 files changed, 276 insertions(+), 30 deletions(-) create mode 100644 pkg/dpf/config/resolved_output.go diff --git a/docs/user/dpf-sentry.md b/docs/user/dpf-sentry.md index c975af6d..9a716cbe 100644 --- a/docs/user/dpf-sentry.md +++ b/docs/user/dpf-sentry.md @@ -106,10 +106,12 @@ Copy a successful [discovered configuration](dpf-discovery.md) to below; replace every angle-bracket placeholder before generating. Do not edit the inventory to turn an unknown or conflicting observation into a fact. -Every selected device must retain successful discovery evidence under -`observations.bmc-administrator`: `state: observed` and -`reasonCode: BMC_ADMIN_VERIFIED`, with an explicit BMC endpoint and no ARM SSH -endpoint. Generation accepts the exact observed model `BlueField-4`, or `B4240` +Every selected device must retain successful discovery evidence: +`bmcAdministratorVerified: true` in typed `v1alpha3` inventory, with an explicit +BMC endpoint and no ARM SSH endpoint. Legacy `v1alpha1`/`v1alpha2` inventory is +also accepted when its retained administrator observation is `state: observed` +with `reasonCode: BMC_ADMIN_VERIFIED`; missing or failed proof is never inferred. +Generation accepts the exact observed model `BlueField-4`, or `B4240` from the reviewed `bluefield-card-v1` reader, and preserves that measured model. Unselected records do not need to satisfy this selection gate. If older inventory or a retained bundle lacks this proof, rerun discovery with DPU BMC administrator @@ -118,7 +120,8 @@ directory. Do not add the proof by editing YAML. Saved privileges are an observation; deployment still verifies live DPF access and its own prerequisites. Firmware selection uses the PSID measured by discovery. Supported BF4 BMCs can -report it through Redfish; review `observations.psid` in the inventory. Existing +report it through Redfish; review `psid` and any `issues` for that field in typed +inventory (`observations.psid` in legacy inventory). Existing host SSH measurements can also provide the PSID. All available measurements must agree. A malformed, conflicting, changing or incompletely verified Redfish PSID blocks generation even when another source reports a value. Resolve the recorded cause and @@ -235,8 +238,10 @@ The bundle includes native resources, chart archives, effective values and validation-only chart projections. DPF owns chart deployment; Launch Kit does not apply the projections a second time. `.l8k/bundle.yaml` identifies the DPF target and binds the files; `.l8k/resolved-config.yaml` retains the effective -inputs. Preserve the bundle unchanged. To change desired intent, generate a new -bundle directory. Repeating identical generation at the same path is allowed; +inputs using typed selected-device facts in resolved format `v1alpha2`. The +bundle index remains `v1alpha1`; valid legacy resolved `v1alpha1` metadata is +accepted with the same administrator and PSID checks. Preserve the bundle +unchanged. To change desired intent, generate a new bundle directory. Repeating identical generation at the same path is allowed; different content does not overwrite an existing bundle. ## Check admission, then deploy diff --git a/pkg/dpf/bundle/bundle_test.go b/pkg/dpf/bundle/bundle_test.go index 05ddd40f..602fe9ad 100644 --- a/pkg/dpf/bundle/bundle_test.go +++ b/pkg/dpf/bundle/bundle_test.go @@ -36,6 +36,20 @@ func fixture() (Index, []File) { } var _ = Describe("validated immutable membership", func() { + It("versions resolved metadata independently of the bundle index", func() { + for _, version := range []string{"v1alpha1", "v1alpha2", "v99"} { + index, files := fixture() + files[0].Data = []byte(strings.Replace(string(files[0].Data), "v1alpha1", version, 1)) + b, err := New(index, files) + if version == "v99" { + Expect(err).To(MatchError(ContainSubstring("unsupported DPF resolved configuration envelope"))) + } else { + Expect(err).NotTo(HaveOccurred()) + Expect(b.Index().FormatVersion).To(Equal("v1alpha1")) + } + } + }) + It("retains exact bytes and isolates constructor and consumer mutations", func() { index, files := fixture() b, err := New(index, files) diff --git a/pkg/dpf/bundle/validate.go b/pkg/dpf/bundle/validate.go index bfd690dd..f565804a 100644 --- a/pkg/dpf/bundle/validate.go +++ b/pkg/dpf/bundle/validate.go @@ -210,7 +210,8 @@ func validateResolved(data []byte) error { if err := decodeOne(data, &envelope); err != nil { return fmt.Errorf("decode DPF resolved configuration: %w", err) } - if envelope.FormatVersion != FormatVersion || envelope.Target != Target || envelope.Config.Kind != yaml.MappingNode { + // The index envelope remains v1alpha1; resolved metadata has its own schema. + if (envelope.FormatVersion != "v1alpha1" && envelope.FormatVersion != "v1alpha2") || envelope.Target != Target || envelope.Config.Kind != yaml.MappingNode { return fmt.Errorf("unsupported DPF resolved configuration envelope") } return nil diff --git a/pkg/dpf/config/generation.go b/pkg/dpf/config/generation.go index 3a40e6bd..f22a6239 100644 --- a/pkg/dpf/config/generation.go +++ b/pkg/dpf/config/generation.go @@ -167,7 +167,7 @@ func ResolveGeneration(in Input) (Generation, error) { g.ManagementUID = *in.Site.Management.ClusterUID } inv := in.Site.Inventory - if inv == nil || inv.FormatVersion != inventory.FormatVersion { + if inv == nil || (inv.FormatVersion != inventory.FormatVersion && inv.FormatVersion != "v1alpha1" && inv.FormatVersion != "v1alpha2") { return g, fmt.Errorf("generation requires a supported discovery inventory") } if g.ManagementUID == "" && inv.Management != nil { @@ -331,7 +331,7 @@ func resolveProvisioning(g *Generation, inv *inventory.Inventory) error { // Validate source inventory and retained bundle devices by the same contract. // Discovery evidence is a prerequisite; deploy still verifies live DPF access. func validateProvisioningDevice(d inventory.Device) error { - if d.ObservationState != "observed" || (d.Model != "BlueField-4" && (d.Model != "B4240" || d.ReaderAdapter != "bluefield-card-v1")) { + if d.ObservationState != "observed" || d.ReasonCode != "" || (d.Model != "BlueField-4" && (d.Model != "B4240" || d.ReaderAdapter != "bluefield-card-v1")) { return fmt.Errorf("device %s requires observed BlueField-4 identity; B4240 requires the bluefield-card-v1 reader", d.ID) } admin := d.Observations["bmc-administrator"] diff --git a/pkg/dpf/config/generation_test.go b/pkg/dpf/config/generation_test.go index a5e73949..89c4b208 100644 --- a/pkg/dpf/config/generation_test.go +++ b/pkg/dpf/config/generation_test.go @@ -15,6 +15,46 @@ import ( const generationPSID = "MT_0000000010" var _ = Describe("DPF generation configuration", func() { + It("accepts supported inventory versions without upgrading failed identity", func() { + for _, version := range []string{"v1alpha1", "v1alpha2", "v1alpha3"} { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.FormatVersion = version + in = decodeGenerationInput(in) + _, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred(), version) + in.Site.Inventory.DPUs[0].ReasonCode = "IDENTITY_CHANGED" + _, err = ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("requires observed BlueField-4 identity"))) + } + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.FormatVersion = "v99" + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("supported discovery inventory"))) + }) + + It("uses named typed Redfish PSID without requiring an SSH measurement", func() { + in := validGenerationInput(GinkgoT().TempDir()) + d := &in.Site.Inventory.DPUs[0] + d.Model, d.ReaderAdapter = "B4240", "bluefield-card-v1" + d.Observations["psid"] = inventory.Observation{State: "observed", ReasonCode: "PSID_OBSERVED", Tool: "redfish", Values: []string{generationPSID}} + d.Runtime = nil + in.Site.Inventory.Hosts[0].Runtime = nil + encoded, err := yaml.Marshal(in.Site) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(ContainSubstring("psid: " + generationPSID)) + Expect(string(encoded)).NotTo(ContainSubstring("observations:")) + in, err = Decode(encoded, in.Path) + Expect(err).NotTo(HaveOccurred()) + g, err := ResolveGeneration(in) + Expect(err).NotTo(HaveOccurred()) + Expect(g.DevicePSIDs["dpu-1"]).To(Equal(generationPSID)) + sidecar, err := EncodeResolved(g) + Expect(err).NotTo(HaveOccurred()) + restored, err := DecodeResolved(sidecar) + Expect(err).NotTo(HaveOccurred()) + Expect(restored.Devices[0].Observations["psid"].ReasonCode).To(Equal("PSID_OBSERVED")) + Expect(restored.DevicePSIDs["dpu-1"]).To(Equal(generationPSID)) + }) It("preserves the chart directory through a relocated discovery output", func() { in := validGenerationInput(GinkgoT().TempDir()) in.Site.Sentry.ChartDirectory = "charts/sentry" @@ -429,17 +469,17 @@ var _ = Describe("DPF generation configuration", func() { func validGenerationInput(base string) Input { psid := inventory.Observation{State: "observed", Values: []string{generationPSID}} dpu := inventory.Device{ - ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "BlueField-4", ObservationState: "observed", + ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "BlueField-4", ReaderAdapter: "bluefield-card-v1", ObservationState: "observed", BMCEndpoint: "https://192.0.2.21", Observations: map[string]inventory.Observation{"bmc-administrator": {State: "observed", ReasonCode: "BMC_ADMIN_VERIFIED"}}, Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{ - SerialNumber: "BF4-SERIAL-1", PSID: psid, + PCIAddress: "0000:21:00.0", Domain: "arm-pci", SerialNumber: "BF4-SERIAL-1", PSID: psid, }}}}, } host := inventory.Device{ - ID: "host-1", + ID: "host-1", ObservationState: "observed", SSHEndpoint: "ssh://192.0.2.10", ReaderAdapter: "linux-host-ssh-v1", Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{ - SerialNumber: "BF4-SERIAL-1", PSID: psid, + PCIAddress: "0000:21:00.0", Domain: "host-pci", SerialNumber: "BF4-SERIAL-1", PSID: psid, }}}}, } site := Site{} diff --git a/pkg/dpf/config/resolved.go b/pkg/dpf/config/resolved.go index b53b58ac..85ddfbf6 100644 --- a/pkg/dpf/config/resolved.go +++ b/pkg/dpf/config/resolved.go @@ -16,7 +16,8 @@ import ( "k8s.io/apimachinery/pkg/util/validation" ) -// Resolved is deployment metadata, not another user input/defaulting path. +// Resolved is the legacy v1alpha1 deployment metadata. New bundles use the typed +// v1alpha2 codec in EncodeResolved; this type remains for reading legacy bundles. type Resolved struct { FormatVersion string `yaml:"formatVersion"` Target string `yaml:"target"` @@ -27,7 +28,11 @@ func EncodeResolved(g Generation) ([]byte, error) { if err := ValidateGeneration(g); err != nil { return nil, err } - return yaml.Marshal(Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: g}) + wire, err := generationToOutput(g) + if err != nil { + return nil, err + } + return yaml.Marshal(resolvedOutput{FormatVersion: "v1alpha2", Target: "dpf", Config: wire}) } func DecodeResolved(b []byte) (Generation, error) { @@ -42,18 +47,38 @@ func DecodeResolved(b []byte) (Generation, error) { if err := validateKeys(&syntax, map[*yaml.Node]bool{}, map[*yaml.Node]bool{}, 0); err != nil { return r.Config, err } + var header struct { + FormatVersion string `yaml:"formatVersion"` + Target string `yaml:"target"` + } + if err := syntax.Decode(&header); err != nil { + return r.Config, fmt.Errorf("invalid resolved DPF configuration: %w", err) + } + if header.Target != "dpf" || (header.FormatVersion != "v1alpha1" && header.FormatVersion != "v1alpha2") { + return r.Config, fmt.Errorf("unsupported resolved DPF format or target") + } d := yaml.NewDecoder(bytes.NewReader(b)) d.KnownFields(true) - if err := d.Decode(&r); err != nil { - return r.Config, fmt.Errorf("invalid resolved DPF configuration: %w", err) + switch header.FormatVersion { + case "v1alpha1": + if err := d.Decode(&r); err != nil { + return r.Config, fmt.Errorf("invalid resolved DPF configuration: %w", err) + } + case "v1alpha2": + var wire resolvedOutput + if err := d.Decode(&wire); err != nil { + return r.Config, fmt.Errorf("invalid resolved DPF configuration: %w", err) + } + var err error + r.Config, err = wire.Config.generation() + if err != nil { + return r.Config, err + } } var extra any if err := d.Decode(&extra); err != io.EOF { return r.Config, fmt.Errorf("resolved DPF configuration must be one document") } - if r.Target != "dpf" || r.FormatVersion != "v1alpha1" { - return r.Config, fmt.Errorf("unsupported resolved DPF format or target") - } return r.Config, ValidateGeneration(r.Config) } diff --git a/pkg/dpf/config/resolved_output.go b/pkg/dpf/config/resolved_output.go new file mode 100644 index 00000000..fe6d1010 --- /dev/null +++ b/pkg/dpf/config/resolved_output.go @@ -0,0 +1,77 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + +// resolvedOutput versions the selected-device wire shape independently of the +// bundle index. Private collector observations never enter new sidecars. +type resolvedOutput struct { + FormatVersion string `yaml:"formatVersion"` + Target string `yaml:"target"` + Config generationOutput `yaml:"config"` +} + +type generationOutput struct { + ManagementKubernetesVersion string `yaml:"managementKubernetesVersion"` + Name string `yaml:"name"` + Profile string `yaml:"profile"` + Release string `yaml:"release"` + Namespace string `yaml:"namespace"` + ManagementUID string `yaml:"managementUID"` + KubeconfigRef FileRef `yaml:"kubeconfigRef"` + Context string `yaml:"context"` + Provisioning Provisioning `yaml:"provisioning"` + Sentry Sentry `yaml:"sentry"` + Devices []inventory.DeviceOutput `yaml:"devices"` + DevicePSIDs map[string]string `yaml:"devicePSIDs"` +} + +func generationToOutput(g Generation) (generationOutput, error) { + w := generationOutput{ + ManagementKubernetesVersion: g.ManagementKubernetesVersion, + Name: g.Name, + Profile: g.Profile, + Release: g.Release, + Namespace: g.Namespace, + ManagementUID: g.ManagementUID, + KubeconfigRef: g.KubeconfigRef, + Context: g.Context, + Provisioning: g.Provisioning, + Sentry: g.Sentry, + DevicePSIDs: g.DevicePSIDs, + } + for _, device := range g.Devices { + out, err := inventory.DeviceToOutput(device, "dpus") + if err != nil { + return generationOutput{}, err + } + w.Devices = append(w.Devices, out) + } + return w, nil +} + +func (w generationOutput) generation() (Generation, error) { + g := Generation{ + ManagementKubernetesVersion: w.ManagementKubernetesVersion, + Name: w.Name, + Profile: w.Profile, + Release: w.Release, + Namespace: w.Namespace, + ManagementUID: w.ManagementUID, + KubeconfigRef: w.KubeconfigRef, + Context: w.Context, + Provisioning: w.Provisioning, + Sentry: w.Sentry, + DevicePSIDs: w.DevicePSIDs, + } + for _, device := range w.Devices { + out, err := inventory.DeviceFromOutput(device, "dpus") + if err != nil { + return Generation{}, err + } + g.Devices = append(g.Devices, out) + } + return g, nil +} diff --git a/pkg/dpf/config/resolved_test.go b/pkg/dpf/config/resolved_test.go index 83367ebc..84e60886 100644 --- a/pkg/dpf/config/resolved_test.go +++ b/pkg/dpf/config/resolved_test.go @@ -13,6 +13,59 @@ import ( ) var _ = Describe("resolved DPF generation", func() { + It("writes typed devices without generic collector data and accepts legacy metadata", func() { + g := resolvedGeneration() + g.Devices[0].EvidenceRefs = []string{"collector-evidence-canary"} + g.Devices[0].Observations["internal-only"] = inventory.Observation{State: "observed", Values: []string{"generic-value-canary"}} + encoded, err := EncodeResolved(g) + Expect(err).NotTo(HaveOccurred()) + Expect(string(encoded)).To(ContainSubstring("formatVersion: v1alpha2")) + Expect(string(encoded)).To(ContainSubstring("bmcAdministratorVerified: true")) + for _, forbidden := range []string{"observations:", "readerAdapter:", "observationState:", "evidenceRefs:", "collector-evidence-canary", "generic-value-canary"} { + Expect(string(encoded)).NotTo(ContainSubstring(forbidden)) + } + legacy, err := yaml.Marshal(Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: g}) + Expect(err).NotTo(HaveOccurred()) + decoded, err := DecodeResolved(legacy) + Expect(err).NotTo(HaveOccurred()) + Expect(decoded).To(Equal(g)) + }) + + It("preserves rejected versus missing PSID semantics across the typed handoff", func() { + for _, code := range []string{"PSID_NOT_EXPOSED", "PSID_READER_UNSUPPORTED", "PSID_CONFLICT", "PSID_INVALID", "PSID_CHANGED", "PSID_RECHECK_FAILED", "PSID_NOT_VERIFIED"} { + g := resolvedGeneration() + g.Devices[0].Observations["psid"] = inventory.Observation{State: "unavailable", ReasonCode: code, Tool: "redfish"} + wire, err := generationToOutput(g) + Expect(err).NotTo(HaveOccurred()) + encoded, err := yaml.Marshal(resolvedOutput{FormatVersion: "v1alpha2", Target: "dpf", Config: wire}) + Expect(err).NotTo(HaveOccurred()) + decoded, err := DecodeResolved(encoded) + if code == "PSID_NOT_EXPOSED" || code == "PSID_READER_UNSUPPORTED" { + Expect(err).NotTo(HaveOccurred()) + Expect(decoded.Devices[0].Observations["psid"].ReasonCode).To(Equal(code)) + } else { + Expect(err).To(MatchError(ContainSubstring("rejected Redfish PSID evidence")), code) + } + } + }) + + It("rejects generic or contradictory typed device fields", func() { + g := resolvedGeneration() + wire, err := generationToOutput(g) + Expect(err).NotTo(HaveOccurred()) + wire.Devices[0].Issues = append(wire.Devices[0].Issues, inventory.DeviceIssue{Field: "identity", Code: "IDENTITY_CHANGED"}) + encoded, err := yaml.Marshal(resolvedOutput{FormatVersion: "v1alpha2", Target: "dpf", Config: wire}) + Expect(err).NotTo(HaveOccurred()) + _, err = DecodeResolved(encoded) + Expect(err).To(MatchError(ContainSubstring("failed recheck"))) + + encoded, err = EncodeResolved(g) + Expect(err).NotTo(HaveOccurred()) + document := strings.Replace(string(encoded), "bmcAdministratorVerified: true", "bmcAdministratorVerified: true\n observations: {}", 1) + _, err = DecodeResolved([]byte(document)) + Expect(err).To(MatchError(ContainSubstring("field observations not found"))) + }) + It("rejects retained PSID selection that contradicts the Redfish observation", func() { generation := resolvedGeneration() d := &generation.Devices[0] @@ -72,7 +125,8 @@ var _ = Describe("resolved DPF generation", func() { decoded, err := DecodeResolved(encoded) Expect(err).NotTo(HaveOccurred()) - Expect(decoded).To(Equal(generation)) + Expect(SameIntent(decoded, generation)).To(BeTrue()) + Expect(decoded.Devices[0].Runtime.Hardware.Cards[0].PSID.Values).To(Equal([]string{generationPSID})) decoded.Provisioning.Software.PLDM[generationPSID] = testArtifact("mutated.pldm") decoded.Provisioning.Devices[0].DPUNode = "mutated-node" @@ -91,7 +145,7 @@ var _ = Describe("resolved DPF generation", func() { edit func(string) string want string }{ - {name: "unknown format", edit: func(s string) string { return strings.Replace(s, "formatVersion: v1alpha1", "formatVersion: v9", 1) }, want: "unsupported resolved DPF format or target"}, + {name: "unknown format", edit: func(s string) string { return strings.Replace(s, "formatVersion: v1alpha2", "formatVersion: v9", 1) }, want: "unsupported resolved DPF format or target"}, {name: "wrong target", edit: func(s string) string { return strings.Replace(s, "target: dpf", "target: host", 1) }, want: "unsupported resolved DPF format or target"}, {name: "unknown field", edit: func(s string) string { return s + "unexpectedField: true\n" }, want: "field unexpectedField not found"}, {name: "duplicate key", edit: func(s string) string { return strings.Replace(s, "target: dpf\n", "target: dpf\ntarget: host\n", 1) }, want: "duplicate"}, @@ -197,7 +251,7 @@ func resolvedGeneration() Generation { } func cloneResolvedGeneration(g Generation) Generation { - encoded, err := EncodeResolved(g) + encoded, err := yaml.Marshal(Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: g}) ExpectWithOffset(1, err).NotTo(HaveOccurred()) cloned, err := DecodeResolved(encoded) ExpectWithOffset(1, err).NotTo(HaveOccurred()) diff --git a/pkg/dpf/generation/generate.go b/pkg/dpf/generation/generate.go index 1ce92884..7d0a78bb 100644 --- a/pkg/dpf/generation/generate.go +++ b/pkg/dpf/generation/generate.go @@ -69,15 +69,21 @@ func Build(ctx context.Context, in config.Input, overrides configinput.Values) ( // renderBundle is shared by generation and admission of a loaded bundle. It uses // only resolved intent and retained chart bytes, never credentials or source paths. func renderBundle(ctx context.Context, g config.Generation, archives map[string][]byte) (*bundle.Bundle, error) { - p, err := profiles.LoadProfileForTarget(target.DPF, g.Profile) + sidecar, err := config.EncodeResolved(g) if err != nil { return nil, err } - rendered, err := sentry.Render(g, p, archives) + return renderBundleWithSidecar(ctx, g, archives, sidecar) +} + +// Admission preserves the already decoded sidecar bytes, including legacy +// metadata, so a schema upgrade does not change the retained bundle digest. +func renderBundleWithSidecar(ctx context.Context, g config.Generation, archives map[string][]byte, sidecar []byte) (*bundle.Bundle, error) { + p, err := profiles.LoadProfileForTarget(target.DPF, g.Profile) if err != nil { return nil, err } - sidecar, err := config.EncodeResolved(g) + rendered, err := sentry.Render(g, p, archives) if err != nil { return nil, err } @@ -157,7 +163,7 @@ func Configuration(b *bundle.Bundle, userConfig string) (config.Generation, erro } } } - expected, err := renderBundle(context.Background(), g, archives) + expected, err := renderBundleWithSidecar(context.Background(), g, archives, data) if err != nil { return g, fmt.Errorf("validate retained DPF intent: %w", err) } diff --git a/pkg/dpf/generation/generation_test.go b/pkg/dpf/generation/generation_test.go index 8b8d2f38..7fce4525 100644 --- a/pkg/dpf/generation/generation_test.go +++ b/pkg/dpf/generation/generation_test.go @@ -84,6 +84,30 @@ var _ = Describe("offline DPF generation", func() { } }) + It("admits legacy resolved metadata without changing its retained digest", func() { + b, err := Build(context.Background(), in, configinput.Values{}) + Expect(err).NotTo(HaveOccurred()) + files := b.Files() + for i := range files { + if files[i].Path == bundle.ResolvedConfigPath { + g, err := config.DecodeResolved(files[i].Data) + Expect(err).NotTo(HaveOccurred()) + files[i].Data, err = yaml.Marshal(config.Resolved{FormatVersion: "v1alpha1", Target: "dpf", Config: g}) + Expect(err).NotTo(HaveOccurred()) + } + } + index := b.Index() + index.BundleDigest = "" + for i := range index.Files { + index.Files[i].SHA256 = "" + } + legacy, err := bundle.New(index, files) + Expect(err).NotTo(HaveOccurred()) + digest := legacy.Index().BundleDigest + _, err = Configuration(legacy, "") + Expect(err).NotTo(HaveOccurred()) + Expect(legacy.Index().BundleDigest).To(Equal(digest)) + }) It("rejects an old inventory before publication and an old bundle before cluster access", func() { b, err := Build(context.Background(), in, configinput.Values{}) Expect(err).NotTo(HaveOccurred()) @@ -216,8 +240,8 @@ var _ = Describe("offline DPF generation", func() { func generationInput(base, charts string) config.Input { const psid = "MT_0000000010" observed := inventory.Observation{State: "observed", Values: []string{psid}} - dpu := inventory.Device{ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "B4240", ReaderAdapter: "bluefield-card-v1", ObservationState: "observed", BMCEndpoint: "https://192.0.2.21", Observations: map[string]inventory.Observation{"bmc-administrator": {State: "observed", ReasonCode: "BMC_ADMIN_VERIFIED"}}, Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} - host := inventory.Device{ID: "host-1", Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} + dpu := inventory.Device{ID: "dpu-1", SerialNumber: "BF4-SERIAL-1", Model: "B4240", ReaderAdapter: "bluefield-card-v1", ObservationState: "observed", BMCEndpoint: "https://192.0.2.21", Observations: map[string]inventory.Observation{"bmc-administrator": {State: "observed", ReasonCode: "BMC_ADMIN_VERIFIED"}}, Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{PCIAddress: "0000:21:00.0", Domain: "arm-pci", SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} + host := inventory.Device{ID: "host-1", ObservationState: "observed", SSHEndpoint: "ssh://192.0.2.10", ReaderAdapter: "linux-host-ssh-v1", Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{PCIAddress: "0000:21:00.0", Domain: "host-pci", SerialNumber: "BF4-SERIAL-1", PSID: observed}}}}} site := config.Site{} site.DPF.Release = stringPtr("26.8.0") site.Management.KubeconfigRef.File = "missing/management.kubeconfig" From 03898fb4c4710a74b6644fc7da82fde364d69148 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 13 Nov 2026 19:31:56 +0100 Subject: [PATCH 8/9] Describe typed discovery proof in generation guidance Signed-off-by: Alexander Maslennikov --- skills/k8s-launch-kit-generate/SKILL.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/skills/k8s-launch-kit-generate/SKILL.md b/skills/k8s-launch-kit-generate/SKILL.md index b27ebb50..49ef9d6a 100644 --- a/skills/k8s-launch-kit-generate/SKILL.md +++ b/skills/k8s-launch-kit-generate/SKILL.md @@ -19,10 +19,14 @@ explicit selected device/tenant bindings, site networks, SHA-bound BF4 software and four catalogued local chart archives. `l8k schema` exposes `dpf.profiles`, `dpf.generationReleases` and `dpf.charts`. Generation reads no credentials and contacts no cluster. Each selected device requires observed BMC administrator -proof (`bmc-administrator`, `observed`, `BMC_ADMIN_VERIFIED`) from explicit BMC -discovery; ARM SSH cannot substitute. Rediscover older inventories without this -proof and regenerate old bundles into a new directory. The exact model `B4240` -is accepted only from `bluefield-card-v1` and stays unchanged in metadata. +proof (`bmcAdministratorVerified: true` with `method: redfish` in typed inventory) +from explicit BMC discovery; ARM SSH cannot substitute. Legacy inventories must +retain the explicit `bmc-administrator` observation with `observed` and +`BMC_ADMIN_VERIFIED`. Rediscover older inventories without this proof. The exact +model `B4240` requires the supported identity-bound Redfish reader (represented +by `readerAdapter: bluefield-card-v1` in legacy metadata) and stays unchanged. +New resolved metadata uses typed device fields; existing validated legacy bundles +remain readable without rewriting their content or digest. Saved proof does not replace live deployment authentication or qualify legacy BMC firmware for provisioning. Core policy defaults to `llmInspection: false`; enabling LLM inspection requires a real inference Secret. The resolved bundle is From 21a0c15b050983221eed574c62b7a9b3d5dac308 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 13 Nov 2026 19:33:37 +0100 Subject: [PATCH 9/9] Reject legacy host measurements after failed identity checks Signed-off-by: Alexander Maslennikov --- pkg/dpf/config/generation.go | 3 +++ pkg/dpf/config/generation_test.go | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/pkg/dpf/config/generation.go b/pkg/dpf/config/generation.go index f22a6239..9c44218a 100644 --- a/pkg/dpf/config/generation.go +++ b/pkg/dpf/config/generation.go @@ -280,6 +280,9 @@ func resolveProvisioning(g *Generation, inv *inventory.Inventory) error { hosts := 0 for _, h := range inv.Hosts { if h.ID == associations[0].HostID { + if h.ObservationState != "observed" || h.ReasonCode != "" { + return fmt.Errorf("device %s host association requires a successfully observed host identity", binding.ID) + } hosts++ collectRuntimePSIDs(psids, d.SerialNumber, h) } diff --git a/pkg/dpf/config/generation_test.go b/pkg/dpf/config/generation_test.go index 89c4b208..b1d3a333 100644 --- a/pkg/dpf/config/generation_test.go +++ b/pkg/dpf/config/generation_test.go @@ -32,6 +32,27 @@ var _ = Describe("DPF generation configuration", func() { Expect(err).To(MatchError(ContainSubstring("supported discovery inventory"))) }) + It("does not reuse legacy host measurements after a failed identity recheck", func() { + for _, version := range []string{"v1alpha1", "v1alpha2"} { + for _, state := range []string{"unknown", "unavailable", "observed"} { + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.FormatVersion = version + in.Site.Inventory.DPUs[0].Runtime = nil // only the rejected host supplies PSID + h := &in.Site.Inventory.Hosts[0] + h.ObservationState, h.ReasonCode = state, "IDENTITY_CHANGED" + in = decodeGenerationInput(in) + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("successfully observed host identity")), version+"/"+state) + } + } + in := validGenerationInput(GinkgoT().TempDir()) + in.Site.Inventory.FormatVersion = "v1alpha1" + in.Site.Inventory.Hosts[0].ObservationState = "" + in = decodeGenerationInput(in) + _, err := ResolveGeneration(in) + Expect(err).To(MatchError(ContainSubstring("successfully observed host identity"))) + }) + It("uses named typed Redfish PSID without requiring an SSH measurement", func() { in := validGenerationInput(GinkgoT().TempDir()) d := &in.Site.Inventory.DPUs[0]