From cb15c24709b1eda3e2b833f389a51bc2f48bb301 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 29 Oct 2026 11:48:22 +0100 Subject: [PATCH 01/35] feat(dpf): add native Go inventory discovery preview Signed-off-by: Alexander Maslennikov --- README.md | 4 +- THIRD_PARTY_NOTICES | 35 ++ docs/advanced/targets.md | 15 +- docs/llms.txt | 1 + docs/reference/cli.md | 4 +- docs/user/discovery.md | 6 + docs/user/dpf-discovery.md | 159 +++++++ examples/dpf/discovery.yaml | 25 ++ go.mod | 3 +- go.sum | 2 + mkdocs.yml | 1 + pkg/cmd/discover.go | 15 +- pkg/cmd/schema_test.go | 2 +- pkg/cmd/target_cli.go | 38 +- pkg/cmd/target_cli_test.go | 9 +- pkg/config/yaml_patch.go | 65 +-- pkg/dpf/artifacts/replace_other.go | 9 + pkg/dpf/artifacts/replace_windows.go | 19 + pkg/dpf/artifacts/store.go | 310 +++++++++++++ pkg/dpf/artifacts/store_test.go | 109 +++++ pkg/dpf/catalog/catalog.go | 29 ++ pkg/dpf/config/config.go | 510 ++++++++++++++++++++++ pkg/dpf/config/config_test.go | 111 +++++ pkg/dpf/discovery/discover.go | 410 +++++++++++++++++ pkg/dpf/discovery/discover_test.go | 115 +++++ pkg/dpf/discovery/management.go | 212 +++++++++ pkg/dpf/discovery/management_test.go | 128 ++++++ pkg/dpf/discovery/references_unix_test.go | 48 ++ pkg/dpf/inventory/identity.go | 26 ++ pkg/dpf/inventory/probe.go | 31 ++ pkg/dpf/inventory/types.go | 206 +++++++++ pkg/dpf/probe/collect.sh | 107 +++++ pkg/dpf/probe/image_test.go | 128 ++++++ pkg/dpf/probe/probe_test.go | 168 +++++++ pkg/dpf/probe/protocol.go | 137 ++++++ pkg/dpf/probe/runner.go | 490 +++++++++++++++++++++ pkg/dpf/redfish/client.go | 311 +++++++++++++ pkg/dpf/redfish/client_test.go | 106 +++++ pkg/dpf/redfish/inventory.go | 301 +++++++++++++ pkg/dpf/redfish/inventory_test.go | 129 ++++++ pkg/target/dpf/driver.go | 132 ++++++ pkg/target/dpf/driver_test.go | 40 ++ pkg/target/dpf/signals_unix.go | 12 + pkg/target/dpf/signals_windows.go | 9 + pkg/yamlinput/node.go | 108 +++++ skills/k8s-launch-kit-discover/SKILL.md | 2 + 46 files changed, 4732 insertions(+), 105 deletions(-) create mode 100644 docs/user/dpf-discovery.md create mode 100644 examples/dpf/discovery.yaml create mode 100644 pkg/dpf/artifacts/replace_other.go create mode 100644 pkg/dpf/artifacts/replace_windows.go create mode 100644 pkg/dpf/artifacts/store.go create mode 100644 pkg/dpf/artifacts/store_test.go create mode 100644 pkg/dpf/catalog/catalog.go create mode 100644 pkg/dpf/config/config.go create mode 100644 pkg/dpf/config/config_test.go create mode 100644 pkg/dpf/discovery/discover.go create mode 100644 pkg/dpf/discovery/discover_test.go create mode 100644 pkg/dpf/discovery/management.go create mode 100644 pkg/dpf/discovery/management_test.go create mode 100644 pkg/dpf/discovery/references_unix_test.go create mode 100644 pkg/dpf/inventory/identity.go create mode 100644 pkg/dpf/inventory/probe.go create mode 100644 pkg/dpf/inventory/types.go create mode 100644 pkg/dpf/probe/collect.sh create mode 100644 pkg/dpf/probe/image_test.go create mode 100644 pkg/dpf/probe/probe_test.go create mode 100644 pkg/dpf/probe/protocol.go create mode 100644 pkg/dpf/probe/runner.go create mode 100644 pkg/dpf/redfish/client.go create mode 100644 pkg/dpf/redfish/client_test.go create mode 100644 pkg/dpf/redfish/inventory.go create mode 100644 pkg/dpf/redfish/inventory_test.go create mode 100644 pkg/target/dpf/driver.go create mode 100644 pkg/target/dpf/driver_test.go create mode 100644 pkg/target/dpf/signals_unix.go create mode 100644 pkg/target/dpf/signals_windows.go create mode 100644 pkg/yamlinput/node.go diff --git a/README.md b/README.md index ebac40ad..309324cd 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Kubernetes Launch Kit (`l8k`) helps platform teams deploy NVIDIA networking on K Standard SR-IOV, RDMA-shared and host-device profiles enable NIC tuning by default; opt out with `--deploy-nic-configuration-template=false`. See [NIC configuration](docs/reference/configuration.md#nic-configuration-and-naming) for fabric-specific settings and operator ownership. -The current lifecycle uses the `host` target by default. The reserved `dpf` target does not have available lifecycle phases; see [target extension](docs/advanced/targets.md) for its implementation status. +The current lifecycle uses the `host` target by default. The `dpf` target provides an [experimental inventory discovery preview](docs/user/dpf-discovery.md); its provisioning phases remain unavailable. Spectrum-X RA2.4 and RA3.1 support Network Operator 26.10 and newer catalogued releases; RA2.3 is limited to 26.7. Both use a doSPCX bundle and derive per-group platform @@ -162,7 +162,7 @@ Available Commands: version Print the version number Target Selection Flags: - --target string Deployment target: host (default) or dpf (reserved; phases are unavailable until the DPF driver is added) (default "host") + --target string Deployment target: host (default) or dpf (experimental inventory discovery) (default "host") Host Target Common Flags: --config-dir string Directory containing optional l8k-config.yaml and presets/ overrides diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index 2c39dff0..6bdb95c3 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -9197,3 +9197,38 @@ Apache license: See the License for the specific language governing permissions and limitations under the License. + +--- +## github.com/gofrs/flock v0.13.0 + +Copyright (c) 2018-2025, The Gofrs +Copyright (c) 2015-2020, Tim Heckman +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of gofrs nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +--- diff --git a/docs/advanced/targets.md b/docs/advanced/targets.md index ae645bec..a4830d09 100644 --- a/docs/advanced/targets.md +++ b/docs/advanced/targets.md @@ -29,15 +29,7 @@ Passing `--target host` is an explicit synonym: l8k generate --target host ... ``` -The `dpf` target name is reserved for the future DPU-plane workflow. Its -phases are intentionally reported as unavailable until a typed DPF driver is -installed. Launch Kit returns validation exit code `2` instead of falling -through to host behavior: - -```console -$ l8k discover --target dpf -Error: invalid target invocation: target "dpf" does not implement phase "discover" ... -``` +The `dpf` target implements an [experimental inventory discovery preview](../user/dpf-discovery.md). Its software collects candidate management/BMC facts; hardware qualification and management probe-image release gates are explicit. Other DPF phases remain unavailable and cannot fall through to Host execution. Use `l8k schema` to inspect target and phase availability programmatically. @@ -103,8 +95,7 @@ binds through the registry and calls `Operation.Run(cmd.Context())` exactly once. Discover, generate, and the root pipeline reuse the Host application launcher; standalone deploy and validate are Host-owned services outside Cobra. Those services return errors, while `pkg/cmd` remains the only process -termination boundary. The DPF target remains unavailable, so it cannot -accidentally construct a Kubernetes client or execute Host logic. +termination boundary. The DPF discovery branch binds its own typed request before collecting Host flags, and constructs only DPF dependencies. ## Registry guarantees @@ -132,7 +123,7 @@ A target integration should: domain driver. 5. Declare all four phase capabilities and their required logical cluster roles. -6. Implement all four phases before marking them available in the schema. +6. Mark only implemented phases available; document preview and qualification limits. 7. Keep target artifacts separate; host output remains `deployment/network-operator/`. 8. Add omitted-target and explicit-host compatibility tests before routing the diff --git a/docs/llms.txt b/docs/llms.txt index f42cc634..c05d09c4 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -18,6 +18,7 @@ This documentation covers the `l8k` CLI for deploying NVIDIA cloud-native networ ## Deploy - Discover cluster hardware: /k8s-launch-kit/user/discovery/ +- DPF discovery preview: /k8s-launch-kit/user/dpf-discovery/ - Generate and review manifests: /k8s-launch-kit/advanced/generation/ - Deploy manifests: /k8s-launch-kit/advanced/deployment/ - Deploy on OpenShift: /k8s-launch-kit/user/openshift/ diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 229c2a29..cedcbddb 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -33,9 +33,7 @@ with a source filename and document number where applicable. ## Target selection and flag ownership Omitting `--target` selects `host`; adding `--target host` follows the same -code path. `dpf` is a recognized target name whose phases are unavailable in -this build. Selecting it returns validation exit code `2`. This explicit -capability error prevents DPF invocations from falling through to host logic. +code path. `dpf` provides an [experimental discovery preview](../user/dpf-discovery.md). Its other phases remain unavailable and return exit `2`, without falling through to Host logic. Host-only flags are rejected when they are explicitly supplied for another target. Defaults are ignored by this check, including explicit-value flags diff --git a/docs/user/discovery.md b/docs/user/discovery.md index f3e21d83..84b3c4c7 100644 --- a/docs/user/discovery.md +++ b/docs/user/discovery.md @@ -471,3 +471,9 @@ install. Review [secondary-network ownership](../reference/configuration.md#seco before selecting explicit managed/external policy. Unsupported or host-only installers can remain undetected; Multus detection does not imply plugin-suite presence. + +## Independent DPF inventory + +For management-cluster and BMC discovery of the separate DPU estate, use the +[DPF discovery preview](dpf-discovery.md). Its configuration, artifacts and +qualification gates are independent of the Host workflow above. diff --git a/docs/user/dpf-discovery.md b/docs/user/dpf-discovery.md new file mode 100644 index 00000000..36034acd --- /dev/null +++ b/docs/user/dpf-discovery.md @@ -0,0 +1,159 @@ + + +# DPF discovery preview + +`l8k discover --target dpf` inspects an explicitly selected management Kubernetes +cluster and DPU/host BMCs. The DPF estate is independent of Host networking. +The initial management topology requires three control-plane nodes. + +This build exposes an **experimental inventory preview**. The hardware +qualification catalog is empty: reads produce candidate facts and a partial +report, with qualification failures visible and exit `2`. No server/firmware +combination is declared supported by a successful GET. Management probes are +implemented using an existing digest-pinned diagnostic image, with orchestration +and validation in Go. The catalog pins `docker.io/nicolaka/netshoot:v0.16`, but +Kubernetes runtime qualification is pending. The default `management` mode +rejects locally until the catalog enables that qualified image/architecture cohort; +explicitly choose `inventory` for the preview. There is no automatic downgrade. +DPF generate/deploy/validate and the root DPF pipeline remain unavailable. + +## Inputs and invocation + +Supply three bundles: management kubeconfig access, selected DPU BMC origins with +credentials, and host BMC origins with credentials. All addresses must come from +site inventory; the CLI does not scan subnets or derive endpoints from IP/MACs. +A host may contain several selected or unselected cards. + +```yaml +management: + kubeconfigRef: + file: ./credentials/management.kubeconfig + context: management +credentials: + dpu-reader: + username: bmc-reader + passwordRef: + env: DPU_BMC_PASSWORD + host-reader: + username: bmc-reader + passwordRef: + env: HOST_BMC_PASSWORD +discovery: + dpus: + - bmcEndpoint: https://192.0.2.11 + credentialRef: dpu-reader + hosts: + - bmcEndpoint: https://192.0.2.21 + credentialRef: host-reader + probes: + mode: inventory +``` + +```bash +l8k discover --target dpf --user-config dpf-site.yaml \ + --save-cluster-config dpf-discovered.yaml --output json +``` + +The YAML is ordinary CLI configuration. `--target` chooses the schema; generated +facts live under `inventory`. There is no CRD wrapper. Unknown fields, duplicate +keys, wrong scalar types, ambiguous contexts and duplicate endpoints fail before +remote access. Lists and credentials stay in YAML; no password CLI flag exists. + +| Source | Values | +| --- | --- | +| Required site inputs | Explicit kubeconfig file; nonempty DPU and host lists; each origin's credential alias; existing username and env/file password reference | +| Defaultable | Context from the supplied file; verified system trust; collection 10m; request 15s; four BMCs in flight | +| Conditional | Private `discovery.tls.caFile`; endpoint TLS override; `systemURI` for multi-System BMCs | +| Observed | Physical-card/system identities, linked host adapters, component firmware, auth/time unknowns, management nodes/version | +| Release owned | Qualified adapter completeness predicates and the probe image/architecture digests | +| Later phases | Profile/release cohort, DPU API allocation, DHCP/service networks and controller authentication | + +The management kubeconfig must use verified HTTPS and certificate/key or bearer +credentials. File and embedded representations of the same credential/trust field are mutually +exclusive; choose token or tokenFile. Exec/auth-provider plugins, basic and anonymous access are rejected +in this version. Ambient kubeconfig files are never merged. BMC authentication +uses one session login using the supplied username/password; Basic fallback is +not included in this preview. Session writes are restricted to the standard +`/redfish/v1/SessionService/Sessions` collection and its exact returned session +child; alternate OEM session paths are unsupported. No password guessing or credential repair occurs. + +Passwords resolve only from `{env: NAME}` or `{file: path}`. Environment values +are verbatim. UTF-8 files lose at most one final LF/CRLF. File and CA references +are relative to the input YAML and are rebased when saved elsewhere. Private files +are not copied. Endpoint origins require HTTPS; redirects, cross-origin links, +unbounded pages and arbitrary action endpoints are rejected. + +## Observations, reports and refresh + +Physical identity comes from qualified card/system fields, keeping raw serials. +Ports/PCI functions, CPU serials and addresses cannot substitute for card identity. +Every requested endpoint remains represented on failure; unrelated cards remain +outside selection. Unknown firmware/auth/time values stay unknown. Qualification +requires both a reviewed model/firmware tuple and an adapter-specific completeness +predicate; no site input can waive that requirement. + +Complete discovery writes the requested YAML. Partial collection preserves an +existing handoff and writes `partial.yaml` plus `report.json` beneath +`.discovery//`. JSON stdout is one final DPF result, with +progress on stderr; syntax errors before dispatch retain normal Cobra stderr. +The report contains facts, checks, factual diffs, later-phase gaps, errors and +successfully written artifacts. Saved YAML alone contains the factual handoff. +Explicit in-place refresh requires the same input/output path; it keeps +`source-before.yaml`. Stable sidecar locks serialize cooperating writers. External +edits between the final hash check and rename cannot be excluded; avoid concurrent +editing. Output paths must use real directories without symlink aliases. + +| Exit | Meaning | +| --- | --- | +| 0 | Explicit-mode checks, cleanup and publication completed | +| 2 | Invalid input, unqualified required capability or proven mismatch | +| 3 | Required observation, authentication, TLS, API or remote cleanup failure | +| 1 | Cancellation or local publication/encoding error | + +## Management probe implementation and qualification + +Launch Kit builds no discovery image. The release catalog pins the existing +[`nicolaka/netshoot:v0.16`](https://github.com/nicolaka/netshoot/releases/tag/v0.16) +image by manifest and Linux amd64/arm64 digests. A fixed, embedded diagnostic +adapter invokes its `getent`, `curl`, `ip` and `jq` tools; the Go CLI resolves +destinations/public trust and validates the versioned JSON results. No Python, +runtime package installation, downloaded helper or site-supplied command is used. +The image repository can be mirrored without changing its pinned digest. + +The NVIDIA candidate +[`nvcr.io/nvidia/k8s/toolbox`](https://catalog.ngc.nvidia.com/orgs/nvidia/k8s/containers/toolbox/latest/layers) +has suitable tools, but its inspected current image is based on Ubuntu 23.04 and +was built in September 2023. DOCA 3.3.0 base/full runtime images were also inspected; +both lack `curl` and `ip`. Netshoot is the selected existing-image alternative. + +The runner uses an existing namespace, an immutable ConfigMap containing the +public request, public CAs and fixed adapter, and exact run-owned Pods. It creates no namespaces, RBAC, Secrets, +operators or DPF objects. Each management node has separate Pod-network and +host-network probes. Admission, node/Pod UIDs, image identity, request immutability +and full row coverage are checked before evidence is accepted. No BMC credential +or kubeconfig is sent to a probe. Pods run as non-root with a read-only root +filesystem, dropped capabilities and a bounded memory-backed scratch volume. +Each resolved IP remains a separate observation, including failed alternatives. + +Cleanup checks exact UID/resourceVersion and admitted-spec ownership, preserving +foreign modifications. Receipts are journaled atomically before creates and after +UID receipts. Ambiguous creates are inspected by exact name/spec; unresolved +outcomes remain explicit leftovers. SIGKILL can leave resources; consult +`resources.json` and verify current UIDs/ownership before operator recovery. +Automatic cleanup of other runs is not implemented. + +The published manifest/platform digests and local non-root tool execution have +been checked. The opt-in `dpf_probe_image` Go test tag exercises the fixed adapter +inside the existing image against local DNS/TLS/HTTP fixtures; set +`DPF_PROBE_TEST_IMAGE` to the immutable catalog reference when running that test. +It defaults to Linux amd64; set `DPF_PROBE_TEST_PLATFORM=linux/arm64` for arm64. +Local containers do not establish Kubernetes network paths or admission behavior. +Before enabling management mode in the release catalog, exercise host-network +admission, scheduling, successful log collection, cancellation and cleanup on +a controlled cluster, and qualify actual vendor resource graphs, component +versions, physical-card correlation and completeness predicates. Software fixtures +establish parser/control-flow behavior; they do not establish physical hardware +support. Discovery performs no firmware, NTP, trust enrollment or power action. diff --git a/examples/dpf/discovery.yaml b/examples/dpf/discovery.yaml new file mode 100644 index 00000000..b47f6cba --- /dev/null +++ b/examples/dpf/discovery.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +# Synthetic site. Replace endpoints, kubeconfig and account references locally. +management: + kubeconfigRef: + file: ./credentials/management.kubeconfig + context: management +credentials: + dpu-reader: + username: bmc-reader + passwordRef: + env: DPU_BMC_PASSWORD + host-reader: + username: bmc-reader + passwordRef: + env: HOST_BMC_PASSWORD +discovery: + dpus: + - bmcEndpoint: https://192.0.2.11 + credentialRef: dpu-reader + hosts: + - bmcEndpoint: https://192.0.2.21 + credentialRef: host-reader + probes: + mode: inventory diff --git a/go.mod b/go.mod index eac3e486..ef739fae 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Mellanox/nic-configuration-operator v1.3.2-0.20261027094742-6806232e532b github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 + github.com/gofrs/flock v0.13.0 github.com/mitchellh/copystructure v1.2.0 github.com/onsi/ginkgo/v2 v2.33.1 github.com/onsi/gomega v1.44.0 @@ -15,6 +16,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.12.1 go.uber.org/zap v1.28.0 + golang.org/x/sys v0.49.0 golang.org/x/term v0.47.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -122,7 +124,6 @@ require ( golang.org/x/net v0.60.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.24.0 // indirect - golang.org/x/sys v0.49.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.49.0 // indirect diff --git a/go.sum b/go.sum index d55753d0..92624cae 100644 --- a/go.sum +++ b/go.sum @@ -147,6 +147,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= +github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= diff --git a/mkdocs.yml b/mkdocs.yml index 380c60b7..8d0f53d8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -90,6 +90,7 @@ nav: - Networking concepts: advanced/concepts.md - Deploy: - Discover cluster hardware: user/discovery.md + - DPF discovery preview: user/dpf-discovery.md - Generate and review manifests: advanced/generation.md - Deploy manifests: advanced/deployment.md - Deploy on OpenShift: user/openshift.md diff --git a/pkg/cmd/discover.go b/pkg/cmd/discover.go index 435747af..c8c30664 100644 --- a/pkg/cmd/discover.go +++ b/pkg/cmd/discover.go @@ -17,11 +17,13 @@ package cmd import ( + apperrors "github.com/nvidia/k8s-launch-kit/pkg/errors" "github.com/spf13/cobra" "github.com/nvidia/k8s-launch-kit/pkg/configflags" "github.com/nvidia/k8s-launch-kit/pkg/options" "github.com/nvidia/k8s-launch-kit/pkg/target" + dpftarget "github.com/nvidia/k8s-launch-kit/pkg/target/dpf" hosttarget "github.com/nvidia/k8s-launch-kit/pkg/target/host" ) @@ -40,7 +42,11 @@ NicDevice CRs, and torn down when discovery finishes. Discovery groups nodes by hardware, detects east-west vs north-south NICs, and probes OFED-dependent modules. With --user-config, it replaces -clusterConfig and discoveredSecondaryNetwork, then applies explicit CLI overrides.`, +clusterConfig and discoveredSecondaryNetwork, then applies explicit CLI overrides. + +With --target dpf, inspect the independent management/BMC estate using ordinary +YAML. The inventory preview requires discovery.probes.mode: inventory; management +probes and hardware support require qualified release assets.`, Example: ` # Basic discovery l8k discover --kubeconfig ~/.kube/config \ --save-cluster-config ./cluster-config.yaml @@ -67,6 +73,13 @@ clusterConfig and discoveredSecondaryNetwork, then applies explicit CLI override l8k discover --save-cluster-config ./cluster-config.yaml \ --output json 2>/dev/null`, Run: func(cmd *cobra.Command, args []string) { + if targetName == string(target.DPF) && len(args) > 0 { + exitWithError(apperrors.NewValidationError("discover takes no positional arguments", nil, "Use --user-config"), outputFormat) + } + if targetName == string(target.DPF) { + runTargetCommand(cmd, target.Discover, dpftarget.NewDiscoverAdapter(dpftarget.Request{UserConfig: userConfig, OutputPath: saveClusterConfig, ExplicitOutput: cmd.Flags().Changed("save-cluster-config")})) + return + } opts := mustCollectConfigFlags(cmd, discoverConfigOptions) opts.ConfigDir = configDir opts.DiscoverClusterConfig = true diff --git a/pkg/cmd/schema_test.go b/pkg/cmd/schema_test.go index 7fe663ec..c9862508 100644 --- a/pkg/cmd/schema_test.go +++ b/pkg/cmd/schema_test.go @@ -38,7 +38,7 @@ func TestSchemaDescribesTargetsAdditively(t *testing.T) { for _, phase := range target.PublicPhases() { assert.Truef(t, host.Phases[string(phase)].Available, "host phase %s", phase) - assert.Falsef(t, dpf.Phases[string(phase)].Available, "dpf phase %s", phase) + assert.Equalf(t, phase == target.Discover, dpf.Phases[string(phase)].Available, "dpf phase %s", phase) assert.NotEmptyf(t, dpf.Phases[string(phase)].Reason, "dpf phase %s reason", phase) } diff --git a/pkg/cmd/target_cli.go b/pkg/cmd/target_cli.go index d213a481..1aa92591 100644 --- a/pkg/cmd/target_cli.go +++ b/pkg/cmd/target_cli.go @@ -30,6 +30,7 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/configflags" apperrors "github.com/nvidia/k8s-launch-kit/pkg/errors" "github.com/nvidia/k8s-launch-kit/pkg/target" + dpftarget "github.com/nvidia/k8s-launch-kit/pkg/target/dpf" hosttarget "github.com/nvidia/k8s-launch-kit/pkg/target/host" ) @@ -67,7 +68,7 @@ func addTargetFlag(cmd *cobra.Command) { &targetName, "target", string(target.Host), - "Deployment target: host (default) or dpf (reserved; phases are unavailable until the DPF driver is added)", + "Deployment target: host (default) or dpf (experimental inventory discovery)", ) setFlagGroup(cmd, "target", GroupTarget) setFlagTargetScope(cmd, []target.Name{target.Host, target.DPF}, "target") @@ -166,27 +167,13 @@ func (d unavailableTargetDriver) Bind(invocation target.Invocation) (target.Oper } } -func dpfTargetDescriptor() target.Descriptor { - const reason = "the DPF target name and CLI boundary are reserved, but the DPF driver is not available in this build" - unavailable := target.Capability{Available: false, Reason: reason} - return target.Descriptor{ - Name: target.DPF, - Description: "DPU-plane provisioning through NVIDIA DPF Operator", - Phases: map[target.Phase]target.Capability{ - target.Discover: unavailable, - target.Generate: unavailable, - target.Deploy: unavailable, - target.Validate: unavailable, - target.Pipeline: unavailable, - }, - } -} +func dpfTargetDescriptor() target.Descriptor { return dpftarget.Descriptor() } func targetDescriptors() []target.Descriptor { return []target.Descriptor{hosttarget.Descriptor(), dpfTargetDescriptor()} } -func bindTargetCommand(cmd *cobra.Command, phase target.Phase, hostDriver target.Driver) (target.Operation, error) { +func bindTargetCommand(cmd *cobra.Command, phase target.Phase, driver target.Driver) (target.Operation, error) { selected, err := target.ParseName(targetName) if err != nil { return nil, err @@ -195,9 +182,19 @@ func bindTargetCommand(cmd *cobra.Command, phase target.Phase, hostDriver target return nil, err } + hostDriver := target.Driver(unavailableTargetDriver{descriptor: hosttarget.Descriptor()}) + dpfDriver := target.Driver(unavailableTargetDriver{descriptor: dpfTargetDescriptor()}) + switch driver.Descriptor().Name { + case target.Host: + hostDriver = driver + case target.DPF: + dpfDriver = driver + default: + return nil, fmt.Errorf("unexpected driver") + } registry, err := target.NewRegistry( hostDriver, - unavailableTargetDriver{descriptor: dpfTargetDescriptor()}, + dpfDriver, ) if err != nil { return nil, fmt.Errorf("construct target registry: %w", err) @@ -225,8 +222,8 @@ func bindTargetCommand(cmd *cobra.Command, phase target.Phase, hostDriver target }) } -func runTargetCommand(cmd *cobra.Command, phase target.Phase, hostDriver target.Driver) { - operation, err := bindTargetCommand(cmd, phase, hostDriver) +func runTargetCommand(cmd *cobra.Command, phase target.Phase, driver target.Driver) { + operation, err := bindTargetCommand(cmd, phase, driver) if err != nil { var structured *apperrors.StructuredError if errors.As(err, &structured) { @@ -292,6 +289,7 @@ func markDiscoverTargetScopes() { "topology-scheme", "ip-version", "topology-file", "spectrum-x-config", "spectrum-x-configmap-name") markConfigFlagTargetScopes(discoverCmd, configflags.ScopeDiscover) + setFlagTargetScope(discoverCmd, []target.Name{target.Host, target.DPF}, "user-config", "save-cluster-config") } func markGenerateTargetScopes() { diff --git a/pkg/cmd/target_cli_test.go b/pkg/cmd/target_cli_test.go index 7220eade..c80b9099 100644 --- a/pkg/cmd/target_cli_test.go +++ b/pkg/cmd/target_cli_test.go @@ -250,7 +250,7 @@ func TestTargetDescriptors(t *testing.T) { assert.Equal(t, target.DPF, descriptors[1].Name) for _, phase := range target.PublicPhases() { assert.True(t, descriptors[0].Capability(phase).Available) - assert.False(t, descriptors[1].Capability(phase).Available) + assert.Equal(t, phase == target.Discover, descriptors[1].Capability(phase).Available) assert.NotEmpty(t, descriptors[1].Capability(phase).Reason) } } @@ -291,6 +291,9 @@ func TestTargetCLIProcess(t *testing.T) { }) for _, phase := range target.PublicPhases() { + if phase == target.Discover { + continue + } t.Run("dpf "+string(phase)+" capability error", func(t *testing.T) { output, exitCode := runCLIHelper(t, string(phase), "--target", "dpf") assert.Equal(t, 2, exitCode) @@ -314,8 +317,8 @@ func TestTargetCLIProcess(t *testing.T) { t.Run("dpf capability errors preserve json output mode", func(t *testing.T) { output, exitCode := runCLIHelper(t, "discover", "--target", "dpf", "--output", "json") assert.Equal(t, 2, exitCode) - assert.Contains(t, output, `"code": "VALIDATION_ERROR"`) - assert.Contains(t, output, `"success": false`) + assert.Contains(t, output, `"code":"INVALID_CONFIGURATION"`) + assert.Contains(t, output, `"target":"dpf"`) }) t.Run("unknown target is distinct from unavailable phase", func(t *testing.T) { diff --git a/pkg/config/yaml_patch.go b/pkg/config/yaml_patch.go index 6a6fa12f..a96433c3 100644 --- a/pkg/config/yaml_patch.go +++ b/pkg/config/yaml_patch.go @@ -10,6 +10,7 @@ import ( "reflect" "strings" + "github.com/nvidia/k8s-launch-kit/pkg/yamlinput" yaml3 "gopkg.in/yaml.v3" ) @@ -17,68 +18,8 @@ import ( // expanded. YAML merges are shallow: an explicit mapping replaces the entire // inherited mapping at that key. Earlier mappings in a merge sequence win. // Sharing this expansion keeps presence tracking and source patches consistent. -func expandYAMLNode(node *yaml3.Node, active map[*yaml3.Node]bool) (*yaml3.Node, error) { - if node == nil { - return nil, fmt.Errorf("missing YAML node") - } - if active[node] { - return nil, fmt.Errorf("cyclic YAML alias") - } - active[node] = true - defer delete(active, node) - if node.Kind == yaml3.AliasNode { - return expandYAMLNode(node.Alias, active) - } - copy := *node - copy.Anchor = "" - copy.Content = nil - for _, child := range node.Content { - expanded, err := expandYAMLNode(child, active) - if err != nil { - return nil, err - } - copy.Content = append(copy.Content, expanded) - } - if copy.Kind != yaml3.MappingNode { - return ©, nil - } - entries := copy.Content - copy.Content = nil - for i := 0; i+1 < len(entries); i += 2 { - if entries[i].Tag != "!!merge" { - continue - } - mappings := []*yaml3.Node{entries[i+1]} - if entries[i+1].Kind == yaml3.SequenceNode { - mappings = entries[i+1].Content - } - for _, mapping := range mappings { - if mapping.Kind != yaml3.MappingNode { - return nil, fmt.Errorf("YAML merge value must be a mapping or sequence of mappings") - } - for j := 0; j+1 < len(mapping.Content); j += 2 { - putYAMLField(©, mapping.Content[j], mapping.Content[j+1], false) - } - } - } - for i := 0; i+1 < len(entries); i += 2 { - if entries[i].Tag != "!!merge" { - putYAMLField(©, entries[i], entries[i+1], true) - } - } - return ©, nil -} - -func putYAMLField(mapping, key, value *yaml3.Node, replace bool) { - for i := 0; i+1 < len(mapping.Content); i += 2 { - if mapping.Content[i].Value == key.Value { - if replace { - mapping.Content[i], mapping.Content[i+1] = key, value - } - return - } - } - mapping.Content = append(mapping.Content, key, value) +func expandYAMLNode(node *yaml3.Node, _ map[*yaml3.Node]bool) (*yaml3.Node, error) { + return yamlinput.Expand(node) } // PatchConfigYAML updates only the requested YAML paths, preserving all other diff --git a/pkg/dpf/artifacts/replace_other.go b/pkg/dpf/artifacts/replace_other.go new file mode 100644 index 00000000..d1554281 --- /dev/null +++ b/pkg/dpf/artifacts/replace_other.go @@ -0,0 +1,9 @@ +//go:build !windows + +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package artifacts + +import "os" + +func replaceFile(source, dest string) error { return os.Rename(source, dest) } diff --git a/pkg/dpf/artifacts/replace_windows.go b/pkg/dpf/artifacts/replace_windows.go new file mode 100644 index 00000000..d7df0ca3 --- /dev/null +++ b/pkg/dpf/artifacts/replace_windows.go @@ -0,0 +1,19 @@ +//go:build windows + +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package artifacts + +import "golang.org/x/sys/windows" + +func replaceFile(source, dest string) error { + s, err := windows.UTF16PtrFromString(source) + if err != nil { + return err + } + d, err := windows.UTF16PtrFromString(dest) + if err != nil { + return err + } + return windows.MoveFileEx(s, d, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH) +} diff --git a/pkg/dpf/artifacts/store.go b/pkg/dpf/artifacts/store.go new file mode 100644 index 00000000..b1645ffd --- /dev/null +++ b/pkg/dpf/artifacts/store.go @@ -0,0 +1,310 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package artifacts publishes secret-free discovery files on local filesystems. +package artifacts + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "time" + + "github.com/gofrs/flock" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "k8s.io/client-go/tools/clientcmd" +) + +type Snapshot struct { + Exists bool + Data []byte +} +type Store struct { + Output string + RunDir string + Input config.Input + Initial Snapshot + PriorInventory *inventory.Inventory +} + +func New(in config.Input, destination string, explicit bool) (*Store, error) { + if destination == "" { + destination = filepath.Join(filepath.Dir(in.Path), "dpf-discovered.yaml") + } + output, err := filepath.Abs(destination) + if err != nil { + return nil, err + } + input, err := filepath.Abs(in.Path) + if err != nil { + return nil, err + } + if input == output && !explicit { + return nil, fmt.Errorf("in-place refresh requires explicit --save-cluster-config") + } + if err := safePath(output); err != nil { + return nil, err + } + initial, err := snapshot(output) + if err != nil { + return nil, err + } + if initial.Exists && output != input { + old, e := config.Decode(initial.Data, output) + if e != nil || old.Site.Inventory == nil || !sameRequested(old.Site, in.Site) { + return nil, fmt.Errorf("output exists and is not a DPF inventory for the same requested scope") + } + } + // Protect source and referenced private files, including aliases via symlinks/hardlinks. + sourceInfo, err := os.Stat(input) + if err != nil { + return nil, fmt.Errorf("cannot inspect source") + } + outputInfo, _ := os.Stat(output) + if outputInfo != nil && os.SameFile(sourceInfo, outputInfo) && input != output { + return nil, fmt.Errorf("output aliases input") + } + protected := []string{in.Site.Management.KubeconfigRef.File} + for _, c := range in.Site.Credentials { + if c.PasswordRef.File != nil { + protected = append(protected, *c.PasswordRef.File) + } + } + if t := in.Site.Discovery.TLS; t != nil && t.CAFile != nil { + protected = append(protected, *t.CAFile) + } + for _, e := range append(append([]config.Endpoint{}, in.Site.Discovery.DPUs...), in.Site.Discovery.Hosts...) { + if e.TLS != nil && e.TLS.CAFile != nil { + protected = append(protected, *e.TLS.CAFile) + } + } + kubeconfigPath := in.Site.Management.KubeconfigRef.File + if !filepath.IsAbs(kubeconfigPath) { + kubeconfigPath = filepath.Join(filepath.Dir(input), kubeconfigPath) + } + if rawBytes, readErr := config.ReadReference(kubeconfigPath, 16<<20); readErr == nil { + if raw, err := clientcmd.Load(rawBytes); err == nil { + for _, auth := range raw.AuthInfos { + for _, p := range []string{auth.ClientKey, auth.ClientCertificate, auth.TokenFile} { + if p != "" { + if !filepath.IsAbs(p) { + p = filepath.Join(filepath.Dir(kubeconfigPath), p) + } + protected = append(protected, p) + } + } + } + for _, cluster := range raw.Clusters { + p := cluster.CertificateAuthority + if p != "" { + if !filepath.IsAbs(p) { + p = filepath.Join(filepath.Dir(kubeconfigPath), p) + } + protected = append(protected, p) + } + } + } + } + for _, p := range protected { + if !filepath.IsAbs(p) { + p = filepath.Join(filepath.Dir(input), p) + } + abs, _ := filepath.Abs(p) + st, _ := os.Stat(abs) + if abs == output || outputInfo != nil && st != nil && os.SameFile(st, outputInfo) { + return nil, fmt.Errorf("output aliases a private reference") + } + } + entropy := make([]byte, 16) + if _, err := rand.Read(entropy); err != nil { + return nil, err + } + runID := time.Now().UTC().Format("20060102T150405Z") + "-" + hex.EncodeToString(entropy) + runDir := filepath.Join(filepath.Dir(output), trimExtension(filepath.Base(output))+".discovery", runID) + if err := safePath(filepath.Join(runDir, "reserved")); err != nil { + return nil, err + } + if err := os.MkdirAll(runDir, 0700); err != nil { + return nil, err + } + store := &Store{Output: output, RunDir: runDir, Input: in, Initial: initial} + if initial.Exists { + old, e := config.Decode(initial.Data, output) + if e == nil { + store.PriorInventory = old.Site.Inventory + } + } + return store, nil +} +func (s *Store) RunID() string { return filepath.Base(s.RunDir) } +func trimExtension(s string) string { return s[:len(s)-len(filepath.Ext(s))] } +func sameRequested(a, b config.Site) bool { + key := func(s config.Site) string { + targets := []inventory.ScopeEndpoint{} + for _, group := range []struct { + role string + eps []config.Endpoint + }{{"dpus", s.Discovery.DPUs}, {"hosts", s.Discovery.Hosts}} { + for _, e := range group.eps { + origin, err := config.Origin(e.BMCEndpoint) + if err != nil { + return "" + } + selector := "" + if e.SystemURI != nil { + selector = *e.SystemURI + } + targets = append(targets, inventory.ScopeEndpoint{Role: group.role, Endpoint: origin, SystemURI: selector}) + } + } + slices.SortFunc(targets, func(a, b inventory.ScopeEndpoint) int { + return strings.Compare(a.Role+a.Endpoint+a.SystemURI, b.Role+b.Endpoint+b.SystemURI) + }) + return inventory.Digest("requested", targets) + } + return key(a) == key(b) +} +func safePath(p string) error { + for current := p; ; current = filepath.Dir(current) { + st, err := os.Lstat(current) + if err == nil { + if st.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("output path contains a symlink") + } + if current == p && !st.Mode().IsRegular() { + return fmt.Errorf("output must be a regular file") + } + } else if !os.IsNotExist(err) { + return fmt.Errorf("cannot inspect output path") + } + if filepath.Dir(current) == current { + break + } + } + return nil +} +func snapshot(p string) (Snapshot, error) { + st, err := os.Stat(p) + if os.IsNotExist(err) { + return Snapshot{}, nil + } + if err != nil || !st.Mode().IsRegular() || st.Size() > 16<<20 { + return Snapshot{}, fmt.Errorf("cannot inspect bounded output file") + } + b, err := os.ReadFile(p) + return Snapshot{Exists: true, Data: b}, err +} +func (s *Store) Publish(ctx context.Context, report *inventory.Report) error { + if report == nil { + return fmt.Errorf("nil report") + } + lockPath := s.Output + ".lock" + if err := safePath(lockPath); err != nil { + return err + } + lock := flock.New(lockPath) + acquired, err := lock.TryLockContext(ctx, 50*time.Millisecond) + if err != nil || !acquired { + return fmt.Errorf("cannot acquire discovery output lock") + } + defer lock.Unlock() + source, err := os.ReadFile(s.Input.Path) + if err != nil || !bytes.Equal(source, s.Input.Source) { + return fmt.Errorf("source changed during discovery") + } + current, err := snapshot(s.Output) + if err != nil || current.Exists != s.Initial.Exists || !bytes.Equal(current.Data, s.Initial.Data) { + return fmt.Errorf("output changed during discovery") + } + if err := safePath(s.Output); err != nil { + return err + } + if report.Inventory != nil && report.Discovery.Outcome == "complete" && s.PriorInventory != nil { + if s.PriorInventory.ScopeDigest != "" && s.PriorInventory.ScopeDigest != report.Inventory.ScopeDigest || inventory.IdentityConflict(s.PriorInventory, report.Inventory) { + return fmt.Errorf("existing destination inventory conflicts with resolved scope or physical identity") + } + } + if report.Inventory != nil { + dest := s.Output + if report.Discovery.Outcome != "complete" { + dest = filepath.Join(s.RunDir, "partial.yaml") + } + encoded, err := config.Saved(s.Input, report.Inventory, dest) + if err != nil { + return err + } + if dest == s.Input.Path { + backup := filepath.Join(s.RunDir, "source-before.yaml") + if err := AtomicWrite(backup, s.Input.Source); err != nil { + return err + } + report.Artifacts = append(report.Artifacts, inventory.Artifact{Path: backup, Kind: "source-backup"}) + } + if err := AtomicWrite(dest, encoded); err != nil { + return err + } + kind := "inventory" + if dest != s.Output { + kind = "partial-inventory" + } + report.Artifacts = append(report.Artifacts, inventory.Artifact{Path: dest, Kind: kind}) + } + return s.Report(report) +} +func (s *Store) Report(report *inventory.Report) error { + p := filepath.Join(s.RunDir, "report.json") + b, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + if err := AtomicWrite(p, append(b, '\n')); err != nil { + return err + } + report.Artifacts = append(report.Artifacts, inventory.Artifact{Path: p, Kind: "report"}) + return nil +} + +// AtomicWrite uses restrictive same-directory temp files. Locks protect cooperating writers only. +func AtomicWrite(filename string, data []byte) error { + if err := safePath(filename); err != nil { + return err + } + f, err := os.CreateTemp(filepath.Dir(filename), ".l8k-dpf-*") + if err != nil { + return err + } + name := f.Name() + defer os.Remove(name) + if err = f.Chmod(0600); err == nil { + _, err = f.Write(data) + } + if err == nil { + err = f.Sync() + } + err = errors.Join(err, f.Close()) + if err != nil { + return err + } + if runtime.GOOS == "windows" { + return replaceFile(name, filename) + } + if err := os.Rename(name, filename); err != nil { + return err + } + directory, err := os.Open(filepath.Dir(filename)) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/pkg/dpf/artifacts/store_test.go b/pkg/dpf/artifacts/store_test.go new file mode 100644 index 00000000..1d5dd8d7 --- /dev/null +++ b/pkg/dpf/artifacts/store_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package artifacts + +import ( + "context" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "path/filepath" + "testing" +) + +func TestArtifacts(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF artifacts") } + +var _ = Describe("safe publication", func() { + var dir string + var input config.Input + BeforeEach(func() { + var e error + dir, e = os.MkdirTemp("", "dpf-artifacts-") + Expect(e).NotTo(HaveOccurred()) + dir, e = filepath.EvalSymlinks(dir) + Expect(e).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + source := []byte("# site intent\nmanagement:\n kubeconfigRef: {file: credentials/mgmt}\ncredentials: {}\ndiscovery:\n dpus: [{bmcEndpoint: 'https://192.0.2.1', credentialRef: r}]\n hosts: [{bmcEndpoint: 'https://192.0.2.2', credentialRef: r}]\n") + p := filepath.Join(dir, "input.yaml") + Expect(os.WriteFile(p, source, 0600)).To(Succeed()) + input, e = config.Load(p) + Expect(e).NotTo(HaveOccurred()) + }) + report := func(complete bool) *inventory.Report { + outcome := "partial" + if complete { + outcome = "complete" + } + return &inventory.Report{Inventory: &inventory.Inventory{FormatVersion: "v1alpha1"}, Discovery: inventory.Discovery{Outcome: outcome}, Artifacts: []inventory.Artifact{}} + } + It("preserves normal handoff on partial collection and rebases references", func() { + s, e := New(input, "", false) + Expect(e).NotTo(HaveOccurred()) + r := report(false) + Expect(s.Publish(context.Background(), r)).To(Succeed()) + _, e = os.Stat(s.Output) + Expect(os.IsNotExist(e)).To(BeTrue()) + b, e := os.ReadFile(filepath.Join(s.RunDir, "partial.yaml")) + Expect(e).NotTo(HaveOccurred()) + Expect(string(b)).To(ContainSubstring("# site intent")) + Expect(string(b)).To(ContainSubstring("../../credentials/mgmt")) + st, e := os.Stat(filepath.Join(s.RunDir, "report.json")) + Expect(e).NotTo(HaveOccurred()) + Expect(st.Mode().Perm()).To(Equal(os.FileMode(0600))) + }) + It("creates an in-place source backup before replacement", func() { + s, e := New(input, input.Path, true) + Expect(e).NotTo(HaveOccurred()) + Expect(s.Publish(context.Background(), report(true))).To(Succeed()) + b, e := os.ReadFile(filepath.Join(s.RunDir, "source-before.yaml")) + Expect(e).NotTo(HaveOccurred()) + Expect(b).To(Equal(input.Source)) + }) + It("rejects concurrent source edits before replacing output", func() { + s, e := New(input, "", false) + Expect(e).NotTo(HaveOccurred()) + Expect(os.WriteFile(input.Path, []byte("changed"), 0600)).To(Succeed()) + Expect(s.Publish(context.Background(), report(true))).NotTo(Succeed()) + _, e = os.Stat(s.Output) + Expect(os.IsNotExist(e)).To(BeTrue()) + }) + It("rejects existing unrelated output and symlinked run directories", func() { + out := filepath.Join(dir, "out.yaml") + Expect(os.WriteFile(out, []byte("unrelated: true"), 0600)).To(Succeed()) + _, e := New(input, out, true) + Expect(e).To(HaveOccurred()) + Expect(os.Remove(out)).To(Succeed()) + Expect(os.Symlink(dir, filepath.Join(dir, "out.discovery"))).To(Succeed()) + _, e = New(input, out, true) + Expect(e).To(HaveOccurred()) + }) + It("keeps a stable lock and detects a second cooperating writer", func() { + a, e := New(input, "", false) + Expect(e).NotTo(HaveOccurred()) + b, e := New(input, "", false) + Expect(e).NotTo(HaveOccurred()) + Expect(a.Publish(context.Background(), report(true))).To(Succeed()) + Expect(b.Publish(context.Background(), report(true))).NotTo(Succeed()) + _, e = os.Stat(a.Output + ".lock") + Expect(e).NotTo(HaveOccurred()) + }) + It("checks the destination baseline even when source already contains inventory", func() { + output := filepath.Join(dir, "prior.yaml") + prior := &inventory.Inventory{FormatVersion: "v1alpha1", ScopeDigest: "cluster-B"} + saved, e := config.Saved(input, prior, output) + Expect(e).NotTo(HaveOccurred()) + Expect(os.WriteFile(output, saved, 0600)).To(Succeed()) + input.Site.Inventory = &inventory.Inventory{FormatVersion: "v1alpha1", ScopeDigest: "cluster-A"} + store, e := New(input, output, true) + Expect(e).NotTo(HaveOccurred()) + r := report(true) + r.Inventory.ScopeDigest = "cluster-A" + Expect(store.Publish(context.Background(), r)).NotTo(Succeed()) + unchanged, e := os.ReadFile(output) + Expect(e).NotTo(HaveOccurred()) + Expect(unchanged).To(Equal(saved)) + }) + +}) diff --git a/pkg/dpf/catalog/catalog.go b/pkg/dpf/catalog/catalog.go new file mode 100644 index 00000000..3da825a9 --- /dev/null +++ b/pkg/dpf/catalog/catalog.go @@ -0,0 +1,29 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package catalog owns qualified discovery assets. Site inputs cannot waive these gates. +package catalog + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/probe" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" +) + +// ProbeAssets pins the existing netshoot v0.16 image. Registry and local non-root +// tool checks do not qualify Kubernetes admission, so activation remains gated. +// No discovery image is built or published by Launch Kit. +func ProbeAssets() probe.Assets { + return probe.Assets{ + Repository: "docker.io/nicolaka/netshoot", + ManifestDigest: "sha256:b09d9b21381f47a79b3cbcb30da25266dc17186ea00ae65e99fdc51396f48e70", + PlatformDigests: map[string]string{ + "amd64": "sha256:147f2c4cb0aa7a741151bce2bfef71d9e1a04631636e24665ee2a5ed799feb47", + "arm64": "sha256:f460dc5e85890cad42b14973f5724b40831cbb3e1f745470caf6b8cf0d6ec7fe", + }, + Qualified: false, + } +} + +// Hardware is empty until real resource graphs and adapter completeness predicates +// have been qualified for specific models/firmware. Candidate reads remain useful. +func Hardware() []redfish.Qualification { return nil } diff --git a/pkg/dpf/config/config.go b/pkg/dpf/config/config.go new file mode 100644 index 00000000..75e20aef --- /dev/null +++ b/pkg/dpf/config/config.go @@ -0,0 +1,510 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package config loads ordinary DPF site YAML without Host defaults or ambient access. +package config + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "k8s.io/apimachinery/pkg/util/validation" + "net" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/yamlinput" + yaml "gopkg.in/yaml.v3" +) + +type FileRef struct { + File string `yaml:"file"` +} +type PasswordRef struct { + Env *string `yaml:"env,omitempty"` + File *string `yaml:"file,omitempty"` +} +type Credential struct { + Username string `yaml:"username"` + PasswordRef PasswordRef `yaml:"passwordRef"` +} +type TLS struct { + CAFile *string `yaml:"caFile,omitempty"` +} +type Endpoint struct { + Name *string `yaml:"name,omitempty"` + BMCEndpoint string `yaml:"bmcEndpoint"` + CredentialRef string `yaml:"credentialRef"` + SystemURI *string `yaml:"systemURI,omitempty"` + TLS *TLS `yaml:"tls,omitempty"` +} +type PullSecret struct { + Name string `yaml:"name"` +} +type Probes struct { + Mode *string `yaml:"mode,omitempty"` + Namespace *string `yaml:"namespace,omitempty"` + ImageRepository *string `yaml:"imageRepository,omitempty"` + ImagePullSecrets []PullSecret `yaml:"imagePullSecrets,omitempty"` +} +type Limits struct { + Timeout *string `yaml:"timeout,omitempty"` + RequestTimeout *string `yaml:"requestTimeout,omitempty"` + MaxConcurrentBMCs *int `yaml:"maxConcurrentBMCs,omitempty"` +} +type Site struct { + Management struct { + KubeconfigRef FileRef `yaml:"kubeconfigRef"` + Context *string `yaml:"context,omitempty"` + } `yaml:"management"` + Credentials map[string]Credential `yaml:"credentials"` + Discovery struct { + DPUs []Endpoint `yaml:"dpus"` + Hosts []Endpoint `yaml:"hosts"` + TLS *TLS `yaml:"tls,omitempty"` + Probes Probes `yaml:"probes,omitempty"` + Limits Limits `yaml:"limits,omitempty"` + } `yaml:"discovery"` + Inventory *inventory.Inventory `yaml:"inventory,omitempty"` +} +type Input struct { + Site Site + Source []byte + Path string + Document *yaml.Node +} + +// Access is private runtime state. It deliberately cannot be serialized or logged. +type Access struct { + username string + password string + tls *tls.Config + ca []byte +} + +func (a Access) Credentials() (string, string) { return a.username, a.password } +func (a Access) TLSConfig() *tls.Config { return a.tls.Clone() } +func (a Access) PublicCA() []byte { return bytes.Clone(a.ca) } +func (a Access) String() string { return "" } + +type Target struct { + Endpoint string + SystemURI string + InputRef string + Role string + Access Access +} +type Request struct { + Input Input + Targets []Target + Kubeconfig string + Context string + Mode string + Namespace string + ImageRepository string + ImagePullSecrets []string + Timeout time.Duration + RequestTimeout time.Duration + MaxConcurrentBMCs int +} + +func regularFile(p string, limit int64) ([]byte, error) { + before, statErr := os.Stat(p) + if statErr != nil || !before.Mode().IsRegular() || before.Size() > limit { + return nil, fmt.Errorf("reference must be a bounded regular file") + } + f, err := os.Open(p) + if err != nil { + return nil, fmt.Errorf("cannot open referenced file") + } + defer f.Close() + st, err := f.Stat() + if err != nil || !st.Mode().IsRegular() || st.Size() > limit { + return nil, fmt.Errorf("reference must be a bounded regular file") + } + b, err := io.ReadAll(io.LimitReader(f, limit+1)) + if err != nil || int64(len(b)) > limit { + return nil, fmt.Errorf("cannot read bounded file") + } + return b, nil +} +func Load(p string) (Input, error) { + abs, err := filepath.Abs(p) + if err != nil || p == "" || p == "-" { + return Input{}, fmt.Errorf("--user-config must name one local YAML file") + } + b, err := regularFile(abs, 16<<20) + if err != nil { + return Input{}, err + } + in, err := Decode(b, abs) + if err != nil { + return Input{}, fmt.Errorf("%s: %w", abs, err) + } + return in, nil +} +func Decode(b []byte, p string) (Input, error) { + if len(b) > 16<<20 { + return Input{}, fmt.Errorf("YAML input exceeds 16 MiB") + } + dec := yaml.NewDecoder(bytes.NewReader(b)) + var doc yaml.Node + if err := dec.Decode(&doc); err != nil { + return Input{}, fmt.Errorf("invalid YAML") + } + var extra yaml.Node + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return Input{}, fmt.Errorf("expected one YAML document") + } + if len(doc.Content) != 1 || doc.Content[0].Kind != yaml.MappingNode { + return Input{}, fmt.Errorf("expected mapping document") + } + if err := validateKeys(&doc, map[*yaml.Node]bool{}, map[*yaml.Node]bool{}, 0); err != nil { + return Input{}, err + } + if err := validateScalarTypes(doc.Content[0], ""); err != nil { + return Input{}, err + } + expanded, err := yamlinput.Expand(&doc) + if err != nil { + return Input{}, err + } + if err := validateScalarTypes(expanded.Content[0], ""); err != nil { + return Input{}, err + } + encoded, err := yamlinput.Encode(expanded) + if err != nil { + return Input{}, err + } + var site Site + strict := yaml.NewDecoder(bytes.NewReader(encoded)) + strict.KnownFields(true) + if err := strict.Decode(&site); err != nil { + return Input{}, fmt.Errorf("unknown field or invalid type in DPF configuration") + } + return Input{Site: site, Source: bytes.Clone(b), Path: p, Document: expanded}, nil +} +func validateKeys(n *yaml.Node, active, visited map[*yaml.Node]bool, depth int) error { + if n == nil || depth > 64 || active[n] { + return fmt.Errorf("cyclic or excessive YAML aliases") + } + if visited[n] { + return nil + } + visited[n] = true + active[n] = true + defer delete(active, n) + if n.Kind == yaml.AliasNode { + return validateKeys(n.Alias, active, visited, depth+1) + } + if n.Kind == yaml.MappingNode { + seen := map[string]bool{} + for i := 0; i < len(n.Content); i += 2 { + k := n.Content[i] + if (k.Tag != "!!str" && k.Tag != "!!merge") || seen[k.Value] { + return fmt.Errorf("duplicate or non-string key at line %d", k.Line) + } + seen[k.Value] = true + } + } + for _, c := range n.Content { + if err := validateKeys(c, active, visited, depth+1); err != nil { + return err + } + } + return nil +} +func Resolve(in Input) (Request, error) { + r := Request{Input: in, Timeout: 10 * time.Minute, RequestTimeout: 15 * time.Second, MaxConcurrentBMCs: 4, Mode: "management"} + s := in.Site + base := filepath.Dir(in.Path) + if s.Management.KubeconfigRef.File == "" { + return r, fmt.Errorf("/management/kubeconfigRef/file is required") + } + r.Kubeconfig = resolvePath(base, s.Management.KubeconfigRef.File) + if s.Management.Context != nil { + if *s.Management.Context == "" { + return r, fmt.Errorf("/management/context must not be empty") + } + r.Context = *s.Management.Context + } + p := s.Discovery.Probes + if p.Mode != nil { + r.Mode = *p.Mode + } + if r.Mode != "management" && r.Mode != "inventory" { + return r, fmt.Errorf("/discovery/probes/mode must be management or inventory") + } + if p.Namespace != nil { + r.Namespace = *p.Namespace + if r.Namespace == "" || len(r.Namespace) > 63 || !dnsLabel(r.Namespace) { + return r, fmt.Errorf("invalid /discovery/probes/namespace") + } + } + if p.ImageRepository != nil { + r.ImageRepository = *p.ImageRepository + if r.ImageRepository == "" || strings.ContainsAny(r.ImageRepository, "@ \t\n") || strings.Contains(r.ImageRepository, "://") { + return r, fmt.Errorf("invalid probe repository") + } + last := r.ImageRepository[strings.LastIndex(r.ImageRepository, "/")+1:] + if strings.Contains(last, ":") { + return r, fmt.Errorf("probe repository must not have a tag") + } + } + for _, secret := range p.ImagePullSecrets { + if p.Namespace == nil || secret.Name == "" || !dnsLabel(secret.Name) { + return r, fmt.Errorf("imagePullSecrets requires an explicit namespace and valid names") + } + r.ImagePullSecrets = append(r.ImagePullSecrets, secret.Name) + } + l := s.Discovery.Limits + for _, f := range []struct { + value *string + dest *time.Duration + max time.Duration + }{{l.Timeout, &r.Timeout, time.Hour}, {l.RequestTimeout, &r.RequestTimeout, 2 * time.Minute}} { + if f.value != nil { + d, err := time.ParseDuration(*f.value) + if err != nil || d < time.Second || d > f.max { + return r, fmt.Errorf("invalid /discovery/limits duration") + } + *f.dest = d + } + } + if l.MaxConcurrentBMCs != nil { + r.MaxConcurrentBMCs = *l.MaxConcurrentBMCs + } + if r.RequestTimeout > r.Timeout || r.MaxConcurrentBMCs < 1 || r.MaxConcurrentBMCs > 16 { + return r, fmt.Errorf("invalid /discovery/limits") + } + if len(s.Discovery.DPUs) == 0 || len(s.Discovery.Hosts) == 0 || len(s.Discovery.DPUs)+len(s.Discovery.Hosts) > 256 { + return r, fmt.Errorf("discovery requires 1-256 explicit DPU/host endpoints and both roles") + } + seen := map[string]bool{} + accesses := map[string]Access{} + for _, group := range []struct { + role string + endpoints []Endpoint + }{{"dpus", s.Discovery.DPUs}, {"hosts", s.Discovery.Hosts}} { + for i, e := range group.endpoints { + ptr := fmt.Sprintf("/discovery/%s/%d", group.role, i) + origin, err := Origin(e.BMCEndpoint) + if err != nil { + return r, fmt.Errorf("%s/bmcEndpoint: %w", ptr, err) + } + if seen[origin] { + return r, fmt.Errorf("duplicate BMC origin at %s", ptr) + } + seen[origin] = true + selector := "" + if e.SystemURI != nil { + selector = *e.SystemURI + if err := ResourceURI(selector); err != nil { + return r, fmt.Errorf("%s/systemURI: invalid Redfish resource", ptr) + } + } + c, ok := s.Credentials[e.CredentialRef] + if !ok || c.Username == "" { + return r, fmt.Errorf("%s/credentialRef must reference a username/passwordRef", ptr) + } + policy := s.Discovery.TLS + if e.TLS != nil { + policy = e.TLS + } + caPath := "" + if policy != nil && policy.CAFile != nil { + if *policy.CAFile == "" { + return r, fmt.Errorf("empty caFile") + } + caPath = resolvePath(base, *policy.CAFile) + } + cacheKey := e.CredentialRef + "\x00" + caPath + a, ok := accesses[cacheKey] + if !ok { + a, err = resolveAccess(c, caPath, base) + if err != nil { + return r, fmt.Errorf("%s: %w", ptr, err) + } + accesses[cacheKey] = a + } + r.Targets = append(r.Targets, Target{Endpoint: origin, SystemURI: selector, InputRef: ptr, Role: group.role, Access: a}) + } + } + return r, nil +} +func dnsLabel(s string) bool { return len(validation.IsDNS1123Label(s)) == 0 } +func resolvePath(base, p string) string { + if filepath.IsAbs(p) { + return filepath.Clean(p) + } + return filepath.Join(base, p) +} +func resolveAccess(c Credential, caPath, base string) (Access, error) { + p := c.PasswordRef + if (p.Env == nil) == (p.File == nil) { + return Access{}, fmt.Errorf("passwordRef requires exactly one env or file") + } + var password string + if p.Env != nil { + if *p.Env == "" { + return Access{}, fmt.Errorf("empty passwordRef.env") + } + password = os.Getenv(*p.Env) + } else { + if *p.File == "" { + return Access{}, fmt.Errorf("empty passwordRef.file") + } + b, err := regularFile(resolvePath(base, *p.File), 64<<10) + if err != nil { + return Access{}, fmt.Errorf("cannot resolve password file") + } + if !utf8.Valid(b) { + return Access{}, fmt.Errorf("password file must be UTF-8") + } + password = string(b) + if strings.HasSuffix(password, "\n") { + password = strings.TrimSuffix(strings.TrimSuffix(password, "\n"), "\r") + } + } + if password == "" || len(password) > 64<<10 { + return Access{}, fmt.Errorf("password reference is missing, empty or too large") + } + roots, err := x509.SystemCertPool() + if err != nil || roots == nil { + roots = x509.NewCertPool() + } + var ca []byte + if caPath != "" { + ca, err = regularFile(caPath, 1<<20) + if err != nil || !roots.AppendCertsFromPEM(ca) { + return Access{}, fmt.Errorf("invalid CA file") + } + } + return Access{username: c.Username, password: password, tls: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}, ca: ca}, nil +} +func Origin(s string) (string, error) { + u, err := url.Parse(s) + if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" || (u.Path != "" && u.Path != "/") { + return "", fmt.Errorf("expected HTTPS origin without userinfo/path/query/fragment") + } + host := strings.ToLower(u.Hostname()) + port := u.Port() + if port != "" { + n, e := strconv.Atoi(port) + if e != nil || n < 1 || n > 65535 { + return "", fmt.Errorf("invalid port") + } + } + if port == "" || port == "443" { + if strings.Contains(host, ":") { + host = "[" + host + "]" + } + } else { + host = net.JoinHostPort(host, port) + } + return "https://" + host, nil +} +func ResourceURI(s string) error { + u, e := url.Parse(s) + if e != nil || u.IsAbs() || u.Host != "" || !strings.HasPrefix(u.Path, "/redfish/v1/") || u.RawQuery != "" || u.Fragment != "" || strings.Contains(u.Path, "..") || path.Clean(u.Path) != u.Path || u.RawPath != "" { + return fmt.Errorf("invalid resource URI") + } + return nil +} + +// Saved patches only generated inventory and file references whose base changed. +func Saved(in Input, inv *inventory.Inventory, destination string) ([]byte, error) { + doc, err := yamlinput.Expand(in.Document) + if err != nil { + return nil, err + } + var replacement yaml.Node + if err := replacement.Encode(inv); err != nil { + return nil, err + } + yamlinput.Set(doc.Content[0], "inventory", &replacement) + if filepath.Dir(destination) != filepath.Dir(in.Path) { + if err := rebase(doc.Content[0], filepath.Dir(in.Path), filepath.Dir(destination), ""); err != nil { + return nil, err + } + } + return yamlinput.Encode(doc) +} +func rebase(n *yaml.Node, oldBase, newBase, p string) error { + if n.Kind == yaml.MappingNode { + for i := 0; i < len(n.Content); i += 2 { + key := n.Content[i].Value + value := n.Content[i+1] + next := p + "/" + key + if key == "inventory" && p == "" { + continue + } + isFile := key == "caFile" || key == "file" && (strings.HasSuffix(p, "kubeconfigRef") || strings.HasSuffix(p, "passwordRef")) + if isFile && value.Kind == yaml.ScalarNode && value.Tag == "!!str" && value.Value != "" { + abs := resolvePath(oldBase, value.Value) + rel, err := filepath.Rel(newBase, abs) + if err != nil { + return err + } + value.Value = rel + } else if err := rebase(value, oldBase, newBase, next); err != nil { + return err + } + } + } else if n.Kind == yaml.SequenceNode { + for _, v := range n.Content { + if err := rebase(v, oldBase, newBase, p); err != nil { + return err + } + } + } + return nil +} + +func validateScalarTypes(n *yaml.Node, p string) error { + if p == "/inventory" { + return nil + } + if n.Kind == yaml.AliasNode { + return nil + } + switch n.Kind { + case yaml.ScalarNode: + if n.Tag == "!!null" { + return nil + } + expected := "!!str" + if p == "/discovery/limits/maxConcurrentBMCs" { + expected = "!!int" + } + if n.Tag != expected { + return fmt.Errorf("%s has invalid scalar type at line %d", p, n.Line) + } + case yaml.MappingNode: + for i := 0; i < len(n.Content); i += 2 { + if err := validateScalarTypes(n.Content[i+1], p+"/"+n.Content[i].Value); err != nil { + return err + } + } + case yaml.SequenceNode: + for i, v := range n.Content { + if err := validateScalarTypes(v, fmt.Sprintf("%s/%d", p, i)); err != nil { + return err + } + } + } + return nil +} + +// ReadReference rejects streams/devices and bounds local reference reads. +func ReadReference(filename string, limit int64) ([]byte, error) { return regularFile(filename, limit) } diff --git a/pkg/dpf/config/config_test.go b/pkg/dpf/config/config_test.go new file mode 100644 index 00000000..25f55788 --- /dev/null +++ b/pkg/dpf/config/config_test.go @@ -0,0 +1,111 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package config + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestConfig(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF configuration") } + +const seed = `# retained comment +management: + kubeconfigRef: {file: credentials/mgmt} +credentials: + read: + username: reader + passwordRef: {env: DPF_TEST_PASSWORD} +discovery: + dpus: [{bmcEndpoint: 'https://192.0.2.1', credentialRef: read}] + hosts: [{bmcEndpoint: 'https://192.0.2.2', credentialRef: read}] +` + +var _ = Describe("DPF input", func() { + BeforeEach(func() { + Expect(os.Setenv("DPF_TEST_PASSWORD", " canary \n")).To(Succeed()) + DeferCleanup(os.Unsetenv, "DPF_TEST_PASSWORD") + }) + It("uses bounded defaults and keeps private values out of saved YAML", func() { + in, e := Decode([]byte(seed), "/site/input.yaml") + Expect(e).NotTo(HaveOccurred()) + r, e := Resolve(in) + Expect(e).NotTo(HaveOccurred()) + Expect(r.Timeout).To(Equal(10 * time.Minute)) + _, password := r.Targets[0].Access.Credentials() + Expect(password).To(Equal(" canary \n")) + b, e := Saved(in, &inventory.Inventory{FormatVersion: "v1alpha1"}, "/results/out.yaml") + Expect(e).NotTo(HaveOccurred()) + Expect(string(b)).To(ContainSubstring("# retained comment")) + Expect(string(b)).To(ContainSubstring("../site/credentials/mgmt")) + Expect(string(b)).NotTo(ContainSubstring("canary")) + }) + It("rejects duplicate, unknown, cyclic and multiple documents", func() { + for _, input := range []string{seed + "management: {}\n", strings.Replace(seed, "discovery:", "typo:", 1), "a: &x {b: *x}", seed + "---\nfoo: bar\n"} { + _, e := Decode([]byte(input), "input.yaml") + Expect(e).To(HaveOccurred()) + } + }) + It("does not replace explicit zero, empty mode or empty TLS path with defaults", func() { + for _, addition := range []string{" limits: {maxConcurrentBMCs: 0}\n", " probes: {mode: ''}\n", " tls: {caFile: ''}\n"} { + in, e := Decode([]byte(seed+addition), "/site/in.yaml") + Expect(e).NotTo(HaveOccurred()) + _, e = Resolve(in) + Expect(e).To(HaveOccurred()) + } + }) + It("uses null as omitted", func() { + in, e := Decode([]byte(seed+" limits: {maxConcurrentBMCs: null}\n"), "/site/in.yaml") + Expect(e).NotTo(HaveOccurred()) + r, e := Resolve(in) + Expect(e).NotTo(HaveOccurred()) + Expect(r.MaxConcurrentBMCs).To(Equal(4)) + }) + It("normalizes DNS/default ports and rejects paths, userinfo and duplicate origins", func() { + origin, e := Origin("https://EXAMPLE.invalid:443/") + Expect(e).NotTo(HaveOccurred()) + Expect(origin).To(Equal("https://example.invalid")) + for _, v := range []string{"http://x", "https://u:p@x", "https://x/path", "https://x?x=y", "https://x:99999"} { + _, e := Origin(v) + Expect(e).To(HaveOccurred()) + } + in, e := Decode([]byte(strings.Replace(seed, "192.0.2.2", "192.0.2.1:443", 1)), "/site/in.yaml") + Expect(e).NotTo(HaveOccurred()) + _, e = Resolve(in) + Expect(e).To(HaveOccurred()) + }) + It("removes only one final LF or CRLF from file passwords", func() { + dir, e := os.MkdirTemp("", "dpf-config-") + Expect(e).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + for _, value := range []string{" space \r\n", " space \n\n", " space \r"} { + Expect(os.WriteFile(filepath.Join(dir, "password"), []byte(value), 0600)).To(Succeed()) + text := strings.Replace(seed, "env: DPF_TEST_PASSWORD", "file: password", 1) + in, e := Decode([]byte(text), filepath.Join(dir, "in.yaml")) + Expect(e).NotTo(HaveOccurred()) + r, e := Resolve(in) + Expect(e).NotTo(HaveOccurred()) + _, actual := r.Targets[0].Access.Credentials() + expected := value + if strings.HasSuffix(expected, "\n") { + expected = strings.TrimSuffix(strings.TrimSuffix(expected, "\n"), "\r") + } + Expect(actual).To(Equal(expected)) + } + }) +}) + +var _ = Describe("strict scalar regression", func() { + It("rejects boolean and numeric access fields rather than coercing strings", func() { + for _, value := range []string{"false", "123"} { + _, err := Decode([]byte(strings.Replace(seed, "username: reader", "username: "+value, 1)), "/site/in.yaml") + Expect(err).To(HaveOccurred()) + } + }) +}) diff --git a/pkg/dpf/discovery/discover.go b/pkg/dpf/discovery/discover.go new file mode 100644 index 00000000..b1826826 --- /dev/null +++ b/pkg/dpf/discovery/discover.go @@ -0,0 +1,410 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package discovery orchestrates independent management and BMC observations. +package discovery + +import ( + "cmp" + "context" + "reflect" + "slices" + "strings" + "sync" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" +) + +type ProbeRunner interface { + Run(context.Context, config.Request, ManagementClient, *inventory.Inventory) ([]inventory.ProbeResult, inventory.Cleanup, error) +} +type Dependencies struct { + BMC redfish.Collector + Probes ProbeRunner +} + +func NewReport() inventory.Report { + return inventory.Report{SchemaVersion: inventory.FormatVersion, Target: "dpf", Phase: "discover", Execution: "completed", Acceptance: "not-evaluated", Discovery: inventory.Discovery{Outcome: "invalid", Checks: []inventory.Check{}, Cleanup: inventory.Cleanup{State: "complete", Leftovers: []string{}}}, Diff: []inventory.Change{}, PendingInputs: []inventory.Problem{}, Artifacts: []inventory.Artifact{}, Errors: []inventory.Problem{}} +} +func Collect(ctx context.Context, r config.Request, runID string, deps Dependencies) inventory.Report { + report := NewReport() + report.Discovery.Mode = r.Mode + inv := &inventory.Inventory{FormatVersion: inventory.FormatVersion, Collection: inventory.Collection{RunID: runID, StartedAt: time.Now().UTC(), Mode: r.Mode, Collector: "l8k-go-discovery-v1"}, RequestedScope: []inventory.ScopeEndpoint{}, DPUs: []inventory.Device{}, Hosts: []inventory.Device{}, Associations: []inventory.Association{}, Evidence: map[string]inventory.Evidence{}} + report.Inventory = inv + for _, t := range r.Targets { + inv.RequestedScope = append(inv.RequestedScope, inventory.ScopeEndpoint{Role: t.Role, Endpoint: t.Endpoint, SystemURI: t.SystemURI}) + } + slices.SortFunc(inv.RequestedScope, func(a, b inventory.ScopeEndpoint) int { + return strings.Compare(a.Role+a.Endpoint+a.SystemURI, b.Role+b.Endpoint+b.SystemURI) + }) + ctx, cancel := context.WithTimeout(ctx, r.Timeout) + defer cancel() + var mu sync.Mutex + var wg sync.WaitGroup + problem := func(p inventory.Problem) { mu.Lock(); defer mu.Unlock(); report.Errors = append(report.Errors, p) } + mg, accessErr := ManagementAccess(r) + if accessErr != nil { + report.Execution = "failed" + report.Errors = append(report.Errors, inventory.Problem{Code: "INVALID_MANAGEMENT_ACCESS", Message: accessErr.Error(), ExitCode: 2}) + return report + } + wg.Add(1) + go func() { + defer wg.Done() + var err error + inv.Management, err = mg.Observe(ctx) + if err != nil { + problem(inventory.Problem{Code: "MANAGEMENT_INSPECTION_FAILED", Message: err.Error(), InputRef: "/management", ExitCode: 3}) + return + } + mu.Lock() + for id, e := range NewEvidence(time.Now().UTC()) { + inv.Evidence[id] = e + } + mu.Unlock() + }() + sem := make(chan struct{}, r.MaxConcurrentBMCs) + devices := make([]inventory.Device, len(r.Targets)) + for index, target := range r.Targets { + wg.Add(1) + go func(i int, t config.Target) { + defer wg.Done() + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + devices[i] = inventory.Device{RecordKey: inventory.Digest("dpf-access-v1", []string{t.Role, t.Endpoint, t.SystemURI}), InputRef: t.InputRef, BMCEndpoint: t.Endpoint, ObservationState: "unavailable", ReasonCode: "CANCELLED"} + return + } + d, e, err := deps.BMC.Collect(ctx, t, r.RequestTimeout) + devices[i] = d + mu.Lock() + for key, value := range e { + inv.Evidence[key] = value + } + mu.Unlock() + if err != nil { + problem(inventory.Problem{Code: redfish.Code(err), Message: "BMC observation failed; inspect record reason and field path", InputRef: t.InputRef, ExitCode: 3}) + if d.ReasonCode == "CLEANUP_INCOMPLETE" { + mu.Lock() + report.Discovery.Cleanup.State = "incomplete" + report.Discovery.Cleanup.Leftovers = append(report.Discovery.Cleanup.Leftovers, t.Endpoint+": session outcome unknown") + mu.Unlock() + } + } + }(index, target) + } + wg.Wait() + for i, d := range devices { + if r.Targets[i].Role == "dpus" { + inv.DPUs = append(inv.DPUs, d) + } else { + inv.Hosts = append(inv.Hosts, d) + } + } + inv.Associations, report.Discovery.Checks = Correlate(inv.DPUs, inv.Hosts) + selected := map[string]bool{} + for _, a := range inv.Associations { + selected[a.HostAdapterKey] = true + } + for hi := range inv.Hosts { + for ai := range inv.Hosts[hi].Adapters { + if selected[inv.Hosts[hi].Adapters[ai].Key] { + inv.Hosts[hi].Adapters[ai].Selection = "selected" + } + } + } + addCheck := func(id, subject string, ok bool, reason string, refs []string) { + state := "failed" + if ok { + state = "passed" + } + report.Discovery.Checks = append(report.Discovery.Checks, inventory.Check{ID: id, Subject: subject, Vantage: "executor", RequiredPhase: "discover", State: state, ReasonCode: reason, EvidenceRefs: refs}) + } + if inv.Management != nil { + cp := 0 + for _, n := range inv.Management.Nodes { + if n.ControlPlane { + cp++ + } + } + addCheck("dpf.management.control-plane-count", "management", cp == 3, "EXACTLY_THREE_CONTROL_PLANES", inv.Management.EvidenceRefs) + if r.Mode == "management" && cp == 3 { + if deps.Probes == nil { + report.Errors = append(report.Errors, inventory.Problem{Code: "PROBE_ASSETS_UNAVAILABLE", Message: "management probes require qualified installed probe assets", ExitCode: 2}) + } else { + rows, cleanup, err := deps.Probes.Run(ctx, r, mg, inv) + inv.Probes = rows + if cleanup.State != "complete" { + report.Discovery.Cleanup.State = cleanup.State + } + report.Discovery.Cleanup.Leftovers = append(report.Discovery.Cleanup.Leftovers, cleanup.Leftovers...) + if err != nil { + report.Errors = append(report.Errors, inventory.Problem{Code: "PROBE_FAILED", Message: err.Error(), ExitCode: 3}) + } + } + } + fresh, err := mg.Observe(ctx) + if err != nil { + report.Errors = append(report.Errors, inventory.Problem{Code: "MANAGEMENT_RECHECK_FAILED", Message: "management identity recheck unavailable", InputRef: "/management", ExitCode: 3}) + } else if fresh.SystemNamespaceUID != inv.Management.SystemNamespaceUID || !sameNodes(fresh.Nodes, inv.Management.Nodes) { + report.Errors = append(report.Errors, inventory.Problem{Code: "MANAGEMENT_SCOPE_CHANGED", Message: "management identity or node membership changed during collection", ExitCode: 3}) + } + } + report.Discovery.Checks = append(report.Discovery.Checks, ProbeChecks(inv, r.Targets)...) + for _, d := range append(slices.Clone(inv.DPUs), inv.Hosts...) { + addCheck("dpf.device.identity", d.RecordKey, d.ID != "" && d.ObservationState == "observed", "PHYSICAL_IDENTITY_REQUIRED", d.EvidenceRefs) + addCheck("dpf.device.qualification", d.RecordKey, d.Qualification == "qualified" && d.InventoryCompleteness == "complete", "QUALIFIED_ADAPTER_REQUIRED", d.EvidenceRefs) + } + report.Discovery.Outcome = "complete" + for _, c := range report.Discovery.Checks { + if c.RequiredPhase == "discover" && c.State != "passed" { + report.Discovery.Outcome = "partial" + } + } + if len(report.Errors) > 0 || report.Discovery.Cleanup.State != "complete" { + report.Discovery.Outcome = "partial" + } + if ctx.Err() != nil { + report.Execution = "failed" + code := 3 + if ctx.Err() == context.Canceled { + report.Execution = "cancelled" + code = 1 + } + report.Errors = append(report.Errors, inventory.Problem{Code: "COLLECTION_INTERRUPTED", Message: "discovery cancelled or timed out", ExitCode: code}) + } + inv.Collection.FinishedAt = time.Now().UTC() + Canonicalize(inv) + report.Diff = Diff(r.Input.Site.Inventory, inv) + if scopeConflict(r.Input.Site.Inventory, inv) { + report.Discovery.Outcome = "partial" + report.Errors = append(report.Errors, inventory.Problem{Code: "SCOPE_CONFLICT", Message: "resolved scope changed; use a fresh seed and new output", ExitCode: 2}) + } + report.PendingInputs = []inventory.Problem{{Code: "LATER_PHASE_INPUT", Message: "DPU API allocation and selected service profile are evaluated by future generation", InputRef: "/networks/dpuAPI"}} + slices.SortFunc(report.Errors, func(a, b inventory.Problem) int { return strings.Compare(a.InputRef+a.Code, b.InputRef+b.Code) }) + return report +} +func scopeConflict(old, current *inventory.Inventory) bool { + return old != nil && ((old.ScopeDigest != "" && current.ScopeDigest != "" && old.ScopeDigest != current.ScopeDigest) || inventory.IdentityConflict(old, current)) +} +func sameNodes(a, b []inventory.Node) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i].UID != b[i].UID || a[i].Name != b[i].Name { + return false + } + } + return true +} +func Correlate(dpus, hosts []inventory.Device) ([]inventory.Association, []inventory.Check) { + associations := []inventory.Association{} + checks := []inventory.Check{} + seenIDs := map[string]int{} + for _, d := range append(slices.Clone(dpus), hosts...) { + if d.ID != "" { + seenIDs[d.ID]++ + } + } + for _, d := range dpus { + matches := []inventory.Association{} + for hi := range hosts { + h := &hosts[hi] + for ai := range h.Adapters { + a := &h.Adapters[ai] + if d.SerialNumber != "" && a.SerialNumber == d.SerialNumber { + // Selection annotations are applied by the orchestrator after correlation. + matches = append(matches, inventory.Association{DPUID: d.ID, HostID: h.ID, HostAdapterKey: a.Key, SerialNumber: d.SerialNumber, EvidenceRefs: append(slices.Clone(d.EvidenceRefs), a.EvidenceRefs...)}) + } + } + } + state := "failed" + if len(matches) == 1 && d.ID != "" && matches[0].HostID != "" && seenIDs[d.ID] == 1 && seenIDs[matches[0].HostID] == 1 { + state = "passed" + associations = append(associations, matches[0]) + } + checks = append(checks, inventory.Check{ID: "dpf.inventory.association", Subject: d.RecordKey, Vantage: "executor", RequiredPhase: "discover", State: state, ReasonCode: "UNIQUE_PHYSICAL_MATCH_REQUIRED", EvidenceRefs: d.EvidenceRefs}) + } + return associations, checks +} +func Canonicalize(inv *inventory.Inventory) { + sortDevices := func(devices []inventory.Device) { + slices.SortFunc(devices, func(a, b inventory.Device) int { return strings.Compare(a.RecordKey, b.RecordKey) }) + for i := range devices { + slices.SortFunc(devices[i].Adapters, func(a, b inventory.Adapter) int { return strings.Compare(a.Key, b.Key) }) + slices.SortFunc(devices[i].Firmware, func(a, b inventory.Firmware) int { + return strings.Compare(a.Component+a.RawVersion, b.Component+b.RawVersion) + }) + } + } + sortDevices(inv.DPUs) + sortDevices(inv.Hosts) + slices.SortFunc(inv.Associations, func(a, b inventory.Association) int { return strings.Compare(a.DPUID, b.DPUID) }) + inv.Fingerprint = inventory.Digest("dpf-facts-v1", material(inv)) + if inv.Management != nil { + inv.ScopeDigest = inventory.Digest("dpf-scope-v1", struct { + API, UID, Trust, Mode string + Endpoints []inventory.ScopeEndpoint + }{inv.Management.APIEndpoint, inv.Management.SystemNamespaceUID, inv.Management.TrustDigest, inv.Collection.Mode, inv.RequestedScope}) + } +} + +// Dedicated material projection excludes timestamps, readiness, credentials and aliases. +func material(inv *inventory.Inventory) map[string]any { + fields := map[string]any{} + if inv.Management != nil { + fields["management.identity"] = []string{inv.Management.APIEndpoint, inv.Management.SystemNamespaceUID, inv.Management.TrustDigest, inv.Management.KubernetesVersion} + for _, n := range inv.Management.Nodes { + addresses := slices.Clone(n.Addresses) + slices.Sort(addresses) + interfaces := slices.Clone(n.Interfaces) + for i := range interfaces { + interfaces[i].Addresses = slices.Clone(interfaces[i].Addresses) + slices.Sort(interfaces[i].Addresses) + } + slices.SortFunc(interfaces, func(a, b inventory.Interface) int { + return cmp.Or(cmp.Compare(a.Index, b.Index), strings.Compare(a.Name, b.Name)) + }) + routes := slices.Clone(n.Routes) + slices.SortFunc(routes, func(a, b inventory.Route) int { + return cmp.Or(cmp.Compare(a.Table, b.Table), strings.Compare(a.Destination, b.Destination), strings.Compare(a.Gateway, b.Gateway), cmp.Compare(a.InterfaceIndex, b.InterfaceIndex)) + }) + fields["management.node."+n.UID] = []any{n.Name, n.OS, n.Architecture, addresses, interfaces, routes} + + } + } + for _, devices := range [][]inventory.Device{inv.DPUs, inv.Hosts} { + for _, d := range devices { + key := "device." + d.RecordKey + fields[key+".identity"] = []any{d.ID, d.IdentityKey, d.Model, d.ReaderAdapter, d.Qualification, d.InventoryCompleteness} + for _, a := range d.Adapters { + fields[key+".adapter."+a.Key] = []string{a.SerialNumber, a.Model, a.Slot} + } + for _, fw := range d.Firmware { + fields[key+".firmware."+fw.Component+"."+fw.ResourceURI] = []string{fw.State, fw.RawVersion, fw.Activation} + } + } + } + for _, a := range inv.Associations { + fields["association."+a.DPUID] = []string{a.HostID, a.HostAdapterKey, a.SerialNumber} + } + return fields +} +func Diff(old, current *inventory.Inventory) []inventory.Change { + out := []inventory.Change{} + if old == nil { + return append(out, inventory.Change{Path: "inventory", State: "baseline-absent", RequiredPhase: "generate"}) + } + before, after := material(old), material(current) + keys := []string{} + for k := range before { + keys = append(keys, k) + } + for k := range after { + if _, ok := before[k]; !ok { + keys = append(keys, k) + } + } + slices.Sort(keys) + for _, k := range keys { + a, aok := before[k] + b, bok := after[k] + if reflect.DeepEqual(a, b) { + continue + } + state := "changed" + if !aok { + state = "added" + } + if !bok { + state = "unknown" + } + for _, group := range [][]inventory.Device{current.DPUs, current.Hosts} { + for _, d := range group { + if d.ObservationState != "observed" && strings.HasPrefix(k, "device."+d.RecordKey+".") { + state = "unknown" + } + } + } + out = append(out, inventory.Change{Path: k, State: state, Previous: a, Current: b, RequiredPhase: "generate"}) + } + return out +} +func ExitCode(r inventory.Report) int { + code := 0 + for _, e := range r.Errors { + if e.ExitCode == 1 { + return 1 + } + if e.ExitCode == 3 { + code = 3 + } else if code == 0 { + code = e.ExitCode + } + } + if code != 0 { + return code + } + if r.Discovery.Outcome != "complete" { + return 2 + } + return 0 +} + +// ProbeChecks derives required rows from frozen node/endpoint scope, never aggregate counts. +func ProbeChecks(inv *inventory.Inventory, targets []config.Target) []inventory.Check { + if inv.Collection.Mode != "management" { + return []inventory.Check{{ID: "dpf.management.path", Subject: "management", RequiredPhase: "generate", State: "not-requested", ReasonCode: "INVENTORY_MODE", EvidenceRefs: []string{}}} + } + checks := []inventory.Check{} + if inv.Management == nil { + return checks + } + destinations := []string{"management-api"} + for _, t := range targets { + if t.Role == "dpus" { + destinations = append(destinations, inventory.Digest("probe-destination-v1", t.Endpoint)) + } + } + rows := map[string][]inventory.ProbeResult{} + for _, r := range inv.Probes { + key := r.NodeUID + "/" + r.NetworkMode + "/" + r.Destination + rows[key] = append(rows[key], r) + } + for _, n := range inv.Management.Nodes { + for _, mode := range []string{"pod-network", "node-network"} { + for _, destination := range destinations { + key := n.UID + "/" + mode + "/" + destination + state := "unknown" + refs := []string{} + values := rows[key] + if len(values) == 1 { + v := values[0] + if v.EvidenceRef != "" { + if _, ok := inv.Evidence[v.EvidenceRef]; ok { + refs = append(refs, v.EvidenceRef) + } + } + state = "passed" + if len(refs) == 0 || len(v.Addresses) == 0 || v.DNS == "unavailable" { + state = "unknown" + } + for _, a := range v.Addresses { + if a.TCP != "reachable" || a.TLS != "verified" { + state = "failed" + } + } + } else if len(values) > 1 { + state = "failed" + } + checks = append(checks, inventory.Check{ID: "dpf.management.path", Subject: key, Vantage: n.UID + "/" + mode, RequiredPhase: "discover", State: state, ReasonCode: "EXACT_NETWORK_COVERAGE_REQUIRED", EvidenceRefs: refs}) + } + } + } + return checks +} diff --git a/pkg/dpf/discovery/discover_test.go b/pkg/dpf/discovery/discover_test.go new file mode 100644 index 00000000..078953f8 --- /dev/null +++ b/pkg/dpf/discovery/discover_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "encoding/json" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" + "time" +) + +func TestDiscovery(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF discovery checks") } +func estate() inventory.Inventory { + return inventory.Inventory{Collection: inventory.Collection{Mode: "inventory"}, Management: &inventory.Management{APIEndpoint: "https://api.invalid", SystemNamespaceUID: "cluster-a", TrustDigest: "system-roots", KubernetesVersion: "v1", Nodes: []inventory.Node{{Name: "cp", UID: "node-a", Ready: true}}}, RequestedScope: []inventory.ScopeEndpoint{}, DPUs: []inventory.Device{{ID: "dpu-a", RecordKey: "a", SerialNumber: "CARD-A", ObservationState: "observed"}, {ID: "dpu-b", RecordKey: "b", SerialNumber: "CARD-B", ObservationState: "observed"}}, Hosts: []inventory.Device{{ID: "host-a", RecordKey: "host", Adapters: []inventory.Adapter{{Key: "adapter-a", SerialNumber: "CARD-A"}, {Key: "adapter-b", SerialNumber: "CARD-B"}, {Key: "adapter-c", SerialNumber: "CARD-C", Selection: "not-selected"}}}}, Associations: []inventory.Association{}} +} + +var _ = Describe("inventory semantics", func() { + It("maps two selected cards to one host without selecting the extra or mutating input", func() { + v := estate() + before, _ := json.Marshal(v.Hosts) + a, c := Correlate(v.DPUs, v.Hosts) + Expect(a).To(HaveLen(2)) + for _, row := range c { + Expect(row.State).To(Equal("passed")) + } + after, _ := json.Marshal(v.Hosts) + Expect(after).To(Equal(before)) + Expect(v.Hosts[0].Adapters[2].Selection).To(Equal("not-selected")) + }) + It("rejects duplicate endpoint physical identity and ambiguous host matches", func() { + v := estate() + v.Hosts = append(v.Hosts, v.Hosts[0]) + a, c := Correlate(v.DPUs, v.Hosts) + Expect(a).To(BeEmpty()) + Expect(c[0].State).To(Equal("failed")) + }) + It("keeps fingerprints stable across order/time/readiness and changes them for identity", func() { + a := estate() + Canonicalize(&a) + first := a.Fingerprint + a.Collection.StartedAt = time.Now() + a.Management.Nodes[0].Ready = false + a.DPUs[0], a.DPUs[1] = a.DPUs[1], a.DPUs[0] + Canonicalize(&a) + Expect(a.Fingerprint).To(Equal(first)) + a.DPUs[0].ID = "replacement" + Canonicalize(&a) + Expect(a.Fingerprint).NotTo(Equal(first)) + }) + It("blocks endpoint replacement but preserves unknown observations", func() { + old := estate() + cur := estate() + cur.DPUs[0].ID = "replacement" + Expect(inventory.IdentityConflict(&old, &cur)).To(BeTrue()) + cur.DPUs[0].ID = "" + Expect(inventory.IdentityConflict(&old, &cur)).To(BeFalse()) + Expect(Diff(nil, &cur)[0].State).To(Equal("baseline-absent")) + }) + It("applies error precedence without hiding inspection failures", func() { + r := NewReport() + r.Discovery.Outcome = "partial" + Expect(ExitCode(r)).To(Equal(2)) + r.Errors = []inventory.Problem{{ExitCode: 2}, {ExitCode: 3}} + Expect(ExitCode(r)).To(Equal(3)) + r.Errors = append(r.Errors, inventory.Problem{ExitCode: 1}) + Expect(ExitCode(r)).To(Equal(1)) + }) +}) + +var _ = Describe("refresh regressions", func() { + It("canonicalizes nested network ordering without changing the observations", func() { + a := estate() + a.Management.Nodes[0].Interfaces = []inventory.Interface{{Index: 2, Name: "b", Addresses: []string{"192.0.2.2/24", "192.0.2.1/24"}}, {Index: 1, Name: "a"}} + a.Management.Nodes[0].Routes = []inventory.Route{{Destination: "b", Table: 2}, {Destination: "a", Table: 1}} + Canonicalize(&a) + original := a.Fingerprint + b := estate() + b.Management.Nodes[0].Interfaces = []inventory.Interface{{Index: 1, Name: "a"}, {Index: 2, Name: "b", Addresses: []string{"192.0.2.1/24", "192.0.2.2/24"}}} + b.Management.Nodes[0].Routes = []inventory.Route{{Destination: "a", Table: 1}, {Destination: "b", Table: 2}} + Canonicalize(&b) + Expect(b.Fingerprint).To(Equal(original)) + Expect(a.Management.Nodes[0].Interfaces[0].Name).To(Equal("b")) + }) + It("classifies failed current identity observations as unknown", func() { + old := estate() + current := estate() + current.DPUs[0].ObservationState = "unavailable" + current.DPUs[0].ID = "" + changes := Diff(&old, ¤t) + Expect(changes).NotTo(BeEmpty()) + for _, change := range changes { + if change.Path == "device.a.identity" { + Expect(change.State).To(Equal("unknown")) + } + } + }) +}) + +var _ = Describe("scope evidence on refresh", func() { + It("keeps missing scope evidence separate from a proven replacement", func() { + old := estate() + Canonicalize(&old) + current := estate() + current.Management = nil + Canonicalize(¤t) + Expect(current.ScopeDigest).To(BeEmpty()) + Expect(scopeConflict(&old, ¤t)).To(BeFalse()) + current = estate() + current.Management.SystemNamespaceUID = "replacement" + Canonicalize(¤t) + Expect(scopeConflict(&old, ¤t)).To(BeTrue()) + }) +}) diff --git a/pkg/dpf/discovery/management.go b/pkg/dpf/discovery/management.go new file mode 100644 index 00000000..4c96aa73 --- /dev/null +++ b/pkg/dpf/discovery/management.go @@ -0,0 +1,212 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/url" + "slices" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +type ManagementClient struct { + Client kubernetes.Interface + REST *rest.Config + Namespace string +} + +func ManagementAccess(r config.Request) (ManagementClient, error) { + rules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: r.Kubeconfig} + content, err := config.ReadReference(r.Kubeconfig, 16<<20) + if err != nil { + return ManagementClient{}, fmt.Errorf("invalid bounded management kubeconfig") + } + raw, err := clientcmd.Load(content) + if err != nil { + return ManagementClient{}, fmt.Errorf("cannot load explicit management kubeconfig") + } + for _, a := range raw.AuthInfos { + a.LocationOfOrigin = r.Kubeconfig + } + for _, c := range raw.Clusters { + c.LocationOfOrigin = r.Kubeconfig + } + if err := clientcmd.ResolveLocalPaths(raw); err != nil { + return ManagementClient{}, fmt.Errorf("cannot resolve kubeconfig file references") + } + selected := r.Context + if selected == "" { + selected = raw.CurrentContext + if raw.Contexts[selected] == nil && len(raw.Contexts) == 1 { + for k := range raw.Contexts { + selected = k + } + } + } + contextConfig := raw.Contexts[selected] + if contextConfig == nil { + return ManagementClient{}, fmt.Errorf("management context selection is required") + } + auth := raw.AuthInfos[contextConfig.AuthInfo] + cluster := raw.Clusters[contextConfig.Cluster] + if auth == nil || cluster == nil { + return ManagementClient{}, fmt.Errorf("management context has missing auth/cluster") + } + if auth.Exec != nil || auth.AuthProvider != nil || auth.Username != "" || auth.Password != "" { + return ManagementClient{}, fmt.Errorf("management kubeconfig requires certificate/key or bearer token; exec/auth-provider/basic authentication unsupported") + } + if auth.Token != "" && auth.TokenFile != "" || len(auth.ClientCertificateData) > 0 && auth.ClientCertificate != "" || len(auth.ClientKeyData) > 0 && auth.ClientKey != "" || len(cluster.CertificateAuthorityData) > 0 && cluster.CertificateAuthority != "" { + return ManagementClient{}, fmt.Errorf("management auth/trust references must use exactly one file or embedded value") + } + hasToken := auth.Token != "" || auth.TokenFile != "" + hasCertificate := (auth.ClientCertificate != "" || len(auth.ClientCertificateData) > 0) && (auth.ClientKey != "" || len(auth.ClientKeyData) > 0) + if !hasToken && !hasCertificate { + return ManagementClient{}, fmt.Errorf("management certificate/key or bearer token is required") + } + u, err := url.Parse(cluster.Server) + if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || cluster.InsecureSkipTLSVerify { + return ManagementClient{}, fmt.Errorf("management server requires verified HTTPS without userinfo/query/fragment") + } + for _, reference := range []struct { + file *string + data *[]byte + }{{&auth.ClientCertificate, &auth.ClientCertificateData}, {&auth.ClientKey, &auth.ClientKeyData}, {&cluster.CertificateAuthority, &cluster.CertificateAuthorityData}} { + if *reference.file != "" && len(*reference.data) == 0 { + b, e := config.ReadReference(*reference.file, 1<<20) + if e != nil { + return ManagementClient{}, fmt.Errorf("invalid bounded management auth/trust reference") + } + *reference.data = b + *reference.file = "" + } + } + if auth.TokenFile != "" && auth.Token == "" { + b, e := config.ReadReference(auth.TokenFile, 64<<10) + if e != nil { + return ManagementClient{}, fmt.Errorf("invalid bounded token reference") + } + auth.Token = strings.TrimSpace(string(b)) + if auth.Token == "" { + return ManagementClient{}, fmt.Errorf("empty management token reference") + } + auth.TokenFile = "" + } + loader := clientcmd.NewNonInteractiveClientConfig(*raw, selected, &clientcmd.ConfigOverrides{}, rules) + rc, err := loader.ClientConfig() + if err != nil { + return ManagementClient{}, fmt.Errorf("cannot resolve management certificate/token configuration") + } + rc.Timeout = r.RequestTimeout + rc.CAFile = "" + client, err := kubernetes.NewForConfig(rc) + if err != nil { + return ManagementClient{}, fmt.Errorf("cannot construct management client") + } + ns := r.Namespace + if ns == "" { + ns = contextConfig.Namespace + } + if ns == "" { + ns = "default" + } + return ManagementClient{Client: client, REST: rc, Namespace: ns}, nil +} +func (m ManagementClient) Observe(ctx context.Context) (*inventory.Management, error) { + ns, err := m.Client.CoreV1().Namespaces().Get(ctx, "kube-system", metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("management cluster identity read failed") + } + if ns.UID == "" { + return nil, fmt.Errorf("management identity UID is empty") + } + // REST request honors caller cancellation, unlike ServerVersion's context-free API. + var version struct { + GitVersion string `json:"gitVersion"` + } + data, err := m.Client.Discovery().RESTClient().Get().AbsPath("/version").Do(ctx).Raw() + if err != nil || json.Unmarshal(data, &version) != nil || version.GitVersion == "" { + return nil, fmt.Errorf("management version read failed") + } + result := &inventory.Management{APIEndpoint: m.REST.Host, SystemNamespaceUID: string(ns.UID), KubernetesVersion: version.GitVersion, Namespace: m.Namespace, TrustDigest: fmt.Sprintf("sha256:%x", sha256.Sum256(m.REST.CAData)), Nodes: []inventory.Node{}, EvidenceRefs: []string{"management-identity", "management-nodes"}} + if len(m.REST.CAData) == 0 { + result.TrustDigest = "system-roots" + } + if m.REST.ServerName != "" { + result.TrustDigest = inventory.Digest("management-trust-v1", []string{result.TrustDigest, m.REST.ServerName}) + } + cursor := "" + for { + nodes, err := m.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 64, Continue: cursor}) + if err != nil { + return nil, fmt.Errorf("management node inspection failed") + } + for _, n := range nodes.Items { + node := inventory.Node{Name: n.Name, UID: string(n.UID), Architecture: n.Status.NodeInfo.Architecture, OS: n.Status.NodeInfo.OperatingSystem, Unschedulable: n.Spec.Unschedulable, Addresses: []string{}} + _, cp := n.Labels["node-role.kubernetes.io/control-plane"] + _, master := n.Labels["node-role.kubernetes.io/master"] + node.ControlPlane = cp || master + for _, condition := range n.Status.Conditions { + if condition.Type == corev1.NodeReady { + node.Ready = condition.Status == corev1.ConditionTrue + } + } + for _, address := range n.Status.Addresses { + node.Addresses = append(node.Addresses, address.Address) + } + slices.Sort(node.Addresses) + result.Nodes = append(result.Nodes, node) + } + if len(result.Nodes) > 64 { + return nil, fmt.Errorf("management node scope exceeds 64 nodes") + } + cursor = nodes.Continue + if cursor == "" { + break + } + } + result.Observations = map[string]inventory.Observation{} + storage, storageErr := m.Client.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{Limit: 256}) + storageFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE"} + if storageErr == nil && storage.Continue == "" { + storageFact.State = "known" + storageFact.ReasonCode = "" + for _, s := range storage.Items { + storageFact.Values = append(storageFact.Values, s.Name+"/"+s.Provisioner) + } + slices.Sort(storageFact.Values) + } + result.Observations["storageClasses"] = storageFact + groupsFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE"} + data, groupErr := m.Client.Discovery().RESTClient().Get().AbsPath("/apis").Do(ctx).Raw() + var groups metav1.APIGroupList + if groupErr == nil && json.Unmarshal(data, &groups) == nil { + groupsFact.State = "known" + groupsFact.ReasonCode = "" + for _, g := range groups.Groups { + if strings.HasSuffix(g.Name, "dpu.nvidia.com") { + groupsFact.Values = append(groupsFact.Values, g.Name) + } + } + slices.Sort(groupsFact.Values) + } + result.Observations["advertisedDPFAPIGroups"] = groupsFact + slices.SortFunc(result.Nodes, func(a, b inventory.Node) int { return strings.Compare(a.UID, b.UID) }) + return result, nil +} + +// NewEvidence binds API reads to the current collection rather than old success flags. +func NewEvidence(at time.Time) map[string]inventory.Evidence { + return map[string]inventory.Evidence{"management-identity": {Kind: "kubernetes-api", InputRef: "/management", ResourceURI: "/api/v1/namespaces/kube-system", Property: "metadata.uid", ObservedAt: at, Vantage: "executor", Authenticated: true}, "management-nodes": {Kind: "kubernetes-api", InputRef: "/management", ResourceURI: "/api/v1/nodes", ObservedAt: at, Vantage: "executor", Authenticated: true}} +} diff --git a/pkg/dpf/discovery/management_test.go b/pkg/dpf/discovery/management_test.go new file mode 100644 index 00000000..3580a973 --- /dev/null +++ b/pkg/dpf/discovery/management_test.go @@ -0,0 +1,128 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "context" + "encoding/json" + "encoding/pem" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/tools/clientcmd" + clientapi "k8s.io/client-go/tools/clientcmd/api" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "time" +) + +var _ = Describe("explicit management access", func() { + It("freezes CA files and preserves a proxy base path while observing three nodes", func() { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Header.Get("Authorization")).To(Equal("Bearer test-token")) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/prefix/version": + json.NewEncoder(w).Encode(map[string]any{"gitVersion": "v1.34.0"}) + case "/prefix/api/v1/namespaces/kube-system": + w.Write([]byte(`{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kube-system","uid":"cluster-one"}}`)) + case "/prefix/api/v1/nodes": + w.Write([]byte(`{"apiVersion":"v1","kind":"NodeList","metadata":{},"items":[{"metadata":{"name":"cp1","uid":"node1","labels":{"node-role.kubernetes.io/control-plane":""}},"status":{"nodeInfo":{"architecture":"amd64","operatingSystem":"linux"},"conditions":[{"type":"Ready","status":"True"}]}}]}`)) + default: + w.WriteHeader(404) + } + })) + defer server.Close() + dir, e := os.MkdirTemp("", "dpf-management-") + Expect(e).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + caPath := filepath.Join(dir, "ca.pem") + Expect(os.WriteFile(caPath, ca, 0600)).To(Succeed()) + raw := clientapi.Config{CurrentContext: "management", Clusters: map[string]*clientapi.Cluster{"management": {Server: server.URL + "/prefix", CertificateAuthority: caPath}}, AuthInfos: map[string]*clientapi.AuthInfo{"reader": {Token: "test-token"}}, Contexts: map[string]*clientapi.Context{"management": {Cluster: "management", AuthInfo: "reader"}}} + b, e := clientcmd.Write(raw) + Expect(e).NotTo(HaveOccurred()) + filename := filepath.Join(dir, "config") + Expect(os.WriteFile(filename, b, 0600)).To(Succeed()) + mg, e := ManagementAccess(config.Request{Kubeconfig: filename, RequestTimeout: time.Second}) + Expect(e).NotTo(HaveOccurred()) + Expect(mg.REST.CAData).To(Equal(ca)) + Expect(mg.REST.CAFile).To(BeEmpty()) + observed, e := mg.Observe(context.Background()) + Expect(e).NotTo(HaveOccurred()) + Expect(observed.SystemNamespaceUID).To(Equal("cluster-one")) + Expect(observed.TrustDigest).NotTo(Equal("system-roots")) + Expect(observed.Nodes).To(HaveLen(1)) + }) + It("rejects anonymous, exec and HTTP configurations before client construction", func() { + dir, e := os.MkdirTemp("", "dpf-management-") + Expect(e).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + for _, test := range []struct { + server string + auth *clientapi.AuthInfo + }{{"https://example.invalid", &clientapi.AuthInfo{}}, {"http://example.invalid", &clientapi.AuthInfo{Token: "private-token"}}, {"https://example.invalid", &clientapi.AuthInfo{Exec: &clientapi.ExecConfig{Command: "must-never-execute"}}}} { + raw := clientapi.Config{CurrentContext: "m", Clusters: map[string]*clientapi.Cluster{"m": {Server: test.server}}, AuthInfos: map[string]*clientapi.AuthInfo{"r": test.auth}, Contexts: map[string]*clientapi.Context{"m": {Cluster: "m", AuthInfo: "r"}}} + b, e := clientcmd.Write(raw) + Expect(e).NotTo(HaveOccurred()) + filename := filepath.Join(dir, "config") + Expect(os.WriteFile(filename, b, 0600)).To(Succeed()) + _, e = ManagementAccess(config.Request{Kubeconfig: filename}) + Expect(e).To(HaveOccurred()) + Expect(e.Error()).NotTo(ContainSubstring("private-token")) + } + }) +}) + +var _ = Describe("management observation failures", func() { + It("distinguishes unavailable initial/recheck observations from changed identities", func() { + for _, initialFailure := range []bool{true, false} { + var reads atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/v1/namespaces/kube-system": + if reads.Add(1) > 1 || initialFailure { + w.WriteHeader(403) + return + } + w.Write([]byte(`{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kube-system","uid":"cluster-one"}}`)) + case "/version": + w.Write([]byte(`{"gitVersion":"v1.34.0"}`)) + case "/api/v1/nodes": + w.Write([]byte(`{"apiVersion":"v1","kind":"NodeList","items":[]}`)) + default: + w.WriteHeader(404) + } + })) + raw := clientapi.Config{CurrentContext: "m", Clusters: map[string]*clientapi.Cluster{"m": {Server: server.URL, CertificateAuthorityData: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})}}, AuthInfos: map[string]*clientapi.AuthInfo{"r": {Token: "test-token"}}, Contexts: map[string]*clientapi.Context{"m": {Cluster: "m", AuthInfo: "r"}}} + data, err := clientcmd.Write(raw) + Expect(err).NotTo(HaveOccurred()) + filename := filepath.Join(GinkgoT().TempDir(), "management.kubeconfig") + Expect(os.WriteFile(filename, data, 0600)).To(Succeed()) + input := config.Input{} + if initialFailure { + input.Site.Inventory = &inventory.Inventory{ScopeDigest: "known-baseline"} + } + report := Collect(context.Background(), config.Request{Input: input, Kubeconfig: filename, Timeout: 10 * time.Second, RequestTimeout: time.Second, MaxConcurrentBMCs: 1, Mode: "inventory"}, "test", Dependencies{}) + server.Close() + codes := []string{} + for _, problem := range report.Errors { + codes = append(codes, problem.Code) + } + if initialFailure { + Expect(codes).To(ContainElement("MANAGEMENT_INSPECTION_FAILED")) + } else { + Expect(codes).To(ContainElement("MANAGEMENT_RECHECK_FAILED")) + } + Expect(codes).NotTo(ContainElement("SCOPE_CONFLICT")) + Expect(codes).NotTo(ContainElement("MANAGEMENT_SCOPE_CHANGED")) + Expect(report.Discovery.Outcome).To(Equal("partial")) + Expect(ExitCode(report)).To(Equal(3)) + } + }) +}) diff --git a/pkg/dpf/discovery/references_unix_test.go b/pkg/dpf/discovery/references_unix_test.go new file mode 100644 index 00000000..bc7b6be9 --- /dev/null +++ b/pkg/dpf/discovery/references_unix_test.go @@ -0,0 +1,48 @@ +//go:build !windows + +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/tools/clientcmd" + clientapi "k8s.io/client-go/tools/clientcmd/api" + "os" + "path/filepath" + "syscall" + "time" +) + +var _ = Describe("bounded reference files", func() { + It("rejects a kubeconfig FIFO before a blocking read", func() { + dir, e := os.MkdirTemp("", "dpf-fifo-") + Expect(e).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + filename := filepath.Join(dir, "pipe") + Expect(syscall.Mkfifo(filename, 0600)).To(Succeed()) + done := make(chan error, 1) + go func() { _, err := ManagementAccess(config.Request{Kubeconfig: filename}); done <- err }() + Eventually(done, time.Second).Should(Receive(HaveOccurred())) + }) +}) + +var _ = Describe("unambiguous credential references", func() { + It("rejects an embedded token plus FIFO tokenFile before any read", func() { + dir, e := os.MkdirTemp("", "dpf-token-") + Expect(e).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + fifo := filepath.Join(dir, "token") + Expect(syscall.Mkfifo(fifo, 0600)).To(Succeed()) + raw := clientapi.Config{CurrentContext: "m", Clusters: map[string]*clientapi.Cluster{"m": {Server: "https://example.invalid"}}, AuthInfos: map[string]*clientapi.AuthInfo{"a": {Token: "inline-token", TokenFile: fifo}}, Contexts: map[string]*clientapi.Context{"m": {Cluster: "m", AuthInfo: "a"}}} + b, e := clientcmd.Write(raw) + Expect(e).NotTo(HaveOccurred()) + filename := filepath.Join(dir, "config") + Expect(os.WriteFile(filename, b, 0600)).To(Succeed()) + done := make(chan error, 1) + go func() { _, err := ManagementAccess(config.Request{Kubeconfig: filename}); done <- err }() + Eventually(done, time.Second).Should(Receive(HaveOccurred())) + }) +}) diff --git a/pkg/dpf/inventory/identity.go b/pkg/dpf/inventory/identity.go new file mode 100644 index 00000000..b3ef11e2 --- /dev/null +++ b/pkg/dpf/inventory/identity.go @@ -0,0 +1,26 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package inventory + +// IdentityConflict rejects replacement of a previously observed physical binding. +func IdentityConflict(old, current *Inventory) bool { + if old == nil || current == nil { + return false + } + before := map[string]string{} + for _, group := range [][]Device{old.DPUs, old.Hosts} { + for _, d := range group { + if d.ID != "" { + before[d.RecordKey] = d.ID + } + } + } + for _, group := range [][]Device{current.DPUs, current.Hosts} { + for _, d := range group { + if id := before[d.RecordKey]; id != "" && d.ID != "" && id != d.ID { + return true + } + } + } + return false +} diff --git a/pkg/dpf/inventory/probe.go b/pkg/dpf/inventory/probe.go new file mode 100644 index 00000000..88364cc4 --- /dev/null +++ b/pkg/dpf/inventory/probe.go @@ -0,0 +1,31 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package inventory + +type Interface struct { + Name string `json:"name" yaml:"name"` + Index int `json:"index" yaml:"index"` + MTU int `json:"mtu" yaml:"mtu"` + Addresses []string `json:"addresses" yaml:"addresses"` +} +type Route struct { + Destination string `json:"destination" yaml:"destination"` + Gateway string `json:"gateway,omitempty" yaml:"gateway,omitempty"` + InterfaceIndex int `json:"interfaceIndex" yaml:"interfaceIndex"` + Table int `json:"table" yaml:"table"` +} +type ProbeAddress struct { + Address string `json:"address" yaml:"address"` + TCP string `json:"tcp" yaml:"tcp"` + TLS string `json:"tls" yaml:"tls"` + HTTP string `json:"http" yaml:"http"` + HTTPStatus int `json:"httpStatus,omitempty" yaml:"httpStatus,omitempty"` +} +type ProbeResult struct { + NodeUID string `json:"nodeUID" yaml:"nodeUID"` + NetworkMode string `json:"networkMode" yaml:"networkMode"` + Destination string `json:"destination" yaml:"destination"` + DNS string `json:"dns" yaml:"dns"` + Addresses []ProbeAddress `json:"addresses" yaml:"addresses"` + EvidenceRef string `json:"evidenceRef" yaml:"evidenceRef"` +} diff --git a/pkg/dpf/inventory/types.go b/pkg/dpf/inventory/types.go new file mode 100644 index 00000000..ccea0317 --- /dev/null +++ b/pkg/dpf/inventory/types.go @@ -0,0 +1,206 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package inventory defines secret-free DPF observations and check results. +package inventory + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "time" +) + +const FormatVersion = "v1alpha1" + +type Collection struct { + RunID string `json:"runID" yaml:"runID"` + StartedAt time.Time `json:"startedAt" yaml:"startedAt"` + FinishedAt time.Time `json:"finishedAt" yaml:"finishedAt"` + Mode string `json:"mode" yaml:"mode"` + Collector string `json:"collector" yaml:"collector"` +} +type ScopeEndpoint struct { + Role string `json:"role" yaml:"role"` + Endpoint string `json:"endpoint" yaml:"endpoint"` + SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` +} +type Evidence struct { + Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` + PodUID string `json:"podUID,omitempty" yaml:"podUID,omitempty"` + NodeUID string `json:"nodeUID,omitempty" yaml:"nodeUID,omitempty"` + ImageDigest string `json:"imageDigest,omitempty" yaml:"imageDigest,omitempty"` + DNSPolicy string `json:"dnsPolicy,omitempty" yaml:"dnsPolicy,omitempty"` + DestinationEndpoint string `json:"destinationEndpoint,omitempty" yaml:"destinationEndpoint,omitempty"` + Kind string `json:"kind" yaml:"kind"` + InputRef string `json:"inputRef,omitempty" yaml:"inputRef,omitempty"` + ResourceURI string `json:"resourceURI,omitempty" yaml:"resourceURI,omitempty"` + Property string `json:"property,omitempty" yaml:"property,omitempty"` + ObservedAt time.Time `json:"observedAt" yaml:"observedAt"` + Vantage string `json:"vantage" yaml:"vantage"` + Adapter string `json:"adapter,omitempty" yaml:"adapter,omitempty"` + Authenticated bool `json:"authenticated" yaml:"authenticated"` +} +type Firmware struct { + ResourceURI string `json:"resourceURI,omitempty" yaml:"resourceURI,omitempty"` + Component string `json:"component" yaml:"component"` + State string `json:"state" yaml:"state"` + RawVersion string `json:"rawVersion,omitempty" yaml:"rawVersion,omitempty"` + Activation string `json:"activation" yaml:"activation"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` +} +type Authentication struct { + PasswordRead string `json:"passwordRead" yaml:"passwordRead"` + Method string `json:"method,omitempty" yaml:"method,omitempty"` + ClientCertificateSupport string `json:"clientCertificateSupport" yaml:"clientCertificateSupport"` + DPFClientAuthentication string `json:"dpfClientAuthentication" yaml:"dpfClientAuthentication"` +} +type TimeObservation struct { + State string `json:"state" yaml:"state"` + DateTime string `json:"dateTime,omitempty" yaml:"dateTime,omitempty"` + Offset string `json:"offset,omitempty" yaml:"offset,omitempty"` + NTPEnabled *bool `json:"ntpEnabled,omitempty" yaml:"ntpEnabled,omitempty"` + NTPServers []string `json:"ntpServers,omitempty" yaml:"ntpServers,omitempty"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` +} +type Adapter struct { + Key string `json:"adapterKey" yaml:"adapterKey"` + SerialNumber string `json:"serialNumber" yaml:"serialNumber"` + SerialNumberRaw string `json:"serialNumberRaw" yaml:"serialNumberRaw"` + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Slot string `json:"slot,omitempty" yaml:"slot,omitempty"` + Selection string `json:"selection" yaml:"selection"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` +} + +// Device represents every requested endpoint, including failed observations. +type Device struct { + RecordKey string `json:"recordKey" yaml:"recordKey"` + InputRef string `json:"inputRef" yaml:"inputRef"` + BMCEndpoint string `json:"bmcEndpoint" yaml:"bmcEndpoint"` + SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` + ObservationState string `json:"observationState" yaml:"observationState"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + IdentityKey []string `json:"identityKey,omitempty" yaml:"identityKey,omitempty"` + SerialNumber string `json:"serialNumber,omitempty" yaml:"serialNumber,omitempty"` + SerialNumberRaw string `json:"serialNumberRaw,omitempty" yaml:"serialNumberRaw,omitempty"` + SystemUUID string `json:"systemUUID,omitempty" yaml:"systemUUID,omitempty"` + Manufacturer string `json:"manufacturer,omitempty" yaml:"manufacturer,omitempty"` + Model string `json:"model,omitempty" yaml:"model,omitempty"` + ReaderAdapter string `json:"readerAdapter,omitempty" yaml:"readerAdapter,omitempty"` + InventoryCompleteness string `json:"inventoryCompleteness,omitempty" yaml:"inventoryCompleteness,omitempty"` + Qualification string `json:"qualification" yaml:"qualification"` + Firmware []Firmware `json:"firmware" yaml:"firmware"` + Authentication Authentication `json:"authentication" yaml:"authentication"` + Time TimeObservation `json:"time" yaml:"time"` + Adapters []Adapter `json:"adapters,omitempty" yaml:"adapters,omitempty"` + AdvertisedResetTypes []string `json:"advertisedResetTypes,omitempty" yaml:"advertisedResetTypes,omitempty"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` +} +type Node struct { + Interfaces []Interface `json:"interfaces,omitempty" yaml:"interfaces,omitempty"` + Routes []Route `json:"routes,omitempty" yaml:"routes,omitempty"` + Name string `json:"name" yaml:"name"` + UID string `json:"uid" yaml:"uid"` + ControlPlane bool `json:"controlPlane" yaml:"controlPlane"` + Ready bool `json:"ready" yaml:"ready"` + Unschedulable bool `json:"unschedulable" yaml:"unschedulable"` + Architecture string `json:"architecture" yaml:"architecture"` + OS string `json:"os" yaml:"os"` + Addresses []string `json:"addresses" yaml:"addresses"` +} +type Observation struct { + State string `json:"state" yaml:"state"` + Values []string `json:"values" yaml:"values"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` +} +type Management struct { + Observations map[string]Observation `json:"observations,omitempty" yaml:"observations,omitempty"` + APIEndpoint string `json:"apiEndpoint" yaml:"apiEndpoint"` + SystemNamespaceUID string `json:"systemNamespaceUID" yaml:"systemNamespaceUID"` + TrustDigest string `json:"trustDigest" yaml:"trustDigest"` + KubernetesVersion string `json:"kubernetesVersion" yaml:"kubernetesVersion"` + Namespace string `json:"namespace" yaml:"namespace"` + Nodes []Node `json:"nodes" yaml:"nodes"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` +} +type Association struct { + DPUID string `json:"dpuID" yaml:"dpuID"` + HostID string `json:"hostID" yaml:"hostID"` + HostAdapterKey string `json:"hostAdapterKey" yaml:"hostAdapterKey"` + SerialNumber string `json:"serialNumber" yaml:"serialNumber"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` +} +type Inventory struct { + FormatVersion string `json:"formatVersion" yaml:"formatVersion"` + Collection Collection `json:"collection" yaml:"collection"` + RequestedScope []ScopeEndpoint `json:"requestedScope" yaml:"requestedScope"` + ScopeDigest string `json:"scopeDigest,omitempty" yaml:"scopeDigest,omitempty"` + Fingerprint string `json:"fingerprint,omitempty" yaml:"fingerprint,omitempty"` + Management *Management `json:"management,omitempty" yaml:"management,omitempty"` + DPUs []Device `json:"dpus" yaml:"dpus"` + Hosts []Device `json:"hosts" yaml:"hosts"` + Associations []Association `json:"associations" yaml:"associations"` + Probes []ProbeResult `json:"probes,omitempty" yaml:"probes,omitempty"` + Evidence map[string]Evidence `json:"evidence" yaml:"evidence"` +} +type Check struct { + ID string `json:"id"` + Subject string `json:"subject"` + Vantage string `json:"vantage"` + RequiredPhase string `json:"requiredPhase"` + State string `json:"state"` + ReasonCode string `json:"reasonCode"` + EvidenceRefs []string `json:"evidenceRefs"` +} +type Problem struct { + Code string `json:"code"` + Message string `json:"message"` + InputRef string `json:"inputRef,omitempty"` + ExitCode int `json:"-"` +} +type Artifact struct { + Path string `json:"path"` + Kind string `json:"kind"` +} +type Change struct { + Path string `json:"path"` + State string `json:"state"` + Previous any `json:"previous,omitempty"` + Current any `json:"current,omitempty"` + RequiredPhase string `json:"requiredPhase"` +} +type Cleanup struct { + State string `json:"state"` + Leftovers []string `json:"leftovers"` +} +type Discovery struct { + Outcome string `json:"outcome"` + Mode string `json:"mode"` + Checks []Check `json:"checks"` + Cleanup Cleanup `json:"cleanup"` +} +type Report struct { + SchemaVersion string `json:"schemaVersion"` + Target string `json:"target"` + Phase string `json:"phase"` + Execution string `json:"execution"` + Acceptance string `json:"acceptance"` + Inventory *Inventory `json:"inventory,omitempty"` + Discovery Discovery `json:"discovery"` + Diff []Change `json:"diff"` + PendingInputs []Problem `json:"pendingInputs"` + Artifacts []Artifact `json:"artifacts"` + Errors []Problem `json:"errors"` +} + +// Digest hashes an explicit, versioned projection, never raw vendor documents. +func Digest(version string, value any) string { + b, _ := json.Marshal(value) + return fmt.Sprintf("sha256:%x", sha256.Sum256(append([]byte(version+"\n"), b...))) +} +func PhysicalID(role string, key []string) string { + return role + "-" + Digest("dpf-identity-v1", key)[7:] +} diff --git a/pkg/dpf/probe/collect.sh b/pkg/dpf/probe/collect.sh new file mode 100644 index 00000000..d5b93d71 --- /dev/null +++ b/pkg/dpf/probe/collect.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +# Fixed adapter for digest-pinned netshoot tools. No credentials, eval or repairs. +set -euo pipefail +umask 077 +request=$1 +node=$2 +mode=$3 +for tool in jq curl ip getent timeout date sort awk; do command -v "$tool" >/dev/null; done +[[ "$mode" == pod-network || "$mode" == node-network ]] +[[ -n "$node" && -n "$POD_UID" ]] +# Bound all commands, including DNS and network inspection, by the CLI deadline. +if [[ ${4:-} != bounded ]]; then + deadline=$(jq -r '.deadline | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601' "$request") + budget=$((deadline - $(date +%s))) + ((budget > 0 && budget <= 3600)) + exec timeout "$budget" /bin/bash "$0" "$request" "$node" "$mode" bounded +fi +ulimit -f 8192 +started=$(date -u +%Y-%m-%dT%H:%M:%SZ) +per_timeout=$(jq -r '.requestTimeout / 1000000000' "$request") +: > /scratch/results.jsonl +while IFS= read -r destination; do + host=$(jq -r '.hostname' <<<"$destination") + port=$(jq -r '.port' <<<"$destination") + verify_name=$(jq -r '.verifyName' <<<"$destination") + authority=$(jq -r '.authority' <<<"$destination") + url=$(jq -r '.url' <<<"$destination") + ca=$(jq -r '.caFile' <<<"$destination") + dns=resolved + if [[ $(jq -r '.literal' <<<"$destination") == true ]]; then + dns=not-applicable + addresses=$(jq -cn --arg host "$host" '[$host]') + else + # STREAM rows cover both families, deduplicated without a happy-eyeballs fallback. + # getent's lookup failure remains explicit unavailable evidence. + status=0 + timeout "$per_timeout" getent ahosts "$host" > /scratch/dns || status=$? + case "$status" in 0|2|124) ;; *) exit 1 ;; esac + if ((status != 0)); then + dns=unavailable + addresses='[]' + else + addresses=$(awk '$2 == "STREAM" { print $1 }' /scratch/dns | sort -u | jq -Rsc 'split("\n") | map(select(length > 0))') + if [[ $(jq 'length' <<<"$addresses") == 0 ]]; then dns=unavailable; fi + fi + fi + (($(jq 'length' <<<"$addresses") <= 32)) + : > /scratch/addresses.jsonl + while IFS= read -r address; do + resolved=$address + if [[ "$resolved" == *:* ]]; then resolved="[$resolved]"; fi + name=$verify_name + if [[ "$name" == *:* ]]; then name="[$name]"; fi + ca_args=() + if [[ -n "$ca" ]]; then ca_args=(--cacert "$ca"); fi + # Disable curlrc, proxies, redirects and all non-HTTPS protocols. Each IP + # is pinned while retaining the original HTTP authority and TLS identity. + status=0 + curl --disable --silent --show-error --noproxy '*' --proxy '' \ + --proto '=https' --proto-redir '=https' --tlsv1.2 --max-redirs 0 \ + --connect-timeout "$per_timeout" --max-time "$per_timeout" \ + --max-filesize 65536 --retry 0 --output /scratch/body \ + --write-out '%{json}' --resolve "$name:$port:$resolved" \ + --header "Host: $authority" "${ca_args[@]}" --url "$url" \ + > /scratch/curl.json 2>/scratch/curl.stderr || status=$? + # A tool/configuration failure must not masquerade as a failed network path. + case "$status" in 0|5|6|7|18|28|35|52|55|56|60|63|80|92) ;; *) exit 1 ;; esac + jq -e --argjson status "$status" '.exitcode == $status and (.time_connect | type == "number") and (.time_appconnect | type == "number")' /scratch/curl.json >/dev/null + tcp=unavailable + tls=unavailable + http=http-unavailable + if jq -e '.time_connect > 0' /scratch/curl.json >/dev/null; then tcp=reachable; fi + if jq -e '.time_appconnect > 0 and .ssl_verify_result == 0' /scratch/curl.json >/dev/null; then tls=verified; fi + code=$(jq '.response_code' /scratch/curl.json) + if [[ "$tls" == verified ]]; then + if ((code == 401 || code == 403)); then http=authentication-required + elif ((code >= 300 && code < 400)); then http=redirect-refused + elif ((code == 404)); then http=resource-unavailable + elif ((code >= 400)); then http=application-error + elif ((code >= 200 && code < 300)); then + http=invalid-response + if ((status == 0)) && jq -se 'length == 1 and (.[0] | type == "object")' /scratch/body >/dev/null 2>&1; then http=observed; fi + fi + fi + jq -cn --arg address "$address" --arg tcp "$tcp" --arg tls "$tls" --arg http "$http" --argjson code "$code" \ + '{address:$address,tcp:$tcp,tls:$tls,http:$http,httpStatus:$code}' >> /scratch/addresses.jsonl + done < <(jq -r '.[]' <<<"$addresses") + jq -cn --arg node "$node" --arg mode "$mode" --arg dns "$dns" \ + --arg id "$(jq -r '.id' <<<"$destination")" --slurpfile addresses /scratch/addresses.jsonl \ + '{nodeUID:$node,networkMode:$mode,destination:$id,dns:$dns,addresses:$addresses,evidenceRef:""}' >> /scratch/results.jsonl +done < <(jq -c '.destinations[]' "$request") +printf '[]' > /scratch/interfaces.json +printf '[]' > /scratch/routes.json +if [[ "$mode" == node-network ]]; then + ip -j -N address show > /scratch/links.json + ip -j -N -4 route show table all > /scratch/routes4.json + ip -j -N -6 route show table all > /scratch/routes6.json + jq '[.[] | {name:.ifname,index:.ifindex,mtu:.mtu,addresses:[.addr_info[] | "\(.local)/\(.prefixlen)"]}]' /scratch/links.json > /scratch/interfaces.json + jq -n --slurpfile links /scratch/links.json --slurpfile r4 /scratch/routes4.json --slurpfile r6 /scratch/routes6.json \ + '[($r4[0]+$r6[0])[] | . as $r | {destination:(.dst // "default"),gateway:(.gateway // ""),table:((.table // 254)|tonumber),interfaceIndex:([$links[0][] | select(.ifname == $r.dev) | .ifindex][0] // 0)}]' > /scratch/routes.json +fi +jq -cn --arg started "$started" --arg finished "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg node "$node" --arg pod "$POD_UID" --arg mode "$mode" --slurpfile request "$request" \ + --slurpfile results /scratch/results.jsonl --slurpfile interfaces /scratch/interfaces.json --slurpfile routes /scratch/routes.json \ + '{startedAt:$started,finishedAt:$finished,version:$request[0].version,runID:$request[0].runID,nodeUID:$node,podUID:$pod,networkMode:$mode,results:$results,interfaces:$interfaces[0],routes:$routes[0]}' diff --git a/pkg/dpf/probe/image_test.go b/pkg/dpf/probe/image_test.go new file mode 100644 index 00000000..2c776f70 --- /dev/null +++ b/pkg/dpf/probe/image_test.go @@ -0,0 +1,128 @@ +//go:build dpf_probe_image + +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package probe + +import ( + "context" + "encoding/json" + "encoding/pem" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// This explicit integration gate runs existing image tools without a cluster. +// DPF_PROBE_TEST_IMAGE must be the immutable catalog image under review. +var _ = Describe("published image runtime", func() { + It("collects both vantages with real DNS, per-IP TLS, HTTP and read-only routes", func() { + image := os.Getenv("DPF_PROBE_TEST_IMAGE") + Expect(image).To(ContainSubstring("@sha256:")) + platform := os.Getenv("DPF_PROBE_TEST_PLATFORM") + if platform == "" { + platform = "linux/amd64" + } + Expect(platform).To(BeElementOf("linux/amd64", "linux/arm64")) + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/401": + w.WriteHeader(401) + case "/403": + w.WriteHeader(403) + case "/302": + w.Header().Set("Location", "https://must-not-follow.invalid") + w.WriteHeader(302) + case "/404": + w.WriteHeader(404) + case "/500": + w.WriteHeader(500) + case "/invalid": + fmt.Fprint(w, "not JSON") + case "/large": + fmt.Fprint(w, `{"value":"`) + fmt.Fprint(w, string(make([]byte, 70000))) + fmt.Fprint(w, `"}`) + case "/multi": + fmt.Fprint(w, "{}\n{}") + case "/scalar-object": + fmt.Fprint(w, "42\n{}") + default: + fmt.Fprint(w, `{"ok":true}`) + } + })) + listener, err := net.Listen("tcp", "0.0.0.0:0") + Expect(err).NotTo(HaveOccurred()) + server.Listener.Close() + server.Listener = listener + server.StartTLS() + defer server.Close() + port := listener.Addr().(*net.TCPAddr).Port + endpoint := fmt.Sprintf("https://host.docker.internal:%d", port) + ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + req := Request{Version: ProtocolVersion, RunID: "local-image-check", Deadline: time.Now().Add(2 * time.Minute), RequestTimeout: 3 * time.Second, Destinations: []Destination{}} + for _, p := range []string{"/ok", "/401", "/403", "/302", "/404", "/500", "/invalid", "/large", "/multi", "/scalar-object"} { + req.Destinations = append(req.Destinations, Destination{ID: p, Endpoint: endpoint, Path: p, ServerName: "example.com", CA: ca}) + } + req.Destinations = append(req.Destinations, Destination{ID: "untrusted", Endpoint: endpoint, Path: "/ok", ServerName: "example.com"}, Destination{ID: "dns-missing", Endpoint: "https://does-not-exist.invalid", Path: "/version"}) + for _, mode := range []string{"pod-network", "node-network"} { + dir := GinkgoT().TempDir() + Expect(os.Chmod(dir, 0755)).To(Succeed()) + req.Deadline = time.Now().Add(2 * time.Minute) + files, err := requestFiles(req) + Expect(err).NotTo(HaveOccurred()) + for key, value := range files { + Expect(os.WriteFile(filepath.Join(dir, key), []byte(value), 0644)).To(Succeed()) + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + command := exec.CommandContext(ctx, "docker", "run", "--rm", "--platform", platform, "--user", "65532:65532", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--memory", "64m", "--env", "POD_UID=pod-local", "--mount", "type=bind,src="+dir+",dst=/request,readonly", "--tmpfs", "/scratch:rw,size=16777216,uid=65532,gid=65532,mode=0700", "--workdir", "/scratch", "--entrypoint", "/bin/bash", image, "/request/collect.sh", "/request/request.json", "node-local", mode) + output, err := command.CombinedOutput() + cancel() + GinkgoWriter.Printf("image %s %s output: %s\n", image, mode, output) + Expect(err).NotTo(HaveOccurred(), string(output)) + var result Response + Expect(json.Unmarshal(output, &result)).To(Succeed()) + Expect(ValidateResponse(result, req, "node-local", "pod-local", mode)).To(Succeed()) + expected := []string{"observed", "authentication-required", "authentication-required", "redirect-refused", "resource-unavailable", "application-error", "invalid-response", "invalid-response", "invalid-response", "invalid-response"} + for i, state := range expected { + Expect(result.Results[i].Addresses).NotTo(BeEmpty()) + verified := 0 + for _, a := range result.Results[i].Addresses { + if a.TLS == "verified" { + verified++ + Expect(a.TCP).To(Equal("reachable")) + Expect(a.HTTP).To(Equal(state)) + } else { + // Docker Desktop can advertise an unroutable IPv6 host + // alternative. Retain its failure; do not drop that IP. + Expect(a.HTTP).To(Equal("http-unavailable")) + } + } + Expect(verified).To(BeNumerically(">", 0)) + } + Expect(result.Results[10].Addresses).NotTo(BeEmpty()) + reachable := false + for _, a := range result.Results[10].Addresses { + reachable = reachable || a.TCP == "reachable" + Expect(a.TLS).To(Equal("unavailable")) + } + Expect(reachable).To(BeTrue()) + Expect(result.Results[11].DNS).To(Equal("unavailable")) + if mode == "node-network" { + Expect(result.Interfaces).NotTo(BeEmpty()) + Expect(result.Routes).NotTo(BeEmpty()) + } else { + Expect(result.Interfaces).To(BeEmpty()) + Expect(result.Routes).To(BeEmpty()) + } + } + }) +}) diff --git a/pkg/dpf/probe/probe_test.go b/pkg/dpf/probe/probe_test.go new file mode 100644 index 00000000..ef4f0a1e --- /dev/null +++ b/pkg/dpf/probe/probe_test.go @@ -0,0 +1,168 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package probe + +import ( + "context" + "fmt" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/discovery" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + k8stest "k8s.io/client-go/testing" + "testing" + "time" +) + +func TestProbe(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DPF probe ownership and protocol") +} + +var _ = Describe("probe integrity", func() { + It("declares two network vantages without credentials or privileged mounts", func() { + node := inventory.Node{Name: "cp", UID: "node-uid"} + a := Pod("probe", "default", map[string]string{"run": "one"}, node, "pod-network", "repo@sha256:d", "request", nil, 60) + b := Pod("probe", "default", a.Labels, node, "node-network", "repo@sha256:d", "request", nil, 60) + Expect(a.Spec.HostNetwork).To(BeFalse()) + Expect(b.Spec.HostNetwork).To(BeTrue()) + Expect(b.Spec.DNSPolicy).To(Equal(corev1.DNSDefault)) + Expect(*a.Spec.AutomountServiceAccountToken).To(BeFalse()) + Expect(a.Spec.Volumes).To(HaveLen(2)) + Expect(a.Spec.Volumes[0].Secret).To(BeNil()) + Expect(*a.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) + }) + It("accepts harmless lifecycle changes but rejects admission and ownership mutations", func() { + p := Pod("p", "default", nil, inventory.Node{Name: "n", UID: "u"}, "pod-network", "repo@sha256:d", "request", nil, 60) + live := p.DeepCopy() + live.ResourceVersion = "2" + live.Spec.NodeName = "n" + live.Status.Phase = corev1.PodSucceeded + Expect(samePod(p, live)).To(BeTrue()) + live.Spec.HostNetwork = true + Expect(samePod(p, live)).To(BeFalse()) + live = p.DeepCopy() + live.OwnerReferences = []metav1.OwnerReference{{UID: "foreign"}} + Expect(samePod(p, live)).To(BeFalse()) + }) + It("rejects duplicate rows and invalid address/state combinations", func() { + req := Request{Version: ProtocolVersion, RunID: "run", Deadline: time.Now().Add(time.Minute), Destinations: []Destination{{ID: "a"}}} + response := Response{Version: ProtocolVersion, RunID: "run", NodeUID: "node", PodUID: "pod", NetworkMode: "pod-network", StartedAt: time.Now(), FinishedAt: time.Now(), Results: []inventory.ProbeResult{{NodeUID: "node", NetworkMode: "pod-network", Destination: "a", DNS: "resolved", Addresses: []inventory.ProbeAddress{{Address: "192.0.2.1", TCP: "reachable", TLS: "verified", HTTP: "authentication-required"}}}}} + Expect(ValidateResponse(response, req, "node", "pod", "pod-network")).To(Succeed()) + response.Results[0].Addresses[0].TCP = "unavailable" + Expect(ValidateResponse(response, req, "node", "pod", "pod-network")).NotTo(Succeed()) + response.Results = append(response.Results, response.Results[0]) + Expect(ValidateResponse(response, req, "node", "pod", "pod-network")).NotTo(Succeed()) + }) + It("rejects absent image assets before Kubernetes writes", func() { + client := fake.NewClientset() + runner := Runner{} + _, _, e := runner.Run(context.Background(), config.Request{}, discovery.ManagementClient{Client: client}, &inventory.Inventory{}) + Expect(e).To(HaveOccurred()) + Expect(client.Actions()).To(BeEmpty()) + }) + It("rejects request ConfigMap admission mutation and preserves changed data", func() { + client := fake.NewClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}}) + client.PrependReactor("create", "configmaps", func(a k8stest.Action) (bool, runtime.Object, error) { + desired := a.(k8stest.CreateAction).GetObject().(*corev1.ConfigMap).DeepCopy() + desired.UID = "owned" + desired.ResourceVersion = "1" + desired.Data["request.json"] = "redirected" + Expect(client.Tracker().Create(corev1.SchemeGroupVersion.WithResource("configmaps"), desired, "default")).To(Succeed()) + return true, desired, nil + }) + assets := Assets{Qualified: true, Repository: "repo/probe", ManifestDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", PlatformDigests: map[string]string{"amd64": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}} + runner := Runner{Assets: assets, Journal: func([]Receipt) error { return nil }} + inv := &inventory.Inventory{Collection: inventory.Collection{RunID: "run"}, Management: &inventory.Management{Nodes: []inventory.Node{}}, Evidence: map[string]inventory.Evidence{}} + _, _, e := runner.Run(context.Background(), config.Request{Timeout: time.Minute, RequestTimeout: time.Second}, discovery.ManagementClient{Client: client, REST: &rest.Config{Host: "https://api.invalid"}, Namespace: "default"}, inv) + Expect(e).To(MatchError(ContainSubstring("admission changed"))) + for _, a := range client.Actions() { + Expect(a.GetResource().Resource).NotTo(Equal("pods")) + } + }) +}) + +var _ = Describe("probe failure regressions", func() { + It("clears a delete conflict after authoritative owned-object absence", func() { + client := fake.NewClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}}) + client.PrependReactor("create", "configmaps", func(a k8stest.Action) (bool, runtime.Object, error) { + cm := a.(k8stest.CreateAction).GetObject().(*corev1.ConfigMap).DeepCopy() + cm.UID = "owned" + cm.ResourceVersion = "1" + Expect(client.Tracker().Create(corev1.SchemeGroupVersion.WithResource("configmaps"), cm, "default")).To(Succeed()) + return true, cm, nil + }) + client.PrependReactor("delete", "configmaps", func(a k8stest.Action) (bool, runtime.Object, error) { + name := a.(k8stest.DeleteAction).GetName() + Expect(client.Tracker().Delete(corev1.SchemeGroupVersion.WithResource("configmaps"), "default", name)).To(Succeed()) + return true, nil, apierrors.NewConflict(corev1.Resource("configmaps"), name, fmt.Errorf("changed then removed")) + }) + runner := Runner{Assets: Assets{Qualified: true, Repository: "repo/probe", ManifestDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", PlatformDigests: map[string]string{"amd64": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}}, Journal: func([]Receipt) error { return nil }} + inv := &inventory.Inventory{Collection: inventory.Collection{RunID: "conflict"}, Management: &inventory.Management{Nodes: []inventory.Node{{Name: "not-ready", UID: "node", Ready: false}}}} + _, cleanup, err := runner.Run(context.Background(), config.Request{Timeout: time.Minute, RequestTimeout: time.Second}, discovery.ManagementClient{Client: client, REST: &rest.Config{Host: "https://api.invalid"}, Namespace: "default"}, inv) + Expect(err).To(HaveOccurred()) + Expect(cleanup.State).To(Equal("complete")) + Expect(cleanup.Leftovers).To(BeEmpty()) + }) +}) + +var _ = Describe("existing tool image adapter", func() { + It("normalizes IPv6 and separate TLS identity without injecting commands", func() { + req := Request{Version: ProtocolVersion, RunID: "run", Deadline: time.Now().Add(time.Minute), RequestTimeout: time.Second, Destinations: []Destination{{ID: "dpu", Endpoint: "https://[2001:db8::1]:8443", Path: "/redfish/v1/", ServerName: "bmc.example.invalid"}}} + files, err := requestFiles(req) + Expect(err).NotTo(HaveOccurred()) + Expect(files["collect.sh"]).To(Equal(collectorScript)) + Expect(files["request.json"]).To(ContainSubstring(`"hostname":"2001:db8::1"`)) + Expect(files["request.json"]).To(ContainSubstring(`"literal":true`)) + Expect(files["request.json"]).To(ContainSubstring(`"url":"https://bmc.example.invalid:8443/redfish/v1/"`)) + Expect(files["request.json"]).NotTo(ContainSubstring("password")) + req.Destinations[0].ServerName = "bmc.invalid\r\nInjected: yes" + _, err = requestFiles(req) + Expect(err).To(HaveOccurred()) + }) + It("rejects duplicate scope, expired deadlines, bad public trust and oversized maps", func() { + req := Request{Version: ProtocolVersion, RunID: "run", Deadline: time.Now().Add(time.Minute), RequestTimeout: time.Second, Destinations: []Destination{{ID: "api", Endpoint: "https://api.invalid", Path: "/version"}}} + req.Destinations = append(req.Destinations, req.Destinations[0]) + _, err := requestFiles(req) + Expect(err).To(HaveOccurred()) + req.Destinations = req.Destinations[:1] + req.Destinations[0].CA = []byte("not a certificate") + _, err = requestFiles(req) + Expect(err).To(MatchError(ContainSubstring("public probe CA"))) + req.Destinations[0].CA = nil + req.Deadline = time.Now().Add(-time.Second) + _, err = requestFiles(req) + Expect(err).To(HaveOccurred()) + req.Deadline = time.Now().Add(time.Minute) + req.RunID = string(make([]byte, 900<<10)) + _, err = requestFiles(req) + Expect(err).To(MatchError(ContainSubstring("ConfigMap budget"))) + }) + It("does not activate a pinned image before runtime qualification", func() { + assets := Assets{Repository: "repo/probe", ManifestDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", PlatformDigests: map[string]string{"amd64": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, Qualified: false} + Expect(assets.Validate()).To(MatchError(ContainSubstring("runtime qualification"))) + assets.Qualified = true + Expect(assets.Validate()).To(Succeed()) + }) + It("binds the fixed adapter and scratch volume into admission invariants", func() { + pod := Pod("p", "default", nil, inventory.Node{Name: "n", UID: "u"}, "pod-network", "repo@sha256:d", "request", nil, 60) + Expect(pod.Spec.Containers[0].Command).To(Equal([]string{"/bin/bash", "/request/collect.sh"})) + Expect(pod.Spec.Containers[0].WorkingDir).To(Equal("/scratch")) + Expect(*pod.Spec.Containers[0].SecurityContext.ReadOnlyRootFilesystem).To(BeTrue()) + Expect(pod.Spec.Volumes[1].EmptyDir.Medium).To(Equal(corev1.StorageMediumMemory)) + changed := pod.DeepCopy() + changed.Spec.Containers[0].Command = []string{"/bin/bash", "-c", "curl bad.invalid"} + Expect(samePod(pod, changed)).To(BeFalse()) + changed = pod.DeepCopy() + changed.Spec.Volumes[1].EmptyDir = nil + changed.Spec.Volumes[1].HostPath = &corev1.HostPathVolumeSource{Path: "/"} + Expect(samePod(pod, changed)).To(BeFalse()) + }) +}) diff --git a/pkg/dpf/probe/protocol.go b/pkg/dpf/probe/protocol.go new file mode 100644 index 00000000..b26177ba --- /dev/null +++ b/pkg/dpf/probe/protocol.go @@ -0,0 +1,137 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package probe contains the public diagnostic protocol and Kubernetes runner. +package probe + +import ( + "crypto/x509" + _ "embed" + "encoding/json" + "fmt" + "net/netip" + "net/url" + "strconv" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +const ProtocolVersion = "v1alpha1" + +// collectorScript adapts the existing image's tools to our public result format. +// It is fixed release code, never site-supplied commands or a downloaded helper. +// +//go:embed collect.sh +var collectorScript string + +type Destination struct { + ID string `json:"id"` + Endpoint string `json:"endpoint"` + Path string `json:"path"` + ServerName string `json:"serverName,omitempty"` + CA []byte `json:"ca,omitempty"` +} +type Request struct { + RequestTimeout time.Duration `json:"requestTimeout"` + Version string `json:"version"` + RunID string `json:"runID"` + Deadline time.Time `json:"deadline"` + Destinations []Destination `json:"destinations"` +} +type Response struct { + StartedAt time.Time `json:"startedAt"` + FinishedAt time.Time `json:"finishedAt"` + Version string `json:"version"` + RunID string `json:"runID"` + NodeUID string `json:"nodeUID"` + PodUID string `json:"podUID"` + NetworkMode string `json:"networkMode"` + Results []inventory.ProbeResult `json:"results"` + Interfaces []inventory.Interface `json:"interfaces,omitempty"` + Routes []inventory.Route `json:"routes,omitempty"` +} + +type toolDestination struct { + ID string `json:"id"` + Hostname string `json:"hostname"` + Port string `json:"port"` + Literal bool `json:"literal"` + URL string `json:"url"` + Authority string `json:"authority"` + VerifyName string `json:"verifyName"` + CAFile string `json:"caFile"` +} + +// requestFiles resolves URL/trust coordinates in Go. The script consumes JSON +// values as quoted arguments; it never parses URLs or evaluates site input. +func requestFiles(request Request) (map[string]string, error) { + if request.Version != ProtocolVersion || request.RunID == "" || len(request.Destinations) == 0 || len(request.Destinations) > 257 || time.Until(request.Deadline) > time.Hour || time.Until(request.Deadline) <= 0 || request.RequestTimeout < time.Second || request.RequestTimeout > 2*time.Minute { + return nil, fmt.Errorf("invalid probe request scope/deadline") + } + files := map[string]string{"collect.sh": collectorScript} + destinations := []toolDestination{} + seen := map[string]bool{} + for i, destination := range request.Destinations { + u, err := url.Parse(destination.Endpoint) + if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "" || strings.ContainsAny(u.Host, "\r\n") || !strings.HasPrefix(destination.Path, "/") || destination.ID == "" || seen[destination.ID] { + return nil, fmt.Errorf("invalid or duplicate probe destination") + } + seen[destination.ID] = true + port := u.Port() + if port == "" { + port = "443" + } + portNumber, err := strconv.Atoi(port) + if err != nil || portNumber < 1 || portNumber > 65535 { + return nil, fmt.Errorf("invalid destination port") + } + name := destination.ServerName + if name == "" { + name = u.Hostname() + } + if strings.ContainsAny(name, "\r\n /@?#%") { + return nil, fmt.Errorf("invalid TLS server name") + } + host := name + if strings.Contains(host, ":") { + if _, err := netip.ParseAddr(host); err != nil { + return nil, fmt.Errorf("invalid TLS server name") + } + host = "[" + host + "]" + } + probeURL := url.URL{Scheme: "https", Host: host + ":" + port, Path: destination.Path} + _, literalErr := netip.ParseAddr(u.Hostname()) + d := toolDestination{ID: destination.ID, Hostname: u.Hostname(), Port: port, Literal: literalErr == nil, URL: probeURL.String(), Authority: u.Host, VerifyName: name} + if len(destination.CA) > 0 { + if !x509.NewCertPool().AppendCertsFromPEM(destination.CA) { + return nil, fmt.Errorf("invalid public probe CA") + } + key := fmt.Sprintf("ca-%d.pem", i) + files[key] = string(destination.CA) + d.CAFile = "/request/" + key + } + destinations = append(destinations, d) + } + document := struct { + Version string `json:"version"` + RunID string `json:"runID"` + Deadline time.Time `json:"deadline"` + RequestTimeout time.Duration `json:"requestTimeout"` + Destinations []toolDestination `json:"destinations"` + }{request.Version, request.RunID, request.Deadline.UTC(), request.RequestTimeout, destinations} + data, err := json.Marshal(document) + if err != nil { + return nil, err + } + files["request.json"] = string(data) + size := 0 + for key, value := range files { + size += len(key) + len(value) + } + if size > 900<<10 { + return nil, fmt.Errorf("probe request exceeds ConfigMap budget") + } + return files, nil +} diff --git a/pkg/dpf/probe/runner.go b/pkg/dpf/probe/runner.go new file mode 100644 index 00000000..6c238f9c --- /dev/null +++ b/pkg/dpf/probe/runner.go @@ -0,0 +1,490 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package probe + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/artifacts" + "io" + "net/netip" + "net/url" + "path" + "reflect" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/discovery" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// Assets are release-owned. A verified published digest is still subject to +// controlled Kubernetes admission/runtime qualification before CLI activation. +type Assets struct { + Repository string + ManifestDigest string + PlatformDigests map[string]string + Qualified bool +} + +func (a Assets) Validate() error { + if a.Repository == "" || !validDigest(a.ManifestDigest) || len(a.PlatformDigests) == 0 { + return fmt.Errorf("qualified DPF probe assets are not installed in this build") + } + for _, d := range a.PlatformDigests { + if !validDigest(d) { + return fmt.Errorf("invalid platform image digest") + } + } + if !a.Qualified { + return fmt.Errorf("DPF probe image is pinned but Kubernetes runtime qualification is pending") + } + return nil +} +func validDigest(s string) bool { + if len(s) != 71 || !strings.HasPrefix(s, "sha256:") { + return false + } + _, err := hex.DecodeString(s[7:]) + return err == nil +} + +type Receipt struct { + Kind string `json:"kind"` + Namespace string `json:"namespace"` + Name string `json:"name"` + UID types.UID `json:"uid,omitempty"` +} +type Runner struct { + Assets Assets + Journal func([]Receipt) error +} + +func (runner Runner) Run(ctx context.Context, r config.Request, mg discovery.ManagementClient, inv *inventory.Inventory) (rows []inventory.ProbeResult, cleanup inventory.Cleanup, err error) { + cleanup = inventory.Cleanup{State: "complete", Leftovers: []string{}} + if err := runner.Assets.Validate(); err != nil { + return nil, cleanup, err + } + if runner.Journal == nil { + return nil, cleanup, fmt.Errorf("probe resource journal required") + } + ns, err := mg.Client.CoreV1().Namespaces().Get(ctx, mg.Namespace, metav1.GetOptions{}) + if err != nil || ns.DeletionTimestamp != nil { + return nil, cleanup, fmt.Errorf("probe namespace unavailable") + } + request := Request{RequestTimeout: r.RequestTimeout, Version: ProtocolVersion, RunID: inv.Collection.RunID, Deadline: time.Now().Add(r.Timeout), Destinations: []Destination{}} + api, _ := url.Parse(mg.REST.Host) + request.Destinations = append(request.Destinations, Destination{ID: "management-api", Endpoint: api.Scheme + "://" + api.Host, Path: path.Join("/", api.Path, "version"), ServerName: mg.REST.ServerName, CA: mg.REST.CAData}) + for _, t := range r.Targets { + if t.Role == "dpus" { + request.Destinations = append(request.Destinations, Destination{ID: inventory.Digest("probe-destination-v1", t.Endpoint), Endpoint: t.Endpoint, Path: "/redfish/v1/", CA: t.Access.PublicCA()}) + } + } + if deadline, ok := ctx.Deadline(); ok { + request.Deadline = deadline + } + files, err := requestFiles(request) + if err != nil { + return nil, cleanup, err + } + receipts := []Receipt{} + pods := map[string]*corev1.Pod{} + var configMap *corev1.ConfigMap + journal := func(kind, name string, uid types.UID) error { + receipts = append(receipts, Receipt{Kind: kind, Namespace: mg.Namespace, Name: name, UID: uid}) + return runner.Journal(receipts) + } + defer func() { + cleanCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second) + defer cancel() + for i := len(receipts) - 1; i >= 0; i-- { + receipt := receipts[i] + if receipt.UID == "" { + if receipt.Kind == "Pod" { + live, e := mg.Client.CoreV1().Pods(mg.Namespace).Get(cleanCtx, receipt.Name, metav1.GetOptions{}) + if e == nil && samePod(pods[receipt.Name], live) { + receipt.UID = live.UID + } + } else { + live, e := mg.Client.CoreV1().ConfigMaps(mg.Namespace).Get(cleanCtx, receipt.Name, metav1.GetOptions{}) + if e == nil && sameConfigMap(configMap, live) { + receipt.UID = live.UID + } + } + if receipt.UID == "" { + cleanup.State = "incomplete" + cleanup.Leftovers = append(cleanup.Leftovers, receipt.Kind+"/"+receipt.Name+": create outcome unknown") + continue + } + receipts[i] = receipt + _ = runner.Journal(receipts) + } + var cleanupErr error + for { + if cleanCtx.Err() != nil { + cleanupErr = cleanCtx.Err() + break + } + options := metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &receipt.UID}} + if receipt.Kind == "Pod" { + live, e := mg.Client.CoreV1().Pods(mg.Namespace).Get(cleanCtx, receipt.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(e) { + cleanupErr = nil + break + } + if e != nil { + cleanupErr = e + break + } + if live.UID != receipt.UID { + cleanupErr = nil + break + } + if !samePod(pods[receipt.Name], live) { + cleanupErr = fmt.Errorf("probe ownership/spec changed") + break + } + options.Preconditions.ResourceVersion = &live.ResourceVersion + cleanupErr = mg.Client.CoreV1().Pods(mg.Namespace).Delete(cleanCtx, receipt.Name, options) + } else { + live, e := mg.Client.CoreV1().ConfigMaps(mg.Namespace).Get(cleanCtx, receipt.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(e) { + cleanupErr = nil + break + } + if e != nil { + cleanupErr = e + break + } + if live.UID != receipt.UID { + cleanupErr = nil + break + } + if !sameConfigMap(configMap, live) { + cleanupErr = fmt.Errorf("request ownership/data changed") + break + } + options.Preconditions.ResourceVersion = &live.ResourceVersion + cleanupErr = mg.Client.CoreV1().ConfigMaps(mg.Namespace).Delete(cleanCtx, receipt.Name, options) + } + if apierrors.IsNotFound(cleanupErr) { + cleanupErr = nil + break + } + if cleanupErr != nil && !apierrors.IsConflict(cleanupErr) { + break + } + select { + case <-cleanCtx.Done(): + cleanupErr = cleanCtx.Err() + case <-time.After(100 * time.Millisecond): + } + if cleanupErr != nil && !apierrors.IsConflict(cleanupErr) { + break + } + } + if cleanupErr != nil { + cleanup.State = "incomplete" + cleanup.Leftovers = append(cleanup.Leftovers, receipt.Kind+"/"+receipt.Name) + } + } + if cleanup.State != "complete" && err == nil { + err = fmt.Errorf("probe cleanup incomplete; see resource journal") + } + }() + name := "l8k-dpf-" + strings.TrimPrefix(inventory.Digest("run", inv.Collection.RunID), "sha256:")[:16] + labels := map[string]string{"app.kubernetes.io/managed-by": "l8k-dpf-discovery", "l8k.nvidia.com/discovery-run": name} + if err = journal("ConfigMap", name, ""); err != nil { + return nil, cleanup, err + } + desired := &corev1.ConfigMap{Immutable: ptr(true), ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: mg.Namespace, Labels: labels}, Data: files} + configMap = desired + configMap, err = mg.Client.CoreV1().ConfigMaps(mg.Namespace).Create(ctx, desired, metav1.CreateOptions{}) + if err != nil { + configMap = desired + if createRejected(err) { + receipts = receipts[:len(receipts)-1] + } + return nil, cleanup, fmt.Errorf("probe ConfigMap create outcome unavailable; inspect journal") + } + receipts[len(receipts)-1].UID = configMap.UID + if !sameConfigMap(desired, configMap) { + configMap = desired + return nil, cleanup, fmt.Errorf("admission changed probe request data/ownership") + } + if err = runner.Journal(receipts); err != nil { + return nil, cleanup, err + } + for _, node := range inv.Management.Nodes { + if !node.Ready || node.Unschedulable || node.OS != "linux" || runner.Assets.PlatformDigests[node.Architecture] == "" { + return rows, cleanup, fmt.Errorf("management node cannot run the qualified probe") + } + for _, mode := range []string{"pod-network", "node-network"} { + podName := name + "-" + strings.TrimPrefix(inventory.Digest("node", node.UID+mode), "sha256:")[:12] + image := runner.Assets.Repository + if r.ImageRepository != "" { + image = r.ImageRepository + } + image += "@" + runner.Assets.ManifestDigest + desiredPod := Pod(podName, mg.Namespace, labels, node, mode, image, name, r.ImagePullSecrets, int64(r.Timeout/time.Second)) + if err = journal("Pod", podName, ""); err != nil { + return rows, cleanup, err + } + pods[podName] = desiredPod + created, createErr := mg.Client.CoreV1().Pods(mg.Namespace).Create(ctx, desiredPod, metav1.CreateOptions{}) + if createErr != nil { + if createRejected(createErr) { + receipts = receipts[:len(receipts)-1] + } + return rows, cleanup, fmt.Errorf("probe Pod create outcome unavailable; inspect journal") + } + receipts[len(receipts)-1].UID = created.UID + pods[podName] = created + if err = runner.Journal(receipts); err != nil { + return rows, cleanup, err + } + if !samePod(desiredPod, created) { + return rows, cleanup, fmt.Errorf("probe admission changed required invariants") + } + var live *corev1.Pod + for { + live, err = mg.Client.CoreV1().Pods(mg.Namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return rows, cleanup, fmt.Errorf("probe Pod inspection failed") + } + if !samePod(created, live) || live.UID != created.UID { + return rows, cleanup, fmt.Errorf("probe Pod identity/spec changed") + } + if live.Status.Phase == corev1.PodSucceeded { + break + } + if live.Status.Phase == corev1.PodFailed { + return rows, cleanup, fmt.Errorf("probe execution failed") + } + select { + case <-ctx.Done(): + return rows, cleanup, ctx.Err() + case <-time.After(200 * time.Millisecond): + } + } + currentNode, err := mg.Client.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) + if err != nil || string(currentNode.UID) != node.UID || live.Spec.NodeName != node.Name { + return rows, cleanup, fmt.Errorf("probe node identity changed") + } + if len(live.Status.ContainerStatuses) != 1 || !strings.HasSuffix(live.Status.ContainerStatuses[0].ImageID, runner.Assets.PlatformDigests[node.Architecture]) { + return rows, cleanup, fmt.Errorf("probe pulled image digest differs from qualified asset") + } + stream, err := mg.Client.CoreV1().Pods(mg.Namespace).GetLogs(podName, &corev1.PodLogOptions{Container: "probe", LimitBytes: ptr(int64(4 << 20))}).Stream(ctx) + if err != nil { + return rows, cleanup, fmt.Errorf("probe log access failed") + } + body, readErr := io.ReadAll(io.LimitReader(stream, (4<<20)+1)) + stream.Close() + if readErr != nil || len(body) > 4<<20 { + return rows, cleanup, fmt.Errorf("probe result too large") + } + var result Response + if json.Unmarshal(body, &result) != nil || ValidateResponse(result, request, node.UID, string(live.UID), mode) != nil { + return rows, cleanup, fmt.Errorf("probe result binding/coverage invalid") + } + frozenMap, mapErr := mg.Client.CoreV1().ConfigMaps(mg.Namespace).Get(ctx, name, metav1.GetOptions{}) + if mapErr != nil || frozenMap.UID != configMap.UID || !sameConfigMap(configMap, frozenMap) { + return rows, cleanup, fmt.Errorf("probe request identity changed") + } + if mode == "node-network" { + for i := range inv.Management.Nodes { + if inv.Management.Nodes[i].UID == node.UID { + inv.Management.Nodes[i].Interfaces = result.Interfaces + inv.Management.Nodes[i].Routes = result.Routes + } + } + } + for _, row := range result.Results { + id := inventory.Digest("probe-evidence-v1", []string{node.UID, mode, row.Destination}) + row.EvidenceRef = id + inv.Evidence[id] = inventory.Evidence{Kind: "connectivity-probe", ObservedAt: result.FinishedAt, Vantage: node.UID + "/" + mode, ResourceURI: "pods/" + string(live.UID), Namespace: mg.Namespace, PodUID: string(live.UID), NodeUID: node.UID, ImageDigest: runner.Assets.PlatformDigests[node.Architecture], DNSPolicy: string(live.Spec.DNSPolicy), DestinationEndpoint: destinationEndpoint(request, row.Destination)} + rows = append(rows, row) + if row.DNS == "unavailable" || len(row.Addresses) == 0 { + return rows, cleanup, fmt.Errorf("required probe DNS coverage unavailable") + } + for _, a := range row.Addresses { + if a.TCP != "reachable" || a.TLS != "verified" { + return rows, cleanup, fmt.Errorf("required probe TCP/TLS path failed") + } + } + } + } + } + return rows, cleanup, nil +} +func ptr[T any](value T) *T { return &value } +func Pod(name, namespace string, labels map[string]string, node inventory.Node, mode, image, configMap string, secrets []string, seconds int64) *corev1.Pod { + container := corev1.Container{ + Name: "probe", Image: image, + Command: []string{"/bin/bash", "/request/collect.sh"}, + Args: []string{"/request/request.json", node.UID, mode}, + WorkingDir: "/scratch", + Env: []corev1.EnvVar{{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}}}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr(false), ReadOnlyRootFilesystem: ptr(true), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("32Mi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("64Mi")}, + }, + VolumeMounts: []corev1.VolumeMount{{Name: "request", MountPath: "/request", ReadOnly: true}, {Name: "scratch", MountPath: "/scratch"}}, + } + spec := corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: ptr(false), ActiveDeadlineSeconds: &seconds, + HostNetwork: mode == "node-network", DNSPolicy: corev1.DNSClusterFirst, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr(true), RunAsUser: ptr(int64(65532)), + FSGroup: ptr(int64(65532)), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{node.Name}}}}}, + }, + }}, + Tolerations: []corev1.Toleration{ + {Key: "node-role.kubernetes.io/control-plane", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + {Key: "node-role.kubernetes.io/master", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}, + }, + Containers: []corev1.Container{container}, + Volumes: []corev1.Volume{{Name: "request", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: configMap}}}}, {Name: "scratch", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: ptr(resource.MustParse("16Mi"))}}}}, + } + if mode == "node-network" { + spec.DNSPolicy = corev1.DNSDefault + } + for _, secret := range secrets { + spec.ImagePullSecrets = append(spec.ImagePullSecrets, corev1.LocalObjectReference{Name: secret}) + } + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: labels}, Spec: spec} +} +func samePod(expected, actual *corev1.Pod) bool { + if expected == nil || actual == nil { + return false + } + a, b := expected.Spec.DeepCopy(), actual.Spec.DeepCopy() + // Only known harmless server/scheduler defaults are excluded from comparison. + normalize := func(s *corev1.PodSpec) { + s.NodeName = "" + tolerations := []corev1.Toleration{} + for _, t := range s.Tolerations { + standard := (t.Key == "node.kubernetes.io/not-ready" || t.Key == "node.kubernetes.io/unreachable") && t.Operator == corev1.TolerationOpExists && t.Effect == corev1.TaintEffectNoExecute && t.TolerationSeconds != nil && *t.TolerationSeconds == 300 + if !standard { + tolerations = append(tolerations, t) + } + } + s.Tolerations = tolerations + s.ServiceAccountName = "" + s.DeprecatedServiceAccount = "" + s.SchedulerName = "" + s.TerminationGracePeriodSeconds = nil + s.EnableServiceLinks = nil + s.PreemptionPolicy = nil + s.Priority = nil + + for i := range s.Containers { + s.Containers[i].ImagePullPolicy = "" + s.Containers[i].TerminationMessagePath = "" + s.Containers[i].TerminationMessagePolicy = "" + for j := range s.Containers[i].Env { + if src := s.Containers[i].Env[j].ValueFrom; src != nil && src.FieldRef != nil { + src.FieldRef.APIVersion = "" + } + } + } + for i := range s.Volumes { + if s.Volumes[i].ConfigMap != nil { + s.Volumes[i].ConfigMap.DefaultMode = nil + } + } + } + normalize(a) + normalize(b) + return reflect.DeepEqual(a, b) && reflect.DeepEqual(expected.Labels, actual.Labels) && reflect.DeepEqual(expected.OwnerReferences, actual.OwnerReferences) && reflect.DeepEqual(expected.Annotations, actual.Annotations) +} +func ValidateResponse(out Response, req Request, node, pod, mode string) error { + if out.Version != ProtocolVersion || out.RunID != req.RunID || out.NodeUID != node || out.PodUID != pod || out.NetworkMode != mode || len(out.Results) != len(req.Destinations) { + return fmt.Errorf("probe binding mismatch") + } + if out.StartedAt.IsZero() || out.FinishedAt.Before(out.StartedAt) || out.FinishedAt.After(req.Deadline.Add(time.Minute)) { + return fmt.Errorf("invalid probe time bounds") + } + expected := map[string]bool{} + for _, d := range req.Destinations { + expected[d.ID] = true + } + for _, r := range out.Results { + if !expected[r.Destination] || r.NodeUID != node || r.NetworkMode != mode { + return fmt.Errorf("unexpected or duplicate row") + } + if err := validateRow(r); err != nil { + return err + } + delete(expected, r.Destination) + } + return nil +} + +// FileJournal contains no auth material and records intended creates before dispatch. +func FileJournal(filename string) func([]Receipt) error { + return func(receipts []Receipt) error { + b, err := json.MarshalIndent(receipts, "", " ") + if err != nil { + return err + } + return artifacts.AtomicWrite(filename, b) + } +} + +func sameConfigMap(a, b *corev1.ConfigMap) bool { + return a != nil && b != nil && a.Immutable != nil && b.Immutable != nil && *a.Immutable && *b.Immutable && reflect.DeepEqual(a.Data, b.Data) && reflect.DeepEqual(a.BinaryData, b.BinaryData) && reflect.DeepEqual(a.Labels, b.Labels) && reflect.DeepEqual(a.OwnerReferences, b.OwnerReferences) && reflect.DeepEqual(a.Annotations, b.Annotations) +} +func destinationEndpoint(request Request, id string) string { + for _, d := range request.Destinations { + if d.ID == id { + return d.Endpoint + } + } + return "" +} +func validateRow(r inventory.ProbeResult) error { + if r.DNS != "resolved" && r.DNS != "not-applicable" && r.DNS != "unavailable" { + return fmt.Errorf("invalid DNS state") + } + if len(r.Addresses) > 32 { + return fmt.Errorf("address limit exceeded") + } + seen := map[string]bool{} + for _, a := range r.Addresses { + if _, err := netip.ParseAddr(a.Address); err != nil || seen[a.Address] { + return fmt.Errorf("invalid/duplicate IP address") + } + seen[a.Address] = true + if a.TCP != "reachable" && a.TCP != "unavailable" || a.TLS != "verified" && a.TLS != "unavailable" || a.TLS == "verified" && a.TCP != "reachable" { + return fmt.Errorf("invalid TCP/TLS state") + } + switch a.HTTP { + case "observed", "authentication-required", "redirect-refused", "resource-unavailable", "application-error", "http-unavailable", "invalid-response": + default: + return fmt.Errorf("invalid HTTP state") + } + } + return nil +} + +func createRejected(err error) bool { + return apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) || apierrors.IsAlreadyExists(err) || apierrors.IsInvalid(err) +} diff --git a/pkg/dpf/redfish/client.go b/pkg/dpf/redfish/client.go new file mode 100644 index 00000000..9e0b8222 --- /dev/null +++ b/pkg/dpf/redfish/client.go @@ -0,0 +1,311 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package redfish implements a bounded GET/session-only Redfish transport. +package redfish + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" +) + +type Document map[string]any + +// Error deliberately omits response bodies, headers and private credentials. +type Error struct { + Code string + URI string + Status int +} + +func (e *Error) Error() string { + return fmt.Sprintf("Redfish %s at %s (HTTP %d)", e.Code, e.URI, e.Status) +} +func Code(err error) string { + var e *Error + if errors.As(err, &e) { + return e.Code + } + if errors.Is(err, context.Canceled) { + return "CANCELLED" + } + if errors.Is(err, context.DeadlineExceeded) { + return "BMC_TIMEOUT" + } + return "BMC_UNAVAILABLE" +} + +// Reader intentionally has no POST/PATCH/action method available to adapters. +type Reader interface { + Get(context.Context, string) (Document, error) + Collection(context.Context, string) ([]Resource, error) +} +type Resource struct { + URI string + Document Document +} +type Client struct { + origin string + http *http.Client + access config.Access + token string + session string + sessionUncertain bool + reads int + bytes int +} + +func New(target config.Target, timeout time.Duration) *Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.TLSClientConfig = target.Access.TLSConfig() + transport.TLSHandshakeTimeout = timeout + transport.ResponseHeaderTimeout = timeout + return &Client{origin: target.Endpoint, access: target.Access, http: &http.Client{Transport: transport, Timeout: timeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}} +} +func (c *Client) guard(ref string) (string, error) { + u, e := url.Parse(ref) + if e != nil { + return "", &Error{Code: "UNSAFE_LINK"} + } + if u.Host != "" && !u.IsAbs() { + return "", &Error{Code: "CROSS_ORIGIN_LINK"} + } + if u.IsAbs() { + o, err := config.Origin(u.Scheme + "://" + u.Host) + if err != nil || o != c.origin { + return "", &Error{Code: "CROSS_ORIGIN_LINK"} + } + } + if u.User != nil || u.Fragment != "" || u.Opaque != "" || u.RawPath != "" || !strings.HasPrefix(u.Path, "/redfish/v1/") || strings.Contains(u.Path, "..") || path.Clean(u.Path) != strings.TrimSuffix(u.Path, "/") { + return "", &Error{Code: "UNSAFE_LINK"} + } + // Queries are accepted only as explicit pagination $skip/$skiptoken parameters. + for key := range u.Query() { + if key != "$skip" && key != "$skiptoken" { + return "", &Error{Code: "UNSAFE_QUERY"} + } + } + u.Scheme = "" + u.Host = "" + return u.String(), nil +} +func (c *Client) request(ctx context.Context, method, ref string, body []byte) (Document, http.Header, error) { + uri, e := c.guard(ref) + if e != nil { + return nil, nil, e + } + req, e := http.NewRequestWithContext(ctx, method, c.origin+uri, bytes.NewReader(body)) + if e != nil { + return nil, nil, &Error{Code: "REQUEST_INVALID", URI: uri} + } + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if c.token != "" { + req.Header.Set("X-Auth-Token", c.token) + } + response, e := c.http.Do(req) + if e != nil { + if ctx.Err() != nil { + return nil, nil, ctx.Err() + } + var verification *tls.CertificateVerificationError + if errors.As(e, &verification) { + return nil, nil, &Error{Code: "TLS_VERIFICATION_FAILED", URI: uri} + } + return nil, nil, &Error{Code: "TRANSPORT_FAILED", URI: uri} + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + code := "HTTP_FAILED" + if response.StatusCode == 401 { + code = "AUTHENTICATION_FAILED" + } + if response.StatusCode == 403 { + code = "ACCESS_DENIED" + } + return nil, nil, &Error{Code: code, URI: uri, Status: response.StatusCode} + } + b, e := io.ReadAll(io.LimitReader(response.Body, (4<<20)+1)) + c.bytes += len(b) + if e != nil || len(b) > 4<<20 || c.bytes > 32<<20 { + return nil, nil, &Error{Code: "RESPONSE_LIMIT", URI: uri} + } + var d Document + if len(b) != 0 { + if e := json.Unmarshal(b, &d); e != nil || d == nil { + return nil, nil, &Error{Code: "INVALID_RESPONSE", URI: uri} + } + } + return d, response.Header.Clone(), nil +} +func (c *Client) Get(ctx context.Context, ref string) (Document, error) { + c.reads++ + if c.reads > 256 { + return nil, &Error{Code: "RESOURCE_LIMIT"} + } + for attempt := 0; attempt < 3; attempt++ { + d, _, err := c.request(ctx, http.MethodGet, ref, nil) + if err == nil { + return d, nil + } + var remote *Error + retry := errors.As(err, &remote) && (remote.Code == "TRANSPORT_FAILED" || remote.Status == 429 || remote.Status == 502 || remote.Status == 503 || remote.Status == 504) + if !retry || attempt == 2 { + return nil, err + } + timer := time.NewTimer(time.Duration(attempt+1) * 100 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + return nil, &Error{Code: "BMC_UNAVAILABLE"} +} +func (c *Client) Open(ctx context.Context) (Document, error) { + root, err := c.Get(ctx, "/redfish/v1/") + if err != nil { + return nil, err + } + service := Link(root, "SessionService") + if service == "" { + return nil, &Error{Code: "SESSION_AUTH_UNSUPPORTED"} + } + service, err = c.guard(service) + if err != nil { + return nil, err + } + sd, err := c.Get(ctx, service) + if err != nil { + return nil, err + } + collection := Link(sd, "Sessions") + if collection == "" { + return nil, &Error{Code: "SESSION_AUTH_UNSUPPORTED"} + } + collection, err = c.guard(collection) + if err != nil { + return nil, err + } + if service != "/redfish/v1/SessionService" || collection != "/redfish/v1/SessionService/Sessions" { + return nil, &Error{Code: "SESSION_URI_UNSUPPORTED"} + } + username, password := c.access.Credentials() + body, _ := json.Marshal(map[string]string{"UserName": username, "Password": password}) + // Never retry an ambiguous login. The token and session URI are never serialized. + c.sessionUncertain = true + _, header, err := c.request(ctx, http.MethodPost, collection, body) + if err != nil { + var failure *Error + if errors.As(err, &failure) && failure.Status >= 400 && failure.Status < 500 && failure.Status != 408 { + c.sessionUncertain = false + } + } + clear(body) + if err != nil { + return nil, err + } + token := header.Get("X-Auth-Token") + session, err := c.guard(header.Get("Location")) + if err != nil || token == "" || !strings.HasPrefix(session, collection+"/") || strings.Contains(strings.TrimPrefix(session, collection+"/"), "/") || strings.Contains(session, "?") { + return nil, &Error{Code: "INVALID_SESSION_RECEIPT"} + } + c.token = token + c.session = session + c.sessionUncertain = false + return root, nil +} +func (c *Client) Close(ctx context.Context) error { + defer func() { c.token = ""; c.http.CloseIdleConnections() }() + if c.session != "" { + _, _, err := c.request(ctx, http.MethodDelete, c.session, nil) + return err + } + if c.sessionUncertain { + return &Error{Code: "SESSION_OUTCOME_UNKNOWN"} + } + return nil +} +func (c *Client) Collection(ctx context.Context, ref string) ([]Resource, error) { + var members []Resource + seen := map[string]bool{} + count := -1 + initial, err := c.guard(ref) + if err != nil { + return nil, err + } + base := strings.Split(initial, "?")[0] + for page := 0; ref != ""; page++ { + if page >= 32 || seen[ref] { + return nil, &Error{Code: "PAGINATION_LIMIT"} + } + seen[ref] = true + normalized, err := c.guard(ref) + if err != nil || strings.Split(normalized, "?")[0] != base { + return nil, &Error{Code: "UNSAFE_PAGINATION"} + } + d, err := c.Get(ctx, ref) + if err != nil { + return nil, err + } + if n, ok := d["Members@odata.count"].(float64); ok { + if n < 0 || n != float64(int(n)) || count >= 0 && count != int(n) { + return nil, &Error{Code: "COLLECTION_CHANGED"} + } + count = int(n) + } + list, ok := d["Members"].([]any) + if !ok { + return nil, &Error{Code: "INCOMPLETE_COLLECTION"} + } + for _, item := range list { + link := Link(Document{"member": item}, "member") + link, err = c.guard(link) + if err != nil { + return nil, err + } + if link == "" || seen["member:"+link] { + return nil, &Error{Code: "DUPLICATE_MEMBER"} + } + seen["member:"+link] = true + value, err := c.Get(ctx, link) + if err != nil { + return nil, err + } + members = append(members, Resource{URI: link, Document: value}) + } + ref = Text(d, "Members@odata.nextLink") + } + if count < 0 || count != len(members) { + return nil, &Error{Code: "INCOMPLETE_COLLECTION"} + } + return members, nil +} +func Text(d Document, k string) string { s, _ := d[k].(string); return s } +func Object(d Document, k string) Document { + switch v := d[k].(type) { + case Document: + return v + case map[string]any: + return Document(v) + default: + return nil + } +} +func Link(d Document, k string) string { return Text(Object(d, k), "@odata.id") } diff --git a/pkg/dpf/redfish/client_test.go b/pkg/dpf/redfish/client_test.go new file mode 100644 index 00000000..5449863a --- /dev/null +++ b/pkg/dpf/redfish/client_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "encoding/json" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestRedfish(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF Redfish") } + +var _ = Describe("GET/session-only transport", func() { + It("allows the service root and logs out exactly its own session", func() { + var methods []string + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/redfish/v1/": + json.NewEncoder(w).Encode(Document{"SessionService": Document{"@odata.id": "/redfish/v1/SessionService"}}) + case "/redfish/v1/SessionService": + json.NewEncoder(w).Encode(Document{"Sessions": Document{"@odata.id": "/redfish/v1/SessionService/Sessions"}}) + case "/redfish/v1/SessionService/Sessions": + Expect(r.Method).To(Equal("POST")) + w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/own") + w.Header().Set("X-Auth-Token", "private-token") + w.WriteHeader(201) + w.Write([]byte(`{}`)) + case "/redfish/v1/SessionService/Sessions/own": + Expect(r.Method).To(Equal("DELETE")) + Expect(r.Header.Get("X-Auth-Token")).To(Equal("private-token")) + w.WriteHeader(204) + default: + w.WriteHeader(404) + } + })) + defer s.Close() + c := &Client{origin: s.URL, http: s.Client()} + _, e := c.Open(context.Background()) + Expect(e).NotTo(HaveOccurred()) + Expect(c.Close(context.Background())).To(Succeed()) + Expect(methods).To(Equal([]string{"GET /redfish/v1/", "GET /redfish/v1/SessionService", "POST /redfish/v1/SessionService/Sessions", "DELETE /redfish/v1/SessionService/Sessions/own"})) + }) + It("rejects unsafe links before credentials leave the process", func() { + c := &Client{origin: "https://example.invalid"} + for _, u := range []string{"https://elsewhere.invalid/redfish/v1/Systems", "http://example.invalid/redfish/v1/Systems", "/redfish/v1/%2e%2e/Accounts", "/redfish/v1/../Accounts", "/redfish/v1/Systems?secret=1", "//elsewhere.invalid/redfish/v1/Systems"} { + _, e := c.guard(u) + Expect(e).To(HaveOccurred()) + } + }) + It("follows only bounded same-collection pages and checks count", func() { + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/redfish/v1/Systems": + if r.URL.RawQuery == "" { + json.NewEncoder(w).Encode(Document{"Members@odata.count": 2, "Members": []any{Document{"@odata.id": "/redfish/v1/Systems/a"}}, "Members@odata.nextLink": "/redfish/v1/Systems?$skip=1"}) + } else { + json.NewEncoder(w).Encode(Document{"Members@odata.count": 2, "Members": []any{Document{"@odata.id": "/redfish/v1/Systems/b"}}}) + } + default: + w.Write([]byte(`{"UUID":"example"}`)) + } + })) + defer s.Close() + c := &Client{origin: s.URL, http: s.Client()} + members, e := c.Collection(context.Background(), "/redfish/v1/Systems") + Expect(e).NotTo(HaveOccurred()) + Expect(members).To(HaveLen(2)) + }) + It("never retries authorization failures or prints response secrets", func() { + calls := 0 + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + w.WriteHeader(401) + w.Write([]byte("private-password")) + })) + defer s.Close() + c := &Client{origin: s.URL, http: s.Client()} + _, e := c.Get(context.Background(), "/redfish/v1/Systems") + Expect(e).To(HaveOccurred()) + Expect(e.Error()).NotTo(ContainSubstring("private-password")) + Expect(calls).To(Equal(1)) + }) + It("stops read retries at cancellation", func() { + ctx, cancel := context.WithCancel(context.Background()) + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { cancel(); w.WriteHeader(503) })) + defer s.Close() + c := &Client{origin: s.URL, http: s.Client()} + start := time.Now() + _, e := c.Get(ctx, "/redfish/v1/Systems") + Expect(e).To(HaveOccurred()) + Expect(time.Since(start)).To(BeNumerically("<", time.Second)) + }) + It("rejects placeholder physical serials", func() { + for _, v := range []string{"", "unknown", "N/A", "00000", "to be filled by o.e.m."} { + Expect(ValidSerial(v)).To(BeFalse()) + } + Expect(ValidSerial(" 000ABC-01 ")).To(BeTrue()) + }) +}) diff --git a/pkg/dpf/redfish/inventory.go b/pkg/dpf/redfish/inventory.go new file mode 100644 index 00000000..6becfc6e --- /dev/null +++ b/pkg/dpf/redfish/inventory.go @@ -0,0 +1,301 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "fmt" + "reflect" + "regexp" + "slices" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +// Qualification is supplied by the installed, reviewed hardware catalog, never site YAML. +// An empty catalog collects candidate facts while keeping completeness unqualified. +type Qualification struct { + VerifyCompleteness func(context.Context, Reader, Resource) error + Adapter string + Model string + BMCFirmware string +} +type Collector struct{ Qualifications []Qualification } + +func ValidSerial(raw string) bool { + s := strings.TrimSpace(raw) + switch strings.ToLower(s) { + case "", "unknown", "n/a", "none", "to be filled by o.e.m.", "default string": + return false + } + return strings.Trim(s, "0- ") != "" +} +func (co Collector) Collect(ctx context.Context, target config.Target, timeout time.Duration) (device inventory.Device, evidence map[string]inventory.Evidence, err error) { + device = inventory.Device{RecordKey: inventory.Digest("dpf-access-v1", []string{target.Role, target.Endpoint, target.SystemURI}), InputRef: target.InputRef, BMCEndpoint: target.Endpoint, ObservationState: "unavailable", Qualification: "unqualified", InventoryCompleteness: "unknown", Authentication: inventory.Authentication{PasswordRead: "unknown", ClientCertificateSupport: "unknown", DPFClientAuthentication: "not-evaluated"}, Time: inventory.TimeObservation{State: "unknown", ReasonCode: "PROPERTY_NOT_EXPOSED"}, Firmware: []inventory.Firmware{}, EvidenceRefs: []string{}} + evidence = map[string]inventory.Evidence{} + expectedSerials := map[string]string{} + c := New(target, timeout) + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if cleanupErr := c.Close(cleanupCtx); cleanupErr != nil { + if err == nil { + err = cleanupErr + } else { + err = fmt.Errorf("%w; BMC session cleanup incomplete", err) + } + device.ReasonCode = "CLEANUP_INCOMPLETE" + } + }() + root, err := c.Open(ctx) + if err != nil { + device.ReasonCode = Code(err) + return device, evidence, err + } + device.Authentication.PasswordRead = "verified" + device.Authentication.Method = "session" + add := func(uri, property string) string { + id := inventory.Digest("dpf-evidence-v1", []string{device.RecordKey, uri, property}) + evidence[id] = inventory.Evidence{Kind: "redfish", InputRef: target.InputRef, ResourceURI: uri, Property: property, ObservedAt: time.Now().UTC(), Vantage: "executor", Adapter: device.ReaderAdapter, Authenticated: true} + device.EvidenceRefs = append(device.EvidenceRefs, id) + return id + } + systems, err := c.Collection(ctx, Link(root, "Systems")) + if err != nil { + return device, evidence, err + } + var system Resource + if target.SystemURI != "" { + for _, s := range systems { + if s.URI == target.SystemURI { + system = s + } + } + } else if len(systems) == 1 { + system = systems[0] + } + if system.URI == "" { + device.ObservationState = "ambiguous" + return device, evidence, &Error{Code: "SYSTEM_SELECTION_REQUIRED"} + } + device.SystemURI = system.URI + device.Manufacturer = Text(system.Document, "Manufacturer") + device.Model = Text(system.Document, "Model") + device.SystemUUID = Text(system.Document, "UUID") + adapter := "" + maker := strings.ToLower(device.Manufacturer) + switch { + case strings.Contains(maker, "nvidia") || strings.Contains(maker, "mellanox"): + adapter = "bluefield-card-v1" + case strings.Contains(maker, "hpe") || strings.Contains(maker, "hewlett"): + adapter = "hpe-standard-v1" + case strings.Contains(maker, "dell"): + adapter = "dell-standard-v1" + } + device.ReaderAdapter = adapter + action := Object(Object(system.Document, "Actions"), "#ComputerSystem.Reset") + if values, ok := action["ResetType@Redfish.AllowableValues"].([]any); ok { + for _, v := range values { + if value, ok := v.(string); ok { + device.AdvertisedResetTypes = append(device.AdvertisedResetTypes, value) + } + } + } + add(system.URI, "UUID") + if target.Role == "dpus" { + if adapter != "bluefield-card-v1" { + return device, evidence, &Error{Code: "UNSUPPORTED_DPU_IDENTITY"} + } + chassis, er := c.Collection(ctx, Link(root, "Chassis")) + if er != nil { + return device, evidence, er + } + var cards []Resource + for _, ch := range chassis { + model := Text(ch.Document, "Model") + if strings.Contains(strings.ToLower(model), "bluefield") && ValidSerial(Text(ch.Document, "SerialNumber")) { + if c.linkedToSystem(ch.Document, system.URI) { + cards = append(cards, ch) + } + } + } + if len(cards) != 1 { + return device, evidence, &Error{Code: "CARD_IDENTITY_AMBIGUOUS"} + } + device.SerialNumberRaw = Text(cards[0].Document, "SerialNumber") + expectedSerials[cards[0].URI] = device.SerialNumberRaw + device.SerialNumber = strings.TrimSpace(device.SerialNumberRaw) + device.Model = Text(cards[0].Document, "Model") + device.IdentityKey = []string{"nvidia", device.SerialNumber} + device.ID = inventory.PhysicalID("dpu", device.IdentityKey) + add(cards[0].URI, "SerialNumber") + } else { + if adapter != "hpe-standard-v1" && adapter != "dell-standard-v1" { + return device, evidence, &Error{Code: "UNSUPPORTED_HOST_ADAPTER"} + } + if !validHostUUID(device.SystemUUID) { + return device, evidence, &Error{Code: "HOST_IDENTITY_MISSING"} + } + device.IdentityKey = []string{"system-uuid", strings.ToLower(device.SystemUUID)} + device.ID = inventory.PhysicalID("host", device.IdentityKey) + // NetworkInterfaces link the selected System to physical NetworkAdapters. + interfaces, er := c.Collection(ctx, Link(system.Document, "NetworkInterfaces")) + if er != nil { + return device, evidence, er + } + seen := map[string]bool{} + for _, iface := range interfaces { + uri := Link(Object(iface.Document, "Links"), "NetworkAdapter") + if uri == "" { + return device, evidence, &Error{Code: "ADAPTER_LINK_MISSING"} + } + uri, er = c.guard(uri) + if er != nil { + return device, evidence, er + } + if seen[uri] { + continue + } + seen[uri] = true + card, er := c.Get(ctx, uri) + if er != nil { + return device, evidence, er + } + raw := Text(card, "SerialNumber") + expectedSerials[uri] = raw + if !ValidSerial(raw) { + return device, evidence, &Error{Code: "CARD_SERIAL_MISSING"} + } + // Retain every physical network card; PCI/vendor omission cannot prove absence. + serial := strings.TrimSpace(raw) + device.Adapters = append(device.Adapters, inventory.Adapter{Key: inventory.Digest("dpf-adapter-v1", []string{device.ID, serial}), SerialNumber: serial, SerialNumberRaw: raw, Model: Text(card, "Model"), Selection: "not-selected", EvidenceRefs: []string{add(uri, "SerialNumber")}}) + } + again, er := c.Collection(ctx, Link(system.Document, "NetworkInterfaces")) + if er != nil || !sameMembers(interfaces, again) { + return device, evidence, &Error{Code: "INVENTORY_CHANGED"} + } + device.InventoryCompleteness = "unqualified" + } + bmcFirmware := "" + if uri := Link(root, "Managers"); uri != "" { + managers, er := c.Collection(ctx, uri) + if er == nil && len(managers) == 1 { + manager := managers[0] + bmcFirmware = Text(manager.Document, "FirmwareVersion") + if bmcFirmware != "" { + device.Firmware = append(device.Firmware, inventory.Firmware{ResourceURI: manager.URI, Component: "bmc", State: "known", RawVersion: bmcFirmware, Activation: "unknown", EvidenceRefs: []string{add(manager.URI, "FirmwareVersion")}}) + } + if npURI := Link(manager.Document, "NetworkProtocol"); npURI != "" { + if np, readErr := c.Get(ctx, npURI); readErr == nil { + ntp := Object(np, "NTP") + if enabled, ok := ntp["ProtocolEnabled"].(bool); ok { + device.Time.NTPEnabled = &enabled + } + if servers, ok := ntp["NTPServers"].([]any); ok { + for _, v := range servers { + if server, ok := v.(string); ok { + device.Time.NTPServers = append(device.Time.NTPServers, server) + } + } + } + add(npURI, "NTP") + } + } + + device.Time.DateTime = Text(manager.Document, "DateTime") + device.Time.Offset = Text(manager.Document, "DateTimeLocalOffset") + if device.Time.DateTime != "" { + device.Time.State = "known" + device.Time.ReasonCode = "" + } + add(manager.URI, "DateTime") + } + } + found := map[string]bool{} + for _, existing := range device.Firmware { + found[existing.Component] = true + } + if uri := Link(root, "UpdateService"); uri != "" { + update, er := c.Get(ctx, uri) + if er == nil { + fw, er := c.Collection(ctx, Link(update, "FirmwareInventory")) + if er == nil { + for _, f := range fw { + name := strings.ToLower(Text(f.Document, "Name")) + component := "other" + for _, candidate := range []string{"bmc", "cec", "atf", "nic", "golden"} { + if strings.Contains(name, candidate) { + component = candidate + break + } + } + if component == "atf" { + component = "atf-uefi" + } + version := Text(f.Document, "Version") + if version == "" { + continue + } + found[component] = true + device.Firmware = append(device.Firmware, inventory.Firmware{ResourceURI: f.URI, Component: component, State: "known", RawVersion: version, Activation: "unknown", EvidenceRefs: []string{add(f.URI, "Version")}}) + } + } + } + } + for _, component := range []string{"bmc", "cec", "atf-uefi", "nic", "dpu-os", "golden"} { + if !found[component] { + device.Firmware = append(device.Firmware, inventory.Firmware{Component: component, State: "unknown", ReasonCode: "PROPERTY_NOT_EXPOSED", Activation: "unknown", EvidenceRefs: []string{}}) + } + } + for _, q := range co.Qualifications { + if q.Adapter == adapter && q.Model == device.Model && q.BMCFirmware == bmcFirmware && bmcFirmware != "" && q.VerifyCompleteness != nil && q.VerifyCompleteness(ctx, c, system) == nil { + device.Qualification = "qualified" + device.InventoryCompleteness = "complete" + } + } + for uri, raw := range expectedSerials { + current, e := c.Get(ctx, uri) + if e != nil || Text(current, "SerialNumber") != raw { + return device, evidence, &Error{Code: "IDENTITY_CHANGED"} + } + } + refreshed, er := c.Get(ctx, system.URI) + if er != nil || Text(refreshed, "UUID") != device.SystemUUID || Text(refreshed, "SerialNumber") != Text(system.Document, "SerialNumber") { + return device, evidence, &Error{Code: "IDENTITY_CHANGED"} + } + device.ObservationState = "observed" + return device, evidence, nil +} +func sameMembers(a, b []Resource) bool { + keys := func(r []Resource) []string { + s := make([]string, 0, len(r)) + for _, v := range r { + s = append(s, v.URI) + } + slices.Sort(s) + return s + } + return reflect.DeepEqual(keys(a), keys(b)) +} + +func (c *Client) linkedToSystem(card Document, system string) bool { + links := Object(card, "Links") + if values, ok := links["ComputerSystems"].([]any); ok { + for _, value := range values { + uri, err := c.guard(Link(Document{"v": value}, "v")) + if err == nil && uri == system { + return true + } + } + } + return false +} + +var hostUUIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +func validHostUUID(s string) bool { + return hostUUIDPattern.MatchString(s) && strings.Trim(s, "0-") != "" && strings.Trim(strings.ToLower(s), "f-") != "" +} diff --git a/pkg/dpf/redfish/inventory_test.go b/pkg/dpf/redfish/inventory_test.go new file mode 100644 index 00000000..cc29cea2 --- /dev/null +++ b/pkg/dpf/redfish/inventory_test.go @@ -0,0 +1,129 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "encoding/json" + "encoding/pem" + "fmt" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "time" +) + +var _ = Describe("physical card identity", func() { + It("records the linked card and rejects a replacement at the same URI", func() { + for _, scenario := range []struct{ replace, absolute bool }{{false, false}, {true, false}, {false, true}, {true, true}} { + replace := scenario.replace + cardReads := 0 + writes := []string{} + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + writes = append(writes, r.Method+" "+r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + var doc any + switch r.URL.Path { + case "/redfish/v1/": + doc = Document{"SessionService": Document{"@odata.id": "/redfish/v1/SessionService"}, "Systems": Document{"@odata.id": "/redfish/v1/Systems"}, "Chassis": Document{"@odata.id": "/redfish/v1/Chassis"}} + case "/redfish/v1/SessionService": + doc = Document{"Sessions": Document{"@odata.id": "/redfish/v1/SessionService/Sessions"}} + case "/redfish/v1/SessionService/Sessions": + w.Header().Set("X-Auth-Token", "private-session") + w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/owned") + w.WriteHeader(201) + doc = Document{} + case "/redfish/v1/SessionService/Sessions/owned": + w.WriteHeader(204) + return + case "/redfish/v1/Systems": + doc = Document{"Members@odata.count": 1, "Members": []any{Document{"@odata.id": "/redfish/v1/Systems/selected"}}} + case "/redfish/v1/Systems/selected": + doc = Document{"UUID": "11111111-1111-4111-8111-111111111111", "Manufacturer": "NVIDIA", "Model": "BlueField-3"} + case "/redfish/v1/Chassis": + doc = Document{"Members@odata.count": 1, "Members": []any{Document{"@odata.id": "/redfish/v1/Chassis/Card"}}} + case "/redfish/v1/Chassis/Card": + cardReads++ + serial := "CARD-A" + if replace && cardReads > 1 { + serial = "CARD-B" + } + doc = Document{"Model": "BlueField-3", "SerialNumber": serial, "Links": Document{"ComputerSystems": []any{Document{"@odata.id": "/redfish/v1/Systems/selected"}}}} + default: + w.WriteHeader(404) + return + } + if scenario.absolute { + absoluteLinks(doc, "https://"+r.Host) + } + json.NewEncoder(w).Encode(doc) + })) + dir, e := os.MkdirTemp("", "dpf-card-") + Expect(e).NotTo(HaveOccurred()) + ca := filepath.Join(dir, "ca.pem") + Expect(os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0600)).To(Succeed()) + Expect(os.Setenv("DPF_CARD_PASSWORD", "private-password")).To(Succeed()) + source := fmt.Sprintf("management:\n kubeconfigRef: {file: unused}\ncredentials:\n read: {username: reader, passwordRef: {env: DPF_CARD_PASSWORD}}\ndiscovery:\n tls: {caFile: '%s'}\n dpus: [{bmcEndpoint: '%s', credentialRef: read, systemURI: /redfish/v1/Systems/selected}]\n hosts: [{bmcEndpoint: 'https://192.0.2.1', credentialRef: read}]\n", ca, server.URL) + in, e := config.Decode([]byte(source), filepath.Join(dir, "site.yaml")) + Expect(e).NotTo(HaveOccurred()) + req, e := config.Resolve(in) + Expect(e).NotTo(HaveOccurred()) + device, evidence, e := (Collector{}).Collect(context.Background(), req.Targets[0], time.Second) + if replace { + Expect(e).To(HaveOccurred()) + Expect(Code(e)).To(Equal("IDENTITY_CHANGED")) + } else { + Expect(e).NotTo(HaveOccurred()) + Expect(device.SerialNumber).To(Equal("CARD-A")) + Expect(device.ObservationState).To(Equal("observed")) + Expect(device.Qualification).To(Equal("unqualified")) + Expect(evidence).NotTo(BeEmpty()) + } + Expect(writes).To(Equal([]string{"POST /redfish/v1/SessionService/Sessions", "DELETE /redfish/v1/SessionService/Sessions/owned"})) + server.Close() + os.RemoveAll(dir) + os.Unsetenv("DPF_CARD_PASSWORD") + } + }) + It("uses ComputerSystem linkage rather than the first chassis", func() { + card := Document{"Links": Document{"ComputerSystems": []any{Document{"@odata.id": "/redfish/v1/Systems/selected"}}}} + Expect((&Client{origin: "https://bmc.invalid"}).linkedToSystem(card, "/redfish/v1/Systems/selected")).To(BeTrue()) + Expect((&Client{origin: "https://bmc.invalid"}).linkedToSystem(card, "/redfish/v1/Systems/other")).To(BeFalse()) + card["Links"] = Document{"ComputerSystems": []any{Document{"@odata.id": "https://foreign.invalid/redfish/v1/Systems/selected"}}} + Expect((&Client{origin: "https://bmc.invalid"}).linkedToSystem(card, "/redfish/v1/Systems/selected")).To(BeFalse()) + }) +}) + +var _ = Describe("host UUID syntax", func() { + It("rejects placeholders before assigning identity", func() { + for _, value := range []string{"unknown", "default", "00000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff", "bad-uuid"} { + Expect(validHostUUID(value)).To(BeFalse()) + } + Expect(validHostUUID("11111111-1111-4111-8111-111111111111")).To(BeTrue()) + }) +}) + +func absoluteLinks(value any, origin string) { + switch v := value.(type) { + case Document: + for key, item := range v { + if key == "@odata.id" { + if s, ok := item.(string); ok { + v[key] = origin + s + } + } else { + absoluteLinks(item, origin) + } + } + case []any: + for _, item := range v { + absoluteLinks(item, origin) + } + } +} diff --git a/pkg/target/dpf/driver.go b/pkg/target/dpf/driver.go new file mode 100644 index 00000000..661c0307 --- /dev/null +++ b/pkg/target/dpf/driver.go @@ -0,0 +1,132 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package dpf binds the independent DPU-plane discovery CLI to typed services. +package dpf + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/artifacts" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/catalog" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/discovery" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/probe" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" + apperrors "github.com/nvidia/k8s-launch-kit/pkg/errors" + "github.com/nvidia/k8s-launch-kit/pkg/target" +) + +// Descriptor exposes the inventory preview, keeping provisioning unavailable. +func Descriptor() target.Descriptor { + phases := map[target.Phase]target.Capability{} + for _, p := range append(target.PublicPhases(), target.Pipeline) { + phases[p] = target.Capability{Available: false, Reason: "DPF provisioning is not implemented in this build"} + } + phases[target.Discover] = target.Capability{Available: true, Reason: "experimental inventory discovery; management probes require qualified release assets; hardware catalog qualification pending"} + return target.Descriptor{Name: target.DPF, Description: "Independent DPF inventory discovery preview", Phases: phases} +} + +type Request struct { + UserConfig string + OutputPath string + ExplicitOutput bool + Stdout io.Writer + Stderr io.Writer +} +type driver struct{ request Request } + +func NewDiscoverAdapter(r Request) target.Driver { return driver{request: r} } +func (d driver) Descriptor() target.Descriptor { return Descriptor() } +func (d driver) Bind(inv target.Invocation) (target.Operation, error) { + if inv.Phase != target.Discover { + return nil, &target.PhaseUnavailableError{Name: target.DPF, Phase: inv.Phase, Reason: Descriptor().Capability(inv.Phase).Reason} + } + if inv.Execution.DryRun { + return nil, fmt.Errorf("DPF discovery uses discovery.probes.mode: inventory to omit probes") + } + r := d.request + if r.Stdout == nil { + r.Stdout = os.Stdout + } + if r.Stderr == nil { + r.Stderr = os.Stderr + } + return target.OperationFunc(func(parent context.Context) error { + ctx, stop := signal.NotifyContext(parent, terminationSignals()...) + defer stop() + report := discovery.NewReport() + var store *artifacts.Store + fail := func(code int, message string) { + report.Execution = "failed" + report.Errors = append(report.Errors, inventory.Problem{Code: "INVALID_CONFIGURATION", Message: message, ExitCode: code}) + } + input, err := config.Load(r.UserConfig) + if err != nil { + fail(2, err.Error()) + } else { + resolved, err := config.Resolve(input) + if err != nil { + fail(2, err.Error()) + } else if _, err = discovery.ManagementAccess(resolved); err != nil { + fail(2, err.Error()) + } else { + // Unqualified image assets reject management mode before any remote operation. + assets := catalog.ProbeAssets() + if resolved.Mode == "management" && assets.Validate() != nil { + fail(2, "management probes unavailable until the pinned diagnostic image passes Kubernetes runtime qualification; explicitly select inventory mode for the read-only preview") + } else { + store, err = artifacts.New(input, r.OutputPath, r.ExplicitOutput) + if err != nil { + fail(1, err.Error()) + } else { + if !inv.Output.Quiet { + fmt.Fprintln(r.Stderr, "Inspecting explicitly selected management and BMC scope") + } + deps := discovery.Dependencies{BMC: redfish.Collector{Qualifications: catalog.Hardware()}, Probes: probe.Runner{Assets: assets, Journal: probe.FileJournal(filepath.Join(store.RunDir, "resources.json"))}} + if resolved.Input.Site.Inventory == nil { + resolved.Input.Site.Inventory = store.PriorInventory + } + report = discovery.Collect(ctx, resolved, store.RunID(), deps) + publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + err = store.Publish(publishCtx, &report) + cancel() + if err != nil { + report.Execution = "failed" + report.Errors = append(report.Errors, inventory.Problem{Code: "PUBLICATION_FAILED", Message: err.Error(), ExitCode: 1}) + _ = store.Report(&report) + } + } + } + } + } + if inv.Output.Format == "json" { + if err := json.NewEncoder(r.Stdout).Encode(report); err != nil { + fmt.Fprintln(r.Stderr, "cannot encode DPF discovery result") + return apperrors.MarkReported(apperrors.NewGeneralError("cannot encode DPF discovery result", nil)) + } + } else { + fmt.Fprintf(r.Stdout, "DPF discovery: %s (%s)\n", report.Discovery.Outcome, report.Discovery.Mode) + for _, c := range report.Discovery.Checks { + if c.State != "passed" { + fmt.Fprintf(r.Stdout, "%s %s: %s (%s)\n", c.ID, c.Subject, c.State, c.ReasonCode) + } + } + for _, e := range report.Errors { + fmt.Fprintf(r.Stdout, "%s: %s\n", e.Code, e.Message) + } + for _, a := range report.Artifacts { + fmt.Fprintf(r.Stdout, "%s: %s\n", a.Kind, a.Path) + } + } + return apperrors.NewExitStatus(discovery.ExitCode(report)) + }), nil +} diff --git a/pkg/target/dpf/driver_test.go b/pkg/target/dpf/driver_test.go new file mode 100644 index 00000000..2f8857bd --- /dev/null +++ b/pkg/target/dpf/driver_test.go @@ -0,0 +1,40 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package dpf + +import ( + "bytes" + "context" + "encoding/json" + "github.com/nvidia/k8s-launch-kit/pkg/errors" + "github.com/nvidia/k8s-launch-kit/pkg/target" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" +) + +func TestDriver(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF target") } + +var _ = Describe("target boundary", func() { + It("reports one JSON document for invalid input and returns exit two", func() { + var stdout, stderr bytes.Buffer + d := NewDiscoverAdapter(Request{Stdout: &stdout, Stderr: &stderr}) + op, e := d.Bind(target.Invocation{Target: target.DPF, Phase: target.Discover, Output: target.OutputOptions{Format: "json"}}) + Expect(e).NotTo(HaveOccurred()) + e = op.Run(context.Background()) + code, ok := errors.IsExitStatus(e) + Expect(ok).To(BeTrue()) + Expect(code).To(Equal(2)) + var result map[string]any + Expect(json.Unmarshal(stdout.Bytes(), &result)).To(Succeed()) + Expect(result["target"]).To(Equal("dpf")) + Expect(result["execution"]).To(Equal("failed")) + }) + It("advertises only discovery and rejects other phases before binding", func() { + for _, p := range []target.Phase{target.Generate, target.Deploy, target.Validate, target.Pipeline} { + _, e := NewDiscoverAdapter(Request{}).Bind(target.Invocation{Target: target.DPF, Phase: p}) + Expect(e).To(HaveOccurred()) + Expect(Descriptor().Capability(p).Available).To(BeFalse()) + } + }) +}) diff --git a/pkg/target/dpf/signals_unix.go b/pkg/target/dpf/signals_unix.go new file mode 100644 index 00000000..d8b24489 --- /dev/null +++ b/pkg/target/dpf/signals_unix.go @@ -0,0 +1,12 @@ +//go:build !windows + +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package dpf + +import ( + "os" + "syscall" +) + +func terminationSignals() []os.Signal { return []os.Signal{os.Interrupt, syscall.SIGTERM} } diff --git a/pkg/target/dpf/signals_windows.go b/pkg/target/dpf/signals_windows.go new file mode 100644 index 00000000..e6e508ee --- /dev/null +++ b/pkg/target/dpf/signals_windows.go @@ -0,0 +1,9 @@ +//go:build windows + +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package dpf + +import "os" + +func terminationSignals() []os.Signal { return []os.Signal{os.Interrupt} } diff --git a/pkg/yamlinput/node.go b/pkg/yamlinput/node.go new file mode 100644 index 00000000..641ff7c4 --- /dev/null +++ b/pkg/yamlinput/node.go @@ -0,0 +1,108 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package yamlinput provides target-neutral source-preserving YAML operations. +package yamlinput + +import ( + "bytes" + "fmt" + yaml3 "gopkg.in/yaml.v3" +) + +func Expand(node *yaml3.Node) (*yaml3.Node, error) { + budget := 200000 + return expand(node, map[*yaml3.Node]bool{}, &budget) +} +func expand(node *yaml3.Node, active map[*yaml3.Node]bool, budget *int) (*yaml3.Node, error) { + *budget-- + if *budget < 0 { + return nil, fmt.Errorf("expanded YAML exceeds node limit") + } + if node == nil { + return nil, fmt.Errorf("missing YAML node") + } + if active[node] { + return nil, fmt.Errorf("cyclic YAML alias") + } + active[node] = true + defer delete(active, node) + if node.Kind == yaml3.AliasNode { + return expand(node.Alias, active, budget) + } + copy := *node + copy.Anchor = "" + copy.Content = nil + for _, child := range node.Content { + expanded, err := expand(child, active, budget) + if err != nil { + return nil, err + } + copy.Content = append(copy.Content, expanded) + } + if copy.Kind != yaml3.MappingNode { + return ©, nil + } + entries := copy.Content + copy.Content = nil + for i := 0; i+1 < len(entries); i += 2 { + if entries[i].Tag != "!!merge" { + continue + } + mappings := []*yaml3.Node{entries[i+1]} + if entries[i+1].Kind == yaml3.SequenceNode { + mappings = entries[i+1].Content + } + for _, mapping := range mappings { + if mapping.Kind != yaml3.MappingNode { + return nil, fmt.Errorf("YAML merge value must be a mapping or sequence of mappings") + } + for j := 0; j+1 < len(mapping.Content); j += 2 { + putYAMLField(©, mapping.Content[j], mapping.Content[j+1], false) + } + } + } + for i := 0; i+1 < len(entries); i += 2 { + if entries[i].Tag != "!!merge" { + putYAMLField(©, entries[i], entries[i+1], true) + } + } + return ©, nil +} + +func putYAMLField(mapping, key, value *yaml3.Node, replace bool) { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key.Value { + if replace { + mapping.Content[i], mapping.Content[i+1] = key, value + } + return + } + } + mapping.Content = append(mapping.Content, key, value) +} + +func Encode(node *yaml3.Node) ([]byte, error) { + var b bytes.Buffer + e := yaml3.NewEncoder(&b) + e.SetIndent(2) + if err := e.Encode(node); err != nil { + return nil, err + } + if err := e.Close(); err != nil { + return nil, err + } + return b.Bytes(), nil +} +func Set(mapping *yaml3.Node, key string, value *yaml3.Node) { + for i := 0; i < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key { + value.HeadComment = mapping.Content[i+1].HeadComment + value.LineComment = mapping.Content[i+1].LineComment + value.FootComment = mapping.Content[i+1].FootComment + mapping.Content[i+1] = value + return + } + } + mapping.Content = append(mapping.Content, &yaml3.Node{Kind: yaml3.ScalarNode, Tag: "!!str", Value: key}, value) +} diff --git a/skills/k8s-launch-kit-discover/SKILL.md b/skills/k8s-launch-kit-discover/SKILL.md index 15803eae..36892d7e 100644 --- a/skills/k8s-launch-kit-discover/SKILL.md +++ b/skills/k8s-launch-kit-discover/SKILL.md @@ -11,6 +11,8 @@ metadata: > **PREREQUISITE:** Read `../k8s-launch-kit-shared/SKILL.md` for install paths, global flags, and output modes. +For the independent DPF preview, read [DPF discovery](../../docs/user/dpf-discovery.md) and use its explicit management/BMC YAML. Keep the empty hardware/probe-image release gates visible; do not run Host discovery against the DPF estate. + This workflow uses the default `host` target; `--target host` is equivalent. The CLI snapshots discovery arguments and runs the concrete Host discovery operation through the target registry; command syntax and artifacts are From 6b4fd58e6303ede6d069c43bdd72b452c90dec1b Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 5 Nov 2026 10:13:43 +0100 Subject: [PATCH 02/35] feat(dpf): allow explicit insecure BMC TLS discovery Signed-off-by: Alexander Maslennikov --- README.md | 3 ++ docs/user/dpf-discovery.md | 31 ++++++++++++++ pkg/dpf/config/config.go | 26 +++++++++--- pkg/dpf/config/config_test.go | 54 +++++++++++++++++++++++++ pkg/dpf/discovery/discover.go | 2 +- pkg/dpf/discovery/discover_test.go | 36 +++++++++++++++++ pkg/dpf/inventory/types.go | 7 ++-- pkg/dpf/redfish/client_test.go | 32 +++++++++++++++ pkg/target/dpf/driver.go | 5 +++ skills/k8s-launch-kit-discover/SKILL.md | 8 ++++ 10 files changed, 194 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 309324cd..96492f68 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ Standard SR-IOV, RDMA-shared and host-device profiles enable NIC tuning by defau The current lifecycle uses the `host` target by default. The `dpf` target provides an [experimental inventory discovery preview](docs/user/dpf-discovery.md); its provisioning phases remain unavailable. +DPF BMC discovery verifies TLS by default; lab endpoints can explicitly opt in to +`tls.insecureSkipVerify: true`, recorded in discovery reports. See the preview guide +for its scope and security implications. Spectrum-X RA2.4 and RA3.1 support Network Operator 26.10 and newer catalogued releases; RA2.3 is limited to 26.7. Both use a doSPCX bundle and derive per-group platform diff --git a/docs/user/dpf-discovery.md b/docs/user/dpf-discovery.md index 36034acd..04d73263 100644 --- a/docs/user/dpf-discovery.md +++ b/docs/user/dpf-discovery.md @@ -86,6 +86,37 @@ are relative to the input YAML and are rebased when saved elsewhere. Private fil are not copied. Endpoint origins require HTTPS; redirects, cross-origin links, unbounded pages and arbitrary action endpoints are rejected. +For BMCs with untrusted certificates, explicitly opt in per endpoint: + +```yaml +discovery: + dpus: + - bmcEndpoint: https://192.0.2.11 + credentialRef: dpu-reader + tls: + insecureSkipVerify: true +``` + +This fragment extends the complete input above. `insecureSkipVerify` is a YAML +boolean, defaults to `false`, and skips certificate-chain and hostname verification +for executor-to-BMC Redfish connections only. HTTPS remains required, but the +server is not authenticated and BMC credentials can be exposed to impersonation. +Prefer a verified certificate/CA through `caFile` when available. + +The same field is accepted under `discovery.tls` as a shared BMC policy. An +endpoint `tls` block replaces the entire shared policy; `{insecureSkipVerify: false}` +restores system-trust verification for that endpoint. Omitted/null values use the +policy's default. `caFile` and `insecureSkipVerify: true` cannot be combined in the +effective policy. TLS policies are independent even when endpoints share credentials. + +The CLI warns on stderr unless quiet output is requested and records +`tlsInsecureSkipVerify: true` on the affected `inventory.requestedScope` entries +in JSON and saved YAML, including failed observations. This records the selected +policy, not successful TLS or hardware verification. Changing the policy changes +the scope digest: use a fresh seed and new output when switching policies. +Management kubeconfig TLS and management-network probe TLS checks remain verified; +this BMC option does not relax them or qualify hardware. + ## Observations, reports and refresh Physical identity comes from qualified card/system fields, keeping raw serials. diff --git a/pkg/dpf/config/config.go b/pkg/dpf/config/config.go index 75e20aef..fedc3e81 100644 --- a/pkg/dpf/config/config.go +++ b/pkg/dpf/config/config.go @@ -39,7 +39,8 @@ type Credential struct { PasswordRef PasswordRef `yaml:"passwordRef"` } type TLS struct { - CAFile *string `yaml:"caFile,omitempty"` + CAFile *string `yaml:"caFile,omitempty"` + InsecureSkipVerify *bool `yaml:"insecureSkipVerify,omitempty"` } type Endpoint struct { Name *string `yaml:"name,omitempty"` @@ -96,6 +97,7 @@ func (a Access) Credentials() (string, string) { return a.username, a.password } func (a Access) TLSConfig() *tls.Config { return a.tls.Clone() } func (a Access) PublicCA() []byte { return bytes.Clone(a.ca) } func (a Access) String() string { return "" } +func (a Access) InsecureSkipVerify() bool { return a.tls != nil && a.tls.InsecureSkipVerify } type Target struct { Endpoint string @@ -291,7 +293,11 @@ func Resolve(in Input) (Request, error) { return r, fmt.Errorf("discovery requires 1-256 explicit DPU/host endpoints and both roles") } seen := map[string]bool{} - accesses := map[string]Access{} + type accessKey struct { + credential, caPath string + insecure bool + } + accesses := map[accessKey]Access{} for _, group := range []struct { role string endpoints []Endpoint @@ -328,10 +334,14 @@ func Resolve(in Input) (Request, error) { } caPath = resolvePath(base, *policy.CAFile) } - cacheKey := e.CredentialRef + "\x00" + caPath + insecure := policy != nil && policy.InsecureSkipVerify != nil && *policy.InsecureSkipVerify + if insecure && caPath != "" { + return r, fmt.Errorf("%s/tls: caFile and insecureSkipVerify: true are mutually exclusive", ptr) + } + cacheKey := accessKey{credential: e.CredentialRef, caPath: caPath, insecure: insecure} a, ok := accesses[cacheKey] if !ok { - a, err = resolveAccess(c, caPath, base) + a, err = resolveAccess(c, caPath, base, insecure) if err != nil { return r, fmt.Errorf("%s: %w", ptr, err) } @@ -349,7 +359,7 @@ func resolvePath(base, p string) string { } return filepath.Join(base, p) } -func resolveAccess(c Credential, caPath, base string) (Access, error) { +func resolveAccess(c Credential, caPath, base string, insecure bool) (Access, error) { p := c.PasswordRef if (p.Env == nil) == (p.File == nil) { return Access{}, fmt.Errorf("passwordRef requires exactly one env or file") @@ -390,7 +400,8 @@ func resolveAccess(c Credential, caPath, base string) (Access, error) { return Access{}, fmt.Errorf("invalid CA file") } } - return Access{username: c.Username, password: password, tls: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}, ca: ca}, nil + // Certificate verification is disabled only by explicit site configuration. + return Access{username: c.Username, password: password, tls: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots, InsecureSkipVerify: insecure}, ca: ca}, nil } func Origin(s string) (string, error) { u, err := url.Parse(s) @@ -487,6 +498,9 @@ func validateScalarTypes(n *yaml.Node, p string) error { if p == "/discovery/limits/maxConcurrentBMCs" { expected = "!!int" } + if strings.HasSuffix(p, "/tls/insecureSkipVerify") { + expected = "!!bool" + } if n.Tag != expected { return fmt.Errorf("%s has invalid scalar type at line %d", p, n.Line) } diff --git a/pkg/dpf/config/config_test.go b/pkg/dpf/config/config_test.go index 25f55788..9f324597 100644 --- a/pkg/dpf/config/config_test.go +++ b/pkg/dpf/config/config_test.go @@ -67,6 +67,60 @@ var _ = Describe("DPF input", func() { Expect(e).NotTo(HaveOccurred()) Expect(r.MaxConcurrentBMCs).To(Equal(4)) }) + It("keeps BMC verification enabled unless explicitly disabled and preserves presence", func() { + for _, value := range []string{"", "false", "null", "true"} { + text := seed + if value != "" { + text += " tls: {insecureSkipVerify: " + value + "}\n" + } + in, err := Decode([]byte(text), "/site/in.yaml") + Expect(err).NotTo(HaveOccurred()) + r, err := Resolve(in) + Expect(err).NotTo(HaveOccurred()) + for _, target := range r.Targets { + Expect(target.Access.InsecureSkipVerify()).To(Equal(value == "true")) + Expect(target.Access.TLSConfig().InsecureSkipVerify).To(Equal(value == "true")) + } + b, err := Saved(in, &inventory.Inventory{FormatVersion: "v1alpha1"}, "/results/out.yaml") + Expect(err).NotTo(HaveOccurred()) + if value == "" { + Expect(string(b)).NotTo(ContainSubstring("insecureSkipVerify")) + } else { + Expect(string(b)).To(ContainSubstring("insecureSkipVerify: " + value)) + } + } + }) + It("isolates cached TLS policies and lets endpoint false replace global true", func() { + for _, role := range []string{"dpus", "hosts"} { + text := seed + " tls: {insecureSkipVerify: true}\n" + lines := strings.Split(text, "\n") + for i, line := range lines { + if strings.HasPrefix(line, " "+role+":") { + lines[i] = strings.Replace(line, "credentialRef: read", "credentialRef: read, tls: {insecureSkipVerify: false}", 1) + } + } + in, err := Decode([]byte(strings.Join(lines, "\n")), "/site/in.yaml") + Expect(err).NotTo(HaveOccurred()) + r, err := Resolve(in) + Expect(err).NotTo(HaveOccurred()) + for _, target := range r.Targets { + Expect(target.Access.InsecureSkipVerify()).To(Equal(target.Role != role)) + clone := target.Access.TLSConfig() + clone.InsecureSkipVerify = !clone.InsecureSkipVerify + Expect(target.Access.InsecureSkipVerify()).To(Equal(target.Role != role)) + } + } + }) + It("rejects nonboolean insecure settings and contradictory CA trust", func() { + for _, value := range []string{"'true'", "1", "''"} { + _, err := Decode([]byte(seed+" tls: {insecureSkipVerify: "+value+"}\n"), "/site/in.yaml") + Expect(err).To(HaveOccurred()) + } + in, err := Decode([]byte(seed+" tls: {insecureSkipVerify: true, caFile: ca.pem}\n"), "/site/in.yaml") + Expect(err).NotTo(HaveOccurred()) + _, err = Resolve(in) + Expect(err).To(MatchError(ContainSubstring("mutually exclusive"))) + }) It("normalizes DNS/default ports and rejects paths, userinfo and duplicate origins", func() { origin, e := Origin("https://EXAMPLE.invalid:443/") Expect(e).NotTo(HaveOccurred()) diff --git a/pkg/dpf/discovery/discover.go b/pkg/dpf/discovery/discover.go index b1826826..6f80bd2f 100644 --- a/pkg/dpf/discovery/discover.go +++ b/pkg/dpf/discovery/discover.go @@ -35,7 +35,7 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc inv := &inventory.Inventory{FormatVersion: inventory.FormatVersion, Collection: inventory.Collection{RunID: runID, StartedAt: time.Now().UTC(), Mode: r.Mode, Collector: "l8k-go-discovery-v1"}, RequestedScope: []inventory.ScopeEndpoint{}, DPUs: []inventory.Device{}, Hosts: []inventory.Device{}, Associations: []inventory.Association{}, Evidence: map[string]inventory.Evidence{}} report.Inventory = inv for _, t := range r.Targets { - inv.RequestedScope = append(inv.RequestedScope, inventory.ScopeEndpoint{Role: t.Role, Endpoint: t.Endpoint, SystemURI: t.SystemURI}) + inv.RequestedScope = append(inv.RequestedScope, inventory.ScopeEndpoint{Role: t.Role, Endpoint: t.Endpoint, SystemURI: t.SystemURI, TLSInsecureSkipVerify: t.Access.InsecureSkipVerify()}) } slices.SortFunc(inv.RequestedScope, func(a, b inventory.ScopeEndpoint) int { return strings.Compare(a.Role+a.Endpoint+a.SystemURI, b.Role+b.Endpoint+b.SystemURI) diff --git a/pkg/dpf/discovery/discover_test.go b/pkg/dpf/discovery/discover_test.go index 078953f8..b3fa3719 100644 --- a/pkg/dpf/discovery/discover_test.go +++ b/pkg/dpf/discovery/discover_test.go @@ -3,10 +3,13 @@ package discovery import ( + "context" "encoding/json" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "os" "testing" "time" ) @@ -99,6 +102,39 @@ var _ = Describe("refresh regressions", func() { }) var _ = Describe("scope evidence on refresh", func() { + It("records the resolved TLS policy even when collection fails before remote access", func() { + Expect(os.Setenv("DPF_SCOPE_TEST_PASSWORD", "private-password")).To(Succeed()) + DeferCleanup(os.Unsetenv, "DPF_SCOPE_TEST_PASSWORD") + input, err := config.Decode([]byte(`management: + kubeconfigRef: {file: missing-kubeconfig} +credentials: + reader: {username: reader, passwordRef: {env: DPF_SCOPE_TEST_PASSWORD}} +discovery: + dpus: [{bmcEndpoint: 'https://192.0.2.1', credentialRef: reader, tls: {insecureSkipVerify: true}}] + hosts: [{bmcEndpoint: 'https://192.0.2.2', credentialRef: reader}] + probes: {mode: inventory} +`), GinkgoT().TempDir()+"/site.yaml") + Expect(err).NotTo(HaveOccurred()) + req, err := config.Resolve(input) + Expect(err).NotTo(HaveOccurred()) + report := Collect(context.Background(), req, "test", Dependencies{}) + Expect(report.Inventory.RequestedScope).To(HaveLen(2)) + Expect(report.Inventory.RequestedScope[0].TLSInsecureSkipVerify).To(BeTrue()) + Expect(report.Inventory.RequestedScope[1].TLSInsecureSkipVerify).To(BeFalse()) + Expect(report.Errors[0].Code).To(Equal("INVALID_MANAGEMENT_ACCESS")) + }) + It("retains insecure BMC policy in saved scope and rejects a silent trust-policy change", func() { + old := estate() + old.RequestedScope = []inventory.ScopeEndpoint{{Role: "dpus", Endpoint: "https://192.0.2.1", TLSInsecureSkipVerify: false}} + Canonicalize(&old) + current := estate() + current.RequestedScope = []inventory.ScopeEndpoint{{Role: "dpus", Endpoint: "https://192.0.2.1", TLSInsecureSkipVerify: true}} + Canonicalize(¤t) + Expect(scopeConflict(&old, ¤t)).To(BeTrue()) + data, err := json.Marshal(current) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring(`"tlsInsecureSkipVerify":true`)) + }) It("keeps missing scope evidence separate from a proven replacement", func() { old := estate() Canonicalize(&old) diff --git a/pkg/dpf/inventory/types.go b/pkg/dpf/inventory/types.go index ccea0317..f3874b17 100644 --- a/pkg/dpf/inventory/types.go +++ b/pkg/dpf/inventory/types.go @@ -21,9 +21,10 @@ type Collection struct { Collector string `json:"collector" yaml:"collector"` } type ScopeEndpoint struct { - Role string `json:"role" yaml:"role"` - Endpoint string `json:"endpoint" yaml:"endpoint"` - SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` + Role string `json:"role" yaml:"role"` + Endpoint string `json:"endpoint" yaml:"endpoint"` + SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` + TLSInsecureSkipVerify bool `json:"tlsInsecureSkipVerify,omitempty" yaml:"tlsInsecureSkipVerify,omitempty"` } type Evidence struct { Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` diff --git a/pkg/dpf/redfish/client_test.go b/pkg/dpf/redfish/client_test.go index 5449863a..b00e4d05 100644 --- a/pkg/dpf/redfish/client_test.go +++ b/pkg/dpf/redfish/client_test.go @@ -5,10 +5,14 @@ package redfish import ( "context" "encoding/json" + "fmt" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "net/http" "net/http/httptest" + "os" + "sync/atomic" "testing" "time" ) @@ -16,6 +20,34 @@ import ( func TestRedfish(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF Redfish") } var _ = Describe("GET/session-only transport", func() { + It("requires explicit insecure opt-in for an untrusted server and still rejects foreign origins", func() { + var requests atomic.Int32 + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Write([]byte(`{"RedfishVersion":"1.0.0"}`)) + })) + defer s.Close() + Expect(os.Setenv("DPF_TLS_TEST_PASSWORD", "private-password")).To(Succeed()) + DeferCleanup(os.Unsetenv, "DPF_TLS_TEST_PASSWORD") + for _, policy := range []string{"", ", tls: {insecureSkipVerify: false}", ", tls: {insecureSkipVerify: true}"} { + source := fmt.Sprintf("management:\n kubeconfigRef: {file: unused}\ncredentials:\n read: {username: reader, passwordRef: {env: DPF_TLS_TEST_PASSWORD}}\ndiscovery:\n dpus: [{bmcEndpoint: '%s', credentialRef: read%s}]\n hosts: [{bmcEndpoint: 'https://192.0.2.1', credentialRef: read}]\n", s.URL, policy) + in, err := config.Decode([]byte(source), "/site/input.yaml") + Expect(err).NotTo(HaveOccurred()) + req, err := config.Resolve(in) + Expect(err).NotTo(HaveOccurred()) + c := New(req.Targets[0], time.Second) + _, err = c.Get(context.Background(), "/redfish/v1/") + if req.Targets[0].Access.InsecureSkipVerify() { + Expect(err).NotTo(HaveOccurred()) + } else { + Expect(Code(err)).To(Equal("TLS_VERIFICATION_FAILED")) + Expect(requests.Load()).To(BeZero()) + } + _, err = c.Get(context.Background(), "https://foreign.invalid/redfish/v1/") + Expect(Code(err)).To(Equal("CROSS_ORIGIN_LINK")) + } + Expect(requests.Load()).To(Equal(int32(1))) + }) It("allows the service root and logs out exactly its own session", func() { var methods []string s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/target/dpf/driver.go b/pkg/target/dpf/driver.go index 661c0307..a5455522 100644 --- a/pkg/target/dpf/driver.go +++ b/pkg/target/dpf/driver.go @@ -90,6 +90,11 @@ func (d driver) Bind(inv target.Invocation) (target.Operation, error) { } else { if !inv.Output.Quiet { fmt.Fprintln(r.Stderr, "Inspecting explicitly selected management and BMC scope") + for _, endpoint := range resolved.Targets { + if endpoint.Access.InsecureSkipVerify() { + fmt.Fprintf(r.Stderr, "Warning: BMC TLS certificate verification disabled for %s\n", endpoint.Endpoint) + } + } } deps := discovery.Dependencies{BMC: redfish.Collector{Qualifications: catalog.Hardware()}, Probes: probe.Runner{Assets: assets, Journal: probe.FileJournal(filepath.Join(store.RunDir, "resources.json"))}} if resolved.Input.Site.Inventory == nil { diff --git a/skills/k8s-launch-kit-discover/SKILL.md b/skills/k8s-launch-kit-discover/SKILL.md index 36892d7e..ab82cf50 100644 --- a/skills/k8s-launch-kit-discover/SKILL.md +++ b/skills/k8s-launch-kit-discover/SKILL.md @@ -13,6 +13,14 @@ metadata: For the independent DPF preview, read [DPF discovery](../../docs/user/dpf-discovery.md) and use its explicit management/BMC YAML. Keep the empty hardware/probe-image release gates visible; do not run Host discovery against the DPF estate. +DPF BMC TLS verification defaults to enabled. For an explicitly requested lab +bypass, set the endpoint's `tls.insecureSkipVerify: true`; the shared +`discovery.tls` policy also supports it. An endpoint `tls` block replaces that +policy, including an explicit `false`. Do not combine `caFile` with `true`. +Retain stderr warnings and `inventory.requestedScope[].tlsInsecureSkipVerify` in +the report. A policy change requires a fresh seed/output; it does not weaken +management kubeconfig or network-probe TLS verification. + This workflow uses the default `host` target; `--target host` is equivalent. The CLI snapshots discovery arguments and runs the concrete Host discovery operation through the target registry; command syntax and artifacts are From f8b1c939ca34fca12d4d0133b29e05c3f479ea49 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 5 Nov 2026 10:15:00 +0100 Subject: [PATCH 03/35] fix(dpf): use advertised public Redfish session links Signed-off-by: Alexander Maslennikov --- docs/user/dpf-discovery.md | 8 ++- pkg/dpf/redfish/client.go | 38 ++++++++----- pkg/dpf/redfish/client_test.go | 76 +++++++++++++++++++++++++ skills/k8s-launch-kit-discover/SKILL.md | 4 ++ 4 files changed, 110 insertions(+), 16 deletions(-) diff --git a/docs/user/dpf-discovery.md b/docs/user/dpf-discovery.md index 04d73263..5f35e90a 100644 --- a/docs/user/dpf-discovery.md +++ b/docs/user/dpf-discovery.md @@ -76,9 +76,13 @@ credentials. File and embedded representations of the same credential/trust fiel exclusive; choose token or tokenFile. Exec/auth-provider plugins, basic and anonymous access are rejected in this version. Ambient kubeconfig files are never merged. BMC authentication uses one session login using the supplied username/password; Basic fallback is -not included in this preview. Session writes are restricted to the standard +not included in this preview. Login uses the public service root's `Links.Sessions` +when advertised, avoiding an unauthenticated read of a protected `SessionService`. +If absent, the client reads the advertised standard `SessionService` to locate +its sessions collection. Session writes are restricted to the standard `/redfish/v1/SessionService/Sessions` collection and its exact returned session -child; alternate OEM session paths are unsupported. No password guessing or credential repair occurs. +child; same-origin absolute links and optional trailing slashes are supported. +Alternate OEM session paths are unsupported. No password guessing or credential repair occurs. Passwords resolve only from `{env: NAME}` or `{file: path}`. Environment values are verbatim. UTF-8 files lose at most one final LF/CRLF. File and CA references diff --git a/pkg/dpf/redfish/client.go b/pkg/dpf/redfish/client.go index 9e0b8222..595321a9 100644 --- a/pkg/dpf/redfish/client.go +++ b/pkg/dpf/redfish/client.go @@ -183,19 +183,27 @@ func (c *Client) Open(ctx context.Context) (Document, error) { if err != nil { return nil, err } - service := Link(root, "SessionService") - if service == "" { - return nil, &Error{Code: "SESSION_AUTH_UNSUPPORTED"} - } - service, err = c.guard(service) - if err != nil { - return nil, err - } - sd, err := c.Get(ctx, service) - if err != nil { - return nil, err + // The public root advertises login even when SessionService itself requires + // authentication. Never require an authenticated GET before creating a session. + collection := Link(Object(root, "Links"), "Sessions") + if collection == "" { + service := Link(root, "SessionService") + if service == "" { + return nil, &Error{Code: "SESSION_AUTH_UNSUPPORTED"} + } + service, err = c.guard(service) + if err != nil { + return nil, err + } + if strings.TrimSuffix(service, "/") != "/redfish/v1/SessionService" { + return nil, &Error{Code: "SESSION_URI_UNSUPPORTED"} + } + sd, err := c.Get(ctx, service) + if err != nil { + return nil, err + } + collection = Link(sd, "Sessions") } - collection := Link(sd, "Sessions") if collection == "" { return nil, &Error{Code: "SESSION_AUTH_UNSUPPORTED"} } @@ -203,7 +211,8 @@ func (c *Client) Open(ctx context.Context) (Document, error) { if err != nil { return nil, err } - if service != "/redfish/v1/SessionService" || collection != "/redfish/v1/SessionService/Sessions" { + collectionBase := strings.TrimSuffix(collection, "/") + if collectionBase != "/redfish/v1/SessionService/Sessions" { return nil, &Error{Code: "SESSION_URI_UNSUPPORTED"} } username, password := c.access.Credentials() @@ -223,7 +232,8 @@ func (c *Client) Open(ctx context.Context) (Document, error) { } token := header.Get("X-Auth-Token") session, err := c.guard(header.Get("Location")) - if err != nil || token == "" || !strings.HasPrefix(session, collection+"/") || strings.Contains(strings.TrimPrefix(session, collection+"/"), "/") || strings.Contains(session, "?") { + sessionID := strings.TrimPrefix(strings.TrimSuffix(session, "/"), collectionBase+"/") + if err != nil || token == "" || !strings.HasPrefix(session, collectionBase+"/") || sessionID == "" || strings.Contains(sessionID, "/") || strings.Contains(session, "?") { return nil, &Error{Code: "INVALID_SESSION_RECEIPT"} } c.token = token diff --git a/pkg/dpf/redfish/client_test.go b/pkg/dpf/redfish/client_test.go index b00e4d05..eee91998 100644 --- a/pkg/dpf/redfish/client_test.go +++ b/pkg/dpf/redfish/client_test.go @@ -20,6 +20,82 @@ import ( func TestRedfish(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF Redfish") } var _ = Describe("GET/session-only transport", func() { + It("logs in through public root session links without reading the protected service", func() { + for _, scenario := range []struct{ slash, absolute bool }{{false, false}, {true, false}, {false, true}, {true, true}} { + collection := "/redfish/v1/SessionService/Sessions" + receipt := collection + "/owned" + if scenario.slash { + collection += "/" + receipt += "/" + } + var methods []string + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + prefix := "" + if scenario.absolute { + prefix = "https://" + r.Host + } + switch r.URL.Path { + case "/redfish/v1/": + json.NewEncoder(w).Encode(Document{"SessionService": Document{"@odata.id": "/redfish/v1/SessionService"}, "Links": Document{"Sessions": Document{"@odata.id": prefix + collection}}}) + case collection: + Expect(r.Method).To(Equal("POST")) + w.Header().Set("Location", prefix+receipt) + w.Header().Set("X-Auth-Token", "private-token") + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{}`)) + case receipt: + Expect(r.Method).To(Equal("DELETE")) + Expect(r.Header.Get("X-Auth-Token")).To(Equal("private-token")) + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusUnauthorized) + } + })) + c := &Client{origin: s.URL, http: s.Client()} + _, err := c.Open(context.Background()) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Close(context.Background())).To(Succeed()) + Expect(methods).To(Equal([]string{"GET /redfish/v1/", "POST " + collection, "DELETE " + receipt})) + s.Close() + } + }) + It("rejects foreign, OEM and query-bearing session links without attempting login", func() { + for _, link := range []string{"https://foreign.invalid/redfish/v1/SessionService/Sessions", "/redfish/v1/Oem/Sessions", "/redfish/v1/SessionService/Sessions?$skip=1"} { + var writes atomic.Int32 + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + } + json.NewEncoder(w).Encode(Document{"Links": Document{"Sessions": Document{"@odata.id": link}}}) + })) + c := &Client{origin: s.URL, http: s.Client()} + _, err := c.Open(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(writes.Load()).To(BeZero()) + Expect(c.Close(context.Background())).To(Succeed()) + s.Close() + } + }) + It("does not accept the session collection itself as a cleanup receipt", func() { + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/") + w.Header().Set("X-Auth-Token", "private-token") + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{}`)) + return + } + Expect(r.Method).NotTo(Equal(http.MethodDelete)) + json.NewEncoder(w).Encode(Document{"Links": Document{"Sessions": Document{"@odata.id": "/redfish/v1/SessionService/Sessions/"}}}) + })) + defer s.Close() + c := &Client{origin: s.URL, http: s.Client()} + _, err := c.Open(context.Background()) + Expect(Code(err)).To(Equal("INVALID_SESSION_RECEIPT")) + Expect(Code(c.Close(context.Background()))).To(Equal("SESSION_OUTCOME_UNKNOWN")) + }) It("requires explicit insecure opt-in for an untrusted server and still rejects foreign origins", func() { var requests atomic.Int32 s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/skills/k8s-launch-kit-discover/SKILL.md b/skills/k8s-launch-kit-discover/SKILL.md index ab82cf50..b46ad823 100644 --- a/skills/k8s-launch-kit-discover/SKILL.md +++ b/skills/k8s-launch-kit-discover/SKILL.md @@ -20,6 +20,10 @@ policy, including an explicit `false`. Do not combine `caFile` with `true`. Retain stderr warnings and `inventory.requestedScope[].tlsInsecureSkipVerify` in the report. A policy change requires a fresh seed/output; it does not weaken management kubeconfig or network-probe TLS verification. +Redfish login prefers the public root's `Links.Sessions`, including standard +HPE trailing-slash links. A protected `SessionService` GET is not required when +that login link is present; credentials are sent only to the same-origin standard +session collection, and cleanup uses its exact returned session child. This workflow uses the default `host` target; `--target host` is equivalent. The CLI snapshots discovery arguments and runs the concrete Host discovery From 3c42c013f1e31bd273e0bafd6a522ad183310a04 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 5 Nov 2026 18:25:02 +0100 Subject: [PATCH 04/35] feat(dpf): discover Zero Trust inventory through explicit SSH sources Signed-off-by: Alexander Maslennikov --- README.md | 2 + docs/user/dpf-discovery.md | 127 +++++++-- go.mod | 2 +- openspec/specs/outcomes-evidence/spec.md | 6 + pkg/dpf/artifacts/store.go | 18 +- pkg/dpf/config/config.go | 105 +++++--- pkg/dpf/config/scope.go | 17 ++ pkg/dpf/config/ssh.go | 104 ++++++++ pkg/dpf/config/ssh_test.go | 94 +++++++ pkg/dpf/discovery/discover.go | 130 +++++++--- pkg/dpf/discovery/discover_test.go | 8 +- pkg/dpf/discovery/inventory_mode_test.go | 102 ++++++++ pkg/dpf/discovery/observations.go | 73 ++++++ pkg/dpf/inventory/types.go | 97 ++++--- pkg/dpf/redfish/client.go | 18 +- pkg/dpf/redfish/diagnostics.go | 72 ++++++ pkg/dpf/redfish/diagnostics_test.go | 73 ++++++ pkg/dpf/redfish/host_inventory.go | 278 ++++++++++++++++++++ pkg/dpf/redfish/host_inventory_test.go | 137 ++++++++++ pkg/dpf/redfish/inventory.go | 64 ++--- pkg/dpf/sshprobe/collect.go | 311 +++++++++++++++++++++++ pkg/dpf/sshprobe/collect_test.go | 189 ++++++++++++++ pkg/dpf/sshprobe/parse.go | 240 +++++++++++++++++ pkg/dpf/sshprobe/transport.go | 201 +++++++++++++++ pkg/dpf/sshprobe/transport_test.go | 246 ++++++++++++++++++ pkg/target/dpf/driver.go | 15 +- skills/k8s-launch-kit-discover/SKILL.md | 8 +- skills/k8s-launch-kit-shared/SKILL.md | 6 +- 28 files changed, 2560 insertions(+), 183 deletions(-) create mode 100644 pkg/dpf/config/scope.go create mode 100644 pkg/dpf/config/ssh.go create mode 100644 pkg/dpf/config/ssh_test.go create mode 100644 pkg/dpf/discovery/inventory_mode_test.go create mode 100644 pkg/dpf/discovery/observations.go create mode 100644 pkg/dpf/redfish/diagnostics.go create mode 100644 pkg/dpf/redfish/diagnostics_test.go create mode 100644 pkg/dpf/redfish/host_inventory.go create mode 100644 pkg/dpf/redfish/host_inventory_test.go create mode 100644 pkg/dpf/sshprobe/collect.go create mode 100644 pkg/dpf/sshprobe/collect_test.go create mode 100644 pkg/dpf/sshprobe/parse.go create mode 100644 pkg/dpf/sshprobe/transport.go create mode 100644 pkg/dpf/sshprobe/transport_test.go diff --git a/README.md b/README.md index 70c0b24e..dd151d97 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ Standard SR-IOV, RDMA-shared and host-device profiles enable NIC tuning by defau The current lifecycle uses the `host` target by default. The `dpf` target provides an [experimental inventory discovery preview](docs/user/dpf-discovery.md); its provisioning phases remain unavailable. +DPF inventory can use explicit host/ARM SSH or BMC Redfish sources without an existing +management cluster. Inventory success is separate from provisioning readiness. DPF BMC discovery verifies TLS by default; lab endpoints can explicitly opt in to `tls.insecureSkipVerify: true`, recorded in discovery reports. See the preview guide for its scope and security implications. diff --git a/docs/user/dpf-discovery.md b/docs/user/dpf-discovery.md index 5f35e90a..1fc7e64a 100644 --- a/docs/user/dpf-discovery.md +++ b/docs/user/dpf-discovery.md @@ -5,27 +5,37 @@ SPDX-License-Identifier: Apache-2.0 # DPF discovery preview -`l8k discover --target dpf` inspects an explicitly selected management Kubernetes -cluster and DPU/host BMCs. The DPF estate is independent of Host networking. -The initial management topology requires three control-plane nodes. - -This build exposes an **experimental inventory preview**. The hardware -qualification catalog is empty: reads produce candidate facts and a partial -report, with qualification failures visible and exit `2`. No server/firmware -combination is declared supported by a successful GET. Management probes are -implemented using an existing digest-pinned diagnostic image, with orchestration -and validation in Go. The catalog pins `docker.io/nicolaka/netshoot:v0.16`, but -Kubernetes runtime qualification is pending. The default `management` mode -rejects locally until the catalog enables that qualified image/architecture cohort; -explicitly choose `inventory` for the preview. There is no automatic downgrade. -DPF generate/deploy/validate and the root DPF pipeline remain unavailable. +`l8k discover --target dpf` collects hardware inventory from explicitly selected +host/DPU Redfish or SSH endpoints. The DPF estate is independent of Host networking. +Inventory mode can run before Kubernetes exists; an explicitly supplied management +kubeconfig adds management observations and must work if supplied. + +This build exposes an **experimental inventory preview**. Inventory success means +that selected sources were read, physical identities and unique host/card mappings +were established, and cleanup/publication succeeded. It does **not** mean DPF +provisioning readiness or production hardware qualification. Missing optional card +metadata remains visible, including cards without serials. The hardware qualification +catalog remains empty; its failed qualification checks are required for future +`generate`, not for inventory collection. Overall deployment `acceptance` remains +`not-evaluated`, even when `discovery.outcome` is `complete` and the command exits zero. + +Management probes use an existing digest-pinned diagnostic image with Go +orchestration. The catalog pins `docker.io/nicolaka/netshoot:v0.16`, but Kubernetes +runtime qualification is pending. Default `management` mode rejects locally until +that cohort is qualified; explicitly select `inventory`. There is no automatic +downgrade. Management mode retains the three-control-plane topology requirement; +inventory merely reports that later-phase check. DPF generate/deploy/validate and +the root DPF pipeline remain unavailable. ## Inputs and invocation -Supply three bundles: management kubeconfig access, selected DPU BMC origins with -credentials, and host BMC origins with credentials. All addresses must come from -site inventory; the CLI does not scan subnets or derive endpoints from IP/MACs. -A host may contain several selected or unselected cards. +Supply explicit host endpoints and, optionally in inventory mode, DPU endpoints. +Each endpoint selects exactly one source: Redfish (`bmcEndpoint`) or SSH (`ssh`). +A host may contain several selected or unselected cards. With no explicit DPU +endpoints, BlueField cards are derived from host observations; their BMC/ARM +endpoints remain unknown. With explicit DPU endpoints, every selected source must +succeed and its card identity must match exactly one host adapter. Sources are not +silently substituted after access failures. The CLI does not scan subnets. ```yaml management: @@ -64,9 +74,9 @@ remote access. Lists and credentials stay in YAML; no password CLI flag exists. | Source | Values | | --- | --- | -| Required site inputs | Explicit kubeconfig file; nonempty DPU and host lists; each origin's credential alias; existing username and env/file password reference | -| Defaultable | Context from the supplied file; verified system trust; collection 10m; request 15s; four BMCs in flight | -| Conditional | Private `discovery.tls.caFile`; endpoint TLS override; `systemURI` for multi-System BMCs | +| Required site inputs | Nonempty host list; each selected endpoint and credential alias; username and password reference or SSH private-key reference | +| Defaultable | Context from a supplied kubeconfig; verified system trust; SSH port22; SSH sudo false; collection 10m; request 15s; four endpoint collectors in flight (`maxConcurrentBMCs`) | +| Conditional | Kubeconfig and explicit DPUs for management mode; SSH known-hosts file; approved noninteractive sudo for protected reads; private Redfish CA; `systemURI` for multi-System BMCs | | Observed | Physical-card/system identities, linked host adapters, component firmware, auth/time unknowns, management nodes/version | | Release owned | Qualified adapter completeness predicates and the probe image/architecture digests | | Later phases | Profile/release cohort, DPU API allocation, DHCP/service networks and controller authentication | @@ -85,7 +95,7 @@ child; same-origin absolute links and optional trailing slashes are supported. Alternate OEM session paths are unsupported. No password guessing or credential repair occurs. Passwords resolve only from `{env: NAME}` or `{file: path}`. Environment values -are verbatim. UTF-8 files lose at most one final LF/CRLF. File and CA references +are verbatim. UTF-8 files lose at most one final LF/CRLF. File, SSH known-hosts and CA references are relative to the input YAML and are rebased when saved elsewhere. Private files are not copied. Endpoint origins require HTTPS; redirects, cross-origin links, unbounded pages and arbitrary action endpoints are rejected. @@ -121,10 +131,71 @@ the scope digest: use a fresh seed and new output when switching policies. Management kubeconfig TLS and management-network probe TLS checks remain verified; this BMC option does not relax them or qualify hardware. +## Optional SSH inventory in Zero Trust + +SSH collects observations without using host RShim or DPU Redfish. Run the CLI +from a management node that can reach the configured endpoints. This version uses +direct SSH; it does not read ambient SSH configuration, agents or ProxyJump rules. +Existing jump connectivity must be provided through the chosen execution location. + +```yaml +credentials: + host-os: + username: inventory + passwordRef: {env: HOST_SSH_PASSWORD} + arm-os: + username: ubuntu + privateKeyRef: {file: ./credentials/arm-key} +discovery: + hosts: + - credentialRef: host-os + ssh: + address: 192.0.2.21:22 + knownHostsFile: ./credentials/known_hosts + sudo: true + dpus: + - credentialRef: arm-os + ssh: + address: 192.0.2.11:22 + knownHostsFile: ./credentials/known_hosts + sudo: true + probes: {mode: inventory} +``` + +This example needs no management kubeconfig or Redfish credential. Each SSH +credential uses exactly one password reference or unencrypted private-key file +reference. SSH host keys must match the explicitly supplied known-hosts file; +unknown/changed keys fail. Establish that trust through your site's trusted +inventory/onboarding process. `tls.insecureSkipVerify` never applies to SSH. + +Host collection reads the system UUID and NVIDIA network PCI/VPD inventory, +including per-function network-device/driver/NUMA facts where available. It does +not claim an all-vendor hardware census. DPU ARM identity requires agreement +between system DMI serial and the NVIDIA BlueField built-in IPMI FRU serial. +OS interfaces, routes, DNS and local BMC LAN address are additional observations. +Supported remote tools must already exist; discovery installs nothing and invokes +only fixed read commands through Go SSH, optionally with `sudo -n`. It prompts for +no sudo password, changes no privileges and uses no remote Python helper. + +ARM discovery needs a booted, reachable OS with sufficient permissions. Local +`ipmitool lan print` can report the BMC endpoint without Redfish. The endpoint is +recorded as `observed-local-ipmi`; discovery does not connect or send credentials +to that newly observed address. Missing optional tools/data remain unknown. It +never infers a BMC address from an ARM IP or a serial from a hostname. + +Host and ARM PCI addresses belong to different domains; correlation uses physical +serial identity, not matching PCI addresses. Host-reported identity alone does not +attest trusted DPU state. Source evidence, SSH trust digest and any conflicts stay +in the output. SSH inventory success does not establish Redfish authorization, +controller mTLS enrollment or provisioning readiness. Those checks remain separate. + ## Observations, reports and refresh -Physical identity comes from qualified card/system fields, keeping raw serials. -Ports/PCI functions, CPU serials and addresses cannot substitute for card identity. +Physical identity comes from source-specific card/system fields, keeping raw serials +and evidence. Ports/PCI functions, CPU serials and addresses cannot substitute for +card identity. HPE collection follows explicit System/chassis/PCI relationships; +shared-chassis records require an ownership join. Partial adapters retain their +PCI/slot facts without fabricated serials. No supported release tuple is implied. Every requested endpoint remains represented on failure; unrelated cards remain outside selection. Unknown firmware/auth/time values stay unknown. Qualification requires both a reviewed model/firmware tuple and an adapter-specific completeness @@ -135,7 +206,11 @@ existing handoff and writes `partial.yaml` plus `report.json` beneath `.discovery//`. JSON stdout is one final DPF result, with progress on stderr; syntax errors before dispatch retain normal Cobra stderr. The report contains facts, checks, factual diffs, later-phase gaps, errors and -successfully written artifacts. Saved YAML alone contains the factual handoff. +successfully written artifacts. Redfish errors retain sanitized method/resource, +HTTP status and observation/cleanup stage. Cleanup failures retain the exact +validated session URI when known, never its token. Verify ownership before manual +recovery; an unknown cleanup result is not success. Authentication at session +creation does not establish permission to read protected inventory. Saved YAML alone contains the factual handoff. Explicit in-place refresh requires the same input/output path; it keeps `source-before.yaml`. Stable sidecar locks serialize cooperating writers. External edits between the final hash check and rename cannot be excluded; avoid concurrent @@ -144,7 +219,7 @@ editing. Output paths must use real directories without symlink aliases. | Exit | Meaning | | --- | --- | | 0 | Explicit-mode checks, cleanup and publication completed | -| 2 | Invalid input, unqualified required capability or proven mismatch | +| 2 | Invalid input, incomplete required identity/mapping, unavailable management mode or proven mismatch | | 3 | Required observation, authentication, TLS, API or remote cleanup failure | | 1 | Cancellation or local publication/encoding error | diff --git a/go.mod b/go.mod index f6c2fdc4..be089658 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.12.1 go.uber.org/zap v1.28.0 + golang.org/x/crypto v0.57.0 golang.org/x/sys v0.49.0 golang.org/x/term v0.47.0 gopkg.in/yaml.v2 v2.4.0 @@ -119,7 +120,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.57.0 // indirect golang.org/x/mod v0.41.0 // indirect golang.org/x/net v0.60.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect diff --git a/openspec/specs/outcomes-evidence/spec.md b/openspec/specs/outcomes-evidence/spec.md index 3cda26c5..3f67396e 100644 --- a/openspec/specs/outcomes-evidence/spec.md +++ b/openspec/specs/outcomes-evidence/spec.md @@ -34,6 +34,12 @@ A result or gate extension MUST define command execution success separately from - **WHEN** a resource classifier returns in-progress - **THEN** aggregation applies the documented mode policy and does not describe that resource as ready. +#### Scenario: DPF inventory without management readiness + +- **WHEN** explicit DPF inventory sources establish physical identities and unique host/card associations, and required observation, cleanup and publication succeed +- **THEN** inventory discovery can complete without DPU Redfish or a management Kubernetes cluster; release qualification and provisioning authentication remain separate, visible later-phase checks. +- **AND** unavailable selected sources and ambiguous identities still fail inventory; an observed BMC address never authorizes sending credentials to it automatically. + ### Requirement: Output compatibility Output extensions SHALL preserve command-specific framing, existing public field meanings, diagnostic separation and exit mapping unless a compatibility change is explicit. Text, JSON and HTML MUST describe compatible evidence without assuming identical envelopes. diff --git a/pkg/dpf/artifacts/store.go b/pkg/dpf/artifacts/store.go index b1645ffd..87d0e818 100644 --- a/pkg/dpf/artifacts/store.go +++ b/pkg/dpf/artifacts/store.go @@ -76,6 +76,9 @@ func New(in config.Input, destination string, explicit bool) (*Store, error) { } protected := []string{in.Site.Management.KubeconfigRef.File} for _, c := range in.Site.Credentials { + if c.PrivateKeyRef != nil { + protected = append(protected, c.PrivateKeyRef.File) + } if c.PasswordRef.File != nil { protected = append(protected, *c.PasswordRef.File) } @@ -84,6 +87,9 @@ func New(in config.Input, destination string, explicit bool) (*Store, error) { protected = append(protected, *t.CAFile) } for _, e := range append(append([]config.Endpoint{}, in.Site.Discovery.DPUs...), in.Site.Discovery.Hosts...) { + if e.SSH != nil { + protected = append(protected, e.SSH.KnownHostsFile) + } if e.TLS != nil && e.TLS.CAFile != nil { protected = append(protected, *e.TLS.CAFile) } @@ -116,6 +122,9 @@ func New(in config.Input, destination string, explicit bool) (*Store, error) { } } for _, p := range protected { + if p == "" { + continue + } if !filepath.IsAbs(p) { p = filepath.Join(filepath.Dir(input), p) } @@ -157,6 +166,9 @@ func sameRequested(a, b config.Site) bool { }{{"dpus", s.Discovery.DPUs}, {"hosts", s.Discovery.Hosts}} { for _, e := range group.eps { origin, err := config.Origin(e.BMCEndpoint) + if e.SSH != nil { + origin, err = config.SSHOrigin(e.SSH.Address) + } if err != nil { return "" } @@ -164,7 +176,11 @@ func sameRequested(a, b config.Site) bool { if e.SystemURI != nil { selector = *e.SystemURI } - targets = append(targets, inventory.ScopeEndpoint{Role: group.role, Endpoint: origin, SystemURI: selector}) + scope := inventory.ScopeEndpoint{Role: group.role, Endpoint: origin, SystemURI: selector} + if e.SSH != nil { + scope.SSHSudo = e.SSH.Sudo != nil && *e.SSH.Sudo + } + targets = append(targets, scope) } } slices.SortFunc(targets, func(a, b inventory.ScopeEndpoint) int { diff --git a/pkg/dpf/config/config.go b/pkg/dpf/config/config.go index fedc3e81..840ebe14 100644 --- a/pkg/dpf/config/config.go +++ b/pkg/dpf/config/config.go @@ -35,14 +35,21 @@ type PasswordRef struct { File *string `yaml:"file,omitempty"` } type Credential struct { - Username string `yaml:"username"` - PasswordRef PasswordRef `yaml:"passwordRef"` + Username string `yaml:"username"` + PasswordRef PasswordRef `yaml:"passwordRef,omitempty"` + PrivateKeyRef *FileRef `yaml:"privateKeyRef,omitempty"` } type TLS struct { CAFile *string `yaml:"caFile,omitempty"` InsecureSkipVerify *bool `yaml:"insecureSkipVerify,omitempty"` } +type SSH struct { + Address string `yaml:"address"` + KnownHostsFile string `yaml:"knownHostsFile"` + Sudo *bool `yaml:"sudo,omitempty"` +} type Endpoint struct { + SSH *SSH `yaml:"ssh,omitempty"` Name *string `yaml:"name,omitempty"` BMCEndpoint string `yaml:"bmcEndpoint"` CredentialRef string `yaml:"credentialRef"` @@ -100,6 +107,7 @@ func (a Access) String() string { return "" } func (a Access) InsecureSkipVerify() bool { return a.tls != nil && a.tls.InsecureSkipVerify } type Target struct { + SSH *SSHAccess Endpoint string SystemURI string InputRef string @@ -230,10 +238,9 @@ func Resolve(in Input) (Request, error) { r := Request{Input: in, Timeout: 10 * time.Minute, RequestTimeout: 15 * time.Second, MaxConcurrentBMCs: 4, Mode: "management"} s := in.Site base := filepath.Dir(in.Path) - if s.Management.KubeconfigRef.File == "" { - return r, fmt.Errorf("/management/kubeconfigRef/file is required") + if s.Management.KubeconfigRef.File != "" { + r.Kubeconfig = resolvePath(base, s.Management.KubeconfigRef.File) } - r.Kubeconfig = resolvePath(base, s.Management.KubeconfigRef.File) if s.Management.Context != nil { if *s.Management.Context == "" { return r, fmt.Errorf("/management/context must not be empty") @@ -247,6 +254,12 @@ func Resolve(in Input) (Request, error) { if r.Mode != "management" && r.Mode != "inventory" { return r, fmt.Errorf("/discovery/probes/mode must be management or inventory") } + if r.Mode == "management" && r.Kubeconfig == "" { + return r, fmt.Errorf("/management/kubeconfigRef/file is required in management mode") + } + if r.Kubeconfig == "" && s.Management.Context != nil { + return r, fmt.Errorf("/management/context requires kubeconfigRef") + } if p.Namespace != nil { r.Namespace = *p.Namespace if r.Namespace == "" || len(r.Namespace) > 63 || !dnsLabel(r.Namespace) { @@ -289,8 +302,8 @@ func Resolve(in Input) (Request, error) { if r.RequestTimeout > r.Timeout || r.MaxConcurrentBMCs < 1 || r.MaxConcurrentBMCs > 16 { return r, fmt.Errorf("invalid /discovery/limits") } - if len(s.Discovery.DPUs) == 0 || len(s.Discovery.Hosts) == 0 || len(s.Discovery.DPUs)+len(s.Discovery.Hosts) > 256 { - return r, fmt.Errorf("discovery requires 1-256 explicit DPU/host endpoints and both roles") + if len(s.Discovery.Hosts) == 0 || (r.Mode == "management" && len(s.Discovery.DPUs) == 0) || len(s.Discovery.DPUs)+len(s.Discovery.Hosts) > 256 { + return r, fmt.Errorf("discovery requires explicit hosts, at most 256 endpoints, and explicit DPUs in management mode") } seen := map[string]bool{} type accessKey struct { @@ -304,6 +317,32 @@ func Resolve(in Input) (Request, error) { }{{"dpus", s.Discovery.DPUs}, {"hosts", s.Discovery.Hosts}} { for i, e := range group.endpoints { ptr := fmt.Sprintf("/discovery/%s/%d", group.role, i) + if e.SSH != nil { + if r.Mode != "inventory" { + return r, fmt.Errorf("%s/ssh requires inventory mode; management probes require BMC targets", ptr) + } + if e.BMCEndpoint != "" || e.SystemURI != nil || e.TLS != nil { + return r, fmt.Errorf("%s must select exactly one SSH or Redfish source", ptr) + } + origin, err := SSHOrigin(e.SSH.Address) + if err != nil { + return r, fmt.Errorf("%s/ssh/address: %w", ptr, err) + } + if seen[origin] { + return r, fmt.Errorf("duplicate SSH endpoint at %s", ptr) + } + seen[origin] = true + c, ok := s.Credentials[e.CredentialRef] + if !ok { + return r, fmt.Errorf("%s/credentialRef is unknown", ptr) + } + access, err := resolveSSHAccess(c, *e.SSH, base) + if err != nil { + return r, fmt.Errorf("%s/ssh: %w", ptr, err) + } + r.Targets = append(r.Targets, Target{Endpoint: origin, InputRef: ptr, Role: group.role, SSH: access}) + continue + } origin, err := Origin(e.BMCEndpoint) if err != nil { return r, fmt.Errorf("%s/bmcEndpoint: %w", ptr, err) @@ -360,26 +399,46 @@ func resolvePath(base, p string) string { return filepath.Join(base, p) } func resolveAccess(c Credential, caPath, base string, insecure bool) (Access, error) { - p := c.PasswordRef + if c.PrivateKeyRef != nil { + return Access{}, fmt.Errorf("redfish requires passwordRef, not privateKeyRef") + } + password, err := resolvePassword(c.PasswordRef, base) + if err != nil { + return Access{}, err + } + roots, err := x509.SystemCertPool() + if err != nil || roots == nil { + roots = x509.NewCertPool() + } + var ca []byte + if caPath != "" { + ca, err = regularFile(caPath, 1<<20) + if err != nil || !roots.AppendCertsFromPEM(ca) { + return Access{}, fmt.Errorf("invalid CA file") + } + } + return Access{username: c.Username, password: password, tls: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots, InsecureSkipVerify: insecure}, ca: ca}, nil +} +func resolvePassword(p PasswordRef, base string) (string, error) { if (p.Env == nil) == (p.File == nil) { - return Access{}, fmt.Errorf("passwordRef requires exactly one env or file") + return "", fmt.Errorf("passwordRef requires exactly one env or file") } var password string if p.Env != nil { if *p.Env == "" { - return Access{}, fmt.Errorf("empty passwordRef.env") + return "", fmt.Errorf("empty passwordRef.env") } password = os.Getenv(*p.Env) } else { if *p.File == "" { - return Access{}, fmt.Errorf("empty passwordRef.file") + return "", fmt.Errorf("empty passwordRef.file") } b, err := regularFile(resolvePath(base, *p.File), 64<<10) if err != nil { - return Access{}, fmt.Errorf("cannot resolve password file") + return "", fmt.Errorf("cannot resolve password file") } if !utf8.Valid(b) { - return Access{}, fmt.Errorf("password file must be UTF-8") + return "", fmt.Errorf("password file must be UTF-8") } password = string(b) if strings.HasSuffix(password, "\n") { @@ -387,21 +446,9 @@ func resolveAccess(c Credential, caPath, base string, insecure bool) (Access, er } } if password == "" || len(password) > 64<<10 { - return Access{}, fmt.Errorf("password reference is missing, empty or too large") - } - roots, err := x509.SystemCertPool() - if err != nil || roots == nil { - roots = x509.NewCertPool() - } - var ca []byte - if caPath != "" { - ca, err = regularFile(caPath, 1<<20) - if err != nil || !roots.AppendCertsFromPEM(ca) { - return Access{}, fmt.Errorf("invalid CA file") - } + return "", fmt.Errorf("password reference is missing, empty or too large") } - // Certificate verification is disabled only by explicit site configuration. - return Access{username: c.Username, password: password, tls: &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots, InsecureSkipVerify: insecure}, ca: ca}, nil + return password, nil } func Origin(s string) (string, error) { u, err := url.Parse(s) @@ -460,7 +507,7 @@ func rebase(n *yaml.Node, oldBase, newBase, p string) error { if key == "inventory" && p == "" { continue } - isFile := key == "caFile" || key == "file" && (strings.HasSuffix(p, "kubeconfigRef") || strings.HasSuffix(p, "passwordRef")) + isFile := key == "caFile" || key == "knownHostsFile" || key == "file" && (strings.HasSuffix(p, "kubeconfigRef") || strings.HasSuffix(p, "passwordRef") || strings.HasSuffix(p, "privateKeyRef")) if isFile && value.Kind == yaml.ScalarNode && value.Tag == "!!str" && value.Value != "" { abs := resolvePath(oldBase, value.Value) rel, err := filepath.Rel(newBase, abs) @@ -498,7 +545,7 @@ func validateScalarTypes(n *yaml.Node, p string) error { if p == "/discovery/limits/maxConcurrentBMCs" { expected = "!!int" } - if strings.HasSuffix(p, "/tls/insecureSkipVerify") { + if strings.HasSuffix(p, "/tls/insecureSkipVerify") || strings.HasSuffix(p, "/ssh/sudo") { expected = "!!bool" } if n.Tag != expected { diff --git a/pkg/dpf/config/scope.go b/pkg/dpf/config/scope.go new file mode 100644 index 00000000..24c072be --- /dev/null +++ b/pkg/dpf/config/scope.go @@ -0,0 +1,17 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + +// Scope records access policy without credentials. The SSH trust snapshot is +// hashed so changing a trust file invalidates an earlier discovery scope. +func Scope(t Target) inventory.ScopeEndpoint { + s := inventory.ScopeEndpoint{Role: t.Role, Endpoint: t.Endpoint, SystemURI: t.SystemURI, TLSInsecureSkipVerify: t.Access.InsecureSkipVerify()} + if t.SSH != nil { + s.SSHTrustDigest = inventory.Digest("dpf-ssh-trust-v1", t.SSH.KnownHosts()) + s.SSHSudo = t.SSH.Sudo + } + return s +} diff --git a/pkg/dpf/config/ssh.go b/pkg/dpf/config/ssh.go new file mode 100644 index 00000000..88eff692 --- /dev/null +++ b/pkg/dpf/config/ssh.go @@ -0,0 +1,104 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "bytes" + "fmt" + "net" + "net/url" + "strconv" + "strings" + + "golang.org/x/crypto/ssh" +) + +// SSHAccess holds a snapshot of explicit authentication and host trust. It is +// runtime-only: saved configuration retains references, never their contents. +type SSHAccess struct { + username, password string + privateKey, knownHosts []byte + Sudo bool +} + +func (a *SSHAccess) Credentials() (string, string) { + if a == nil { + return "", "" + } + return a.username, a.password +} +func (a *SSHAccess) PrivateKey() []byte { + if a == nil { + return nil + } + return bytes.Clone(a.privateKey) +} +func (a *SSHAccess) KnownHosts() []byte { + if a == nil { + return nil + } + return bytes.Clone(a.knownHosts) +} +func (a *SSHAccess) String() string { return "" } +func (a *SSHAccess) GoString() string { return a.String() } + +// SSHOrigin accepts a host or host:port, with brackets for IPv6. It deliberately +// accepts no userinfo, path, SSH configuration alias, or proxy command. +func SSHOrigin(address string) (string, error) { + if address == "" || strings.HasSuffix(address, ":") || strings.ContainsAny(address, " \t\r\n/@?#\\") || strings.Contains(address, "://") { + return "", fmt.Errorf("expected SSH host or host:port") + } + u, err := url.Parse("ssh://" + address) + if err != nil || u.Hostname() == "" || u.User != nil || u.Path != "" { + return "", fmt.Errorf("invalid SSH address") + } + host := strings.ToLower(u.Hostname()) + if strings.Contains(host, ":") && net.ParseIP(host) == nil { + return "", fmt.Errorf("IPv6 addresses require brackets") + } + port := u.Port() + if port == "" { + port = "22" + } + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return "", fmt.Errorf("invalid SSH port") + } + return "ssh://" + net.JoinHostPort(host, strconv.Itoa(n)), nil +} + +func resolveSSHAccess(c Credential, endpoint SSH, base string) (*SSHAccess, error) { + if c.Username == "" || strings.ContainsAny(c.Username, "\x00\r\n") { + return nil, fmt.Errorf("SSH credential requires a username") + } + a := &SSHAccess{username: c.Username, Sudo: endpoint.Sudo != nil && *endpoint.Sudo} + if c.PrivateKeyRef != nil { + if c.PasswordRef.Env != nil || c.PasswordRef.File != nil || c.PrivateKeyRef.File == "" { + return nil, fmt.Errorf("SSH requires exactly one passwordRef or privateKeyRef") + } + key, err := regularFile(resolvePath(base, c.PrivateKeyRef.File), 1<<20) + if err != nil { + return nil, fmt.Errorf("cannot resolve SSH private key") + } + if _, err := ssh.ParsePrivateKey(key); err != nil { + return nil, fmt.Errorf("invalid or encrypted SSH private key") + } + a.privateKey = key + } else { + password, err := resolvePassword(c.PasswordRef, base) + if err != nil { + return nil, err + } + a.password = password + } + if endpoint.KnownHostsFile == "" { + return nil, fmt.Errorf("knownHostsFile is required") + } + known, err := regularFile(resolvePath(base, endpoint.KnownHostsFile), 1<<20) + if err != nil || len(bytes.TrimSpace(known)) == 0 { + return nil, fmt.Errorf("cannot resolve nonempty SSH knownHostsFile") + } + a.knownHosts = known + return a, nil +} diff --git a/pkg/dpf/config/ssh_test.go b/pkg/dpf/config/ssh_test.go new file mode 100644 index 00000000..87e9d904 --- /dev/null +++ b/pkg/dpf/config/ssh_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +var _ = Describe("explicit SSH input", func() { + var dir, document string + BeforeEach(func() { + dir = GinkgoT().TempDir() + public, _, err := ed25519.GenerateKey(rand.Reader) + Expect(err).NotTo(HaveOccurred()) + key, err := ssh.NewPublicKey(public) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "known_hosts"), []byte(knownhosts.Line([]string{"host.example"}, key)+"\n"), 0600)).To(Succeed()) + Expect(os.Setenv("SSH_TEST_PASSWORD", "secret-canary")).To(Succeed()) + DeferCleanup(os.Unsetenv, "SSH_TEST_PASSWORD") + document = `credentials: + reader: {username: reader, passwordRef: {env: SSH_TEST_PASSWORD}} +discovery: + hosts: + - credentialRef: reader + ssh: {address: host.example, knownHostsFile: known_hosts, sudo: false} + probes: {mode: inventory} +` + }) + It("resolves without Kubernetes or DPU endpoints, snapshots trust and preserves references", func() { + in, err := Decode([]byte(document), filepath.Join(dir, "site.yaml")) + Expect(err).NotTo(HaveOccurred()) + r, err := Resolve(in) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Kubeconfig).To(BeEmpty()) + Expect(r.Targets).To(HaveLen(1)) + t := r.Targets[0] + Expect(t.Endpoint).To(Equal("ssh://host.example:22")) + Expect(t.SSH.Sudo).To(BeFalse()) + _, password := t.SSH.Credentials() + Expect(password).To(Equal("secret-canary")) + raw, err := json.Marshal(t.SSH) + Expect(err).NotTo(HaveOccurred()) + Expect(string(raw)).NotTo(ContainSubstring("canary")) + Expect(fmt.Sprintf("%#v", t.SSH)).NotTo(ContainSubstring("canary")) + before := Scope(t).SSHTrustDigest + changed := t.SSH.KnownHosts() + changed[0] = 'X' + Expect(Scope(t).SSHTrustDigest).To(Equal(before)) + out, err := Saved(in, nil, filepath.Join(dir, "result", "out.yaml")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(out)).To(ContainSubstring("../known_hosts")) + Expect(string(out)).To(ContainSubstring("sudo: false")) + Expect(string(out)).NotTo(ContainSubstring("secret-canary")) + }) + It("rejects ambiguous transports, auth, invalid presence and management-mode SSH", func() { + for _, text := range []string{ + strings.Replace(document, "ssh:", "bmcEndpoint: https://host.example\n ssh:", 1), + strings.Replace(document, "ssh:", "tls: {insecureSkipVerify: true}\n ssh:", 1), + strings.Replace(document, "knownHostsFile: known_hosts", "knownHostsFile: ''", 1), + strings.Replace(document, "sudo: false", "sudo: 'false'", 1), + strings.Replace(document, "mode: inventory", "mode: management", 1), + strings.Replace(document, "passwordRef: {env: SSH_TEST_PASSWORD}", "passwordRef: {env: SSH_TEST_PASSWORD}, privateKeyRef: {file: key}", 1), + } { + in, err := Decode([]byte(text), filepath.Join(dir, "site.yaml")) + if err == nil { + _, err = Resolve(in) + } + Expect(err).To(HaveOccurred()) + } + }) + It("canonicalizes only bounded address forms", func() { + for input, want := range map[string]string{"HOST.EXAMPLE": "ssh://host.example:22", "host.example:2222": "ssh://host.example:2222", "[2001:db8::1]": "ssh://[2001:db8::1]:22"} { + got, err := SSHOrigin(input) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(want)) + } + for _, bad := range []string{"", "user@host", "host/path", "host:0", "host:65536", "host:", "host?x", "ssh://host", "host\ncommand"} { + _, err := SSHOrigin(bad) + Expect(err).To(HaveOccurred()) + } + }) +}) diff --git a/pkg/dpf/discovery/discover.go b/pkg/dpf/discovery/discover.go index 6f80bd2f..89b7ecdc 100644 --- a/pkg/dpf/discovery/discover.go +++ b/pkg/dpf/discovery/discover.go @@ -7,6 +7,7 @@ package discovery import ( "cmp" "context" + "errors" "reflect" "slices" "strings" @@ -16,13 +17,18 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sshprobe" ) type ProbeRunner interface { Run(context.Context, config.Request, ManagementClient, *inventory.Inventory) ([]inventory.ProbeResult, inventory.Cleanup, error) } +type Collector interface { + Collect(context.Context, config.Target, time.Duration) (inventory.Device, map[string]inventory.Evidence, error) +} type Dependencies struct { - BMC redfish.Collector + SSH Collector + BMC Collector Probes ProbeRunner } @@ -35,7 +41,7 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc inv := &inventory.Inventory{FormatVersion: inventory.FormatVersion, Collection: inventory.Collection{RunID: runID, StartedAt: time.Now().UTC(), Mode: r.Mode, Collector: "l8k-go-discovery-v1"}, RequestedScope: []inventory.ScopeEndpoint{}, DPUs: []inventory.Device{}, Hosts: []inventory.Device{}, Associations: []inventory.Association{}, Evidence: map[string]inventory.Evidence{}} report.Inventory = inv for _, t := range r.Targets { - inv.RequestedScope = append(inv.RequestedScope, inventory.ScopeEndpoint{Role: t.Role, Endpoint: t.Endpoint, SystemURI: t.SystemURI, TLSInsecureSkipVerify: t.Access.InsecureSkipVerify()}) + inv.RequestedScope = append(inv.RequestedScope, config.Scope(t)) } slices.SortFunc(inv.RequestedScope, func(a, b inventory.ScopeEndpoint) int { return strings.Compare(a.Role+a.Endpoint+a.SystemURI, b.Role+b.Endpoint+b.SystemURI) @@ -45,27 +51,31 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc var mu sync.Mutex var wg sync.WaitGroup problem := func(p inventory.Problem) { mu.Lock(); defer mu.Unlock(); report.Errors = append(report.Errors, p) } - mg, accessErr := ManagementAccess(r) - if accessErr != nil { - report.Execution = "failed" - report.Errors = append(report.Errors, inventory.Problem{Code: "INVALID_MANAGEMENT_ACCESS", Message: accessErr.Error(), ExitCode: 2}) - return report - } - wg.Add(1) - go func() { - defer wg.Done() - var err error - inv.Management, err = mg.Observe(ctx) - if err != nil { - problem(inventory.Problem{Code: "MANAGEMENT_INSPECTION_FAILED", Message: err.Error(), InputRef: "/management", ExitCode: 3}) - return - } - mu.Lock() - for id, e := range NewEvidence(time.Now().UTC()) { - inv.Evidence[id] = e + var mg ManagementClient + if r.Kubeconfig != "" { + var accessErr error + mg, accessErr = ManagementAccess(r) + if accessErr != nil { + report.Execution = "failed" + report.Errors = append(report.Errors, inventory.Problem{Code: "INVALID_MANAGEMENT_ACCESS", Message: accessErr.Error(), ExitCode: 2}) + return report } - mu.Unlock() - }() + wg.Add(1) + go func() { + defer wg.Done() + var err error + inv.Management, err = mg.Observe(ctx) + if err != nil { + problem(inventory.Problem{Code: "MANAGEMENT_INSPECTION_FAILED", Message: err.Error(), InputRef: "/management", ExitCode: 3}) + return + } + mu.Lock() + defer mu.Unlock() + for id, e := range NewEvidence(time.Now().UTC()) { + inv.Evidence[id] = e + } + }() + } sem := make(chan struct{}, r.MaxConcurrentBMCs) devices := make([]inventory.Device, len(r.Targets)) for index, target := range r.Targets { @@ -79,7 +89,16 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc devices[i] = inventory.Device{RecordKey: inventory.Digest("dpf-access-v1", []string{t.Role, t.Endpoint, t.SystemURI}), InputRef: t.InputRef, BMCEndpoint: t.Endpoint, ObservationState: "unavailable", ReasonCode: "CANCELLED"} return } - d, e, err := deps.BMC.Collect(ctx, t, r.RequestTimeout) + collector := deps.BMC + if t.SSH != nil { + collector = deps.SSH + } + if collector == nil { + problem(inventory.Problem{Code: "COLLECTOR_UNAVAILABLE", Message: "selected discovery source is unavailable", InputRef: t.InputRef, ExitCode: 3}) + devices[i] = inventory.Device{RecordKey: inventory.Digest("dpf-access-v1", []string{t.Role, t.Endpoint, t.SystemURI}), InputRef: t.InputRef, ObservationState: "unavailable", ReasonCode: "COLLECTOR_UNAVAILABLE"} + return + } + d, e, err := collector.Collect(ctx, t, r.RequestTimeout) devices[i] = d mu.Lock() for key, value := range e { @@ -87,11 +106,26 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc } mu.Unlock() if err != nil { - problem(inventory.Problem{Code: redfish.Code(err), Message: "BMC observation failed; inspect record reason and field path", InputRef: t.InputRef, ExitCode: 3}) + if t.SSH != nil { + code := d.ReasonCode + if code == "" { + code = "SSH_OBSERVATION_FAILED" + } + stage := "observation" + var diagnostic *sshprobe.Error + if errors.As(err, &diagnostic) { + stage = diagnostic.Stage + } + problem(inventory.Problem{Code: code, Message: "SSH observation failed", InputRef: t.InputRef, Stage: stage, ExitCode: 3}) + } else { + for _, diagnostic := range redfish.Diagnostics(err) { + problem(inventory.Problem{Code: diagnostic.Code, Message: "Redfish operation failed", InputRef: t.InputRef, Method: diagnostic.Method, ResourceURI: diagnostic.URI, HTTPStatus: diagnostic.Status, Stage: diagnostic.Stage, ExitCode: 3}) + } + } if d.ReasonCode == "CLEANUP_INCOMPLETE" { mu.Lock() report.Discovery.Cleanup.State = "incomplete" - report.Discovery.Cleanup.Leftovers = append(report.Discovery.Cleanup.Leftovers, t.Endpoint+": session outcome unknown") + report.Discovery.Cleanup.Leftovers = append(report.Discovery.Cleanup.Leftovers, t.Endpoint+redfish.CleanupReceipt(err)+": session cleanup incomplete") mu.Unlock() } } @@ -105,6 +139,13 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc inv.Hosts = append(inv.Hosts, d) } } + requestedDPUs := false + for _, t := range r.Targets { + requestedDPUs = requestedDPUs || t.Role == "dpus" + } + if !requestedDPUs { + inv.DPUs = HostCards(inv.Hosts) + } inv.Associations, report.Discovery.Checks = Correlate(inv.DPUs, inv.Hosts) selected := map[string]bool{} for _, a := range inv.Associations { @@ -117,12 +158,12 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc } } } - addCheck := func(id, subject string, ok bool, reason string, refs []string) { + addCheck := func(id, subject string, ok bool, reason string, refs []string, phase string) { state := "failed" if ok { state = "passed" } - report.Discovery.Checks = append(report.Discovery.Checks, inventory.Check{ID: id, Subject: subject, Vantage: "executor", RequiredPhase: "discover", State: state, ReasonCode: reason, EvidenceRefs: refs}) + report.Discovery.Checks = append(report.Discovery.Checks, inventory.Check{ID: id, Subject: subject, Vantage: "executor", RequiredPhase: phase, State: state, ReasonCode: reason, EvidenceRefs: refs}) } if inv.Management != nil { cp := 0 @@ -131,7 +172,11 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc cp++ } } - addCheck("dpf.management.control-plane-count", "management", cp == 3, "EXACTLY_THREE_CONTROL_PLANES", inv.Management.EvidenceRefs) + phase := "generate" + if r.Mode == "management" { + phase = "discover" + } + addCheck("dpf.management.control-plane-count", "management", cp == 3, "EXACTLY_THREE_CONTROL_PLANES", inv.Management.EvidenceRefs, phase) if r.Mode == "management" && cp == 3 { if deps.Probes == nil { report.Errors = append(report.Errors, inventory.Problem{Code: "PROBE_ASSETS_UNAVAILABLE", Message: "management probes require qualified installed probe assets", ExitCode: 2}) @@ -156,8 +201,17 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc } report.Discovery.Checks = append(report.Discovery.Checks, ProbeChecks(inv, r.Targets)...) for _, d := range append(slices.Clone(inv.DPUs), inv.Hosts...) { - addCheck("dpf.device.identity", d.RecordKey, d.ID != "" && d.ObservationState == "observed", "PHYSICAL_IDENTITY_REQUIRED", d.EvidenceRefs) - addCheck("dpf.device.qualification", d.RecordKey, d.Qualification == "qualified" && d.InventoryCompleteness == "complete", "QUALIFIED_ADAPTER_REQUIRED", d.EvidenceRefs) + addCheck("dpf.device.identity", d.RecordKey, d.ID != "" && d.ObservationState == "observed", "PHYSICAL_IDENTITY_REQUIRED", d.EvidenceRefs, "discover") + addCheck("dpf.device.qualification", d.RecordKey, d.Qualification == "qualified" && d.InventoryCompleteness == "complete", "QUALIFIED_ADAPTER_REQUIRED", d.EvidenceRefs, "generate") + } + addCheck("dpf.inventory.dpus", "inventory", len(inv.DPUs) > 0, "DPU_IDENTITY_REQUIRED", nil, "discover") + for _, d := range inv.DPUs { + state, reason := "not-evaluated", "REDFISH_AUTHORIZATION_NOT_EVALUATED" + if d.Authentication.PasswordRead == "verified" { + state, reason = "passed", "INVENTORY_READ_VERIFIED" + } + report.Discovery.Checks = append(report.Discovery.Checks, inventory.Check{ID: "dpf.management.redfish-read", Subject: d.RecordKey, RequiredPhase: "deploy", State: state, ReasonCode: reason, EvidenceRefs: d.EvidenceRefs}) + report.Discovery.Checks = append(report.Discovery.Checks, inventory.Check{ID: "dpf.management.dpf-authentication", Subject: d.RecordKey, RequiredPhase: "deploy", State: "not-evaluated", ReasonCode: "DPF_AUTHENTICATION_NOT_EVALUATED", EvidenceRefs: []string{}}) } report.Discovery.Outcome = "complete" for _, c := range report.Discovery.Checks { @@ -169,6 +223,7 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc report.Discovery.Outcome = "partial" } if ctx.Err() != nil { + report.Discovery.Outcome = "partial" report.Execution = "failed" code := 3 if ctx.Err() == context.Canceled { @@ -179,6 +234,9 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc } inv.Collection.FinishedAt = time.Now().UTC() Canonicalize(inv) + if r.Kubeconfig != "" && inv.Management == nil { + inv.ScopeDigest = "" + } report.Diff = Diff(r.Input.Site.Inventory, inv) if scopeConflict(r.Input.Site.Inventory, inv) { report.Discovery.Outcome = "partial" @@ -217,7 +275,7 @@ func Correlate(dpus, hosts []inventory.Device) ([]inventory.Association, []inven h := &hosts[hi] for ai := range h.Adapters { a := &h.Adapters[ai] - if d.SerialNumber != "" && a.SerialNumber == d.SerialNumber { + if d.ObservationState == "observed" && h.ObservationState == "observed" && d.SerialNumber != "" && a.SerialNumber == d.SerialNumber { // Selection annotations are applied by the orchestrator after correlation. matches = append(matches, inventory.Association{DPUID: d.ID, HostID: h.ID, HostAdapterKey: a.Key, SerialNumber: d.SerialNumber, EvidenceRefs: append(slices.Clone(d.EvidenceRefs), a.EvidenceRefs...)}) } @@ -246,6 +304,10 @@ func Canonicalize(inv *inventory.Inventory) { sortDevices(inv.Hosts) slices.SortFunc(inv.Associations, func(a, b inventory.Association) int { return strings.Compare(a.DPUID, b.DPUID) }) inv.Fingerprint = inventory.Digest("dpf-facts-v1", material(inv)) + inv.ScopeDigest = inventory.Digest("dpf-infrastructure-scope-v1", struct { + Mode string + Endpoints []inventory.ScopeEndpoint + }{inv.Collection.Mode, inv.RequestedScope}) if inv.Management != nil { inv.ScopeDigest = inventory.Digest("dpf-scope-v1", struct { API, UID, Trust, Mode string @@ -283,7 +345,11 @@ func material(inv *inventory.Inventory) map[string]any { key := "device." + d.RecordKey fields[key+".identity"] = []any{d.ID, d.IdentityKey, d.Model, d.ReaderAdapter, d.Qualification, d.InventoryCompleteness} for _, a := range d.Adapters { - fields[key+".adapter."+a.Key] = []string{a.SerialNumber, a.Model, a.Slot} + fields[key+".adapter."+a.Key] = []any{a.SerialNumber, a.Model, a.Slot, a.PCIAddress, a.VendorID, a.DeviceID, a.IdentityState, a.ReasonCode, canonicalFunctions(a.PCIFunctions)} + } + fields[key+".endpoints"] = []string{d.SSHEndpoint, d.BMCEndpoint, d.BMCAddressState} + if d.Runtime != nil { + fields[key+".runtime"] = canonicalRuntime(*d.Runtime) } for _, fw := range d.Firmware { fields[key+".firmware."+fw.Component+"."+fw.ResourceURI] = []string{fw.State, fw.RawVersion, fw.Activation} diff --git a/pkg/dpf/discovery/discover_test.go b/pkg/dpf/discovery/discover_test.go index b3fa3719..bb3c25a4 100644 --- a/pkg/dpf/discovery/discover_test.go +++ b/pkg/dpf/discovery/discover_test.go @@ -16,7 +16,7 @@ import ( func TestDiscovery(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF discovery checks") } func estate() inventory.Inventory { - return inventory.Inventory{Collection: inventory.Collection{Mode: "inventory"}, Management: &inventory.Management{APIEndpoint: "https://api.invalid", SystemNamespaceUID: "cluster-a", TrustDigest: "system-roots", KubernetesVersion: "v1", Nodes: []inventory.Node{{Name: "cp", UID: "node-a", Ready: true}}}, RequestedScope: []inventory.ScopeEndpoint{}, DPUs: []inventory.Device{{ID: "dpu-a", RecordKey: "a", SerialNumber: "CARD-A", ObservationState: "observed"}, {ID: "dpu-b", RecordKey: "b", SerialNumber: "CARD-B", ObservationState: "observed"}}, Hosts: []inventory.Device{{ID: "host-a", RecordKey: "host", Adapters: []inventory.Adapter{{Key: "adapter-a", SerialNumber: "CARD-A"}, {Key: "adapter-b", SerialNumber: "CARD-B"}, {Key: "adapter-c", SerialNumber: "CARD-C", Selection: "not-selected"}}}}, Associations: []inventory.Association{}} + return inventory.Inventory{Collection: inventory.Collection{Mode: "inventory"}, Management: &inventory.Management{APIEndpoint: "https://api.invalid", SystemNamespaceUID: "cluster-a", TrustDigest: "system-roots", KubernetesVersion: "v1", Nodes: []inventory.Node{{Name: "cp", UID: "node-a", Ready: true}}}, RequestedScope: []inventory.ScopeEndpoint{}, DPUs: []inventory.Device{{ID: "dpu-a", RecordKey: "a", SerialNumber: "CARD-A", ObservationState: "observed"}, {ID: "dpu-b", RecordKey: "b", SerialNumber: "CARD-B", ObservationState: "observed"}}, Hosts: []inventory.Device{{ObservationState: "observed", ID: "host-a", RecordKey: "host", Adapters: []inventory.Adapter{{Key: "adapter-a", SerialNumber: "CARD-A"}, {Key: "adapter-b", SerialNumber: "CARD-B"}, {Key: "adapter-c", SerialNumber: "CARD-C", Selection: "not-selected"}}}}, Associations: []inventory.Association{}} } var _ = Describe("inventory semantics", func() { @@ -135,14 +135,14 @@ discovery: Expect(err).NotTo(HaveOccurred()) Expect(string(data)).To(ContainSubstring(`"tlsInsecureSkipVerify":true`)) }) - It("keeps missing scope evidence separate from a proven replacement", func() { + It("distinguishes infrastructure-only scope from management scope", func() { old := estate() Canonicalize(&old) current := estate() current.Management = nil Canonicalize(¤t) - Expect(current.ScopeDigest).To(BeEmpty()) - Expect(scopeConflict(&old, ¤t)).To(BeFalse()) + Expect(current.ScopeDigest).NotTo(BeEmpty()) + Expect(scopeConflict(&old, ¤t)).To(BeTrue()) current = estate() current.Management.SystemNamespaceUID = "replacement" Canonicalize(¤t) diff --git a/pkg/dpf/discovery/inventory_mode_test.go b/pkg/dpf/discovery/inventory_mode_test.go new file mode 100644 index 00000000..673443d9 --- /dev/null +++ b/pkg/dpf/discovery/inventory_mode_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package discovery + +import ( + "context" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type fixtureCollector struct { + fail bool + cancel context.CancelFunc +} + +func (f fixtureCollector) Collect(_ context.Context, t config.Target, _ time.Duration) (inventory.Device, map[string]inventory.Evidence, error) { + d := inventory.Device{ID: "host-one", RecordKey: t.InputRef, InputRef: t.InputRef, ObservationState: "observed", Qualification: "unqualified", InventoryCompleteness: "partial", EvidenceRefs: []string{"source"}} + if t.Role == "hosts" { + d.Adapters = []inventory.Adapter{{Key: "card-one", SerialNumber: "CARD-ONE", Model: "BlueField-4", EvidenceRefs: []string{"source"}}} + } else { + d.ID = "dpu-one" + d.SerialNumber = "CARD-ONE" + d.SSHEndpoint = t.Endpoint + } + if f.fail { + d.ObservationState = "unavailable" + d.ReasonCode = "CLEANUP_INCOMPLETE" + return d, nil, &redfish.ObservationError{Primary: &redfish.Error{Code: "ACCESS_DENIED", URI: "/redfish/v1/Systems", Status: 403, Method: "GET"}, Cleanup: &redfish.Error{Code: "ACCESS_DENIED", URI: "/redfish/v1/SessionService/Sessions/own", Status: 403, Method: "DELETE"}, SessionURI: "/redfish/v1/SessionService/Sessions/own"} + } + if f.cancel != nil { + f.cancel() + } + return d, map[string]inventory.Evidence{"source": {Kind: "fixture", Authenticated: true}}, nil +} +func inventoryRequest() config.Request { + return config.Request{Mode: "inventory", Timeout: time.Second, RequestTimeout: time.Second, MaxConcurrentBMCs: 2, Targets: []config.Target{{Role: "hosts", Endpoint: "https://host.invalid", InputRef: "/discovery/hosts/0"}, {Role: "dpus", Endpoint: "ssh://arm.invalid:22", InputRef: "/discovery/dpus/0", SSH: &config.SSHAccess{}}}} +} + +var _ = Describe("independent inventory acceptance", func() { + It("completes identity and mapping without Kubernetes or DPU Redfish, retaining qualification and readiness gaps", func() { + r := Collect(context.Background(), inventoryRequest(), "run", Dependencies{BMC: fixtureCollector{}, SSH: fixtureCollector{}}) + Expect(ExitCode(r)).To(Equal(0)) + Expect(r.Inventory.Management).To(BeNil()) + Expect(r.Inventory.Associations).To(HaveLen(1)) + Expect(r.Acceptance).To(Equal("not-evaluated")) + Expect(r.Inventory.ScopeDigest).NotTo(BeEmpty()) + Expect(r.Discovery.Checks).To(ContainElement(And(HaveField("ID", "dpf.device.qualification"), HaveField("State", "failed"), HaveField("RequiredPhase", "generate")))) + Expect(r.Discovery.Checks).To(ContainElement(And(HaveField("ID", "dpf.management.dpf-authentication"), HaveField("State", "not-evaluated"), HaveField("RequiredPhase", "deploy")))) + }) + It("derives only host-reported card identity without inventing a BMC or ARM binding", func() { + req := inventoryRequest() + req.Targets = req.Targets[:1] + r := Collect(context.Background(), req, "run", Dependencies{BMC: fixtureCollector{}}) + Expect(ExitCode(r)).To(Equal(0)) + Expect(r.Inventory.DPUs).To(HaveLen(1)) + Expect(r.Inventory.Associations).To(HaveLen(1)) + Expect(r.Inventory.DPUs[0].BMCEndpoint).To(BeEmpty()) + Expect(r.Inventory.DPUs[0].SSHEndpoint).To(BeEmpty()) + Expect(r.Inventory.DPUs[0].ReaderAdapter).To(Equal("host-card-v1")) + }) + It("retains observation and cleanup failures instead of substituting another source", func() { + r := Collect(context.Background(), inventoryRequest(), "run", Dependencies{BMC: fixtureCollector{fail: true}, SSH: fixtureCollector{}}) + Expect(ExitCode(r)).To(Equal(3)) + Expect(r.Inventory.Associations).To(BeEmpty()) + Expect(r.Discovery.Cleanup.State).To(Equal("incomplete")) + Expect(r.Errors).To(ContainElement(And(HaveField("HTTPStatus", 403), HaveField("Method", "GET"), HaveField("Stage", "observation")))) + Expect(r.Errors).To(ContainElement(And(HaveField("HTTPStatus", 403), HaveField("Method", "DELETE"), HaveField("Stage", "cleanup")))) + Expect(r.Discovery.Cleanup.Leftovers).To(ContainElement(ContainSubstring("/Sessions/own"))) + }) + It("does not leave a completed handoff after cancellation of otherwise valid observations", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + req := inventoryRequest() + req.Targets = req.Targets[:1] + r := Collect(ctx, req, "run", Dependencies{BMC: fixtureCollector{cancel: cancel}}) + Expect(r.Inventory.Associations).To(HaveLen(1)) + Expect(r.Discovery.Outcome).To(Equal("partial")) + Expect(r.Execution).To(Equal("cancelled")) + Expect(ExitCode(r)).To(Equal(1)) + }) + + It("keeps unknown-serial BlueFields and rejects fabricated or ambiguous associations", func() { + h := inventory.Device{ID: "host", RecordKey: "h", ObservationState: "observed", Adapters: []inventory.Adapter{{Key: "one", Model: "BlueField-4"}}} + d := HostCards([]inventory.Device{h}) + Expect(d).To(HaveLen(1)) + Expect(d[0].ID).To(BeEmpty()) + a, c := Correlate(d, []inventory.Device{h}) + Expect(a).To(BeEmpty()) + Expect(c[0].State).To(Equal("failed")) + h.Adapters[0].SerialNumber = "SAME" + h.Adapters = append(h.Adapters, inventory.Adapter{Key: "two", Model: "BlueField-4", SerialNumber: "SAME"}) + a, c = Correlate(HostCards([]inventory.Device{h}), []inventory.Device{h}) + Expect(a).To(BeEmpty()) + Expect(c[0].State).To(Equal("failed")) + }) +}) diff --git a/pkg/dpf/discovery/observations.go b/pkg/dpf/discovery/observations.go new file mode 100644 index 00000000..580542b0 --- /dev/null +++ b/pkg/dpf/discovery/observations.go @@ -0,0 +1,73 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +package discovery + +import ( + "cmp" + "slices" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" +) + +// HostCards preserves host-reported BlueField identity when no independent DPU +// endpoint was requested. It does not manufacture ARM or BMC endpoint bindings. +func HostCards(hosts []inventory.Device) []inventory.Device { + devices := []inventory.Device{} + for _, host := range hosts { + if host.ObservationState != "observed" { + continue + } + for _, card := range host.Adapters { + if !strings.Contains(strings.ToLower(card.Model), "bluefield") { + continue + } + d := inventory.Device{ + RecordKey: inventory.Digest("dpf-host-card-v1", []string{host.ID, card.Key}), + InputRef: host.InputRef, Model: card.Model, SerialNumber: card.SerialNumber, + SerialNumberRaw: card.SerialNumberRaw, ReaderAdapter: "host-card-v1", + ObservationState: "observed", Qualification: "unqualified", InventoryCompleteness: "partial", + Authentication: inventory.Authentication{PasswordRead: "not-evaluated", ClientCertificateSupport: "unknown", DPFClientAuthentication: "not-evaluated"}, + BMCAddressState: "unknown", EvidenceRefs: slices.Clone(card.EvidenceRefs), + } + if redfish.ValidSerial(card.SerialNumber) { + d.IdentityKey = []string{"nvidia", card.SerialNumber} + d.ID = inventory.PhysicalID("dpu", d.IdentityKey) + } else { + d.ReasonCode = "CARD_SERIAL_MISSING" + } + devices = append(devices, d) + } + } + return devices +} + +func canonicalRuntime(r inventory.RuntimeObservation) inventory.RuntimeObservation { + r.Interfaces = slices.Clone(r.Interfaces) + for i := range r.Interfaces { + r.Interfaces[i].Addresses = slices.Clone(r.Interfaces[i].Addresses) + slices.Sort(r.Interfaces[i].Addresses) + } + slices.SortFunc(r.Interfaces, func(a, b inventory.Interface) int { + return cmp.Or(cmp.Compare(a.Index, b.Index), strings.Compare(a.Name, b.Name)) + }) + r.Routes = slices.Clone(r.Routes) + slices.SortFunc(r.Routes, func(a, b inventory.Route) int { + return cmp.Or(cmp.Compare(a.Table, b.Table), strings.Compare(a.Destination, b.Destination), strings.Compare(a.Gateway, b.Gateway), cmp.Compare(a.InterfaceIndex, b.InterfaceIndex)) + }) + r.DNS = slices.Clone(r.DNS) + slices.Sort(r.DNS) + return r +} + +func canonicalFunctions(functions []inventory.PCIFunction) []inventory.PCIFunction { + out := slices.Clone(functions) + for i := range out { + out[i].Netdevs = slices.Clone(out[i].Netdevs) + slices.Sort(out[i].Netdevs) + } + slices.SortFunc(out, func(a, b inventory.PCIFunction) int { return strings.Compare(a.Address, b.Address) }) + return out +} diff --git a/pkg/dpf/inventory/types.go b/pkg/dpf/inventory/types.go index f3874b17..b3210f1e 100644 --- a/pkg/dpf/inventory/types.go +++ b/pkg/dpf/inventory/types.go @@ -21,6 +21,8 @@ type Collection struct { Collector string `json:"collector" yaml:"collector"` } type ScopeEndpoint struct { + SSHTrustDigest string `json:"sshTrustDigest,omitempty" yaml:"sshTrustDigest,omitempty"` + SSHSudo bool `json:"sshSudo,omitempty" yaml:"sshSudo,omitempty"` Role string `json:"role" yaml:"role"` Endpoint string `json:"endpoint" yaml:"endpoint"` SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` @@ -65,40 +67,65 @@ type TimeObservation struct { NTPServers []string `json:"ntpServers,omitempty" yaml:"ntpServers,omitempty"` ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` } +type PCIFunction struct { + Address string `json:"address" yaml:"address"` + Driver string `json:"driver,omitempty" yaml:"driver,omitempty"` + NUMANode string `json:"numaNode,omitempty" yaml:"numaNode,omitempty"` + Netdevs []string `json:"netdevs,omitempty" yaml:"netdevs,omitempty"` +} type Adapter struct { - Key string `json:"adapterKey" yaml:"adapterKey"` - SerialNumber string `json:"serialNumber" yaml:"serialNumber"` - SerialNumberRaw string `json:"serialNumberRaw" yaml:"serialNumberRaw"` - Model string `json:"model,omitempty" yaml:"model,omitempty"` - Slot string `json:"slot,omitempty" yaml:"slot,omitempty"` - Selection string `json:"selection" yaml:"selection"` - EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` + PCIFunctions []PCIFunction `json:"pciFunctions,omitempty" yaml:"pciFunctions,omitempty"` + PCIAddress string `json:"pciAddress,omitempty" yaml:"pciAddress,omitempty"` + VendorID string `json:"vendorID,omitempty" yaml:"vendorID,omitempty"` + DeviceID string `json:"deviceID,omitempty" yaml:"deviceID,omitempty"` + IdentityState string `json:"identityState,omitempty" yaml:"identityState,omitempty"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + Key string `json:"adapterKey" yaml:"adapterKey"` + SerialNumber string `json:"serialNumber" yaml:"serialNumber"` + SerialNumberRaw string `json:"serialNumberRaw" yaml:"serialNumberRaw"` + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Slot string `json:"slot,omitempty" yaml:"slot,omitempty"` + Selection string `json:"selection" yaml:"selection"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` +} + +// RuntimeObservation contains OS-visible facts, separate from BMC management readiness. +type RuntimeObservation struct { + Hostname string `json:"hostname,omitempty" yaml:"hostname,omitempty"` + OS string `json:"os,omitempty" yaml:"os,omitempty"` + Interfaces []Interface `json:"interfaces,omitempty" yaml:"interfaces,omitempty"` + Routes []Route `json:"routes,omitempty" yaml:"routes,omitempty"` + DNS []string `json:"dns,omitempty" yaml:"dns,omitempty"` + Observations map[string]Observation `json:"observations,omitempty" yaml:"observations,omitempty"` } // Device represents every requested endpoint, including failed observations. type Device struct { - RecordKey string `json:"recordKey" yaml:"recordKey"` - InputRef string `json:"inputRef" yaml:"inputRef"` - BMCEndpoint string `json:"bmcEndpoint" yaml:"bmcEndpoint"` - SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` - ObservationState string `json:"observationState" yaml:"observationState"` - ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - IdentityKey []string `json:"identityKey,omitempty" yaml:"identityKey,omitempty"` - SerialNumber string `json:"serialNumber,omitempty" yaml:"serialNumber,omitempty"` - SerialNumberRaw string `json:"serialNumberRaw,omitempty" yaml:"serialNumberRaw,omitempty"` - SystemUUID string `json:"systemUUID,omitempty" yaml:"systemUUID,omitempty"` - Manufacturer string `json:"manufacturer,omitempty" yaml:"manufacturer,omitempty"` - Model string `json:"model,omitempty" yaml:"model,omitempty"` - ReaderAdapter string `json:"readerAdapter,omitempty" yaml:"readerAdapter,omitempty"` - InventoryCompleteness string `json:"inventoryCompleteness,omitempty" yaml:"inventoryCompleteness,omitempty"` - Qualification string `json:"qualification" yaml:"qualification"` - Firmware []Firmware `json:"firmware" yaml:"firmware"` - Authentication Authentication `json:"authentication" yaml:"authentication"` - Time TimeObservation `json:"time" yaml:"time"` - Adapters []Adapter `json:"adapters,omitempty" yaml:"adapters,omitempty"` - AdvertisedResetTypes []string `json:"advertisedResetTypes,omitempty" yaml:"advertisedResetTypes,omitempty"` - EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` + SSHEndpoint string `json:"sshEndpoint,omitempty" yaml:"sshEndpoint,omitempty"` + BMCAddressState string `json:"bmcAddressState,omitempty" yaml:"bmcAddressState,omitempty"` + Runtime *RuntimeObservation `json:"runtime,omitempty" yaml:"runtime,omitempty"` + RecordKey string `json:"recordKey" yaml:"recordKey"` + InputRef string `json:"inputRef" yaml:"inputRef"` + BMCEndpoint string `json:"bmcEndpoint" yaml:"bmcEndpoint"` + SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` + ObservationState string `json:"observationState" yaml:"observationState"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + IdentityKey []string `json:"identityKey,omitempty" yaml:"identityKey,omitempty"` + SerialNumber string `json:"serialNumber,omitempty" yaml:"serialNumber,omitempty"` + SerialNumberRaw string `json:"serialNumberRaw,omitempty" yaml:"serialNumberRaw,omitempty"` + SystemUUID string `json:"systemUUID,omitempty" yaml:"systemUUID,omitempty"` + Manufacturer string `json:"manufacturer,omitempty" yaml:"manufacturer,omitempty"` + Model string `json:"model,omitempty" yaml:"model,omitempty"` + ReaderAdapter string `json:"readerAdapter,omitempty" yaml:"readerAdapter,omitempty"` + InventoryCompleteness string `json:"inventoryCompleteness,omitempty" yaml:"inventoryCompleteness,omitempty"` + Qualification string `json:"qualification" yaml:"qualification"` + Firmware []Firmware `json:"firmware" yaml:"firmware"` + Authentication Authentication `json:"authentication" yaml:"authentication"` + Time TimeObservation `json:"time" yaml:"time"` + Adapters []Adapter `json:"adapters,omitempty" yaml:"adapters,omitempty"` + AdvertisedResetTypes []string `json:"advertisedResetTypes,omitempty" yaml:"advertisedResetTypes,omitempty"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` } type Node struct { Interfaces []Interface `json:"interfaces,omitempty" yaml:"interfaces,omitempty"` @@ -157,10 +184,14 @@ type Check struct { EvidenceRefs []string `json:"evidenceRefs"` } type Problem struct { - Code string `json:"code"` - Message string `json:"message"` - InputRef string `json:"inputRef,omitempty"` - ExitCode int `json:"-"` + Method string `json:"method,omitempty"` + ResourceURI string `json:"resourceURI,omitempty"` + HTTPStatus int `json:"httpStatus,omitempty"` + Stage string `json:"stage,omitempty"` + Code string `json:"code"` + Message string `json:"message"` + InputRef string `json:"inputRef,omitempty"` + ExitCode int `json:"-"` } type Artifact struct { Path string `json:"path"` diff --git a/pkg/dpf/redfish/client.go b/pkg/dpf/redfish/client.go index 595321a9..eb70e4f1 100644 --- a/pkg/dpf/redfish/client.go +++ b/pkg/dpf/redfish/client.go @@ -26,12 +26,13 @@ type Document map[string]any // Error deliberately omits response bodies, headers and private credentials. type Error struct { Code string + Method string URI string Status int } func (e *Error) Error() string { - return fmt.Sprintf("Redfish %s at %s (HTTP %d)", e.Code, e.URI, e.Status) + return fmt.Sprintf("Redfish %s: %s %s (HTTP %d)", e.Code, e.Method, e.URI, e.Status) } func Code(err error) string { var e *Error @@ -89,7 +90,7 @@ func (c *Client) guard(ref string) (string, error) { return "", &Error{Code: "CROSS_ORIGIN_LINK"} } } - if u.User != nil || u.Fragment != "" || u.Opaque != "" || u.RawPath != "" || !strings.HasPrefix(u.Path, "/redfish/v1/") || strings.Contains(u.Path, "..") || path.Clean(u.Path) != strings.TrimSuffix(u.Path, "/") { + if len(ref) > 2048 || u.User != nil || u.Fragment != "" || u.Opaque != "" || u.RawPath != "" || !strings.HasPrefix(u.Path, "/redfish/v1/") || strings.Contains(u.Path, "..") || path.Clean(u.Path) != strings.TrimSuffix(u.Path, "/") { return "", &Error{Code: "UNSAFE_LINK"} } // Queries are accepted only as explicit pagination $skip/$skiptoken parameters. @@ -102,7 +103,15 @@ func (c *Client) guard(ref string) (string, error) { u.Host = "" return u.String(), nil } -func (c *Client) request(ctx context.Context, method, ref string, body []byte) (Document, http.Header, error) { +func (c *Client) request(ctx context.Context, method, ref string, body []byte) (doc Document, headers http.Header, err error) { + defer func() { + var failure *Error + if errors.As(err, &failure) { + failure.Method = method + // Pagination tokens can carry opaque private state. Retain only the path. + failure.URI = strings.Split(failure.URI, "?")[0] + } + }() uri, e := c.guard(ref) if e != nil { return nil, nil, e @@ -217,7 +226,8 @@ func (c *Client) Open(ctx context.Context) (Document, error) { } username, password := c.access.Credentials() body, _ := json.Marshal(map[string]string{"UserName": username, "Password": password}) - // Never retry an ambiguous login. The token and session URI are never serialized. + // Never retry an ambiguous login. The token is never serialized; only a validated + // non-secret session receipt may be reported when cleanup fails. c.sessionUncertain = true _, header, err := c.request(ctx, http.MethodPost, collection, body) if err != nil { diff --git a/pkg/dpf/redfish/diagnostics.go b/pkg/dpf/redfish/diagnostics.go new file mode 100644 index 00000000..564da2d0 --- /dev/null +++ b/pkg/dpf/redfish/diagnostics.go @@ -0,0 +1,72 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import "errors" + +// Diagnostic contains only bounded public request metadata, never headers or bodies. +type Diagnostic struct { + Code string + Method string + URI string + Status int + Stage string +} + +// ObservationError retains independent observation and exact-session cleanup errors. +// SessionURI is a validated resource receipt, not a session token. +type ObservationError struct { + Primary error + Cleanup error + SessionURI string +} + +func (e *ObservationError) Error() string { + if e.Primary != nil { + return e.Primary.Error() + "; BMC session cleanup incomplete" + } + return "BMC session cleanup incomplete: " + e.Cleanup.Error() +} +func (e *ObservationError) Unwrap() error { + if e.Primary != nil { + return e.Primary + } + return e.Cleanup +} + +// Diagnostics preserves a failed observation even when cleanup also fails. +func Diagnostics(err error) []Diagnostic { + if err == nil { + return nil + } + var result []Diagnostic + appendError := func(err error, stage string) { + if err == nil { + return + } + d := Diagnostic{Code: Code(err), Stage: stage} + var remote *Error + if errors.As(err, &remote) { + d.Method, d.URI, d.Status = remote.Method, remote.URI, remote.Status + } + result = append(result, d) + } + var observation *ObservationError + if errors.As(err, &observation) { + appendError(observation.Primary, "observation") + appendError(observation.Cleanup, "cleanup") + } else { + appendError(err, "observation") + } + return result +} + +// CleanupReceipt returns only the exact validated receipt for a failed cleanup. +// An empty result means that no usable receipt is available. +func CleanupReceipt(err error) string { + var observation *ObservationError + if errors.As(err, &observation) { + return observation.SessionURI + } + return "" +} diff --git a/pkg/dpf/redfish/diagnostics_test.go b/pkg/dpf/redfish/diagnostics_test.go new file mode 100644 index 00000000..c5492295 --- /dev/null +++ b/pkg/dpf/redfish/diagnostics_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("observation and cleanup diagnostics", func() { + It("retains independent forbidden read and DELETE metadata with the exact recovery receipt", func() { + var writes []string + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes = append(writes, r.Method+" "+r.URL.Path) + } + switch r.URL.Path { + case "/redfish/v1/": + json.NewEncoder(w).Encode(Document{"Links": Document{"Sessions": Document{"@odata.id": "/redfish/v1/SessionService/Sessions"}}, "Systems": Document{"@odata.id": "/redfish/v1/Systems"}}) + case "/redfish/v1/SessionService/Sessions": + w.Header().Set("X-Auth-Token", "secret-token") + w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/owned") + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{}`)) + default: + w.WriteHeader(http.StatusForbidden) + } + })) + defer s.Close() + Expect(os.Setenv("DPF_DIAGNOSTIC_PASSWORD", "secret-password")).To(Succeed()) + defer os.Unsetenv("DPF_DIAGNOSTIC_PASSWORD") + source := fmt.Sprintf("management:\n kubeconfigRef: {file: unused}\ncredentials:\n read: {username: reader, passwordRef: {env: DPF_DIAGNOSTIC_PASSWORD}}\ndiscovery:\n tls: {insecureSkipVerify: true}\n dpus: [{bmcEndpoint: '%s', credentialRef: read}]\n hosts: [{bmcEndpoint: 'https://192.0.2.1', credentialRef: read}]\n", s.URL) + in, err := config.Decode([]byte(source), "/site/input.yaml") + Expect(err).NotTo(HaveOccurred()) + req, err := config.Resolve(in) + Expect(err).NotTo(HaveOccurred()) + device, _, err := (Collector{}).Collect(context.Background(), req.Targets[0], time.Second) + Expect(err).To(HaveOccurred()) + Expect(device.Authentication.PasswordRead).NotTo(Equal("verified")) + Expect(device.ReasonCode).To(Equal("CLEANUP_INCOMPLETE")) + Expect(Code(err)).To(Equal("ACCESS_DENIED")) + Expect(Diagnostics(err)).To(Equal([]Diagnostic{ + {Code: "ACCESS_DENIED", Method: "GET", URI: "/redfish/v1/Systems", Status: 403, Stage: "observation"}, + {Code: "ACCESS_DENIED", Method: "DELETE", URI: "/redfish/v1/SessionService/Sessions/owned", Status: 403, Stage: "cleanup"}, + })) + Expect(CleanupReceipt(err)).To(Equal("/redfish/v1/SessionService/Sessions/owned")) + Expect(err.Error()).NotTo(ContainSubstring("secret-")) + Expect(writes).To(Equal([]string{"POST /redfish/v1/SessionService/Sessions", "DELETE /redfish/v1/SessionService/Sessions/owned"})) + }) + It("omits query tokens and response bodies from diagnostics", func() { + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(`{"secret":"private-body"}`)) + })) + defer s.Close() + c := &Client{origin: s.URL, http: s.Client()} + _, err := c.Get(context.Background(), "/redfish/v1/Systems?$skiptoken=private-query") + Expect(Diagnostics(err)).To(Equal([]Diagnostic{{Code: "ACCESS_DENIED", Method: "GET", URI: "/redfish/v1/Systems", Status: 403, Stage: "observation"}})) + Expect(strings.Contains(err.Error(), "private")).To(BeFalse()) + Expect(CleanupReceipt(err)).To(BeEmpty()) + Expect(Diagnostics(nil)).To(BeEmpty()) + }) +}) diff --git a/pkg/dpf/redfish/host_inventory.go b/pkg/dpf/redfish/host_inventory.go new file mode 100644 index 00000000..b0842bac --- /dev/null +++ b/pkg/dpf/redfish/host_inventory.go @@ -0,0 +1,278 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "fmt" + "reflect" + "slices" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +type hostCard struct { + resource Resource + owned bool + pci *Resource +} + +// collectHostAdapters joins only advertised resources of the selected System. +// Chassis membership alone is insufficient when a chassis is shared or its +// ownership is unknown; HPE's System PCI DeviceResourceId provides that join. +func collectHostAdapters(ctx context.Context, c *Client, system Resource, hostID string, hpe bool, add func(string, string) string) ([]inventory.Adapter, error) { + cards := map[string]*hostCard{} + collections := map[string][]Resource{} + observed := map[string]Document{system.URI: hostIdentityFields(system.Document)} + readCollection := func(uri string) ([]Resource, error) { + members, err := c.Collection(ctx, uri) + if err == nil { + collections[uri] = members + for _, member := range members { + observed[member.URI] = hostIdentityFields(member.Document) + } + } + return members, err + } + putCard := func(resource Resource, owned bool) error { + key := strings.TrimSuffix(resource.URI, "/") + if existing := cards[key]; existing != nil { + if !reflect.DeepEqual(hostIdentityFields(existing.resource.Document), hostIdentityFields(resource.Document)) { + return &Error{Code: "INVENTORY_CHANGED", URI: resource.URI} + } + existing.owned = existing.owned || owned + } else { + cards[key] = &hostCard{resource: resource, owned: owned} + } + observed[resource.URI] = hostIdentityFields(resource.Document) + return nil + } + if uri := Link(system.Document, "NetworkInterfaces"); uri != "" { + interfaces, err := readCollection(uri) + if err != nil { + return nil, err + } + for _, iface := range interfaces { + uri := Link(Object(iface.Document, "Links"), "NetworkAdapter") + if uri == "" { + return nil, &Error{Code: "ADAPTER_LINK_MISSING", URI: iface.URI} + } + uri, err = c.guard(uri) + if err != nil { + return nil, err + } + card, err := c.Get(ctx, uri) + if err != nil { + return nil, err + } + if err := putCard(Resource{URI: uri, Document: card}, true); err != nil { + return nil, err + } + } + } + if hpe { + chassisLinks, err := c.resourceLinks(Object(system.Document, "Links"), "Chassis") + if err != nil { + return nil, err + } + for _, uri := range chassisLinks { + chassis, err := c.Get(ctx, uri) + if err != nil { + return nil, err + } + observed[uri] = hostIdentityFields(chassis) + owners, err := c.resourceLinks(Object(chassis, "Links"), "ComputerSystems") + if err != nil { + return nil, err + } + exclusive := len(owners) == 1 && strings.TrimSuffix(owners[0], "/") == strings.TrimSuffix(system.URI, "/") + networkURI := Link(chassis, "NetworkAdapters") + if networkURI == "" { + continue + } + adapters, err := readCollection(networkURI) + if err != nil { + return nil, err + } + for _, resource := range adapters { + if err := putCard(resource, exclusive); err != nil { + return nil, err + } + } + } + pciLinks, err := c.resourceLinks(Object(Object(Object(system.Document, "Oem"), "Hpe"), "Links"), "PCIDevices") + if err != nil { + return nil, err + } + for _, uri := range pciLinks { + devices, err := readCollection(uri) + if err != nil { + return nil, err + } + for _, pci := range devices { + class, hasClass := integer(pci.Document, "ClassCode", 255) + if (!hasClass || class != 2) && Text(pci.Document, "DeviceType") != "NIC" { + continue + } + id := Text(pci.Document, "DeviceResourceId") + var matches []*hostCard + if id != "" { + for _, card := range cards { + if Text(card.resource.Document, "Id") == id && card.pci == nil { + matches = append(matches, card) + } else if Text(card.resource.Document, "Id") == id && card.pci != nil { + return nil, &Error{Code: "ADAPTER_PCI_AMBIGUOUS", URI: pci.URI} + } + } + } + if len(matches) > 1 { + return nil, &Error{Code: "ADAPTER_PCI_AMBIGUOUS", URI: pci.URI} + } + if len(matches) == 1 { + matches[0].owned = true + matches[0].pci = &pci + } else { + // A PCI record is still useful when no NetworkAdapter exists. + cards[strings.TrimSuffix(pci.URI, "/")] = &hostCard{resource: pci, owned: true, pci: &pci} + } + } + } + } + if len(collections) == 0 { + return nil, &Error{Code: "HOST_INVENTORY_UNAVAILABLE", URI: system.URI} + } + var adapters []inventory.Adapter + serials := map[string]bool{} + for _, key := range sortedKeys(cards) { + card := cards[key] + if !card.owned { + continue + } + raw := Text(card.resource.Document, "SerialNumber") + serial := "" + state, reason := "unknown", "CARD_SERIAL_MISSING" + adapterKey := inventory.Digest("dpf-adapter-observation-v1", []string{hostID, key}) + if ValidSerial(raw) { + serial = strings.TrimSpace(raw) + if serials[serial] { + return nil, &Error{Code: "CARD_IDENTITY_AMBIGUOUS", URI: card.resource.URI} + } + serials[serial] = true + state, reason = "known", "" + adapterKey = inventory.Digest("dpf-adapter-v1", []string{hostID, serial}) + } + model := Text(card.resource.Document, "Model") + if model == "" { + model = Text(card.resource.Document, "Name") + } + a := inventory.Adapter{Key: adapterKey, SerialNumber: serial, SerialNumberRaw: raw, Model: model, IdentityState: state, ReasonCode: reason, Selection: "not-selected", Slot: Text(Object(Object(card.resource.Document, "Location"), "PartLocation"), "ServiceLabel"), EvidenceRefs: []string{add(card.resource.URI, "SerialNumber"), add(card.resource.URI, "Model")}} + if card.pci != nil { + pci := card.pci.Document + a.PCIAddress = hpePCIAddress(pci) + a.VendorID = hexID(pci, "VendorID") + a.DeviceID = hexID(pci, "DeviceID") + if slot := Text(pci, "LocationString"); slot != "" { + a.Slot = slot + } + for _, property := range []string{"DeviceResourceId", "SegmentNumber", "BusNumber", "DeviceNumber", "FunctionNumber", "VendorID", "DeviceID", "LocationString"} { + a.EvidenceRefs = append(a.EvidenceRefs, add(card.pci.URI, property)) + } + } + adapters = append(adapters, a) + } + // Recheck both collection membership and the identity/ownership fields used + // in joins. Telemetry counters are intentionally outside this snapshot. + for _, uri := range sortedKeys(collections) { + again, err := c.Collection(ctx, uri) + if err != nil { + return adapters, err + } + if !sameMembers(collections[uri], again) { + return adapters, &Error{Code: "INVENTORY_CHANGED", URI: uri} + } + } + for _, uri := range sortedKeys(observed) { + again, err := c.Get(ctx, uri) + if err != nil { + return adapters, err + } + if !reflect.DeepEqual(observed[uri], hostIdentityFields(again)) { + return adapters, &Error{Code: "IDENTITY_CHANGED", URI: uri} + } + } + return adapters, nil +} + +func sortedKeys[T any](m map[string]T) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} + +// resourceLinks supports the array used by HPE OEM PCI links and standard links. +func (c *Client) resourceLinks(d Document, key string) ([]string, error) { + value, exists := d[key] + if !exists || value == nil { + return nil, nil + } + values, array := value.([]any) + if !array { + values = []any{value} + } + var links []string + seen := map[string]bool{} + for _, v := range values { + uri, err := c.guard(Link(Document{"v": v}, "v")) + if err != nil { + return nil, err + } + canonical := strings.TrimSuffix(uri, "/") + if seen[canonical] { + return nil, &Error{Code: "DUPLICATE_MEMBER", URI: canonical} + } + seen[canonical] = true + links = append(links, uri) + } + return links, nil +} + +func hostIdentityFields(d Document) Document { + result := Document{} + for _, key := range []string{"Id", "SerialNumber", "Model", "Name", "Links", "NetworkInterfaces", "NetworkAdapters", "Location", "DeviceResourceId", "ClassCode", "DeviceType", "SegmentNumber", "BusNumber", "DeviceNumber", "FunctionNumber", "VendorID", "DeviceID", "LocationString"} { + if value, ok := d[key]; ok { + result[key] = value + } + } + if pci := Object(Object(Object(d, "Oem"), "Hpe"), "Links")["PCIDevices"]; pci != nil { + result["HpePCIDevices"] = pci + } + return result +} +func integer(d Document, key string, max int) (int, bool) { + n, ok := d[key].(float64) + return int(n), ok && n >= 0 && n <= float64(max) && n == float64(int(n)) +} +func hpePCIAddress(d Document) string { + values := make([]int, 4) + for i, field := range []struct { + key string + max int + }{{"SegmentNumber", 65535}, {"BusNumber", 255}, {"DeviceNumber", 31}, {"FunctionNumber", 7}} { + value, ok := integer(d, field.key, field.max) + if !ok { + return "" + } + values[i] = value + } + return fmt.Sprintf("%04x:%02x:%02x.%x", values[0], values[1], values[2], values[3]) +} +func hexID(d Document, field string) string { + if value, ok := integer(d, field, 65535); ok { + return fmt.Sprintf("%04x", value) + } + return "" +} diff --git a/pkg/dpf/redfish/host_inventory_test.go b/pkg/dpf/redfish/host_inventory_test.go new file mode 100644 index 00000000..c6518311 --- /dev/null +++ b/pkg/dpf/redfish/host_inventory_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func hostGraph() map[string]Document { + link := func(uri string) Document { return Document{"@odata.id": uri} } + collection := func(uris ...string) Document { + members := []any{} + for _, uri := range uris { + members = append(members, link(uri)) + } + return Document{"Members@odata.count": len(uris), "Members": members} + } + pci := func(id string, bus int, slot, name string) Document { + return Document{"ClassCode": 2, "DeviceType": "NIC", "DeviceResourceId": id, "SegmentNumber": 0, "BusNumber": bus, "DeviceNumber": 0, "FunctionNumber": 0, "VendorID": 5555, "DeviceID": 41695, "LocationString": slot, "Name": name} + } + return map[string]Document{ + "/redfish/v1/Systems/host": {"NetworkInterfaces": link("/redfish/v1/Systems/host/NetworkInterfaces"), "Links": Document{"Chassis": []any{link("/redfish/v1/Chassis/enclosure")}}, "Oem": Document{"Hpe": Document{"Links": Document{"PCIDevices": []any{link("/redfish/v1/Systems/host/PCIDevices")}}}}}, + "/redfish/v1/Systems/host/NetworkInterfaces": collection("/redfish/v1/Systems/host/NetworkInterfaces/embedded"), + "/redfish/v1/Systems/host/NetworkInterfaces/embedded": {"Links": Document{"NetworkAdapter": link("/redfish/v1/Chassis/enclosure/NetworkAdapters/embedded/")}}, + "/redfish/v1/Chassis/enclosure": {"NetworkAdapters": link("/redfish/v1/Chassis/enclosure/NetworkAdapters")}, + "/redfish/v1/Chassis/enclosure/NetworkAdapters": collection("/redfish/v1/Chassis/enclosure/NetworkAdapters/embedded", "/redfish/v1/Chassis/enclosure/NetworkAdapters/dpu"), + "/redfish/v1/Chassis/enclosure/NetworkAdapters/embedded": {"Id": "embedded", "Model": "Embedded NIC", "SerialNumber": "NIC-A"}, + "/redfish/v1/Chassis/enclosure/NetworkAdapters/dpu": {"Id": "dpu", "Model": "BlueField-4", "SerialNumber": "DPU-A"}, + "/redfish/v1/Systems/host/PCIDevices": collection("/redfish/v1/Systems/host/PCIDevices/dpu", "/redfish/v1/Systems/host/PCIDevices/other"), + "/redfish/v1/Systems/host/PCIDevices/dpu": pci("dpu", 129, "Slot 5", "BlueField-4"), + "/redfish/v1/Systems/host/PCIDevices/other": pci("", 33, "Slot 1", "ConnectX"), + } +} + +func graphClient(docs map[string]Document, mutate func(string, int, Document)) (*Client, func()) { + reads := map[string]int{} + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + uri := strings.TrimSuffix(r.URL.Path, "/") + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + doc, ok := docs[uri] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + reads[uri]++ + if mutate != nil { + mutate(uri, reads[uri], doc) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(doc) + })) + return &Client{origin: s.URL, http: s.Client()}, s.Close +} + +var _ = Describe("host inventory ownership and partial identity", func() { + It("joins HPE chassis cards to System PCI records and retains cards without serials", func() { + c, closeServer := graphClient(hostGraph(), nil) + defer closeServer() + system, err := c.Get(context.Background(), "/redfish/v1/Systems/host") + Expect(err).NotTo(HaveOccurred()) + cards, err := collectHostAdapters(context.Background(), c, Resource{URI: "/redfish/v1/Systems/host", Document: system}, "host-a", true, func(uri, prop string) string { return uri + "#" + prop }) + Expect(err).NotTo(HaveOccurred()) + Expect(cards).To(HaveLen(3)) + Expect(cards[0].SerialNumber).To(Equal("DPU-A")) + Expect(cards[0].PCIAddress).To(Equal("0000:81:00.0")) + Expect(cards[0].Slot).To(Equal("Slot 5")) + Expect(cards[0].VendorID).To(Equal("15b3")) + Expect(cards[0].DeviceID).To(Equal("a2df")) + Expect(cards[0].IdentityState).To(Equal("known")) + Expect(cards[0].EvidenceRefs).To(ContainElement("/redfish/v1/Systems/host/PCIDevices/dpu#DeviceResourceId")) + Expect(cards[2].SerialNumber).To(BeEmpty()) + Expect(cards[2].ReasonCode).To(Equal("CARD_SERIAL_MISSING")) + Expect(cards[2].IdentityState).To(Equal("unknown")) + Expect(cards[2].PCIAddress).To(Equal("0000:21:00.0")) + Expect(cards[2].Key).NotTo(BeEmpty()) + }) + It("does not assign an uncorrelated shared-chassis adapter to the selected host", func() { + docs := hostGraph() + docs["/redfish/v1/Systems/host/PCIDevices/dpu"]["DeviceResourceId"] = "not-a-chassis-card" + c, closeServer := graphClient(docs, nil) + defer closeServer() + system, err := c.Get(context.Background(), "/redfish/v1/Systems/host") + Expect(err).NotTo(HaveOccurred()) + cards, err := collectHostAdapters(context.Background(), c, Resource{URI: "/redfish/v1/Systems/host", Document: system}, "host-a", true, func(uri, prop string) string { return uri + "#" + prop }) + Expect(err).NotTo(HaveOccurred()) + for _, card := range cards { + Expect(card.SerialNumber).NotTo(Equal("DPU-A")) + } + }) + It("rejects duplicate physical serials and ambiguous PCI-to-adapter joins", func() { + for _, duplicateID := range []bool{false, true} { + docs := hostGraph() + if duplicateID { + docs["/redfish/v1/Chassis/enclosure/NetworkAdapters/embedded"]["Id"] = "dpu" + } else { + docs["/redfish/v1/Chassis/enclosure/NetworkAdapters/embedded"]["SerialNumber"] = "DPU-A" + } + c, closeServer := graphClient(docs, nil) + system, err := c.Get(context.Background(), "/redfish/v1/Systems/host") + Expect(err).NotTo(HaveOccurred()) + _, err = collectHostAdapters(context.Background(), c, Resource{URI: "/redfish/v1/Systems/host", Document: system}, "host-a", true, func(uri, prop string) string { return uri + "#" + prop }) + Expect(err).To(HaveOccurred()) + if duplicateID { + Expect(Code(err)).To(Equal("ADAPTER_PCI_AMBIGUOUS")) + } else { + Expect(Code(err)).To(Equal("CARD_IDENTITY_AMBIGUOUS")) + } + closeServer() + } + }) + It("detects card replacement during collection", func() { + c, closeServer := graphClient(hostGraph(), func(uri string, reads int, doc Document) { + if uri == "/redfish/v1/Chassis/enclosure/NetworkAdapters/dpu" && reads > 1 { + doc["SerialNumber"] = "DPU-REPLACED" + } + }) + defer closeServer() + system, err := c.Get(context.Background(), "/redfish/v1/Systems/host") + Expect(err).NotTo(HaveOccurred()) + _, err = collectHostAdapters(context.Background(), c, Resource{URI: "/redfish/v1/Systems/host", Document: system}, "host-a", true, func(uri, prop string) string { return uri + "#" + prop }) + Expect(Code(err)).To(Equal("IDENTITY_CHANGED")) + }) + It("does not guess a PCI domain or address when components are unavailable", func() { + Expect(hpePCIAddress(Document{"BusNumber": float64(42)})).To(BeEmpty()) + Expect(hexID(Document{"VendorID": "15b3"}, "VendorID")).To(BeEmpty()) + }) +}) diff --git a/pkg/dpf/redfish/inventory.go b/pkg/dpf/redfish/inventory.go index 6becfc6e..7d735ce6 100644 --- a/pkg/dpf/redfish/inventory.go +++ b/pkg/dpf/redfish/inventory.go @@ -4,7 +4,6 @@ package redfish import ( "context" - "fmt" "reflect" "regexp" "slices" @@ -42,20 +41,18 @@ func (co Collector) Collect(ctx context.Context, target config.Target, timeout t cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) defer cancel() if cleanupErr := c.Close(cleanupCtx); cleanupErr != nil { - if err == nil { - err = cleanupErr - } else { - err = fmt.Errorf("%w; BMC session cleanup incomplete", err) - } + err = &ObservationError{Primary: err, Cleanup: cleanupErr, SessionURI: c.session} device.ReasonCode = "CLEANUP_INCOMPLETE" } + if err != nil && device.ReasonCode == "" { + device.ReasonCode = Code(err) + } }() root, err := c.Open(ctx) if err != nil { device.ReasonCode = Code(err) return device, evidence, err } - device.Authentication.PasswordRead = "verified" device.Authentication.Method = "session" add := func(uri, property string) string { id := inventory.Digest("dpf-evidence-v1", []string{device.RecordKey, uri, property}) @@ -67,6 +64,7 @@ func (co Collector) Collect(ctx context.Context, target config.Target, timeout t if err != nil { return device, evidence, err } + device.Authentication.PasswordRead = "verified" var system Resource if target.SystemURI != "" { for _, s := range systems { @@ -141,41 +139,9 @@ func (co Collector) Collect(ctx context.Context, target config.Target, timeout t } device.IdentityKey = []string{"system-uuid", strings.ToLower(device.SystemUUID)} device.ID = inventory.PhysicalID("host", device.IdentityKey) - // NetworkInterfaces link the selected System to physical NetworkAdapters. - interfaces, er := c.Collection(ctx, Link(system.Document, "NetworkInterfaces")) - if er != nil { - return device, evidence, er - } - seen := map[string]bool{} - for _, iface := range interfaces { - uri := Link(Object(iface.Document, "Links"), "NetworkAdapter") - if uri == "" { - return device, evidence, &Error{Code: "ADAPTER_LINK_MISSING"} - } - uri, er = c.guard(uri) - if er != nil { - return device, evidence, er - } - if seen[uri] { - continue - } - seen[uri] = true - card, er := c.Get(ctx, uri) - if er != nil { - return device, evidence, er - } - raw := Text(card, "SerialNumber") - expectedSerials[uri] = raw - if !ValidSerial(raw) { - return device, evidence, &Error{Code: "CARD_SERIAL_MISSING"} - } - // Retain every physical network card; PCI/vendor omission cannot prove absence. - serial := strings.TrimSpace(raw) - device.Adapters = append(device.Adapters, inventory.Adapter{Key: inventory.Digest("dpf-adapter-v1", []string{device.ID, serial}), SerialNumber: serial, SerialNumberRaw: raw, Model: Text(card, "Model"), Selection: "not-selected", EvidenceRefs: []string{add(uri, "SerialNumber")}}) - } - again, er := c.Collection(ctx, Link(system.Document, "NetworkInterfaces")) - if er != nil || !sameMembers(interfaces, again) { - return device, evidence, &Error{Code: "INVENTORY_CHANGED"} + device.Adapters, err = collectHostAdapters(ctx, c, system, device.ID, adapter == "hpe-standard-v1", add) + if err != nil { + return device, evidence, err } device.InventoryCompleteness = "unqualified" } @@ -258,13 +224,19 @@ func (co Collector) Collect(ctx context.Context, target config.Target, timeout t } for uri, raw := range expectedSerials { current, e := c.Get(ctx, uri) - if e != nil || Text(current, "SerialNumber") != raw { - return device, evidence, &Error{Code: "IDENTITY_CHANGED"} + if e != nil { + return device, evidence, e + } + if Text(current, "SerialNumber") != raw { + return device, evidence, &Error{Code: "IDENTITY_CHANGED", URI: uri} } } refreshed, er := c.Get(ctx, system.URI) - if er != nil || Text(refreshed, "UUID") != device.SystemUUID || Text(refreshed, "SerialNumber") != Text(system.Document, "SerialNumber") { - return device, evidence, &Error{Code: "IDENTITY_CHANGED"} + if er != nil { + return device, evidence, er + } + if Text(refreshed, "UUID") != device.SystemUUID || Text(refreshed, "SerialNumber") != Text(system.Document, "SerialNumber") { + return device, evidence, &Error{Code: "IDENTITY_CHANGED", URI: system.URI} } device.ObservationState = "observed" return device, evidence, nil diff --git a/pkg/dpf/sshprobe/collect.go b/pkg/dpf/sshprobe/collect.go new file mode 100644 index 00000000..3fba48cf --- /dev/null +++ b/pkg/dpf/sshprobe/collect.go @@ -0,0 +1,311 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "context" + "path" + "strconv" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +type Collector struct{} + +type runner interface { + Run(context.Context, string, string) (string, error) +} + +func (Collector) Collect(ctx context.Context, target config.Target, timeout time.Duration) (device inventory.Device, evidence map[string]inventory.Evidence, err error) { + device = emptyDevice(target) + evidence = map[string]inventory.Evidence{} + if target.SSH == nil || (target.Role != "hosts" && target.Role != "dpus") { + device.ReasonCode = "SSH_CONFIGURATION_INVALID" + return device, evidence, &Error{Code: device.ReasonCode, Stage: "target"} + } + user, password := target.SSH.Credentials() + c, err := connect(ctx, target.Endpoint, credentials{user: user, password: password, key: target.SSH.PrivateKey(), knownHosts: target.SSH.KnownHosts()}, timeout) + if err != nil { + device.ReasonCode = Code(err) + return device, evidence, err + } + defer c.Close() + return collect(ctx, target, c) +} + +func emptyDevice(target config.Target) inventory.Device { + return inventory.Device{ + RecordKey: inventory.Digest("dpf-access-v1", []string{target.Role, target.Endpoint, target.SystemURI}), + InputRef: target.InputRef, SSHEndpoint: target.Endpoint, ObservationState: "unavailable", + Qualification: "unqualified", InventoryCompleteness: "unknown", BMCAddressState: "not-evaluated", + Authentication: inventory.Authentication{Method: "ssh", PasswordRead: "not-evaluated", ClientCertificateSupport: "unknown", DPFClientAuthentication: "not-evaluated"}, + Firmware: []inventory.Firmware{}, EvidenceRefs: []string{}, + Time: inventory.TimeObservation{State: "unknown", ReasonCode: "NOT_EVALUATED"}, + Runtime: &inventory.RuntimeObservation{Observations: map[string]inventory.Observation{}}, + } +} + +func collect(ctx context.Context, target config.Target, remote runner) (device inventory.Device, evidence map[string]inventory.Evidence, err error) { + device = emptyDevice(target) + evidence = map[string]inventory.Evidence{} + defer func() { + if err != nil { + device.ReasonCode = Code(err) + } + }() + device.ReaderAdapter = "linux-host-ssh-v1" + if target.Role == "dpus" { + device.ReaderAdapter = "bluefield-arm-ssh-v1" + } + privilege := "" + if target.SSH != nil && target.SSH.Sudo { + privilege = "sudo -n " + } + add := func(stage, property string) string { + id := inventory.Digest("dpf-evidence-v1", []string{device.RecordKey, stage, property}) + evidence[id] = inventory.Evidence{Kind: "ssh", InputRef: target.InputRef, ResourceURI: stage, Property: property, ObservedAt: time.Now().UTC(), Vantage: target.Role + "-os", Adapter: device.ReaderAdapter, Authenticated: true} + device.EvidenceRefs = append(device.EvidenceRefs, id) + return id + } + read := func(stage, command string) (string, error) { return remote.Run(ctx, stage, command) } + if target.Role == "dpus" { + dmi, e := read("arm-dmi", privilege+"dmidecode -s system-serial-number") + if e != nil { + return device, evidence, e + } + fru, e := read("arm-fru", privilege+"ipmitool fru print 0") + if e != nil { + return device, evidence, e + } + serial, model, maker, e := armIdentity(dmi, fru) + if e != nil { + return device, evidence, e + } + device.SerialNumber, device.SerialNumberRaw = serial, strings.TrimSpace(dmi) + device.Model, device.Manufacturer = model, maker + device.IdentityKey = []string{"nvidia", serial} + device.ID = inventory.PhysicalID("dpu", device.IdentityKey) + add("arm-dmi", "SerialNumber") + add("arm-fru", "Product Serial") + device.BMCAddressState = "unavailable" + lan, e := read("arm-ipmi-lan", privilege+"ipmitool lan print") + if e == nil { + device.BMCEndpoint, e = bmcAddress(lan) + } + if e == nil { + device.BMCAddressState = "observed-local-ipmi" + add("arm-ipmi-lan", "IP Address") + device.Runtime.Observations["bmcAddress"] = inventory.Observation{State: "observed", Values: []string{device.BMCEndpoint}} + } else { + device.Runtime.Observations["bmcAddress"] = inventory.Observation{State: "unavailable", ReasonCode: Code(e)} + } + } else { + uuid, e := read("host-dmi", privilege+"dmidecode -s system-uuid") + if e != nil { + return device, evidence, e + } + uuid = strings.TrimSpace(uuid) + if !validUUID(uuid) { + return device, evidence, &Error{Code: "SSH_HOST_IDENTITY_INVALID", Stage: "host-dmi"} + } + device.SystemUUID = uuid + device.IdentityKey = []string{"system-uuid", strings.ToLower(uuid)} + device.ID = inventory.PhysicalID("host", device.IdentityKey) + add("host-dmi", "UUID") + pci, e := read("host-pci", "lspci -Dnn -d 15b3:") + if e != nil { + return device, evidence, e + } + cards, e := pciCards(pci) + if e != nil { + return device, evidence, e + } + pciEvidence := add("host-pci", "PCIAddress") + for _, card := range cards { + a := inventory.Adapter{Key: inventory.Digest("dpf-adapter-pci-v1", []string{device.ID, card.address[:len(card.address)-2]}), PCIAddress: card.address, VendorID: card.vendor, DeviceID: card.device, Model: card.model, IdentityState: "partial", ReasonCode: "CARD_SERIAL_MISSING", Selection: "not-selected", EvidenceRefs: []string{pciEvidence}} + text, e := read("host-vpd:"+card.address, privilege+"lspci -D -vv -s "+card.address) + if e == nil { + serial, slot, parseErr := vpd(text) + if parseErr != nil { + return device, evidence, parseErr + } + a.SerialNumber, a.SerialNumberRaw, a.Slot = serial, serial, slot + a.EvidenceRefs = append(a.EvidenceRefs, add("host-vpd:"+card.address, "SerialNumber")) + if serial != "" { + a.IdentityState = "observed" + a.ReasonCode = "" + } + } else { + a.ReasonCode = Code(e) + } + for _, bdf := range card.functions { + a.PCIFunctions = append(a.PCIFunctions, collectPF(ctx, remote, bdf, device.Runtime, add)) + } + device.Adapters = append(device.Adapters, a) + } + device.Runtime.Observations["pciScope"] = inventory.Observation{State: "observed", Values: []string{"NVIDIA network PCI functions; grouped by host-local PCI slot"}} + } + collectRuntime(ctx, remote, device.Runtime, add) + // Recheck physical identity after optional observations, so disconnects and + // device changes cannot publish stale identity as a successful observation. + if target.Role == "dpus" { + dmi, e := read("arm-dmi-recheck", privilege+"dmidecode -s system-serial-number") + if e != nil { + return device, evidence, e + } + fru, e := read("arm-fru-recheck", privilege+"ipmitool fru print 0") + if e != nil { + return device, evidence, e + } + serial, model, maker, e := armIdentity(dmi, fru) + if e != nil { + return device, evidence, e + } + if serial != device.SerialNumber || model != device.Model || maker != device.Manufacturer { + return device, evidence, &Error{Code: "INVENTORY_CHANGED", Stage: "arm-identity-recheck"} + } + } else { + uuid, e := read("host-dmi-recheck", privilege+"dmidecode -s system-uuid") + if e != nil { + return device, evidence, e + } + if strings.TrimSpace(uuid) != device.SystemUUID { + return device, evidence, &Error{Code: "INVENTORY_CHANGED", Stage: "host-identity-recheck"} + } + pci, e := read("host-pci-recheck", "lspci -Dnn -d 15b3:") + if e != nil { + return device, evidence, e + } + cards, e := pciCards(pci) + if e != nil { + return device, evidence, e + } + if len(cards) != len(device.Adapters) { + return device, evidence, &Error{Code: "INVENTORY_CHANGED", Stage: "host-pci-recheck"} + } + for i, card := range cards { + a := device.Adapters[i] + if card.address != a.PCIAddress || card.vendor != a.VendorID || card.device != a.DeviceID || card.model != a.Model { + return device, evidence, &Error{Code: "INVENTORY_CHANGED", Stage: "host-pci-recheck"} + } + if len(card.functions) != len(a.PCIFunctions) { + return device, evidence, &Error{Code: "INVENTORY_CHANGED", Stage: "host-pci-functions-recheck"} + } + for j, bdf := range card.functions { + if bdf != a.PCIFunctions[j].Address { + return device, evidence, &Error{Code: "INVENTORY_CHANGED", Stage: "host-pci-functions-recheck"} + } + } + if a.SerialNumber == "" { + continue + } + text, e := read("host-vpd-recheck:"+card.address, privilege+"lspci -D -vv -s "+card.address) + if e != nil { + return device, evidence, e + } + serial, slot, e := vpd(text) + if e != nil { + return device, evidence, e + } + if serial != a.SerialNumber || slot != a.Slot { + return device, evidence, &Error{Code: "INVENTORY_CHANGED", Stage: "host-vpd-recheck"} + } + } + } + if ctx.Err() != nil { + return device, evidence, &Error{Code: "SSH_TIMEOUT_OR_CANCELLED", Stage: "collection"} + } + device.ObservationState = "observed" + device.InventoryCompleteness = "observed" + for _, a := range device.Adapters { + if a.IdentityState != "observed" { + device.InventoryCompleteness = "partial" + } + } + return device, evidence, nil +} + +func collectPF(ctx context.Context, remote runner, bdf string, runtime *inventory.RuntimeObservation, add func(string, string) string) inventory.PCIFunction { + pf := inventory.PCIFunction{Address: bdf, Netdevs: []string{}} + base := "/sys/bus/pci/devices/" + bdf + for _, p := range []struct{ field, command string }{{"driver", "readlink " + base + "/driver"}, {"numa", "cat " + base + "/numa_node"}, {"netdevs", "ls -1 " + base + "/net"}} { + stage := "host-pf:" + bdf + ":" + p.field + text, err := remote.Run(ctx, stage, p.command) + if err == nil { + text = strings.TrimSpace(text) + switch p.field { + case "driver": + if text == "" || strings.ContainsAny(text, "\r\n\t ") { + err = &Error{Code: "SSH_RUNTIME_INVALID", Stage: stage} + } else { + pf.Driver = path.Base(text) + } + case "numa": + n, e := strconv.Atoi(text) + if e != nil || n < -1 { + err = &Error{Code: "SSH_RUNTIME_INVALID", Stage: stage} + } else { + pf.NUMANode = text + } + case "netdevs": + if text != "" { + pf.Netdevs = strings.Split(text, "\n") + } + } + } + if err != nil { + runtime.Observations[stage] = inventory.Observation{State: "unavailable", ReasonCode: Code(err)} + } else { + runtime.Observations[stage] = inventory.Observation{State: "observed"} + add(stage, p.field) + } + } + return pf +} + +func collectRuntime(ctx context.Context, remote runner, runtime *inventory.RuntimeObservation, add func(string, string) string) { + for _, probe := range []struct{ stage, command string }{ + {"hostname", "hostname"}, {"os", "cat /etc/os-release"}, {"interfaces", "ip -j address show"}, {"routes", "ip -j -4 route show"}, {"dns", "cat /etc/resolv.conf"}, + } { + text, err := remote.Run(ctx, probe.stage, probe.command) + if err == nil { + switch probe.stage { + case "hostname": + runtime.Hostname = strings.TrimSpace(text) + if runtime.Hostname == "" || strings.ContainsAny(runtime.Hostname, "\r\n\t ") { + runtime.Hostname = "" + err = &Error{Code: "SSH_RUNTIME_INVALID", Stage: probe.stage} + } + case "os": + for _, line := range strings.Split(text, "\n") { + if value, ok := strings.CutPrefix(line, "PRETTY_NAME="); ok { + runtime.OS = strings.Trim(value, "\"") + } + } + if runtime.OS == "" { + err = &Error{Code: "SSH_RUNTIME_INVALID", Stage: probe.stage} + } + case "interfaces": + runtime.Interfaces, err = interfaces(text) + case "routes": + runtime.Routes, err = routes(text, runtime.Interfaces) + case "dns": + for _, line := range strings.Split(text, "\n") { + f := strings.Fields(line) + if len(f) >= 2 && f[0] == "nameserver" { + runtime.DNS = append(runtime.DNS, f[1]) + } + } + } + } + if err != nil { + runtime.Observations[probe.stage] = inventory.Observation{State: "unavailable", ReasonCode: Code(err)} + } else { + runtime.Observations[probe.stage] = inventory.Observation{State: "observed"} + add(probe.stage, probe.stage) + } + } +} diff --git a/pkg/dpf/sshprobe/collect_test.go b/pkg/dpf/sshprobe/collect_test.go new file mode 100644 index 00000000..d798c32a --- /dev/null +++ b/pkg/dpf/sshprobe/collect_test.go @@ -0,0 +1,189 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "context" + "strings" + "testing" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSSHProbe(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF SSH discovery") } + +const testUUID = "12345678-1234-1234-1234-123456789abc" +const testSerial = "CARD000123" +const testFRU = `FRU Device Description : Builtin FRU Device (ID 0) + Board Mfg : Nvidia + Board Product : B4240 + Board Serial : CARD000123 + Product Manufacturer : Nvidia + Product Name : BlueField-4 + Product Serial : CARD000123 +` +const testLAN = `Set in Progress : Set Complete +Auth Type Enable : Callback : + : User : + : Operator : +IP Address Source : DHCP Address +IP Address : 192.0.2.42 +MAC Address : 02:00:00:00:00:42 +` +const testPCI = `0000:21:00.0 Ethernet controller [0200]: Vendor BlueField-4 [15b3:a2df] +0000:21:00.1 Ethernet controller [0200]: Vendor BlueField-4 [15b3:a2df] +0000:41:00.0 Ethernet controller [0200]: Vendor ConnectX-9 [15b3:1025] +` + +type fakeRunner struct { + values map[string]string + errors map[string]error + calls []string +} + +func (r *fakeRunner) Run(_ context.Context, stage, command string) (string, error) { + r.calls = append(r.calls, stage+" "+command) + if err, ok := r.errors[stage]; ok { + return "", err + } + if value, ok := r.values[stage]; ok { + return value, nil + } + return "", &Error{Code: "SSH_COMMAND_FAILED", Stage: stage} +} + +func armRunner() *fakeRunner { + return &fakeRunner{values: map[string]string{ + "arm-dmi": testSerial + "\n", "arm-fru": testFRU, "arm-ipmi-lan": testLAN, + "arm-dmi-recheck": testSerial + "\n", "arm-fru-recheck": testFRU, + "hostname": "arm.example\n", "os": "PRETTY_NAME=\"Example Linux\"\n", + "interfaces": `[{"ifindex":2,"ifname":"oob_net0","mtu":1500,"addr_info":[{"local":"192.0.2.41","prefixlen":24}]}]`, + "routes": `[{"dst":"default","gateway":"192.0.2.1","dev":"oob_net0"}]`, + "dns": "nameserver 192.0.2.53\n", + }, errors: map[string]error{}, calls: []string{}} +} + +func hostRunner() *fakeRunner { + return &fakeRunner{values: map[string]string{ + "host-dmi": testUUID, "host-dmi-recheck": testUUID, + "host-pci": testPCI, "host-pci-recheck": testPCI, + "host-vpd:0000:21:00.0": "Physical Slot: 5\n [SN] Serial number: " + testSerial, + "host-vpd-recheck:0000:21:00.0": "Physical Slot: 5\n [SN] Serial number: " + testSerial, + "host-vpd:0000:41:00.0": "Capabilities: ", + }, errors: map[string]error{}, calls: []string{}} +} + +var _ = Describe("SSH inventory observations", func() { + It("corroborates ARM identity and observes a BMC address without contacting it", func() { + r := armRunner() + d, e, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22", InputRef: "/discovery/dpus/0", SSH: &config.SSHAccess{Sudo: true}}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.SerialNumber).To(Equal(testSerial)) + Expect(d.BMCEndpoint).To(Equal("https://192.0.2.42")) + Expect(d.BMCAddressState).To(Equal("observed-local-ipmi")) + Expect(d.ObservationState).To(Equal("observed")) + Expect(d.Qualification).To(Equal("unqualified")) + Expect(d.Authentication.DPFClientAuthentication).To(Equal("not-evaluated")) + Expect(d.Runtime.Interfaces).To(HaveLen(1)) + Expect(d.Runtime.Routes[0].InterfaceIndex).To(Equal(2)) + Expect(d.Runtime.Routes[0].Destination).To(Equal("0.0.0.0/0")) + Expect(e).NotTo(BeEmpty()) + Expect(strings.Join(r.calls, "\n")).NotTo(ContainSubstring("192.0.2.42")) + Expect(r.calls).To(ContainElement("arm-fru sudo -n ipmitool fru print 0")) + }) + It("preserves ARM identity with an unavailable optional BMC address", func() { + r := armRunner() + r.errors["arm-ipmi-lan"] = &Error{Code: "SSH_COMMAND_FAILED", Stage: "arm-ipmi-lan"} + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22"}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.ID).NotTo(BeEmpty()) + Expect(d.BMCEndpoint).To(BeEmpty()) + Expect(d.Runtime.Observations["bmcAddress"].State).To(Equal("unavailable")) + }) + It("rejects differing ARM identity sources without using hostname as identity", func() { + r := armRunner() + r.values["arm-dmi"] = "OTHER-CARD" + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22"}, r) + Expect(Code(err)).To(Equal("SSH_IDENTITY_CONFLICT")) + Expect(d.ID).To(BeEmpty()) + Expect(d.ObservationState).To(Equal("unavailable")) + }) + It("rejects a physical identity change during collection", func() { + r := armRunner() + r.values["arm-dmi-recheck"] = "NEW-CARD" + r.values["arm-fru-recheck"] = strings.ReplaceAll(testFRU, testSerial, "NEW-CARD") + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22"}, r) + Expect(Code(err)).To(Equal("INVENTORY_CHANGED")) + Expect(d.ObservationState).To(Equal("unavailable")) + }) + It("groups PCI functions by slot and retains cards with unknown serials", func() { + r := hostRunner() + d, _, err := collect(context.Background(), config.Target{Role: "hosts", Endpoint: "ssh://192.0.2.11:22"}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Adapters).To(HaveLen(2)) + Expect(d.SystemUUID).To(Equal(testUUID)) + Expect(d.Adapters[0].SerialNumber).To(Equal(testSerial)) + Expect(d.Adapters[0].Slot).To(Equal("5")) + Expect(d.Adapters[1].SerialNumber).To(BeEmpty()) + Expect(d.Adapters[1].IdentityState).To(Equal("partial")) + Expect(d.InventoryCompleteness).To(Equal("partial")) + Expect(d.Runtime.Observations["interfaces"].State).To(Equal("unavailable")) + }) + It("does not combine separate PCI slots with the same serial", func() { + r := hostRunner() + r.values["host-vpd:0000:41:00.0"] = "[SN] Serial number: " + testSerial + r.values["host-vpd-recheck:0000:41:00.0"] = "[SN] Serial number: " + testSerial + d, _, err := collect(context.Background(), config.Target{Role: "hosts", Endpoint: "ssh://192.0.2.11:22"}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Adapters).To(HaveLen(2)) + Expect(d.Adapters[0].Key).NotTo(Equal(d.Adapters[1].Key)) + }) + It("retains distinct per-function driver, NUMA and netdev observations", func() { + r := hostRunner() + for i, bdf := range []string{"0000:21:00.0", "0000:21:00.1"} { + r.values["host-pf:"+bdf+":driver"] = "../../../../bus/pci/drivers/mlx5_core\n" + r.values["host-pf:"+bdf+":numa"] = "0\n" + r.values["host-pf:"+bdf+":netdevs"] = []string{"hostpf0\n", "hostpf1\n"}[i] + } + d, _, err := collect(context.Background(), config.Target{Role: "hosts", Endpoint: "ssh://192.0.2.11:22"}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Adapters[0].PCIFunctions).To(HaveLen(2)) + Expect(d.Adapters[0].PCIFunctions[0].Netdevs).To(Equal([]string{"hostpf0"})) + Expect(d.Adapters[0].PCIFunctions[1].Netdevs).To(Equal([]string{"hostpf1"})) + Expect(d.Adapters[0].PCIFunctions[0].Driver).To(Equal("mlx5_core")) + Expect(d.Adapters[0].PCIFunctions[0].NUMANode).To(Equal("0")) + Expect(d.Runtime.Observations["host-pf:0000:41:00.0:driver"].State).To(Equal("unavailable")) + }) + It("rejects host identity or topology changes", func() { + for _, field := range []string{"host-dmi-recheck", "host-pci-recheck", "host-vpd-recheck:0000:21:00.0"} { + r := hostRunner() + r.values[field] = "" + d, _, err := collect(context.Background(), config.Target{Role: "hosts", Endpoint: "ssh://192.0.2.11:22"}, r) + Expect(err).To(HaveOccurred()) + Expect(d.ObservationState).To(Equal("unavailable")) + } + }) + It("rejects malformed and shell-bearing PCI output before any VPD command", func() { + r := hostRunner() + r.values["host-pci"] = "0000:21:00.0;touch /tmp/unwanted Ethernet controller [0200]: Vendor [15b3:a2df]" + _, _, err := collect(context.Background(), config.Target{Role: "hosts", Endpoint: "ssh://192.0.2.11:22"}, r) + Expect(Code(err)).To(Equal("SSH_PCI_INVENTORY_INVALID")) + Expect(strings.Join(r.calls, "\n")).NotTo(ContainSubstring("host-vpd")) + }) + It("rejects ambiguous FRUs, invalid identities and invalid BMC addresses", func() { + _, _, _, err := armIdentity(testSerial, testFRU+"Product Serial: OTHER\n") + Expect(err).To(HaveOccurred()) + for _, serial := range []string{"unknown", "000000", "\n", "line\nbreak"} { + Expect(validSerial(serial)).To(BeFalse()) + } + for _, uuid := range []string{"00000000-0000-0000-0000-000000000000", "ffffffff-ffff-ffff-ffff-ffffffffffff", "example"} { + Expect(validUUID(uuid)).To(BeFalse()) + } + for _, ip := range []string{"0.0.0.0", "127.0.0.1", "224.0.0.1", "192.0.2.1; echo bad"} { + _, err := bmcAddress(strings.ReplaceAll(testLAN, "192.0.2.42", ip)) + Expect(err).To(HaveOccurred()) + } + }) +}) diff --git a/pkg/dpf/sshprobe/parse.go b/pkg/dpf/sshprobe/parse.go new file mode 100644 index 00000000..feb6a8d4 --- /dev/null +++ b/pkg/dpf/sshprobe/parse.go @@ -0,0 +1,240 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "encoding/json" + "net" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +var ( + pciLine = regexp.MustCompile(`(?i)^([0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]) (.+) \[([0-9a-f]{4}):([0-9a-f]{4})\](?: .*)?$`) + uuidPattern = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) +) + +func validSerial(s string) bool { + s = strings.TrimSpace(s) + if s == "" || len(s) > 256 || strings.ContainsAny(s, "\x00\r\n\t") { + return false + } + switch strings.ToLower(s) { + case "unknown", "n/a", "none", "default string", "to be filled by o.e.m.": + return false + } + return strings.Trim(s, "0- ") != "" +} + +func validUUID(s string) bool { + return uuidPattern.MatchString(s) && strings.Trim(strings.ToLower(s), "0-") != "" && strings.Trim(strings.ToLower(s), "f-") != "" +} + +// fields parses fixed-label utility output. Duplicate labels with different +// values are rejected so a later FRU or injected line cannot replace identity. +func fields(text string) (map[string]string, error) { + out := map[string]string{} + for _, line := range strings.Split(text, "\n") { + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + k, v = strings.TrimSpace(k), strings.TrimSpace(v) + if k == "" { + continue + } + if before, exists := out[k]; exists && before != v { + return nil, &Error{Code: "SSH_IDENTITY_AMBIGUOUS", Stage: "duplicate-field"} + } + out[k] = v + } + return out, nil +} + +func armIdentity(dmi, fru string) (serial, model, manufacturer string, err error) { + f, err := fields(fru) + if err != nil { + return "", "", "", err + } + serial = strings.TrimSpace(dmi) + model, manufacturer = f["Product Name"], f["Product Manufacturer"] + if !validSerial(serial) || !validSerial(f["Product Serial"]) || serial != f["Product Serial"] { + return "", "", "", &Error{Code: "SSH_IDENTITY_CONFLICT", Stage: "arm-dmi-fru"} + } + if !strings.EqualFold(manufacturer, "nvidia") || !strings.Contains(strings.ToLower(model), "bluefield") { + return "", "", "", &Error{Code: "SSH_DPU_IDENTITY_UNSUPPORTED", Stage: "arm-fru"} + } + if board := f["Board Serial"]; board != "" && board != serial { + return "", "", "", &Error{Code: "SSH_IDENTITY_CONFLICT", Stage: "arm-board-fru"} + } + return serial, model, manufacturer, nil +} + +func bmcAddress(text string) (string, error) { + f, err := fields(text) + if err != nil { + return "", err + } + ip := net.ParseIP(f["IP Address"]) + mac, macErr := net.ParseMAC(f["MAC Address"]) + if ip == nil || ip.To4() == nil || ip.IsUnspecified() || ip.IsLoopback() || ip.IsMulticast() || macErr != nil || len(mac) != 6 { + return "", &Error{Code: "SSH_BMC_ADDRESS_UNAVAILABLE", Stage: "arm-ipmi-lan"} + } + return "https://" + ip.String(), nil +} + +type pciCard struct { + address, vendor, device, model string + functions []string +} + +func pciCards(text string) ([]pciCard, error) { + bySlot := map[string]pciCard{} + seen := map[string]bool{} + for _, line := range strings.Split(strings.TrimSpace(text), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + m := pciLine.FindStringSubmatch(line) + if m == nil { + return nil, &Error{Code: "SSH_PCI_INVENTORY_INVALID", Stage: "host-pci"} + } + if !strings.EqualFold(m[3], "15b3") || (!strings.Contains(m[2], "[0200]") && !strings.Contains(m[2], "[0280]")) { + continue + } + bdf := strings.ToLower(m[1]) + if seen[bdf] { + return nil, &Error{Code: "SSH_PCI_INVENTORY_INVALID", Stage: "duplicate-pci-function"} + } + seen[bdf] = true + if len(seen) > 256 { + return nil, &Error{Code: "SSH_PCI_INVENTORY_LIMIT", Stage: "host-pci"} + } + slot := bdf[:len(bdf)-2] + old, ok := bySlot[slot] + functions := append(old.functions, bdf) + if !ok || bdf < old.address { + old = pciCard{address: bdf, vendor: strings.ToLower(m[3]), device: strings.ToLower(m[4]), model: m[2]} + } + old.functions = functions + bySlot[slot] = old + } + if len(bySlot) > 256 { + return nil, &Error{Code: "SSH_PCI_INVENTORY_LIMIT", Stage: "host-pci"} + } + out := make([]pciCard, 0, len(bySlot)) + for _, card := range bySlot { + sort.Strings(card.functions) + out = append(out, card) + } + sort.Slice(out, func(i, j int) bool { return out[i].address < out[j].address }) + return out, nil +} + +func vpd(text string) (serial, slot string, err error) { + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "[SN] Serial number:") { + value := strings.TrimSpace(strings.TrimPrefix(line, "[SN] Serial number:")) + if serial != "" && serial != value { + return "", "", &Error{Code: "SSH_IDENTITY_AMBIGUOUS", Stage: "host-vpd"} + } + serial = value + } + if strings.HasPrefix(line, "Physical Slot:") { + slot = strings.TrimSpace(strings.TrimPrefix(line, "Physical Slot:")) + } + } + if !validSerial(serial) { + serial = "" + } + return serial, slot, nil +} + +func interfaces(text string) ([]inventory.Interface, error) { + var rows []struct { + Name string `json:"ifname"` + Index int `json:"ifindex"` + MTU int `json:"mtu"` + Addresses []struct { + Local string `json:"local"` + Prefix int `json:"prefixlen"` + } `json:"addr_info"` + } + if err := json.Unmarshal([]byte(text), &rows); err != nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "interfaces"} + } + if len(rows) > 4096 { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "interfaces"} + } + out := make([]inventory.Interface, 0, len(rows)) + for _, r := range rows { + if r.Name == "" || r.Index < 1 || r.MTU < 1 { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "interfaces"} + } + item := inventory.Interface{Name: r.Name, Index: r.Index, MTU: r.MTU, Addresses: []string{}} + for _, a := range r.Addresses { + address := a.Local + "/" + strconv.Itoa(a.Prefix) + if _, _, err := net.ParseCIDR(address); err != nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "interfaces"} + } + item.Addresses = append(item.Addresses, address) + } + out = append(out, item) + } + return out, nil +} + +func routes(text string, ifaces []inventory.Interface) ([]inventory.Route, error) { + var rows []struct { + Destination string `json:"dst"` + Gateway string `json:"gateway"` + Device string `json:"dev"` + Table json.RawMessage `json:"table"` + } + if err := json.Unmarshal([]byte(text), &rows); err != nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "routes"} + } + if len(rows) > 8192 { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "routes"} + } + indexes := map[string]int{} + for _, i := range ifaces { + indexes[i.Name] = i.Index + } + out := make([]inventory.Route, 0, len(rows)) + for _, r := range rows { + dst := r.Destination + if dst == "default" { + dst = "0.0.0.0/0" + } + if ip := net.ParseIP(dst); ip != nil { + if ip.To4() != nil { + dst += "/32" + } else { + dst += "/128" + } + } + if _, _, err := net.ParseCIDR(dst); err != nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "routes"} + } + if r.Gateway != "" && net.ParseIP(r.Gateway) == nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "routes"} + } + if indexes[r.Device] == 0 { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "routes"} + } + table := 254 + if len(r.Table) > 0 && string(r.Table) != "\"main\"" { + if err := json.Unmarshal(r.Table, &table); err != nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID", Stage: "routes"} + } + } + out = append(out, inventory.Route{Destination: dst, Gateway: r.Gateway, InterfaceIndex: indexes[r.Device], Table: table}) + } + return out, nil +} diff --git a/pkg/dpf/sshprobe/transport.go b/pkg/dpf/sshprobe/transport.go new file mode 100644 index 00000000..5dcafeb4 --- /dev/null +++ b/pkg/dpf/sshprobe/transport.go @@ -0,0 +1,201 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package sshprobe observes explicitly selected Linux hosts and DPU ARM endpoints. +// It never installs software, changes configuration, or connects to an address +// learned from the remote system. +package sshprobe + +import ( + "bytes" + "context" + "errors" + "fmt" + "net" + "net/url" + "os" + "sync" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +const maxOutput = 1 << 20 + +// Error carries a stable diagnostic without remote output or credential material. +type Error struct { + Code string + Stage string +} + +func (e *Error) Error() string { return e.Code + ": " + e.Stage } + +// Code classifies errors without exposing SSH library or remote error text. +func Code(err error) string { + var e *Error + if errors.As(err, &e) { + return e.Code + } + return "SSH_OBSERVATION_FAILED" +} + +type credentials struct { + user, password string + key, knownHosts []byte +} + +type client struct { + ssh *ssh.Client + stop func() bool + timeout time.Duration +} + +// hostKeyCallback loads a stable snapshot through the upstream known_hosts +// parser, preserving its hashed-host, wildcard, CA and revocation semantics. +// The temporary file contains public host keys only and is removed before dial. +func hostKeyCallback(data []byte) (ssh.HostKeyCallback, error) { + if len(bytes.TrimSpace(data)) == 0 || len(data) > maxOutput { + return nil, &Error{Code: "SSH_TRUST_INVALID", Stage: "known-hosts"} + } + f, err := os.CreateTemp("", "l8k-known-hosts-*") + if err != nil { + return nil, &Error{Code: "SSH_TRUST_INVALID", Stage: "known-hosts"} + } + defer os.Remove(f.Name()) + if _, err = f.Write(data); err != nil { + _ = f.Close() + return nil, &Error{Code: "SSH_TRUST_INVALID", Stage: "known-hosts"} + } + if err = f.Close(); err != nil { + return nil, &Error{Code: "SSH_TRUST_INVALID", Stage: "known-hosts"} + } + callback, err := knownhosts.New(f.Name()) + if err != nil { + return nil, &Error{Code: "SSH_TRUST_INVALID", Stage: "known-hosts"} + } + return callback, nil +} + +func connect(ctx context.Context, endpoint string, auth credentials, timeout time.Duration) (*client, error) { + u, err := url.Parse(endpoint) + if err != nil || u.Scheme != "ssh" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" || u.Hostname() == "" || u.Port() == "" || timeout <= 0 { + return nil, &Error{Code: "SSH_CONFIGURATION_INVALID", Stage: "endpoint"} + } + verify, err := hostKeyCallback(auth.knownHosts) + if err != nil { + return nil, err + } + if auth.user == "" || (auth.password == "") == (len(auth.key) == 0) { + return nil, &Error{Code: "SSH_CONFIGURATION_INVALID", Stage: "credential"} + } + var method ssh.AuthMethod + if len(auth.key) > 0 { + signer, parseErr := ssh.ParsePrivateKey(auth.key) + if parseErr != nil { + return nil, &Error{Code: "SSH_CONFIGURATION_INVALID", Stage: "private-key"} + } + method = ssh.PublicKeys(signer) + } else { + method = ssh.Password(auth.password) + } + dialCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + conn, err := (&net.Dialer{}).DialContext(dialCtx, "tcp", u.Host) + if err != nil { + return nil, &Error{Code: "SSH_CONNECT_FAILED", Stage: "dial"} + } + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + failed := true + defer func() { + if failed { + stop() + _ = conn.Close() + } + }() + deadline, _ := dialCtx.Deadline() + if err = conn.SetDeadline(deadline); err != nil { + return nil, &Error{Code: "SSH_CONNECT_FAILED", Stage: "deadline"} + } + trustFailed := false + cfg := &ssh.ClientConfig{User: auth.user, Auth: []ssh.AuthMethod{method}, HostKeyCallback: func(host string, remote net.Addr, key ssh.PublicKey) error { + err := verify(host, remote, key) + trustFailed = err != nil + return err + }} + cc, channels, requests, err := ssh.NewClientConn(conn, u.Host, cfg) + if err != nil { + code := "SSH_AUTHENTICATION_OR_HANDSHAKE_FAILED" + if trustFailed { + code = "SSH_HOST_KEY_REJECTED" + } + return nil, &Error{Code: code, Stage: "handshake"} + } + if err = conn.SetDeadline(time.Time{}); err != nil { + _ = cc.Close() + return nil, &Error{Code: "SSH_CONNECT_FAILED", Stage: "deadline"} + } + failed = false + return &client{ssh: ssh.NewClient(cc, channels, requests), stop: stop, timeout: timeout}, nil +} + +func (c *client) Close() { + c.stop() + _ = c.ssh.Close() +} + +type commandOutput struct { + mu sync.Mutex + stdout, stderr bytes.Buffer + size int + exceeded bool +} +type outputWriter struct { + output *commandOutput + stderr bool +} + +func (w outputWriter) Write(p []byte) (int, error) { + o := w.output + o.mu.Lock() + defer o.mu.Unlock() + if len(p) > maxOutput-o.size { + o.exceeded = true + return 0, fmt.Errorf("SSH output limit exceeded") + } + o.size += len(p) + if w.stderr { + return o.stderr.Write(p) + } + return o.stdout.Write(p) +} + +func (c *client) Run(ctx context.Context, stage, command string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + // Closing the connection also interrupts NewSession/Start if the server stops + // responding. A timed-out connection is not reused for later observations. + stop := context.AfterFunc(ctx, func() { _ = c.ssh.Close() }) + defer stop() + session, err := c.ssh.NewSession() + if err != nil { + return "", &Error{Code: "SSH_COMMAND_FAILED", Stage: stage} + } + defer session.Close() + output := &commandOutput{} + session.Stdout = outputWriter{output: output} + session.Stderr = outputWriter{output: output, stderr: true} + err = session.Run("LC_ALL=C " + command) + output.mu.Lock() + defer output.mu.Unlock() + if output.exceeded { + return "", &Error{Code: "SSH_OUTPUT_LIMIT", Stage: stage} + } + if ctx.Err() != nil { + return "", &Error{Code: "SSH_TIMEOUT_OR_CANCELLED", Stage: stage} + } + if err != nil { + return "", &Error{Code: "SSH_COMMAND_FAILED", Stage: stage} + } + return output.stdout.String(), nil +} diff --git a/pkg/dpf/sshprobe/transport_test.go b/pkg/dpf/sshprobe/transport_test.go new file mode 100644 index 00000000..7d119d7b --- /dev/null +++ b/pkg/dpf/sshprobe/transport_test.go @@ -0,0 +1,246 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/pem" + "fmt" + "net" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +type testServer struct { + listener net.Listener + key ssh.Signer + private ed25519.PrivateKey + mu sync.Mutex + connections []net.Conn + wg sync.WaitGroup + closing bool +} + +func startServer() *testServer { + _, private, err := ed25519.GenerateKey(rand.Reader) + Expect(err).NotTo(HaveOccurred()) + key, err := ssh.NewSignerFromKey(private) + Expect(err).NotTo(HaveOccurred()) + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + s := &testServer{listener: listener, key: key, private: private, connections: []net.Conn{}} + s.wg.Add(1) + go func() { + defer s.wg.Done() + for { + c, err := listener.Accept() + if err != nil { + return + } + s.mu.Lock() + if s.closing { + s.mu.Unlock() + _ = c.Close() + return + } + s.connections = append(s.connections, c) + s.wg.Add(1) + s.mu.Unlock() + go func() { defer s.wg.Done(); defer c.Close(); s.serve(c) }() + } + }() + return s +} + +func (s *testServer) serve(conn net.Conn) { + cfg := &ssh.ServerConfig{PublicKeyCallback: func(meta ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + if meta.User() == "reader" && string(key.Marshal()) == string(s.key.PublicKey().Marshal()) { + return nil, nil + } + return nil, fmt.Errorf("denied") + }, PasswordCallback: func(meta ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { + if meta.User() == "reader" && string(password) == "test-password" { + return nil, nil + } + return nil, fmt.Errorf("denied") + }} + cfg.AddHostKey(s.key) + server, chans, requests, err := ssh.NewServerConn(conn, cfg) + if err != nil { + return + } + defer server.Close() + go ssh.DiscardRequests(requests) + var sessions sync.WaitGroup + defer sessions.Wait() + for channel := range chans { + if channel.ChannelType() != "session" { + _ = channel.Reject(ssh.UnknownChannelType, "unsupported") + continue + } + ch, reqs, err := channel.Accept() + if err != nil { + return + } + sessions.Add(1) + go func() { + defer sessions.Done() + defer ch.Close() + for req := range reqs { + if req.Type != "exec" { + _ = req.Reply(false, nil) + continue + } + var payload struct{ Command string } + if ssh.Unmarshal(req.Payload, &payload) != nil { + _ = req.Reply(false, nil) + return + } + _ = req.Reply(true, nil) + switch payload.Command { + case "LC_ALL=C slow": + for range reqs { + } + return + case "LC_ALL=C flood": + _, _ = ch.Write([]byte(strings.Repeat("x", maxOutput+1))) + case "LC_ALL=C fail": + _, _ = ch.Stderr().Write([]byte("remote sensitive diagnostic")) + _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{7})) + return + default: + _, _ = ch.Write([]byte("observed\n")) + _, _ = ch.Stderr().Write([]byte("warning\n")) + } + _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0})) + return + } + }() + } +} + +func (s *testServer) close() { + s.mu.Lock() + s.closing = true + _ = s.listener.Close() + for _, c := range s.connections { + _ = c.Close() + } + s.mu.Unlock() + s.wg.Wait() +} +func (s *testServer) auth() credentials { + return credentials{user: "reader", password: "test-password", knownHosts: []byte(knownhosts.Line([]string{s.listener.Addr().String()}, s.key.PublicKey()) + "\n")} +} + +var _ = Describe("bounded SSH transport", func() { + var server *testServer + BeforeEach(func() { server = startServer() }) + AfterEach(func() { server.close() }) + It("verifies host identity before authentication and keeps stdout separate", func() { + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), server.auth(), time.Second) + Expect(err).NotTo(HaveOccurred()) + defer c.Close() + out, err := c.Run(context.Background(), "query", "query") + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(Equal("observed\n")) + }) + It("accepts hashed known-hosts entries through the upstream parser", func() { + auth := server.auth() + auth.knownHosts = []byte(knownhosts.HashHostname(knownhosts.Normalize(server.listener.Addr().String())) + " " + strings.TrimSpace(string(ssh.MarshalAuthorizedKey(server.key.PublicKey()))) + "\n") + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(err).NotTo(HaveOccurred()) + c.Close() + }) + It("uses only the explicitly provided private key", func() { + block, err := ssh.MarshalPrivateKey(server.private, "test") + Expect(err).NotTo(HaveOccurred()) + auth := server.auth() + auth.password = "" + auth.key = pem.EncodeToMemory(block) + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(err).NotTo(HaveOccurred()) + c.Close() + auth.key = []byte("invalid-private-key") + _, err = connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(Code(err)).To(Equal("SSH_CONFIGURATION_INVALID")) + Expect(err.Error()).NotTo(ContainSubstring("invalid-private-key")) + }) + It("rejects unknown and changed host keys", func() { + _, private, err := ed25519.GenerateKey(rand.Reader) + Expect(err).NotTo(HaveOccurred()) + other, err := ssh.NewSignerFromKey(private) + Expect(err).NotTo(HaveOccurred()) + for _, entry := range []string{knownhosts.Line([]string{"other.example"}, server.key.PublicKey()), knownhosts.Line([]string{server.listener.Addr().String()}, other.PublicKey())} { + auth := server.auth() + auth.knownHosts = []byte(entry + "\n") + _, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(Code(err)).To(Equal("SSH_HOST_KEY_REJECTED")) + } + }) + It("does not expose rejected credentials or remote stderr", func() { + auth := server.auth() + auth.password = "do-not-print" + _, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring(auth.password)) + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), server.auth(), time.Second) + Expect(err).NotTo(HaveOccurred()) + defer c.Close() + _, err = c.Run(context.Background(), "failure", "fail") + Expect(Code(err)).To(Equal("SSH_COMMAND_FAILED")) + Expect(err.Error()).NotTo(ContainSubstring("sensitive")) + }) + It("bounds command output", func() { + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), server.auth(), time.Second) + Expect(err).NotTo(HaveOccurred()) + defer c.Close() + _, err = c.Run(context.Background(), "large", "flood") + Expect(Code(err)).To(Equal("SSH_OUTPUT_LIMIT")) + }) + It("cancels a command and closes its transport", func() { + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), server.auth(), time.Second) + Expect(err).NotTo(HaveOccurred()) + defer c.Close() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + start := time.Now() + _, err = c.Run(ctx, "slow", "slow") + Expect(Code(err)).To(Equal("SSH_TIMEOUT_OR_CANCELLED")) + Expect(time.Since(start)).To(BeNumerically("<", time.Second)) + }) + It("bounds an unresponsive handshake", func() { + listener, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + defer listener.Close() + done := make(chan struct{}) + go func() { + conn, e := listener.Accept() + if e == nil { + defer conn.Close() + buf := make([]byte, 1024) + for { + if _, e = conn.Read(buf); e != nil { + break + } + } + } + close(done) + }() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + start := time.Now() + _, err = connect(ctx, "ssh://"+listener.Addr().String(), server.auth(), time.Second) + Expect(err).To(HaveOccurred()) + Expect(time.Since(start)).To(BeNumerically("<", time.Second)) + Eventually(done).Should(BeClosed()) + }) +}) diff --git a/pkg/target/dpf/driver.go b/pkg/target/dpf/driver.go index a5455522..fa22b15b 100644 --- a/pkg/target/dpf/driver.go +++ b/pkg/target/dpf/driver.go @@ -21,6 +21,7 @@ import ( "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" "github.com/nvidia/k8s-launch-kit/pkg/dpf/probe" "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/sshprobe" apperrors "github.com/nvidia/k8s-launch-kit/pkg/errors" "github.com/nvidia/k8s-launch-kit/pkg/target" ) @@ -76,7 +77,7 @@ func (d driver) Bind(inv target.Invocation) (target.Operation, error) { resolved, err := config.Resolve(input) if err != nil { fail(2, err.Error()) - } else if _, err = discovery.ManagementAccess(resolved); err != nil { + } else if err = validateManagement(resolved); err != nil { fail(2, err.Error()) } else { // Unqualified image assets reject management mode before any remote operation. @@ -89,14 +90,14 @@ func (d driver) Bind(inv target.Invocation) (target.Operation, error) { fail(1, err.Error()) } else { if !inv.Output.Quiet { - fmt.Fprintln(r.Stderr, "Inspecting explicitly selected management and BMC scope") + fmt.Fprintln(r.Stderr, "Inspecting explicitly selected inventory sources") for _, endpoint := range resolved.Targets { if endpoint.Access.InsecureSkipVerify() { fmt.Fprintf(r.Stderr, "Warning: BMC TLS certificate verification disabled for %s\n", endpoint.Endpoint) } } } - deps := discovery.Dependencies{BMC: redfish.Collector{Qualifications: catalog.Hardware()}, Probes: probe.Runner{Assets: assets, Journal: probe.FileJournal(filepath.Join(store.RunDir, "resources.json"))}} + deps := discovery.Dependencies{SSH: sshprobe.Collector{}, BMC: redfish.Collector{Qualifications: catalog.Hardware()}, Probes: probe.Runner{Assets: assets, Journal: probe.FileJournal(filepath.Join(store.RunDir, "resources.json"))}} if resolved.Input.Site.Inventory == nil { resolved.Input.Site.Inventory = store.PriorInventory } @@ -135,3 +136,11 @@ func (d driver) Bind(inv target.Invocation) (target.Operation, error) { return apperrors.NewExitStatus(discovery.ExitCode(report)) }), nil } + +func validateManagement(r config.Request) error { + if r.Kubeconfig == "" { + return nil + } + _, err := discovery.ManagementAccess(r) + return err +} diff --git a/skills/k8s-launch-kit-discover/SKILL.md b/skills/k8s-launch-kit-discover/SKILL.md index 219bbfa8..dee867b7 100644 --- a/skills/k8s-launch-kit-discover/SKILL.md +++ b/skills/k8s-launch-kit-discover/SKILL.md @@ -11,7 +11,13 @@ metadata: > **PREREQUISITE:** Read `../k8s-launch-kit-shared/SKILL.md` for install paths, global flags, and output modes. -For the independent DPF preview, read [DPF discovery](../../docs/user/dpf-discovery.md) and use its explicit management/BMC YAML. Keep the empty hardware/probe-image release gates visible; do not run Host discovery against the DPF estate. +For the independent DPF preview, read [DPF discovery](../../docs/user/dpf-discovery.md) and use its explicit SSH/Redfish YAML. Inventory mode needs no management kubeconfig unless management observations are requested. Inventory success requires selected identities and unique host/card associations, while hardware qualification and DPF authentication remain later-phase checks. Keep those gaps visible; do not run Host discovery against the DPF estate. + +SSH requires explicit credential references and known-hosts trust; it never inherits +Redfish TLS bypass, ambient keys or SSH configuration. ARM-local IPMI can report a +BMC address but does not authorize connecting to it. A failed selected source is +not silently replaced by another source. Host-only inventory preserves host-reported +BlueFields without inventing ARM/BMC bindings. DPF BMC TLS verification defaults to enabled. For an explicitly requested lab bypass, set the endpoint's `tls.insecureSkipVerify: true`; the shared diff --git a/skills/k8s-launch-kit-shared/SKILL.md b/skills/k8s-launch-kit-shared/SKILL.md index c42d17cd..f75ddac5 100644 --- a/skills/k8s-launch-kit-shared/SKILL.md +++ b/skills/k8s-launch-kit-shared/SKILL.md @@ -67,8 +67,8 @@ The root command `l8k --discover-cluster-config ...` still works for backward-co the `host` target. Existing invocations must omit `--target` unless the user explicitly requests it; `--target host` is an equivalent explicit form. -The `dpf` target name is reserved but its phases are not implemented in this -build. Do not attempt DPF provisioning. Use `l8k schema` and inspect +The `dpf` target exposes experimental inventory discovery through explicit +SSH/Redfish sources. Its provisioning phases remain unavailable. Do not attempt DPF provisioning. Use `l8k schema` and inspect `targets[].phases` before selecting any non-host target. Host-only flags such as `--fabric`, `--kubeconfig`, and the Network Operator/Spectrum-X flags are rejected when explicitly combined with another target. @@ -92,7 +92,7 @@ root-only, `--deploy-timeout` belongs to root/standalone deploy, and | Flag | Description | |------|-------------| -| `--target ` | Lifecycle target: `host` (default); `dpf` is currently unavailable | +| `--target ` | Lifecycle target: `host` (default); `dpf` offers experimental inventory discovery | | `--kubeconfig ` | Path to kubeconfig file (optional — falls back to `$KUBECONFIG` env var) | | `--user-config ` | Path to user-supplied l8k-config.yaml | | `--output ` | Output format: `text` (default), `json` | From 93752e8c33f53915771c77b829db759b0a9e4f98 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 5 Nov 2026 18:32:08 +0100 Subject: [PATCH 05/35] fix(dpf): prefer pinned SSH host key algorithms Signed-off-by: Alexander Maslennikov --- docs/advanced/targets.md | 2 +- docs/architecture/overview.md | 2 +- pkg/dpf/sshprobe/collect.go | 6 +-- pkg/dpf/sshprobe/collect_test.go | 3 +- pkg/dpf/sshprobe/transport.go | 58 ++++++++++++++++++++++++++- pkg/dpf/sshprobe/transport_test.go | 63 +++++++++++++++++++++++++++++- 6 files changed, 125 insertions(+), 9 deletions(-) diff --git a/docs/advanced/targets.md b/docs/advanced/targets.md index a4830d09..2078ed62 100644 --- a/docs/advanced/targets.md +++ b/docs/advanced/targets.md @@ -29,7 +29,7 @@ Passing `--target host` is an explicit synonym: l8k generate --target host ... ``` -The `dpf` target implements an [experimental inventory discovery preview](../user/dpf-discovery.md). Its software collects candidate management/BMC facts; hardware qualification and management probe-image release gates are explicit. Other DPF phases remain unavailable and cannot fall through to Host execution. +The `dpf` target implements an [experimental inventory discovery preview](../user/dpf-discovery.md). Its software collects explicit Redfish or host/ARM SSH inventory, optionally with management Kubernetes observations. Inventory acceptance is independent of DPF provisioning readiness; hardware qualification and management probe-image release gates remain explicit. Other DPF phases remain unavailable and cannot fall through to Host execution. Use `l8k schema` to inspect target and phase availability programmatically. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 160cf2f4..02adf219 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -49,7 +49,7 @@ flowchart TB validatesvc["Standalone validate + report service"] hostpaths["Host config, kubeconfig, preset, artifact resolution"] end - dpf["DPF target extension point\nrecognized; phases unavailable"] + dpf["DPF inventory — pkg/target/dpf + pkg/dpf\nexplicit Redfish / host SSH / ARM OOB SSH sources\nprovisioning phases unavailable"] end subgraph application["Host application orchestration"] diff --git a/pkg/dpf/sshprobe/collect.go b/pkg/dpf/sshprobe/collect.go index 3fba48cf..2d1fd0e7 100644 --- a/pkg/dpf/sshprobe/collect.go +++ b/pkg/dpf/sshprobe/collect.go @@ -125,7 +125,7 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i } pciEvidence := add("host-pci", "PCIAddress") for _, card := range cards { - a := inventory.Adapter{Key: inventory.Digest("dpf-adapter-pci-v1", []string{device.ID, card.address[:len(card.address)-2]}), PCIAddress: card.address, VendorID: card.vendor, DeviceID: card.device, Model: card.model, IdentityState: "partial", ReasonCode: "CARD_SERIAL_MISSING", Selection: "not-selected", EvidenceRefs: []string{pciEvidence}} + a := inventory.Adapter{Key: inventory.Digest("dpf-adapter-pci-v1", []string{device.ID, card.address[:len(card.address)-2]}), PCIAddress: card.address, VendorID: card.vendor, DeviceID: card.device, Model: card.model, IdentityState: "unknown", ReasonCode: "CARD_SERIAL_MISSING", Selection: "not-selected", EvidenceRefs: []string{pciEvidence}} text, e := read("host-vpd:"+card.address, privilege+"lspci -D -vv -s "+card.address) if e == nil { serial, slot, parseErr := vpd(text) @@ -135,7 +135,7 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i a.SerialNumber, a.SerialNumberRaw, a.Slot = serial, serial, slot a.EvidenceRefs = append(a.EvidenceRefs, add("host-vpd:"+card.address, "SerialNumber")) if serial != "" { - a.IdentityState = "observed" + a.IdentityState = "known" a.ReasonCode = "" } } else { @@ -221,7 +221,7 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i device.ObservationState = "observed" device.InventoryCompleteness = "observed" for _, a := range device.Adapters { - if a.IdentityState != "observed" { + if a.IdentityState != "known" { device.InventoryCompleteness = "partial" } } diff --git a/pkg/dpf/sshprobe/collect_test.go b/pkg/dpf/sshprobe/collect_test.go index d798c32a..ee78ee7b 100644 --- a/pkg/dpf/sshprobe/collect_test.go +++ b/pkg/dpf/sshprobe/collect_test.go @@ -127,7 +127,8 @@ var _ = Describe("SSH inventory observations", func() { Expect(d.Adapters[0].SerialNumber).To(Equal(testSerial)) Expect(d.Adapters[0].Slot).To(Equal("5")) Expect(d.Adapters[1].SerialNumber).To(BeEmpty()) - Expect(d.Adapters[1].IdentityState).To(Equal("partial")) + Expect(d.Adapters[0].IdentityState).To(Equal("known")) + Expect(d.Adapters[1].IdentityState).To(Equal("unknown")) Expect(d.InventoryCompleteness).To(Equal("partial")) Expect(d.Runtime.Observations["interfaces"].State).To(Equal("unavailable")) }) diff --git a/pkg/dpf/sshprobe/transport.go b/pkg/dpf/sshprobe/transport.go index 5dcafeb4..115cae60 100644 --- a/pkg/dpf/sshprobe/transport.go +++ b/pkg/dpf/sshprobe/transport.go @@ -77,6 +77,58 @@ func hostKeyCallback(data []byte) (ssh.HostKeyCallback, error) { return callback, nil } +// preferredHostKeyAlgorithms avoids negotiating an unpinned server key when a +// different advertised key is already trusted. Host matching and revocation +// decisions belong to the upstream callback, including hashed host entries. +// Certificates remain available and are verified by that same callback: the +// signing CA's algorithm does not determine the host certificate's key type. +func preferredHostKeyAlgorithms(ctx context.Context, data []byte, verify ssh.HostKeyCallback, address string, remote net.Addr) ([]string, error) { + preferred := map[string]bool{} + seen := map[string]bool{} + for _, line := range bytes.Split(data, []byte{'\n'}) { + if ctx.Err() != nil { + return nil, &Error{Code: "SSH_TIMEOUT_OR_CANCELLED", Stage: "known-host-algorithms"} + } + fields := bytes.Fields(line) + if len(fields) < 3 || fields[0][0] == '#' || fields[0][0] == '@' { + continue + } + // New already validated the file. Parse just its key fields so arbitrary + // multi-word comments do not change ParseKnownHosts field-count handling. + _, _, key, _, _, err := ssh.ParseKnownHosts(bytes.Join(fields[:3], []byte{' '})) + if err != nil { + return nil, &Error{Code: "SSH_TRUST_INVALID", Stage: "known-host-algorithms"} + } + blob := string(key.Marshal()) + if seen[blob] { + continue + } + seen[blob] = true + if verify(address, remote, key) != nil { + continue + } + switch key.Type() { + case ssh.KeyAlgoRSA: + preferred[ssh.KeyAlgoRSASHA256], preferred[ssh.KeyAlgoRSASHA512] = true, true + default: + preferred[key.Type()] = true + } + } + supported := ssh.SupportedAlgorithms().HostKeys + result := make([]string, 0, len(supported)) + for _, a := range supported { + if preferred[a] { + result = append(result, a) + } + } + for _, a := range supported { + if !preferred[a] { + result = append(result, a) + } + } + return result, nil +} + func connect(ctx context.Context, endpoint string, auth credentials, timeout time.Duration) (*client, error) { u, err := url.Parse(endpoint) if err != nil || u.Scheme != "ssh" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" || u.Hostname() == "" || u.Port() == "" || timeout <= 0 { @@ -117,8 +169,12 @@ func connect(ctx context.Context, endpoint string, auth credentials, timeout tim if err = conn.SetDeadline(deadline); err != nil { return nil, &Error{Code: "SSH_CONNECT_FAILED", Stage: "deadline"} } + algorithms, err := preferredHostKeyAlgorithms(dialCtx, auth.knownHosts, verify, u.Host, conn.RemoteAddr()) + if err != nil { + return nil, err + } trustFailed := false - cfg := &ssh.ClientConfig{User: auth.user, Auth: []ssh.AuthMethod{method}, HostKeyCallback: func(host string, remote net.Addr, key ssh.PublicKey) error { + cfg := &ssh.ClientConfig{User: auth.user, Auth: []ssh.AuthMethod{method}, HostKeyAlgorithms: algorithms, HostKeyCallback: func(host string, remote net.Addr, key ssh.PublicKey) error { err := verify(host, remote, key) trustFailed = err != nil return err diff --git a/pkg/dpf/sshprobe/transport_test.go b/pkg/dpf/sshprobe/transport_test.go index 7d119d7b..cb1563b5 100644 --- a/pkg/dpf/sshprobe/transport_test.go +++ b/pkg/dpf/sshprobe/transport_test.go @@ -3,9 +3,13 @@ package sshprobe import ( + "bytes" "context" + "crypto/ecdsa" "crypto/ed25519" + "crypto/elliptic" "crypto/rand" + "crypto/rsa" "encoding/pem" "fmt" "net" @@ -23,20 +27,21 @@ type testServer struct { listener net.Listener key ssh.Signer private ed25519.PrivateKey + extraKeys []ssh.Signer mu sync.Mutex connections []net.Conn wg sync.WaitGroup closing bool } -func startServer() *testServer { +func startServer(extraKeys ...ssh.Signer) *testServer { _, private, err := ed25519.GenerateKey(rand.Reader) Expect(err).NotTo(HaveOccurred()) key, err := ssh.NewSignerFromKey(private) Expect(err).NotTo(HaveOccurred()) listener, err := net.Listen("tcp", "127.0.0.1:0") Expect(err).NotTo(HaveOccurred()) - s := &testServer{listener: listener, key: key, private: private, connections: []net.Conn{}} + s := &testServer{listener: listener, key: key, private: private, extraKeys: extraKeys, connections: []net.Conn{}} s.wg.Add(1) go func() { defer s.wg.Done() @@ -73,6 +78,9 @@ func (s *testServer) serve(conn net.Conn) { return nil, fmt.Errorf("denied") }} cfg.AddHostKey(s.key) + for _, key := range s.extraKeys { + cfg.AddHostKey(key) + } server, chans, requests, err := ssh.NewServerConn(conn, cfg) if err != nil { return @@ -160,6 +168,57 @@ var _ = Describe("bounded SSH transport", func() { Expect(err).NotTo(HaveOccurred()) c.Close() }) + It("selects the pinned Ed25519 key on a server also offering ECDSA and RSA", func() { + ecdsaKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + ecdsaSigner, err := ssh.NewSignerFromKey(ecdsaKey) + Expect(err).NotTo(HaveOccurred()) + rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) + Expect(err).NotTo(HaveOccurred()) + rsaSigner, err := ssh.NewSignerFromKey(rsaKey) + Expect(err).NotTo(HaveOccurred()) + server.close() + server = startServer(ecdsaSigner, rsaSigner) + for _, hashed := range []bool{false, true} { + auth := server.auth() + if hashed { + auth.knownHosts = []byte(knownhosts.HashHostname(knownhosts.Normalize(server.listener.Addr().String())) + " " + strings.TrimSpace(string(ssh.MarshalAuthorizedKey(server.key.PublicKey()))) + "\n") + } + auth.knownHosts = append([]byte(knownhosts.Line([]string{"other.example"}, ecdsaSigner.PublicKey())+"\n"), bytes.TrimSpace(auth.knownHosts)...) + auth.knownHosts = append(auth.knownHosts, []byte(" pinned host key with a multi word comment\n")...) + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(err).NotTo(HaveOccurred()) + c.Close() + } + auth := server.auth() + auth.knownHosts = []byte(knownhosts.Line([]string{server.listener.Addr().String()}, rsaSigner.PublicKey()) + "\n") + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(err).NotTo(HaveOccurred()) + c.Close() + }) + It("preserves host certificate authority verification", func() { + _, caPrivate, err := ed25519.GenerateKey(rand.Reader) + Expect(err).NotTo(HaveOccurred()) + ca, err := ssh.NewSignerFromKey(caPrivate) + Expect(err).NotTo(HaveOccurred()) + certificate := &ssh.Certificate{Key: server.key.PublicKey(), CertType: ssh.HostCert, ValidPrincipals: []string{"127.0.0.1"}, ValidAfter: uint64(time.Now().Add(-time.Minute).Unix()), ValidBefore: uint64(time.Now().Add(time.Minute).Unix())} + Expect(certificate.SignCert(rand.Reader, ca)).To(Succeed()) + signer, err := ssh.NewCertSigner(certificate, server.key) + Expect(err).NotTo(HaveOccurred()) + server.close() + server = startServer(signer) + auth := server.auth() + auth.knownHosts = []byte("@cert-authority " + knownhosts.Line([]string{server.listener.Addr().String()}, ca.PublicKey()) + "\n") + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(err).NotTo(HaveOccurred()) + c.Close() + }) + It("does not promote revoked pinned keys", func() { + auth := server.auth() + auth.knownHosts = append(auth.knownHosts, []byte("@revoked "+knownhosts.Line([]string{server.listener.Addr().String()}, server.key.PublicKey())+"\n")...) + _, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), auth, time.Second) + Expect(Code(err)).To(Equal("SSH_HOST_KEY_REJECTED")) + }) It("uses only the explicitly provided private key", func() { block, err := ssh.MarshalPrivateKey(server.private, "test") Expect(err).NotTo(HaveOccurred()) From 9da5839204f863fafa615ea99fc0939be7019211 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 6 Nov 2026 11:29:49 +0100 Subject: [PATCH 06/35] feat(dpf): enrich discovery evidence and resolve release software BOM Signed-off-by: Alexander Maslennikov --- .../specs/configuration-parameters/spec.md | 14 + openspec/specs/discovery-inventory/spec.md | 28 ++ openspec/specs/workflow-execution/spec.md | 14 + pkg/dpf/addresslookup/lookup.go | 196 ++++++++++++ pkg/dpf/addresslookup/lookup_test.go | 64 ++++ pkg/dpf/artifacts/store.go | 5 + pkg/dpf/config/config.go | 42 ++- pkg/dpf/config/software_test.go | 46 +++ pkg/dpf/discovery/addresses.go | 49 +++ pkg/dpf/discovery/discover.go | 16 +- pkg/dpf/discovery/management.go | 6 +- pkg/dpf/discovery/reasons.go | 28 ++ pkg/dpf/discovery/software.go | 160 ++++++++++ pkg/dpf/discovery/software_test.go | 64 ++++ pkg/dpf/inventory/addresses.go | 17 ++ pkg/dpf/inventory/hardware.go | 62 ++++ pkg/dpf/inventory/types.go | 81 +++-- pkg/dpf/redfish/client.go | 45 ++- pkg/dpf/redfish/inventory.go | 72 +---- pkg/dpf/redfish/software.go | 285 ++++++++++++++++++ pkg/dpf/redfish/software_test.go | 182 +++++++++++ pkg/dpf/releases/releases.go | 230 ++++++++++++++ pkg/dpf/releases/releases.yaml | 108 +++++++ pkg/dpf/releases/releases_test.go | 104 +++++++ pkg/dpf/sshprobe/collect.go | 45 ++- pkg/dpf/sshprobe/diagnostics.go | 107 +++++++ pkg/dpf/sshprobe/hardware.go | 236 +++++++++++++++ pkg/dpf/sshprobe/hardware_test.go | 80 +++++ pkg/dpf/sshprobe/network.go | 224 ++++++++++++++ pkg/dpf/sshprobe/network_test.go | 64 ++++ pkg/dpf/sshprobe/transport.go | 13 +- pkg/dpf/sshprobe/transport_test.go | 23 ++ pkg/target/dpf/driver.go | 2 +- 33 files changed, 2578 insertions(+), 134 deletions(-) create mode 100644 pkg/dpf/addresslookup/lookup.go create mode 100644 pkg/dpf/addresslookup/lookup_test.go create mode 100644 pkg/dpf/config/software_test.go create mode 100644 pkg/dpf/discovery/addresses.go create mode 100644 pkg/dpf/discovery/reasons.go create mode 100644 pkg/dpf/discovery/software.go create mode 100644 pkg/dpf/discovery/software_test.go create mode 100644 pkg/dpf/inventory/addresses.go create mode 100644 pkg/dpf/inventory/hardware.go create mode 100644 pkg/dpf/redfish/software.go create mode 100644 pkg/dpf/redfish/software_test.go create mode 100644 pkg/dpf/releases/releases.go create mode 100644 pkg/dpf/releases/releases.yaml create mode 100644 pkg/dpf/releases/releases_test.go create mode 100644 pkg/dpf/sshprobe/diagnostics.go create mode 100644 pkg/dpf/sshprobe/hardware.go create mode 100644 pkg/dpf/sshprobe/hardware_test.go create mode 100644 pkg/dpf/sshprobe/network.go create mode 100644 pkg/dpf/sshprobe/network_test.go diff --git a/openspec/specs/configuration-parameters/spec.md b/openspec/specs/configuration-parameters/spec.md index bdd87852..4c8faea0 100644 --- a/openspec/specs/configuration-parameters/spec.md +++ b/openspec/specs/configuration-parameters/spec.md @@ -66,3 +66,17 @@ Tag-driven wiring covers root/generate/discover; standalone deploy/validate also - Implementation: [config](../../../pkg/config), [option tags](../../../pkg/options/options.go), [config input](../../../pkg/configinput), [bindings](../../../pkg/configflags), [resolver](../../../pkg/resolve), [Host adapters](../../../pkg/target/host). - Existing regression suites to extend: [resolver](../../../pkg/resolve/resolver_test.go), [YAML input](../../../pkg/config/input_test.go), [Host mapping](../../../pkg/target/host/driver_test.go). These are evidence starting points, not proof for a new field. - User documentation: [configuration](../../../docs/reference/configuration.md). + +### Requirement: DPF software release defaults + +The DPF target SHALL resolve software coordinates from a release-owned BOM, separately from site access inputs and observed running versions. Omitted or null release selection SHALL use the catalog default; explicit empty or unknown releases SHALL fail before remote access. Resolution SHALL preserve source configuration and return isolated catalog values. Minimum requirements, desired targets, model applicability and provenance SHALL remain distinct; unknown requirements SHALL retain reasons and never imply qualification. + +#### Scenario: Explicit unsupported release + +- **WHEN** a DPF input selects a release absent from the embedded catalog +- **THEN** configuration fails without substituting the default release or contacting infrastructure. + +#### Scenario: Inventory hardware is outside the BOM applicability + +- **WHEN** identity collection succeeds for hardware outside the selected release BOM +- **THEN** inventory remains useful and a reasoned later-phase compatibility check is unresolved; no supported tuple is invented. diff --git a/openspec/specs/discovery-inventory/spec.md b/openspec/specs/discovery-inventory/spec.md index 4f55f052..151b512e 100644 --- a/openspec/specs/discovery-inventory/spec.md +++ b/openspec/specs/discovery-inventory/spec.md @@ -166,3 +166,31 @@ There is no universal probe registry or typed provenance/unknown framework. Exis - Implementation: [discovery](../../../pkg/networkoperatorplugin/discovery/discover.go), [GPU topology](../../../pkg/networkoperatorplugin/discovery/gputopology.go), [presets](../../../pkg/presets), [application integration](../../../pkg/app/discover.go). - Existing suites: [discovery](../../../pkg/networkoperatorplugin/discovery/discover_test.go), [topology](../../../pkg/networkoperatorplugin/discovery/gputopology_test.go), [library](../../../pkg/networkoperatorplugin/discovery/library_test.go). - User documentation: [discovery](../../../docs/user/discovery.md), [presets](../../../docs/user/presets.md). + +### Requirement: Reasoned DPF observations + +New and changed discovery checks SHALL retain a stable reason code and safe human-readable explanation for failed, unavailable, unknown, skipped and deferred outcomes. Where known, the observation SHALL identify its subject, vantage, tool or resource, and provenance. Missing tools, denied access, malformed output, timeout, absent properties, unsupported capabilities and conflicts MUST NOT collapse into an unexplained not-measured result. Optional failures SHALL remain separate from required physical identity and association failures. + +#### Scenario: Firmware tool is absent + +- **WHEN** an SSH endpoint lacks an executable needed for an optional firmware observation +- **THEN** the saved inventory identifies the missing tool and reason, without installing it or discarding successfully collected identity. + +#### Scenario: Restricted device rejects a query + +- **WHEN** an installed tool cannot read a restricted device +- **THEN** the collector retains an access/query failure rather than claiming the tool is missing or the hardware absent, without changing privilege settings. + +### Requirement: DPF locator candidates are not identity + +Address lookup SHALL operate only on explicit MAC selectors and bounded declared sources. It SHALL preserve expired leases, ambiguous matches and unavailable sources as reasoned outcomes. Discovered addresses MUST NOT implicitly receive credentials, become selected endpoints or establish physical identity. + +#### Scenario: Sources disagree + +- **WHEN** usable sources associate one requested MAC with different addresses +- **THEN** all candidates retain source provenance and the result is ambiguous; discovery does not choose or connect to one. + +#### Scenario: Lease source cannot be parsed + +- **WHEN** a configured lease file is absent, unreadable, oversized or malformed +- **THEN** its unavailable result retains a safe reason and it contributes no candidates; other source evidence remains separately visible. diff --git a/openspec/specs/workflow-execution/spec.md b/openspec/specs/workflow-execution/spec.md index b08c147f..f3b8e442 100644 --- a/openspec/specs/workflow-execution/spec.md +++ b/openspec/specs/workflow-execution/spec.md @@ -71,3 +71,17 @@ The root pipeline runs discover/generate/deploy, not automatic validation. Exact - Implementation: [target registry](../../../pkg/target), [Host adapters](../../../pkg/target/host), [CLI routing](../../../pkg/cmd/target_cli.go), [application deploy](../../../pkg/app/deploy.go). - Existing suites: [target](../../../pkg/target/target_test.go), [Host driver](../../../pkg/target/host/driver_test.go), [deploy](../../../pkg/target/host/deploy_test.go), [launcher](../../../pkg/target/host/launcher_test.go). - User documentation: [targets](../../../docs/advanced/targets.md). + +### Requirement: Actionable non-success outcomes + +New and changed workflow checks SHALL persist a stable reason code and safe explanation for every failed, unavailable, unknown, skipped or deferred result. The result SHALL distinguish missing prerequisites from an attempted operation failure and identify the subject and source where known. Diagnostic persistence MUST NOT disclose credentials or treat optional unknowns as passes. Existing unrelated baseline paths are not retroactively migrated by this requirement. + +#### Scenario: Prerequisite executable is missing + +- **WHEN** a workflow cannot execute a check because a required tool is unavailable +- **THEN** the report and saved observations identify the executable and explain the missing prerequisite, while required versus optional policy determines whether dependent work may continue. + +#### Scenario: Service does not exist in the current phase + +- **WHEN** a check depends on an operator-created endpoint before that operator is installed +- **THEN** it is deferred with a reason and required phase, not reported as passed or used to fail initial hardware inventory. diff --git a/pkg/dpf/addresslookup/lookup.go b/pkg/dpf/addresslookup/lookup.go new file mode 100644 index 00000000..df787f36 --- /dev/null +++ b/pkg/dpf/addresslookup/lookup.go @@ -0,0 +1,196 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package addresslookup resolves explicitly selected MAC locators without contacting candidates. +package addresslookup + +import ( + "fmt" + "net" + "net/netip" + "slices" + "strconv" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +type Source struct { + File string `yaml:"file"` + Format string `yaml:"format"` +} +type Selector struct { + Name string `yaml:"name"` + Role string `yaml:"role"` + MAC string `yaml:"mac"` +} +type Config struct { + Sources []Source `yaml:"sources,omitempty"` + Devices []Selector `yaml:"devices"` +} + +// Request contains normalized selectors and absolute source paths, no credentials. +type Request struct { + Sources []Source + Devices []Selector +} +type Neighbor struct{ Address, MAC, State, Source, EvidenceRef string } + +func MAC(value string) (string, error) { + m, err := net.ParseMAC(value) + if err != nil || len(m) != 6 || m[0]&1 != 0 || m.String() == "00:00:00:00:00:00" { + return "", fmt.Errorf("expected a unicast six-byte MAC address") + } + return m.String(), nil +} +func Validate(c Config) (Request, error) { + r := Request{Sources: slices.Clone(c.Sources), Devices: slices.Clone(c.Devices)} + if len(r.Devices) == 0 || len(r.Devices) > 256 || len(r.Sources) > 16 { + return r, fmt.Errorf("addressLookup needs 1-256 devices and at most 16 sources") + } + names := map[string]bool{} + for i, d := range r.Devices { + if d.Name == "" || len(d.Name) > 63 || strings.ContainsAny(d.Name, "/\\\n\r\t ") || names[d.Name] { + return r, fmt.Errorf("addressLookup device names must be unique nonempty labels") + } + names[d.Name] = true + if d.Role != "arm" && d.Role != "bmc" { + return r, fmt.Errorf("addressLookup device role must be arm or bmc") + } + mac, err := MAC(d.MAC) + if err != nil { + return r, err + } + r.Devices[i].MAC = mac + } + for _, s := range r.Sources { + if s.File == "" || s.Format != "dnsmasq-leases" { + return r, fmt.Errorf("addressLookup source needs file and format: dnsmasq-leases") + } + } + return r, nil +} + +type lease struct { + mac, address string + expired bool +} + +func leases(data []byte, now time.Time) ([]lease, error) { + if len(data) > 1<<20 { + return nil, fmt.Errorf("lease source exceeds 1 MiB") + } + out := []lease{} + for _, line := range strings.Split(string(data), "\n") { + f := strings.Fields(line) + if len(f) == 0 || strings.HasPrefix(f[0], "#") { + continue + } + // The five-column dnsmasq IPv4 lease format; never guess other formats. + if len(f) != 5 { + return nil, fmt.Errorf("invalid dnsmasq lease record") + } + expiry, err := strconv.ParseInt(f[0], 10, 64) + if err != nil || expiry < 0 { + return nil, fmt.Errorf("invalid lease expiry") + } + mac, err := MAC(f[1]) + if err != nil { + return nil, err + } + ip, err := netip.ParseAddr(f[2]) + if err != nil || !ip.Is4() || !ip.IsGlobalUnicast() { + return nil, fmt.Errorf("invalid lease address") + } + out = append(out, lease{mac, ip.String(), expiry != 0 && expiry <= now.Unix()}) + if len(out) > 4096 { + return nil, fmt.Errorf("too many lease records") + } + } + return out, nil +} + +// Collect uses a bounded reader supplied by the caller. Invalid sources contribute +// no candidates. Valid candidates remain hints even when other sources fail. +func Collect(r Request, now time.Time, read func(string, int64) ([]byte, error), neighbors []Neighbor) (*inventory.AddressLookup, map[string]inventory.Evidence) { + out := &inventory.AddressLookup{Candidates: []inventory.AddressCandidate{}, Observations: map[string]inventory.Observation{}} + evidence := map[string]inventory.Evidence{} + expired := map[string]bool{} + add := func(mac, address, source, ref string) { + for _, d := range r.Devices { + if d.MAC == mac { + out.Candidates = append(out.Candidates, inventory.AddressCandidate{Name: d.Name, Role: d.Role, MAC: mac, Address: address, Source: source, EvidenceRef: ref}) + } + } + } + for i, s := range r.Sources { + key := fmt.Sprintf("source.%d", i) + data, err := read(s.File, 1<<20) + if err != nil { + out.Observations[key] = inventory.Observation{State: "unavailable", ReasonCode: "ADDRESS_SOURCE_UNREADABLE", Message: "Cannot read the configured bounded lease file; check that it exists and is readable."} + continue + } + rows, err := leases(data, now) + if err != nil { + out.Observations[key] = inventory.Observation{State: "unavailable", ReasonCode: "ADDRESS_SOURCE_INVALID", Message: "The lease file is not a bounded supported dnsmasq IPv4 lease table."} + continue + } + ref := inventory.Digest("dpf-address-source-v1", []string{key, inventory.Digest("lease-bytes-v1", data)}) + evidence[ref] = inventory.Evidence{Kind: "lease-file", InputRef: fmt.Sprintf("/discovery/addressLookup/sources/%d", i), ObservedAt: now, Vantage: "executor", Property: "MAC/IP/expiry"} + out.Observations[key] = inventory.Observation{State: "observed", Values: []string{ref}} + for _, row := range rows { + if row.expired { + expired[row.mac] = true + continue + } + add(row.mac, row.address, key, ref) + } + } + for _, n := range neighbors { + mac, err := MAC(n.MAC) + if err != nil { + continue + } + ip, err := netip.ParseAddr(n.Address) + if err != nil || !ip.IsGlobalUnicast() { + continue + } + // Only usable kernel entries; stale entries are not proof of current reachability. + if n.State != "REACHABLE" && n.State != "PERMANENT" { + continue + } + add(mac, ip.String(), n.Source, n.EvidenceRef) + } + slices.SortFunc(out.Candidates, func(a, b inventory.AddressCandidate) int { + return strings.Compare(a.Name+"/"+a.Address+"/"+a.Source, b.Name+"/"+b.Address+"/"+b.Source) + }) + out.Candidates = slices.Compact(out.Candidates) + for _, d := range r.Devices { + ips := map[string]bool{} + for _, c := range out.Candidates { + if c.Name == d.Name { + ips[c.Address] = true + } + } + o := inventory.Observation{State: "unknown", ReasonCode: "ADDRESS_NOT_FOUND", Message: "No usable lease or reachable neighbor matches the requested MAC; provide an address or refresh the source."} + if expired[d.MAC] { + o.ReasonCode = "ADDRESS_LEASE_EXPIRED" + o.Message = "Only expired matching lease evidence is available; refresh the lease source." + } + if len(ips) > 1 { + o.ReasonCode = "ADDRESS_AMBIGUOUS" + o.Message = "Sources report multiple addresses for this MAC; verify the intended endpoint before connecting." + } + if len(ips) == 1 { + o.State = "candidate" + o.ReasonCode = "ADDRESS_UNVERIFIED" + o.Message = "One address candidate was found; verify endpoint identity and configure explicit credentials and trust before connecting." + for ip := range ips { + o.Values = []string{ip} + } + } + out.Observations["device."+d.Name] = o + } + return out, evidence +} diff --git a/pkg/dpf/addresslookup/lookup_test.go b/pkg/dpf/addresslookup/lookup_test.go new file mode 100644 index 00000000..3ba83957 --- /dev/null +++ b/pkg/dpf/addresslookup/lookup_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package addresslookup + +import ( + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "testing" + "time" +) + +func TestLookup(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF address candidates") } + +var _ = Describe("Explicit MAC address lookup", func() { + var r Request + BeforeEach(func() { + var err error + r, err = Validate(Config{Sources: []Source{{File: "leases", Format: "dnsmasq-leases"}}, Devices: []Selector{{Name: "dpu1", Role: "arm", MAC: "02:00:00:00:00:01"}}}) + Expect(err).NotTo(HaveOccurred()) + }) + It("retains usable leases as candidates with provenance, not verified endpoints", func() { + out, ev := Collect(r, time.Unix(100, 0), func(string, int64) ([]byte, error) { return []byte("200 02:00:00:00:00:01 192.0.2.3 ignored *\n"), nil }, nil) + Expect(out.Candidates).To(HaveLen(1)) + Expect(ev).To(HaveLen(1)) + Expect(out.Observations["device.dpu1"].State).To(Equal("candidate")) + Expect(out.Candidates[0].EvidenceRef).NotTo(BeEmpty()) + }) + It("reports expired, missing, unreadable and malformed sources distinctly", func() { + for _, tc := range []struct { + data, code string + err error + }{{"99 02:00:00:00:00:01 192.0.2.3 * *", "ADDRESS_LEASE_EXPIRED", nil}, {"", "ADDRESS_NOT_FOUND", nil}, {"", "ADDRESS_SOURCE_UNREADABLE", fmt.Errorf("secret-bearing failure")}, {"bad data", "ADDRESS_SOURCE_INVALID", nil}} { + out, _ := Collect(r, time.Unix(100, 0), func(string, int64) ([]byte, error) { return []byte(tc.data), tc.err }, nil) + Expect(out.Candidates).To(BeEmpty()) + key := "device.dpu1" + if tc.code == "ADDRESS_SOURCE_UNREADABLE" || tc.code == "ADDRESS_SOURCE_INVALID" { + key = "source.0" + } + Expect(out.Observations[key].ReasonCode).To(Equal(tc.code)) + Expect(out.Observations[key].Message).NotTo(BeEmpty()) + Expect(out.Observations[key].Message).NotTo(ContainSubstring("secret-bearing")) + } + }) + It("retains disagreements and does not select an address", func() { + out, _ := Collect(r, time.Unix(100, 0), func(string, int64) ([]byte, error) { return []byte("200 02:00:00:00:00:01 192.0.2.3 * *"), nil }, []Neighbor{{Address: "192.0.2.4", MAC: r.Devices[0].MAC, State: "REACHABLE", Source: "host", EvidenceRef: "public"}}) + Expect(out.Candidates).To(HaveLen(2)) + Expect(out.Observations["device.dpu1"].ReasonCode).To(Equal("ADDRESS_AMBIGUOUS")) + Expect(out.Observations["device.dpu1"].Values).To(BeEmpty()) + }) + It("rejects unsupported sources and invalid selectors", func() { + for _, c := range []Config{{}, {Devices: []Selector{{Name: "x", Role: "arm", MAC: "ff:ff:ff:ff:ff:ff"}}}, {Devices: []Selector{{Name: "x", Role: "host", MAC: "02:00:00:00:00:01"}}}, {Sources: []Source{{File: "x", Format: "guessed"}}, Devices: r.Devices}} { + _, err := Validate(c) + Expect(err).To(HaveOccurred()) + } + }) + It("ignores stale neighbors and invalidates the whole malformed lease source", func() { + out, _ := Collect(r, time.Unix(100, 0), func(string, int64) ([]byte, error) { + return []byte("200 02:00:00:00:00:01 192.0.2.3 * *\nmalformed"), nil + }, []Neighbor{{Address: "192.0.2.4", MAC: r.Devices[0].MAC, State: "STALE", Source: "host"}}) + Expect(out.Candidates).To(BeEmpty()) + Expect(out.Observations["source.0"].ReasonCode).To(Equal("ADDRESS_SOURCE_INVALID")) + }) +}) diff --git a/pkg/dpf/artifacts/store.go b/pkg/dpf/artifacts/store.go index 87d0e818..9f03f944 100644 --- a/pkg/dpf/artifacts/store.go +++ b/pkg/dpf/artifacts/store.go @@ -75,6 +75,11 @@ func New(in config.Input, destination string, explicit bool) (*Store, error) { return nil, fmt.Errorf("output aliases input") } protected := []string{in.Site.Management.KubeconfigRef.File} + if in.Site.Discovery.AddressLookup != nil { + for _, source := range in.Site.Discovery.AddressLookup.Sources { + protected = append(protected, source.File) + } + } for _, c := range in.Site.Credentials { if c.PrivateKeyRef != nil { protected = append(protected, c.PrivateKeyRef.File) diff --git a/pkg/dpf/config/config.go b/pkg/dpf/config/config.go index 840ebe14..07823d6c 100644 --- a/pkg/dpf/config/config.go +++ b/pkg/dpf/config/config.go @@ -22,7 +22,9 @@ import ( "time" "unicode/utf8" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/addresslookup" "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/releases" "github.com/nvidia/k8s-launch-kit/pkg/yamlinput" yaml "gopkg.in/yaml.v3" ) @@ -71,17 +73,21 @@ type Limits struct { MaxConcurrentBMCs *int `yaml:"maxConcurrentBMCs,omitempty"` } type Site struct { + DPF struct { + Release *string `yaml:"release,omitempty"` + } `yaml:"dpf,omitempty"` Management struct { KubeconfigRef FileRef `yaml:"kubeconfigRef"` Context *string `yaml:"context,omitempty"` } `yaml:"management"` Credentials map[string]Credential `yaml:"credentials"` Discovery struct { - DPUs []Endpoint `yaml:"dpus"` - Hosts []Endpoint `yaml:"hosts"` - TLS *TLS `yaml:"tls,omitempty"` - Probes Probes `yaml:"probes,omitempty"` - Limits Limits `yaml:"limits,omitempty"` + AddressLookup *addresslookup.Config `yaml:"addressLookup,omitempty"` + DPUs []Endpoint `yaml:"dpus"` + Hosts []Endpoint `yaml:"hosts"` + TLS *TLS `yaml:"tls,omitempty"` + Probes Probes `yaml:"probes,omitempty"` + Limits Limits `yaml:"limits,omitempty"` } `yaml:"discovery"` Inventory *inventory.Inventory `yaml:"inventory,omitempty"` } @@ -115,6 +121,8 @@ type Target struct { Access Access } type Request struct { + Software *releases.Release + AddressLookup *addresslookup.Request Input Input Targets []Target Kubeconfig string @@ -238,6 +246,28 @@ func Resolve(in Input) (Request, error) { r := Request{Input: in, Timeout: 10 * time.Minute, RequestTimeout: 15 * time.Second, MaxConcurrentBMCs: 4, Mode: "management"} s := in.Site base := filepath.Dir(in.Path) + release := "" + if s.DPF.Release != nil { + if *s.DPF.Release == "" { + return r, fmt.Errorf("/dpf/release must not be empty") + } + release = *s.DPF.Release + } + software, err := releases.Resolve(release) + if err != nil { + return r, err + } + r.Software = &software + if s.Discovery.AddressLookup != nil { + lookup, err := addresslookup.Validate(*s.Discovery.AddressLookup) + if err != nil { + return r, err + } + for i := range lookup.Sources { + lookup.Sources[i].File = resolvePath(base, lookup.Sources[i].File) + } + r.AddressLookup = &lookup + } if s.Management.KubeconfigRef.File != "" { r.Kubeconfig = resolvePath(base, s.Management.KubeconfigRef.File) } @@ -507,7 +537,7 @@ func rebase(n *yaml.Node, oldBase, newBase, p string) error { if key == "inventory" && p == "" { continue } - isFile := key == "caFile" || key == "knownHostsFile" || key == "file" && (strings.HasSuffix(p, "kubeconfigRef") || strings.HasSuffix(p, "passwordRef") || strings.HasSuffix(p, "privateKeyRef")) + isFile := key == "caFile" || key == "knownHostsFile" || key == "file" && (p == "/discovery/addressLookup/sources" || strings.HasSuffix(p, "kubeconfigRef") || strings.HasSuffix(p, "passwordRef") || strings.HasSuffix(p, "privateKeyRef")) if isFile && value.Kind == yaml.ScalarNode && value.Tag == "!!str" && value.Value != "" { abs := resolvePath(oldBase, value.Value) rel, err := filepath.Rel(newBase, abs) diff --git a/pkg/dpf/config/software_test.go b/pkg/dpf/config/software_test.go new file mode 100644 index 00000000..0d25ada8 --- /dev/null +++ b/pkg/dpf/config/software_test.go @@ -0,0 +1,46 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package config + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/releases" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "os" + "strings" +) + +var _ = Describe("DPF release and address source configuration", func() { + BeforeEach(func() { + Expect(os.Setenv("DPF_TEST_PASSWORD", "private")).To(Succeed()) + DeferCleanup(os.Unsetenv, "DPF_TEST_PASSWORD") + }) + It("defaults omitted/null releases and rejects explicit empty or unknown releases", func() { + for _, value := range []string{"", "dpf: {release: null}\n", "dpf: {release: '26.4.0'}\n"} { + in, e := Decode([]byte(value+seed), "/site/in.yaml") + Expect(e).NotTo(HaveOccurred()) + r, e := Resolve(in) + Expect(e).NotTo(HaveOccurred()) + Expect(r.Software.Version).To(Equal(releases.DefaultRelease)) + } + for _, value := range []string{"''", "'latest'", "'26.8.0'"} { + in, e := Decode([]byte("dpf: {release: "+value+"}\n"+seed), "/site/in.yaml") + Expect(e).NotTo(HaveOccurred()) + _, e = Resolve(in) + Expect(e).To(HaveOccurred()) + } + }) + It("preserves release intent and rebases optional lease source references", func() { + text := strings.Replace(seed, "discovery:\n", "discovery:\n addressLookup:\n sources: [{file: leases.txt, format: dnsmasq-leases}]\n devices: [{name: dpu1, role: arm, mac: '02:00:00:00:00:01'}]\n", 1) + in, e := Decode([]byte(text), "/site/in.yaml") + Expect(e).NotTo(HaveOccurred()) + r, e := Resolve(in) + Expect(e).NotTo(HaveOccurred()) + Expect(r.AddressLookup.Sources[0].File).To(Equal("/site/leases.txt")) + b, e := Saved(in, &inventory.Inventory{Software: r.Software}, "/output/result.yaml") + Expect(e).NotTo(HaveOccurred()) + Expect(string(b)).To(ContainSubstring("../site/leases.txt")) + Expect(string(b)).NotTo(ContainSubstring("dpf:\n")) + }) +}) diff --git a/pkg/dpf/discovery/addresses.go b/pkg/dpf/discovery/addresses.go new file mode 100644 index 00000000..151be42f --- /dev/null +++ b/pkg/dpf/discovery/addresses.go @@ -0,0 +1,49 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/addresslookup" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "slices" + "time" +) + +func collectAddresses(r config.Request, inv *inventory.Inventory) { + if r.AddressLookup == nil { + return + } + neighbors := []addresslookup.Neighbor{} + for _, group := range [][]inventory.Device{inv.Hosts, inv.DPUs} { + for _, d := range group { + if d.Runtime == nil || d.Runtime.Network == nil { + continue + } + ref := "" + for id, e := range inv.Evidence { + if e.InputRef == d.InputRef && e.ResourceURI == "neighbors" { + ref = id + break + } + } + for _, n := range d.Runtime.Network.Neighbors { + state := "" + if slices.Contains(n.State, "REACHABLE") { + state = "REACHABLE" + } else if slices.Contains(n.State, "PERMANENT") { + state = "PERMANENT" + } + // Require the collector's authenticated provenance, not an address alone. + if ref != "" { + neighbors = append(neighbors, addresslookup.Neighbor{Address: n.Address, MAC: n.MAC, State: state, Source: d.RecordKey + "/" + n.Interface, EvidenceRef: ref}) + } + } + } + } + lookup, ev := addresslookup.Collect(*r.AddressLookup, time.Now().UTC(), config.ReadReference, neighbors) + inv.AddressLookup = lookup + for id, e := range ev { + inv.Evidence[id] = e + } +} diff --git a/pkg/dpf/discovery/discover.go b/pkg/dpf/discovery/discover.go index 89b7ecdc..f386cf51 100644 --- a/pkg/dpf/discovery/discover.go +++ b/pkg/dpf/discovery/discover.go @@ -40,6 +40,7 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc report.Discovery.Mode = r.Mode inv := &inventory.Inventory{FormatVersion: inventory.FormatVersion, Collection: inventory.Collection{RunID: runID, StartedAt: time.Now().UTC(), Mode: r.Mode, Collector: "l8k-go-discovery-v1"}, RequestedScope: []inventory.ScopeEndpoint{}, DPUs: []inventory.Device{}, Hosts: []inventory.Device{}, Associations: []inventory.Association{}, Evidence: map[string]inventory.Evidence{}} report.Inventory = inv + inv.Software = r.Software for _, t := range r.Targets { inv.RequestedScope = append(inv.RequestedScope, config.Scope(t)) } @@ -112,11 +113,13 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc code = "SSH_OBSERVATION_FAILED" } stage := "observation" + tool := "" var diagnostic *sshprobe.Error if errors.As(err, &diagnostic) { stage = diagnostic.Stage + tool = diagnostic.Tool } - problem(inventory.Problem{Code: code, Message: "SSH observation failed", InputRef: t.InputRef, Stage: stage, ExitCode: 3}) + problem(inventory.Problem{Code: code, Message: sshprobe.Reason(err), Tool: tool, InputRef: t.InputRef, Stage: stage, ExitCode: 3}) } else { for _, diagnostic := range redfish.Diagnostics(err) { problem(inventory.Problem{Code: diagnostic.Code, Message: "Redfish operation failed", InputRef: t.InputRef, Method: diagnostic.Method, ResourceURI: diagnostic.URI, HTTPStatus: diagnostic.Status, Stage: diagnostic.Stage, ExitCode: 3}) @@ -140,6 +143,7 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc } } requestedDPUs := false + collectAddresses(r, inv) for _, t := range r.Targets { requestedDPUs = requestedDPUs || t.Role == "dpus" } @@ -213,6 +217,10 @@ func Collect(ctx context.Context, r config.Request, runID string, deps Dependenc report.Discovery.Checks = append(report.Discovery.Checks, inventory.Check{ID: "dpf.management.redfish-read", Subject: d.RecordKey, RequiredPhase: "deploy", State: state, ReasonCode: reason, EvidenceRefs: d.EvidenceRefs}) report.Discovery.Checks = append(report.Discovery.Checks, inventory.Check{ID: "dpf.management.dpf-authentication", Subject: d.RecordKey, RequiredPhase: "deploy", State: "not-evaluated", ReasonCode: "DPF_AUTHENTICATION_NOT_EVALUATED", EvidenceRefs: []string{}}) } + report.Discovery.Checks = append(report.Discovery.Checks, SoftwareChecks(inv)...) + for i := range report.Discovery.Checks { + ExplainCheck(&report.Discovery.Checks[i]) + } report.Discovery.Outcome = "complete" for _, c := range report.Discovery.Checks { if c.RequiredPhase == "discover" && c.State != "passed" { @@ -319,6 +327,12 @@ func Canonicalize(inv *inventory.Inventory) { // Dedicated material projection excludes timestamps, readiness, credentials and aliases. func material(inv *inventory.Inventory) map[string]any { fields := map[string]any{} + if inv.Software != nil { + fields["software"] = inv.Software + } + if inv.AddressLookup != nil { + fields["addressLookup"] = inv.AddressLookup + } if inv.Management != nil { fields["management.identity"] = []string{inv.Management.APIEndpoint, inv.Management.SystemNamespaceUID, inv.Management.TrustDigest, inv.Management.KubernetesVersion} for _, n := range inv.Management.Nodes { diff --git a/pkg/dpf/discovery/management.go b/pkg/dpf/discovery/management.go index 4c96aa73..df2def2f 100644 --- a/pkg/dpf/discovery/management.go +++ b/pkg/dpf/discovery/management.go @@ -178,22 +178,24 @@ func (m ManagementClient) Observe(ctx context.Context) (*inventory.Management, e } result.Observations = map[string]inventory.Observation{} storage, storageErr := m.Client.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{Limit: 256}) - storageFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE"} + storageFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE", Message: "The Kubernetes API inspection failed or returned incomplete evidence."} if storageErr == nil && storage.Continue == "" { storageFact.State = "known" storageFact.ReasonCode = "" + storageFact.Message = "" for _, s := range storage.Items { storageFact.Values = append(storageFact.Values, s.Name+"/"+s.Provisioner) } slices.Sort(storageFact.Values) } result.Observations["storageClasses"] = storageFact - groupsFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE"} + groupsFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE", Message: "The Kubernetes API inspection failed or returned incomplete evidence."} data, groupErr := m.Client.Discovery().RESTClient().Get().AbsPath("/apis").Do(ctx).Raw() var groups metav1.APIGroupList if groupErr == nil && json.Unmarshal(data, &groups) == nil { groupsFact.State = "known" groupsFact.ReasonCode = "" + groupsFact.Message = "" for _, g := range groups.Groups { if strings.HasSuffix(g.Name, "dpu.nvidia.com") { groupsFact.Values = append(groupsFact.Values, g.Name) diff --git a/pkg/dpf/discovery/reasons.go b/pkg/dpf/discovery/reasons.go new file mode 100644 index 00000000..b55ddd3f --- /dev/null +++ b/pkg/dpf/discovery/reasons.go @@ -0,0 +1,28 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + +// ExplainCheck completes report diagnostics at the orchestration boundary. Collectors +// retain more specific failure messages; this table describes acceptance predicates. +func ExplainCheck(c *inventory.Check) { + if c == nil || c.State == "passed" || c.Message != "" { + return + } + messages := map[string]string{ + "EXACTLY_THREE_CONTROL_PLANES": "The management topology does not contain exactly three control-plane nodes.", + "PHYSICAL_IDENTITY_REQUIRED": "The selected source did not establish a usable physical identity; inspect its observation and error reasons.", + "QUALIFIED_ADAPTER_REQUIRED": "This hardware/firmware and reader combination has not passed release qualification; inventory success does not establish provisioning support.", + "DPU_IDENTITY_REQUIRED": "No DPU identity was established from the selected sources.", + "UNIQUE_PHYSICAL_MATCH_REQUIRED": "The DPU serial must match exactly one observed host adapter; missing, duplicate or conflicting identity prevents association.", + "REDFISH_AUTHORIZATION_NOT_EVALUATED": "This run did not verify permission to read DPU Redfish inventory; SSH access is a separate authorization path.", + "DPF_AUTHENTICATION_NOT_EVALUATED": "The installed DPF controller authentication path is checked during provisioning, not initial inventory.", + "INVENTORY_MODE": "Inventory mode omits management-network probes; later management readiness remains unevaluated.", + "EXACT_NETWORK_COVERAGE_REQUIRED": "Required management node, network-mode or destination evidence is missing or did not pass; inspect individual probe results.", + } + c.Message = messages[c.ReasonCode] + if c.Message == "" { + c.Message = "The check did not produce passing evidence; inspect the associated source observation and reason code." + } +} diff --git a/pkg/dpf/discovery/software.go b/pkg/dpf/discovery/software.go new file mode 100644 index 00000000..253e5d5e --- /dev/null +++ b/pkg/dpf/discovery/software.go @@ -0,0 +1,160 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "slices" + "strconv" + "strings" +) + +// SoftwareChecks reports release compatibility separately from discovery success. +// Only exact catalog models and identity-bound measurements can satisfy a minimum. +func SoftwareChecks(inv *inventory.Inventory) []inventory.Check { + out := []inventory.Check{} + if inv == nil || inv.Software == nil { + return out + } + for _, d := range inv.DPUs { + model := d.Model + if d.Runtime != nil { + board := d.Runtime.Observations["board-product"] + if board.State == "observed" && len(board.Values) == 1 { + model = board.Values[0] + } + } + hw, ok := inv.Software.RequirementsFor(model) + base := inventory.Check{ID: "dpf.software.applicability", Subject: d.RecordKey, Vantage: "executor", RequiredPhase: "generate", State: "unknown", ReasonCode: "BOM_MODEL_UNSUPPORTED", Message: "The selected release BOM has no exact applicability entry for this observed model; software compatibility remains unverified.", EvidenceRefs: slices.Clone(d.EvidenceRefs)} + if !ok { + out = append(out, base) + continue + } + base.State = "passed" + base.ReasonCode = "BOM_MODEL_MATCH" + base.Message = "Observed model matches a documented BOM entry; hardware qualification remains separate." + out = append(out, base) + for _, req := range hw.Requirements { + c := inventory.Check{ID: "dpf.software." + req.Component, Subject: d.RecordKey, Vantage: "executor", RequiredPhase: "generate", State: "unknown", ReasonCode: req.ReasonCode, Message: req.Message, EvidenceRefs: slices.Clone(d.EvidenceRefs)} + if req.State != "known" { + out = append(out, c) + continue + } + values := []string{} + for _, fw := range d.Firmware { + if fw.Component == req.Component && fw.State == "known" && fw.Activation == "active" { + values = append(values, fw.RawVersion) + c.EvidenceRefs = append(c.EvidenceRefs, fw.EvidenceRefs...) + } + } + if req.Component == "nic" { + for _, group := range [][]inventory.Device{inv.Hosts, []inventory.Device{d}} { + for _, device := range group { + if device.Runtime != nil && device.Runtime.Hardware != nil { + for _, card := range device.Runtime.Hardware.Cards { + if d.SerialNumber != "" && card.SerialNumber == d.SerialNumber && card.Firmware.State == "observed" { + values = append(values, card.Firmware.Values...) + c.EvidenceRefs = append(c.EvidenceRefs, card.EvidenceRefs...) + } + } + } + } + } + } + if req.Scope == "host" && req.Component == "mft" { + for _, a := range inv.Associations { + if a.DPUID == d.ID { + for _, h := range inv.Hosts { + if h.ID == a.HostID && h.Runtime != nil { + v := h.Runtime.Observations["mft-version"] + if v.State == "observed" { + values = append(values, v.Values...) + c.EvidenceRefs = append(c.EvidenceRefs, h.EvidenceRefs...) + } + } + } + } + } + } + slices.Sort(values) + values = slices.Compact(values) + switch len(values) { + case 0: + c.ReasonCode = "SOFTWARE_VERSION_UNAVAILABLE" + c.Message = "No active or current, identity-bound version was measured for this requirement; inspect firmware/tool observation reasons." + case 1: + cmp, valid := compareVersion(values[0], req.Minimum) + if req.Minimum == "" || !valid { + c.ReasonCode = "SOFTWARE_VERSION_UNCOMPARABLE" + c.Message = "The observed version cannot be safely compared with the declared minimum." + } else if cmp < 0 { + c.State = "failed" + c.ReasonCode = "SOFTWARE_BELOW_MINIMUM" + c.Message = "Observed version " + values[0] + " is below the selected release minimum " + req.Minimum + "." + } else { + c.State = "passed" + c.ReasonCode = "SOFTWARE_MINIMUM_MET" + c.Message = "The measured version satisfies the declared minimum; this is not overall deployment qualification." + } + default: + c.ReasonCode = "SOFTWARE_VERSION_CONFLICT" + c.Message = "Identity-bound sources report conflicting versions; no version was selected to satisfy the requirement." + } + out = append(out, c) + } + } + return out +} + +// Version grammar is deliberately narrow: numeric release components separated by +// dots and an optional numeric build suffix. Vendor strings require normalization +// by their specific collector before they can satisfy a minimum. +func compareVersion(a, b string) (int, bool) { + parse := func(v string) ([]uint64, bool) { + if v == "" { + return nil, false + } + out := []uint64{} + for _, s := range strings.Split(strings.ReplaceAll(v, "-", "."), ".") { + if s == "" { + return nil, false + } + for _, r := range s { + if r < '0' || r > '9' { + return nil, false + } + } + n, e := strconv.ParseUint(s, 10, 64) + if e != nil { + return nil, false + } + out = append(out, n) + } + return out, true + } + aa, ok := parse(a) + if !ok { + return 0, false + } + bb, ok := parse(b) + if !ok { + return 0, false + } + n := max(len(aa), len(bb)) + for i := 0; i < n; i++ { + var x, y uint64 + if i < len(aa) { + x = aa[i] + } + if i < len(bb) { + y = bb[i] + } + if x < y { + return -1, true + } + if x > y { + return 1, true + } + } + return 0, true +} diff --git a/pkg/dpf/discovery/software_test.go b/pkg/dpf/discovery/software_test.go new file mode 100644 index 00000000..9225fad3 --- /dev/null +++ b/pkg/dpf/discovery/software_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/releases" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Software BOM readiness", func() { + It("does not confuse inventory collection, software applicability and hardware qualification", func() { + r, err := releases.Resolve("") + Expect(err).NotTo(HaveOccurred()) + inv := &inventory.Inventory{Software: &r, DPUs: []inventory.Device{{RecordKey: "bf4", Model: "BlueField-4"}}} + checks := SoftwareChecks(inv) + Expect(checks).To(HaveLen(1)) + Expect(checks[0].ReasonCode).To(Equal("BOM_MODEL_UNSUPPORTED")) + Expect(checks[0].RequiredPhase).To(Equal("generate")) + }) + It("requires active evidence and retains unknown requirements", func() { + r, err := releases.Resolve("") + Expect(err).NotTo(HaveOccurred()) + d := inventory.Device{RecordKey: "bf3", Model: "B3210", SerialNumber: "MT1234567890", Firmware: []inventory.Firmware{{Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "unknown"}}} + inv := &inventory.Inventory{Software: &r, DPUs: []inventory.Device{d}} + find := func() inventory.Check { + for _, c := range SoftwareChecks(inv) { + if c.ID == "dpf.software.nic" { + return c + } + } + Fail("missing NIC check") + return inventory.Check{} + } + Expect(find().ReasonCode).To(Equal("SOFTWARE_VERSION_UNAVAILABLE")) + inv.DPUs[0].Firmware[0].Activation = "active" + Expect(find().State).To(Equal("passed")) + inv.DPUs[0].Firmware[0].RawVersion = "32.37.9999" + Expect(find().State).To(Equal("failed")) + inv.DPUs[0].Firmware = append(inv.DPUs[0].Firmware, inventory.Firmware{Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "active"}) + Expect(find().ReasonCode).To(Equal("SOFTWARE_VERSION_CONFLICT")) + for _, c := range SoftwareChecks(inv) { + if c.State != "passed" { + Expect(c.ReasonCode).NotTo(BeEmpty()) + Expect(c.Message).NotTo(BeEmpty()) + } + } + }) + It("compares numeric build versions without ordering vendor text", func() { + c, ok := compareVersion("4.33.0-169", "4.33.0-168") + Expect(ok).To(BeTrue()) + Expect(c).To(Equal(1)) + _, ok = compareVersion("unverified", "4.33.0-169") + Expect(ok).To(BeFalse()) + }) + It("explains every nonpassing acceptance predicate", func() { + for _, code := range []string{"EXACTLY_THREE_CONTROL_PLANES", "PHYSICAL_IDENTITY_REQUIRED", "QUALIFIED_ADAPTER_REQUIRED", "DPU_IDENTITY_REQUIRED", "UNIQUE_PHYSICAL_MATCH_REQUIRED", "REDFISH_AUTHORIZATION_NOT_EVALUATED", "DPF_AUTHENTICATION_NOT_EVALUATED", "INVENTORY_MODE", "EXACT_NETWORK_COVERAGE_REQUIRED"} { + c := inventory.Check{State: "unknown", ReasonCode: code} + ExplainCheck(&c) + Expect(c.Message).NotTo(BeEmpty()) + } + }) +}) diff --git a/pkg/dpf/inventory/addresses.go b/pkg/dpf/inventory/addresses.go new file mode 100644 index 00000000..503147cd --- /dev/null +++ b/pkg/dpf/inventory/addresses.go @@ -0,0 +1,17 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package inventory + +// AddressCandidate is an unverified locator, never physical identity or a connection grant. +type AddressCandidate struct { + Name string `json:"name" yaml:"name"` + Role string `json:"role" yaml:"role"` + MAC string `json:"mac" yaml:"mac"` + Address string `json:"address" yaml:"address"` + Source string `json:"source" yaml:"source"` + EvidenceRef string `json:"evidenceRef" yaml:"evidenceRef"` +} +type AddressLookup struct { + Candidates []AddressCandidate `json:"candidates" yaml:"candidates"` + Observations map[string]Observation `json:"observations" yaml:"observations"` +} diff --git a/pkg/dpf/inventory/hardware.go b/pkg/dpf/inventory/hardware.go new file mode 100644 index 00000000..1ed8baa4 --- /dev/null +++ b/pkg/dpf/inventory/hardware.go @@ -0,0 +1,62 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package inventory + +// HardwareObservation keeps PCI-domain-local facts separate from device identity. +// A host or ARM may expose multiple independent cards; PCI addresses are not +// comparable across those domains. +type HardwareObservation struct { + Cards []CardHardware `json:"cards" yaml:"cards"` + RShim Observation `json:"rshim" yaml:"rshim"` +} + +type CardHardware struct { + PCIAddress string `json:"pciAddress" yaml:"pciAddress"` + Domain string `json:"domain" yaml:"domain"` + Model string `json:"model" yaml:"model"` + SerialNumber string `json:"serialNumber,omitempty" yaml:"serialNumber,omitempty"` + Firmware Observation `json:"firmware" yaml:"firmware"` + PSID Observation `json:"psid" yaml:"psid"` + NVConfig map[string]Observation `json:"nvConfig" yaml:"nvConfig"` + Privilege Observation `json:"privilege" yaml:"privilege"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` +} + +type NetworkObservation struct { + Links []NetworkLink `json:"links" yaml:"links"` + Resolver ResolverObservation `json:"resolver" yaml:"resolver"` + Neighbors []Neighbor `json:"neighbors" yaml:"neighbors"` + LLDP []LLDPNeighbor `json:"lldp" yaml:"lldp"` +} + +type NetworkLink struct { + Name string `json:"name" yaml:"name"` + MAC string `json:"mac,omitempty" yaml:"mac,omitempty"` + Master string `json:"master,omitempty" yaml:"master,omitempty"` + Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` +} + +type ResolverObservation struct { + Owner string `json:"owner" yaml:"owner"` + Path string `json:"path,omitempty" yaml:"path,omitempty"` + Search []string `json:"search" yaml:"search"` + PerLink map[string][]string `json:"perLink" yaml:"perLink"` + Domains map[string][]string `json:"domains" yaml:"domains"` + DNSMasqNoResolv *bool `json:"dnsmasqNoResolv,omitempty" yaml:"dnsmasqNoResolv,omitempty"` + DNSMasqServers []string `json:"dnsmasqServers" yaml:"dnsmasqServers"` +} + +// Neighbor is an observed address candidate, never device identity or authority +// to connect. Its state and collection vantage must be retained by consumers. +type Neighbor struct { + Address string `json:"address" yaml:"address"` + MAC string `json:"mac,omitempty" yaml:"mac,omitempty"` + Interface string `json:"interface" yaml:"interface"` + State []string `json:"state" yaml:"state"` +} + +type LLDPNeighbor struct { + Interface string `json:"interface" yaml:"interface"` + Chassis string `json:"chassis,omitempty" yaml:"chassis,omitempty"` + Port string `json:"port,omitempty" yaml:"port,omitempty"` +} diff --git a/pkg/dpf/inventory/types.go b/pkg/dpf/inventory/types.go index b3210f1e..e4b46f33 100644 --- a/pkg/dpf/inventory/types.go +++ b/pkg/dpf/inventory/types.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/releases" "time" ) @@ -45,13 +46,18 @@ type Evidence struct { Authenticated bool `json:"authenticated" yaml:"authenticated"` } type Firmware struct { - ResourceURI string `json:"resourceURI,omitempty" yaml:"resourceURI,omitempty"` - Component string `json:"component" yaml:"component"` - State string `json:"state" yaml:"state"` - RawVersion string `json:"rawVersion,omitempty" yaml:"rawVersion,omitempty"` - Activation string `json:"activation" yaml:"activation"` - ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` - EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` + RawID string `json:"rawID,omitempty" yaml:"rawID,omitempty"` + RawName string `json:"rawName,omitempty" yaml:"rawName,omitempty"` + ActivationReasonCode string `json:"activationReasonCode,omitempty" yaml:"activationReasonCode,omitempty"` + ActivationMessage string `json:"activationMessage,omitempty" yaml:"activationMessage,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` + ResourceURI string `json:"resourceURI,omitempty" yaml:"resourceURI,omitempty"` + Component string `json:"component" yaml:"component"` + State string `json:"state" yaml:"state"` + RawVersion string `json:"rawVersion,omitempty" yaml:"rawVersion,omitempty"` + Activation string `json:"activation" yaml:"activation"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` } type Authentication struct { PasswordRead string `json:"passwordRead" yaml:"passwordRead"` @@ -60,6 +66,7 @@ type Authentication struct { DPFClientAuthentication string `json:"dpfClientAuthentication" yaml:"dpfClientAuthentication"` } type TimeObservation struct { + Message string `json:"message,omitempty" yaml:"message,omitempty"` State string `json:"state" yaml:"state"` DateTime string `json:"dateTime,omitempty" yaml:"dateTime,omitempty"` Offset string `json:"offset,omitempty" yaml:"offset,omitempty"` @@ -91,6 +98,8 @@ type Adapter struct { // RuntimeObservation contains OS-visible facts, separate from BMC management readiness. type RuntimeObservation struct { + Hardware *HardwareObservation `json:"hardware,omitempty" yaml:"hardware,omitempty"` + Network *NetworkObservation `json:"network,omitempty" yaml:"network,omitempty"` Hostname string `json:"hostname,omitempty" yaml:"hostname,omitempty"` OS string `json:"os,omitempty" yaml:"os,omitempty"` Interfaces []Interface `json:"interfaces,omitempty" yaml:"interfaces,omitempty"` @@ -101,31 +110,33 @@ type RuntimeObservation struct { // Device represents every requested endpoint, including failed observations. type Device struct { - SSHEndpoint string `json:"sshEndpoint,omitempty" yaml:"sshEndpoint,omitempty"` - BMCAddressState string `json:"bmcAddressState,omitempty" yaml:"bmcAddressState,omitempty"` - Runtime *RuntimeObservation `json:"runtime,omitempty" yaml:"runtime,omitempty"` - RecordKey string `json:"recordKey" yaml:"recordKey"` - InputRef string `json:"inputRef" yaml:"inputRef"` - BMCEndpoint string `json:"bmcEndpoint" yaml:"bmcEndpoint"` - SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` - ObservationState string `json:"observationState" yaml:"observationState"` - ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` - ID string `json:"id,omitempty" yaml:"id,omitempty"` - IdentityKey []string `json:"identityKey,omitempty" yaml:"identityKey,omitempty"` - SerialNumber string `json:"serialNumber,omitempty" yaml:"serialNumber,omitempty"` - SerialNumberRaw string `json:"serialNumberRaw,omitempty" yaml:"serialNumberRaw,omitempty"` - SystemUUID string `json:"systemUUID,omitempty" yaml:"systemUUID,omitempty"` - Manufacturer string `json:"manufacturer,omitempty" yaml:"manufacturer,omitempty"` - Model string `json:"model,omitempty" yaml:"model,omitempty"` - ReaderAdapter string `json:"readerAdapter,omitempty" yaml:"readerAdapter,omitempty"` - InventoryCompleteness string `json:"inventoryCompleteness,omitempty" yaml:"inventoryCompleteness,omitempty"` - Qualification string `json:"qualification" yaml:"qualification"` - Firmware []Firmware `json:"firmware" yaml:"firmware"` - Authentication Authentication `json:"authentication" yaml:"authentication"` - Time TimeObservation `json:"time" yaml:"time"` - Adapters []Adapter `json:"adapters,omitempty" yaml:"adapters,omitempty"` - AdvertisedResetTypes []string `json:"advertisedResetTypes,omitempty" yaml:"advertisedResetTypes,omitempty"` - EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` + Observations map[string]Observation `json:"observations,omitempty" yaml:"observations,omitempty"` + SSHEndpoint string `json:"sshEndpoint,omitempty" yaml:"sshEndpoint,omitempty"` + BMCAddressState string `json:"bmcAddressState,omitempty" yaml:"bmcAddressState,omitempty"` + Runtime *RuntimeObservation `json:"runtime,omitempty" yaml:"runtime,omitempty"` + RecordKey string `json:"recordKey" yaml:"recordKey"` + InputRef string `json:"inputRef" yaml:"inputRef"` + BMCEndpoint string `json:"bmcEndpoint" yaml:"bmcEndpoint"` + SystemURI string `json:"systemURI,omitempty" yaml:"systemURI,omitempty"` + ObservationState string `json:"observationState" yaml:"observationState"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + ID string `json:"id,omitempty" yaml:"id,omitempty"` + IdentityKey []string `json:"identityKey,omitempty" yaml:"identityKey,omitempty"` + SerialNumber string `json:"serialNumber,omitempty" yaml:"serialNumber,omitempty"` + SerialNumberRaw string `json:"serialNumberRaw,omitempty" yaml:"serialNumberRaw,omitempty"` + SystemUUID string `json:"systemUUID,omitempty" yaml:"systemUUID,omitempty"` + Manufacturer string `json:"manufacturer,omitempty" yaml:"manufacturer,omitempty"` + Model string `json:"model,omitempty" yaml:"model,omitempty"` + ReaderAdapter string `json:"readerAdapter,omitempty" yaml:"readerAdapter,omitempty"` + InventoryCompleteness string `json:"inventoryCompleteness,omitempty" yaml:"inventoryCompleteness,omitempty"` + Qualification string `json:"qualification" yaml:"qualification"` + Firmware []Firmware `json:"firmware" yaml:"firmware"` + Authentication Authentication `json:"authentication" yaml:"authentication"` + Time TimeObservation `json:"time" yaml:"time"` + Adapters []Adapter `json:"adapters,omitempty" yaml:"adapters,omitempty"` + AdvertisedResetTypes []string `json:"advertisedResetTypes,omitempty" yaml:"advertisedResetTypes,omitempty"` + EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` } type Node struct { Interfaces []Interface `json:"interfaces,omitempty" yaml:"interfaces,omitempty"` @@ -140,6 +151,8 @@ type Node struct { Addresses []string `json:"addresses" yaml:"addresses"` } type Observation struct { + Message string `json:"message,omitempty" yaml:"message,omitempty"` + Tool string `json:"tool,omitempty" yaml:"tool,omitempty"` State string `json:"state" yaml:"state"` Values []string `json:"values" yaml:"values"` ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` @@ -162,6 +175,8 @@ type Association struct { EvidenceRefs []string `json:"evidenceRefs" yaml:"evidenceRefs"` } type Inventory struct { + Software *releases.Release `json:"software,omitempty" yaml:"software,omitempty"` + AddressLookup *AddressLookup `json:"addressLookup,omitempty" yaml:"addressLookup,omitempty"` FormatVersion string `json:"formatVersion" yaml:"formatVersion"` Collection Collection `json:"collection" yaml:"collection"` RequestedScope []ScopeEndpoint `json:"requestedScope" yaml:"requestedScope"` @@ -175,6 +190,7 @@ type Inventory struct { Evidence map[string]Evidence `json:"evidence" yaml:"evidence"` } type Check struct { + Message string `json:"message,omitempty"` ID string `json:"id"` Subject string `json:"subject"` Vantage string `json:"vantage"` @@ -184,6 +200,7 @@ type Check struct { EvidenceRefs []string `json:"evidenceRefs"` } type Problem struct { + Tool string `json:"tool,omitempty"` Method string `json:"method,omitempty"` ResourceURI string `json:"resourceURI,omitempty"` HTTPStatus int `json:"httpStatus,omitempty"` diff --git a/pkg/dpf/redfish/client.go b/pkg/dpf/redfish/client.go index eb70e4f1..0c503d73 100644 --- a/pkg/dpf/redfish/client.go +++ b/pkg/dpf/redfish/client.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "path" @@ -132,6 +133,10 @@ func (c *Client) request(ctx context.Context, method, ref string, body []byte) ( if ctx.Err() != nil { return nil, nil, ctx.Err() } + var timeout net.Error + if errors.As(e, &timeout) && timeout.Timeout() { + return nil, nil, &Error{Code: "BMC_TIMEOUT", URI: uri} + } var verification *tls.CertificateVerificationError if errors.As(e, &verification) { return nil, nil, &Error{Code: "TLS_VERIFICATION_FAILED", URI: uri} @@ -147,11 +152,24 @@ func (c *Client) request(ctx context.Context, method, ref string, body []byte) ( if response.StatusCode == 403 { code = "ACCESS_DENIED" } + if response.StatusCode == 404 { + code = "RESOURCE_NOT_FOUND" + } return nil, nil, &Error{Code: code, URI: uri, Status: response.StatusCode} } b, e := io.ReadAll(io.LimitReader(response.Body, (4<<20)+1)) c.bytes += len(b) - if e != nil || len(b) > 4<<20 || c.bytes > 32<<20 { + if e != nil { + if ctx.Err() != nil { + return nil, nil, ctx.Err() + } + var timeout net.Error + if errors.As(e, &timeout) && timeout.Timeout() { + return nil, nil, &Error{Code: "BMC_TIMEOUT", URI: uri} + } + return nil, nil, &Error{Code: "RESPONSE_READ_FAILED", URI: uri} + } + if len(b) > 4<<20 || c.bytes > 32<<20 { return nil, nil, &Error{Code: "RESPONSE_LIMIT", URI: uri} } var d Document @@ -173,7 +191,7 @@ func (c *Client) Get(ctx context.Context, ref string) (Document, error) { return d, nil } var remote *Error - retry := errors.As(err, &remote) && (remote.Code == "TRANSPORT_FAILED" || remote.Status == 429 || remote.Status == 502 || remote.Status == 503 || remote.Status == 504) + retry := errors.As(err, &remote) && (remote.Code == "TRANSPORT_FAILED" || remote.Code == "BMC_TIMEOUT" || remote.Status == 429 || remote.Status == 502 || remote.Status == 503 || remote.Status == 504) if !retry || attempt == 2 { return nil, err } @@ -262,58 +280,61 @@ func (c *Client) Close(ctx context.Context) error { } return nil } + +// Collection returns already read members alongside an error. Required consumers +// must reject the error; optional inventory may retain those members as partial evidence. func (c *Client) Collection(ctx context.Context, ref string) ([]Resource, error) { var members []Resource seen := map[string]bool{} count := -1 initial, err := c.guard(ref) if err != nil { - return nil, err + return members, err } base := strings.Split(initial, "?")[0] for page := 0; ref != ""; page++ { if page >= 32 || seen[ref] { - return nil, &Error{Code: "PAGINATION_LIMIT"} + return members, &Error{Code: "PAGINATION_LIMIT"} } seen[ref] = true normalized, err := c.guard(ref) if err != nil || strings.Split(normalized, "?")[0] != base { - return nil, &Error{Code: "UNSAFE_PAGINATION"} + return members, &Error{Code: "UNSAFE_PAGINATION"} } d, err := c.Get(ctx, ref) if err != nil { - return nil, err + return members, err } if n, ok := d["Members@odata.count"].(float64); ok { if n < 0 || n != float64(int(n)) || count >= 0 && count != int(n) { - return nil, &Error{Code: "COLLECTION_CHANGED"} + return members, &Error{Code: "COLLECTION_CHANGED"} } count = int(n) } list, ok := d["Members"].([]any) if !ok { - return nil, &Error{Code: "INCOMPLETE_COLLECTION"} + return members, &Error{Code: "INCOMPLETE_COLLECTION"} } for _, item := range list { link := Link(Document{"member": item}, "member") link, err = c.guard(link) if err != nil { - return nil, err + return members, err } if link == "" || seen["member:"+link] { - return nil, &Error{Code: "DUPLICATE_MEMBER"} + return members, &Error{Code: "DUPLICATE_MEMBER"} } seen["member:"+link] = true value, err := c.Get(ctx, link) if err != nil { - return nil, err + return members, err } members = append(members, Resource{URI: link, Document: value}) } ref = Text(d, "Members@odata.nextLink") } if count < 0 || count != len(members) { - return nil, &Error{Code: "INCOMPLETE_COLLECTION"} + return members, &Error{Code: "INCOMPLETE_COLLECTION"} } return members, nil } diff --git a/pkg/dpf/redfish/inventory.go b/pkg/dpf/redfish/inventory.go index 7d735ce6..9dd72835 100644 --- a/pkg/dpf/redfish/inventory.go +++ b/pkg/dpf/redfish/inventory.go @@ -145,77 +145,7 @@ func (co Collector) Collect(ctx context.Context, target config.Target, timeout t } device.InventoryCompleteness = "unqualified" } - bmcFirmware := "" - if uri := Link(root, "Managers"); uri != "" { - managers, er := c.Collection(ctx, uri) - if er == nil && len(managers) == 1 { - manager := managers[0] - bmcFirmware = Text(manager.Document, "FirmwareVersion") - if bmcFirmware != "" { - device.Firmware = append(device.Firmware, inventory.Firmware{ResourceURI: manager.URI, Component: "bmc", State: "known", RawVersion: bmcFirmware, Activation: "unknown", EvidenceRefs: []string{add(manager.URI, "FirmwareVersion")}}) - } - if npURI := Link(manager.Document, "NetworkProtocol"); npURI != "" { - if np, readErr := c.Get(ctx, npURI); readErr == nil { - ntp := Object(np, "NTP") - if enabled, ok := ntp["ProtocolEnabled"].(bool); ok { - device.Time.NTPEnabled = &enabled - } - if servers, ok := ntp["NTPServers"].([]any); ok { - for _, v := range servers { - if server, ok := v.(string); ok { - device.Time.NTPServers = append(device.Time.NTPServers, server) - } - } - } - add(npURI, "NTP") - } - } - - device.Time.DateTime = Text(manager.Document, "DateTime") - device.Time.Offset = Text(manager.Document, "DateTimeLocalOffset") - if device.Time.DateTime != "" { - device.Time.State = "known" - device.Time.ReasonCode = "" - } - add(manager.URI, "DateTime") - } - } - found := map[string]bool{} - for _, existing := range device.Firmware { - found[existing.Component] = true - } - if uri := Link(root, "UpdateService"); uri != "" { - update, er := c.Get(ctx, uri) - if er == nil { - fw, er := c.Collection(ctx, Link(update, "FirmwareInventory")) - if er == nil { - for _, f := range fw { - name := strings.ToLower(Text(f.Document, "Name")) - component := "other" - for _, candidate := range []string{"bmc", "cec", "atf", "nic", "golden"} { - if strings.Contains(name, candidate) { - component = candidate - break - } - } - if component == "atf" { - component = "atf-uefi" - } - version := Text(f.Document, "Version") - if version == "" { - continue - } - found[component] = true - device.Firmware = append(device.Firmware, inventory.Firmware{ResourceURI: f.URI, Component: component, State: "known", RawVersion: version, Activation: "unknown", EvidenceRefs: []string{add(f.URI, "Version")}}) - } - } - } - } - for _, component := range []string{"bmc", "cec", "atf-uefi", "nic", "dpu-os", "golden"} { - if !found[component] { - device.Firmware = append(device.Firmware, inventory.Firmware{Component: component, State: "unknown", ReasonCode: "PROPERTY_NOT_EXPOSED", Activation: "unknown", EvidenceRefs: []string{}}) - } - } + bmcFirmware := collectSoftware(ctx, c, root, system, &device, add) for _, q := range co.Qualifications { if q.Adapter == adapter && q.Model == device.Model && q.BMCFirmware == bmcFirmware && bmcFirmware != "" && q.VerifyCompleteness != nil && q.VerifyCompleteness(ctx, c, system) == nil { device.Qualification = "qualified" diff --git a/pkg/dpf/redfish/software.go b/pkg/dpf/redfish/software.go new file mode 100644 index 00000000..7876210d --- /dev/null +++ b/pkg/dpf/redfish/software.go @@ -0,0 +1,285 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +// missing and unavailable deliberately describe our observation, never a guessed +// device capability. Response bodies and error strings are not diagnostic inputs. +func missing(code, message string) inventory.Observation { + return inventory.Observation{State: "unknown", ReasonCode: code, Message: message} +} +func unavailable(err error) inventory.Observation { + reason := Code(err) + message := "The Redfish observation failed; inspect the reason code." + switch reason { + case "ACCESS_DENIED": + message = "The account is not authorized to read this Redfish resource." + case "AUTHENTICATION_FAILED": + message = "The Redfish service rejected authentication for this read." + case "RESOURCE_NOT_FOUND": + message = "The advertised Redfish resource was not found." + case "BMC_TIMEOUT": + message = "The Redfish read exceeded its timeout." + case "CANCELLED": + message = "The Redfish read was cancelled." + case "TRANSPORT_FAILED": + message = "The Redfish connection could not complete." + case "TLS_VERIFICATION_FAILED": + message = "The Redfish server certificate could not be verified." + case "INVALID_RESPONSE", "INCOMPLETE_COLLECTION", "RESPONSE_READ_FAILED": + message = "The Redfish response was malformed or incomplete." + case "RESOURCE_LIMIT", "RESPONSE_LIMIT", "PAGINATION_LIMIT": + message = "The Redfish read exceeded its resource, size or pagination limit." + } + var remote *Error + if errors.As(err, &remote) && remote.Status != 0 { + message += fmt.Sprintf(" HTTP %d.", remote.Status) + } + return inventory.Observation{State: "unavailable", ReasonCode: reason, Message: message} +} + +func collectSoftware(ctx context.Context, reader Reader, root Document, system Resource, device *inventory.Device, add func(string, string) string) string { + if device.Observations == nil { + device.Observations = map[string]inventory.Observation{} + } + device.Observations["client-certificate-support"] = missing("NOT_EVALUATED", "Inventory collection does not test DPF client-certificate authentication.") + manager, managerState := softwareManager(ctx, reader, root) + device.Observations["managers"] = managerState + device.Time = inventory.TimeObservation{State: "unknown", ReasonCode: managerState.ReasonCode, Message: managerState.Message} + if managerState.State == "observed" { + collectManager(ctx, reader, manager, device, add) + } else { + device.Observations["bmc-time"], device.Observations["bmc-ntp"] = managerState, managerState + } + records, state := softwareInventory(ctx, reader, root) + device.Observations["firmware-inventory"] = state + for _, resource := range records { + firmware := firmwareRecord(resource, system.URI, manager.URI, add) + device.Firmware = append(device.Firmware, firmware) + } + for _, component := range []string{"bmc", "cec", "atf", "uefi", "nic", "dpu-os", "dpu-bsp", "golden"} { + found := false + for _, f := range device.Firmware { + found = found || f.Component == component + } + if !found { + reason := state + if state.State == "observed" { + reason = missing("COMPONENT_NOT_EXPOSED", "The firmware inventory does not expose this component.") + } + device.Firmware = append(device.Firmware, inventory.Firmware{Component: component, State: "unknown", ReasonCode: reason.ReasonCode, Message: reason.Message, Activation: "unknown", ActivationReasonCode: "VERSION_UNAVAILABLE", ActivationMessage: "Activation cannot be established without the component observation.", EvidenceRefs: []string{}}) + } + } + return reconcileFirmware(device) +} + +func softwareManager(ctx context.Context, reader Reader, root Document) (Resource, inventory.Observation) { + uri := Link(root, "Managers") + if uri == "" { + return Resource{}, missing("LINK_NOT_EXPOSED", "The service root does not advertise Managers.") + } + managers, err := reader.Collection(ctx, uri) + if err != nil { + return Resource{}, unavailable(err) + } + if len(managers) != 1 { + return Resource{}, missing("MANAGER_SELECTION_AMBIGUOUS", "A unique BMC manager could not be selected; manager firmware and time remain unknown.") + } + return managers[0], inventory.Observation{State: "observed"} +} + +func collectManager(ctx context.Context, reader Reader, manager Resource, device *inventory.Device, add func(string, string) string) { + if version := Text(manager.Document, "FirmwareVersion"); strings.TrimSpace(version) != "" { + device.Firmware = append(device.Firmware, inventory.Firmware{ResourceURI: manager.URI, Component: "bmc", State: "known", RawVersion: version, Activation: "unknown", ActivationReasonCode: "ACTIVATION_NOT_EXPOSED", ActivationMessage: "Manager firmware metadata does not identify an active software inventory image.", EvidenceRefs: []string{add(manager.URI, "FirmwareVersion")}}) + } + datetime := Text(manager.Document, "DateTime") + state := missing("PROPERTY_NOT_EXPOSED", "The manager does not expose DateTime.") + if value, exists := manager.Document["DateTime"]; exists && value != nil && datetime == "" { + state = missing("INVALID_DATETIME", "The manager DateTime is not a nonempty RFC3339 timestamp.") + } + if datetime != "" { + if _, err := time.Parse(time.RFC3339, datetime); err != nil { + state = missing("INVALID_DATETIME", "The manager DateTime is not a valid RFC3339 timestamp.") + } else { + device.Time.DateTime, device.Time.Offset = datetime, Text(manager.Document, "DateTimeLocalOffset") + state = inventory.Observation{State: "observed"} + add(manager.URI, "DateTime") + } + } + device.Time.State, device.Time.ReasonCode, device.Time.Message = state.State, state.ReasonCode, state.Message + if state.State == "observed" { + device.Time.State = "known" + } + device.Observations["bmc-time"] = state + uri := Link(manager.Document, "NetworkProtocol") + state = missing("LINK_NOT_EXPOSED", "The manager does not advertise NetworkProtocol; NTP configuration is unknown.") + if uri != "" { + np, err := reader.Get(ctx, uri) + if err != nil { + state = unavailable(err) + } else { + ntp := Object(np, "NTP") + enabled, hasEnabled := ntp["ProtocolEnabled"].(bool) + servers, hasServers := ntp["NTPServers"].([]any) + validServers := hasServers + for _, value := range servers { + server, ok := value.(string) + if !ok { + validServers = false + break + } + device.Time.NTPServers = append(device.Time.NTPServers, server) + } + if !validServers { + device.Time.NTPServers = nil + } + if hasEnabled { + device.Time.NTPEnabled = &enabled + } + state = missing("PROPERTY_NOT_EXPOSED", "NTP enablement or server configuration is not exposed.") + if (ntp["ProtocolEnabled"] != nil && !hasEnabled) || (ntp["NTPServers"] != nil && !validServers) { + state = missing("PROPERTY_INVALID", "The NTP enablement or server configuration has an invalid type.") + } + if hasEnabled && validServers { + state = inventory.Observation{State: "observed"} + } + add(uri, "NTP") + } + } + device.Observations["bmc-ntp"] = state +} + +func softwareInventory(ctx context.Context, reader Reader, root Document) ([]Resource, inventory.Observation) { + uri := Link(root, "UpdateService") + if uri == "" { + return nil, missing("LINK_NOT_EXPOSED", "The service root does not advertise UpdateService.") + } + update, err := reader.Get(ctx, uri) + if err != nil { + return nil, unavailable(err) + } + uri = Link(update, "FirmwareInventory") + if uri == "" { + return nil, missing("LINK_NOT_EXPOSED", "UpdateService does not advertise FirmwareInventory.") + } + records, err := reader.Collection(ctx, uri) + if err != nil { + return records, unavailable(err) + } + return records, inventory.Observation{State: "observed"} +} + +func componentName(id, name string) string { + normalize := func(s string) string { + return strings.Trim(strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' { + return r + } + return '-' + }, strings.ToLower(s)), "-") + } + // Prefer a recognizable stable ID, then the display name; golden must precede + // component aliases because a golden BMC image is not the running BMC image. + for _, raw := range []string{id, name} { + s := normalize(raw) + words := "-" + s + "-" + switch { + case strings.Contains(words, "-golden-"): + return "golden" + case strings.Contains(words, "-bmc-"): + return "bmc" + case strings.Contains(words, "-cec-"): + return "cec" + case strings.Contains(words, "-atf-") && strings.Contains(words, "-uefi-"): + return "atf-uefi" + case strings.Contains(words, "-atf-"): + return "atf" + case strings.Contains(words, "-uefi-"): + return "uefi" + case strings.Contains(words, "-nic-"): + return "nic" + case s == "dpu-os" || s == "dpu-operating-system": + return "dpu-os" + case s == "dpu-bsp" || s == "bsp": + return "dpu-bsp" + } + } + return "other" +} + +func firmwareRecord(resource Resource, system, manager string, add func(string, string) string) inventory.Firmware { + f := inventory.Firmware{ResourceURI: resource.URI, RawID: Text(resource.Document, "Id"), RawName: Text(resource.Document, "Name"), RawVersion: Text(resource.Document, "Version"), State: "known", Activation: "unknown", ActivationReasonCode: "ACTIVATION_NOT_EXPOSED", ActivationMessage: "No explicit active or staged target link identifies the selected device.", EvidenceRefs: []string{add(resource.URI, "Id,Name,Version")}} + f.Component = componentName(f.RawID, f.RawName) + if strings.TrimSpace(f.RawVersion) == "" { + f.State, f.ReasonCode, f.Message = "unknown", "VERSION_NOT_EXPOSED", "This firmware entry does not expose a nonempty Version." + if value := resource.Document["Version"]; value != nil { + if _, ok := value.(string); !ok { + f.ReasonCode, f.Message = "VERSION_INVALID", "The firmware Version property is not a string." + } + } + } else if f.Component == "other" { + f.ReasonCode, f.Message = "COMPONENT_UNRECOGNIZED", "The raw firmware identity is retained; its component is not recognized." + } + // DMTF SoftwareInventory v1.12 Links.ActiveTargets/StagedTargets describe + // actual activation, unlike Updateable, Status.Enabled, or an image's name. + links := Object(resource.Document, "Links") + matches := func(key string) bool { + targets, ok := links[key].([]any) + if !ok { + return false + } + for _, target := range targets { + uri := Link(Document{"target": target}, "target") + if uri != "" && (uri == system || uri == manager) { + return true + } + } + return false + } + active, staged := matches("ActiveTargets"), matches("StagedTargets") + if active != staged { + f.Activation = "running" + if staged { + f.Activation = "staged" + } + f.ActivationReasonCode, f.ActivationMessage = "", "" + f.EvidenceRefs = append(f.EvidenceRefs, add(resource.URI, "Links.ActiveTargets,Links.StagedTargets")) + } else if active { + f.ActivationReasonCode, f.ActivationMessage = "ACTIVATION_CONFLICT", "The selected target is reported as both active and staged." + } + return f +} + +func reconcileFirmware(device *inventory.Device) string { + versions := map[string]map[string]bool{} + for _, f := range device.Firmware { + if f.State != "known" || f.Activation == "staged" || f.Component == "other" { + continue + } + if versions[f.Component] == nil { + versions[f.Component] = map[string]bool{} + } + versions[f.Component][strings.TrimSpace(f.RawVersion)] = true + } + bmc := "" + for i := range device.Firmware { + f := &device.Firmware[i] + if len(versions[f.Component]) > 1 && f.Activation != "staged" { + f.State, f.ReasonCode, f.Message = "conflict", "FIRMWARE_VERSION_CONFLICT", "Sources report different versions for this component without an explicit staged-image distinction." + device.Observations["firmware."+f.Component] = missing(f.ReasonCode, f.Message) + } + if f.Component == "bmc" && f.State == "known" && f.Activation != "staged" { + bmc = f.RawVersion + } + } + return bmc +} diff --git a/pkg/dpf/redfish/software_test.go b/pkg/dpf/redfish/software_test.go new file mode 100644 index 00000000..4b54021a --- /dev/null +++ b/pkg/dpf/redfish/software_test.go @@ -0,0 +1,182 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package redfish + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type softwareReader struct { + docs map[string]Document + collections map[string][]Resource + failures map[string]error +} + +func (r softwareReader) Get(_ context.Context, uri string) (Document, error) { + return r.docs[uri], r.failures[uri] +} +func (r softwareReader) Collection(_ context.Context, uri string) ([]Resource, error) { + return r.collections[uri], r.failures[uri] +} +func softwareFixture() (softwareReader, Document, Resource) { + return softwareReader{ + docs: map[string]Document{"/redfish/v1/UpdateService": {"FirmwareInventory": Document{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory"}}, "/redfish/v1/Managers/BMC/NetworkProtocol": {"NTP": Document{"ProtocolEnabled": false, "NTPServers": []any{}}}}, + collections: map[string][]Resource{"/redfish/v1/Managers": {{URI: "/redfish/v1/Managers/BMC", Document: Document{"FirmwareVersion": "1.0", "DateTime": "2026-11-06T12:00:00Z", "NetworkProtocol": Document{"@odata.id": "/redfish/v1/Managers/BMC/NetworkProtocol"}}}}}, + failures: map[string]error{}, + }, Document{"Managers": Document{"@odata.id": "/redfish/v1/Managers"}, "UpdateService": Document{"@odata.id": "/redfish/v1/UpdateService"}}, Resource{URI: "/redfish/v1/Systems/DPU", Document: Document{}} +} +func softwareEvidence(uri, property string) string { return uri + "#" + property } + +var _ = Describe("optional Redfish software observations", func() { + It("classifies component IDs and keeps raw versions and resource provenance", func() { + for id, expected := range map[string]string{"BMC_Firmware": "bmc", "DPU_OS": "dpu-os", "DPU_BSP": "dpu-bsp", "CEC_Firmware": "cec", "DPU_ATF": "atf", "DPU_UEFI": "uefi", "NIC_Firmware": "nic", "BMC_Golden_Image": "golden", "unrecognized": "other"} { + f := firmwareRecord(Resource{URI: "/redfish/v1/UpdateService/FirmwareInventory/" + id, Document: Document{"Id": id, "Name": "Inventory entry", "Version": " v1.2 "}}, "", "", softwareEvidence) + Expect(f.Component).To(Equal(expected), id) + Expect(f.RawID).To(Equal(id)) + Expect(f.RawVersion).To(Equal(" v1.2 ")) + Expect(f.EvidenceRefs).To(ContainElement(f.ResourceURI + "#Id,Name,Version")) + Expect(f.Activation).To(Equal("unknown")) + Expect(f.ActivationReasonCode).To(Equal("ACTIVATION_NOT_EXPOSED")) + Expect(f.ActivationMessage).NotTo(BeEmpty()) + } + }) + It("requires explicit target links for activation and does not infer it from names or enabled status", func() { + doc := Document{"Id": "DPU_OS", "Name": "Active image", "Version": "1", "Status": Document{"State": "Enabled"}, "Updateable": true} + record := Resource{URI: "/redfish/v1/UpdateService/FirmwareInventory/os", Document: doc} + Expect(firmwareRecord(record, "/redfish/v1/Systems/selected", "", softwareEvidence).Activation).To(Equal("unknown")) + doc["Links"] = Document{"ActiveTargets": []any{Document{"@odata.id": "/redfish/v1/Systems/other"}}} + Expect(firmwareRecord(record, "/redfish/v1/Systems/selected", "", softwareEvidence).Activation).To(Equal("unknown")) + doc["Links"] = Document{"StagedTargets": []any{Document{"@odata.id": "/redfish/v1/Systems/selected"}}} + Expect(firmwareRecord(record, "/redfish/v1/Systems/selected", "", softwareEvidence).Activation).To(Equal("staged")) + doc["Links"] = Document{"ActiveTargets": []any{Document{"@odata.id": "/redfish/v1/Systems/selected"}}} + Expect(firmwareRecord(record, "/redfish/v1/Systems/selected", "", softwareEvidence).Activation).To(Equal("running")) + Object(doc, "Links")["StagedTargets"] = []any{Document{"@odata.id": "/redfish/v1/Systems/selected"}} + f := firmwareRecord(record, "/redfish/v1/Systems/selected", "", softwareEvidence) + Expect(f.Activation).To(Equal("unknown")) + Expect(f.ActivationReasonCode).To(Equal("ACTIVATION_CONFLICT")) + }) + It("retains a denied source reason and does not equate it with an absent property", func() { + reader, root, system := softwareFixture() + reader.failures["/redfish/v1/UpdateService/FirmwareInventory"] = &Error{Code: "ACCESS_DENIED", Status: 403} + reader.failures["/redfish/v1/Managers/BMC/NetworkProtocol"] = &Error{Code: "ACCESS_DENIED", Status: 403} + d := inventory.Device{} + collectSoftware(context.Background(), reader, root, system, &d, softwareEvidence) + Expect(d.Time.DateTime).To(Equal("2026-11-06T12:00:00Z")) + Expect(d.Observations["bmc-ntp"].ReasonCode).To(Equal("ACCESS_DENIED")) + Expect(d.Observations["bmc-ntp"].Message).To(ContainSubstring("403")) + for _, f := range d.Firmware { + if f.Component != "bmc" { + Expect(f.ReasonCode).To(Equal("ACCESS_DENIED")) + Expect(f.Message).NotTo(BeEmpty()) + } + } + }) + It("preserves available entries in an incomplete firmware collection", func() { + reader, root, system := softwareFixture() + reader.collections["/redfish/v1/UpdateService/FirmwareInventory"] = []Resource{{URI: "/redfish/v1/UpdateService/FirmwareInventory/os", Document: Document{"Id": "DPU_OS", "Version": "24.04"}}} + reader.failures["/redfish/v1/UpdateService/FirmwareInventory"] = &Error{Code: "ACCESS_DENIED", Status: 403} + d := inventory.Device{} + collectSoftware(context.Background(), reader, root, system, &d, softwareEvidence) + Expect(d.Observations["firmware-inventory"].State).To(Equal("unavailable")) + Expect(d.Firmware).To(ContainElement(And(HaveField("Component", "dpu-os"), HaveField("RawVersion", "24.04"), HaveField("State", "known")))) + }) + It("retains conflicting manager and component versions instead of selecting one", func() { + reader, root, system := softwareFixture() + reader.collections["/redfish/v1/UpdateService/FirmwareInventory"] = []Resource{{URI: "/redfish/v1/UpdateService/FirmwareInventory/bmc", Document: Document{"Id": "BMC_Firmware", "Version": "2.0"}}} + d := inventory.Device{} + selected := collectSoftware(context.Background(), reader, root, system, &d, softwareEvidence) + Expect(selected).To(BeEmpty()) + Expect(d.Observations["firmware.bmc"].ReasonCode).To(Equal("FIRMWARE_VERSION_CONFLICT")) + count := 0 + for _, f := range d.Firmware { + if f.Component == "bmc" { + count++ + Expect(f.State).To(Equal("conflict")) + Expect(f.RawVersion).NotTo(BeEmpty()) + } + } + Expect(count).To(Equal(2)) + }) + It("keeps an explicitly staged successor separate from the manager version", func() { + reader, root, system := softwareFixture() + reader.collections["/redfish/v1/UpdateService/FirmwareInventory"] = []Resource{{URI: "/redfish/v1/UpdateService/FirmwareInventory/bmc", Document: Document{"Id": "BMC_Firmware", "Version": "2.0", "Links": Document{"StagedTargets": []any{Document{"@odata.id": "/redfish/v1/Managers/BMC"}}}}}} + d := inventory.Device{} + Expect(collectSoftware(context.Background(), reader, root, system, &d, softwareEvidence)).To(Equal("1.0")) + Expect(d.Firmware).To(ContainElement(And(HaveField("Component", "bmc"), HaveField("RawVersion", "2.0"), HaveField("Activation", "staged")))) + }) + It("does not select the first of multiple managers and preserves explicit false NTP", func() { + reader, root, system := softwareFixture() + d := inventory.Device{} + collectSoftware(context.Background(), reader, root, system, &d, softwareEvidence) + Expect(d.Time.NTPEnabled).NotTo(BeNil()) + Expect(*d.Time.NTPEnabled).To(BeFalse()) + Expect(d.Observations["bmc-ntp"].State).To(Equal("observed")) + reader.collections["/redfish/v1/Managers"] = append(reader.collections["/redfish/v1/Managers"], Resource{URI: "/redfish/v1/Managers/other", Document: Document{}}) + d = inventory.Device{} + Expect(collectSoftware(context.Background(), reader, root, system, &d, softwareEvidence)).To(BeEmpty()) + Expect(d.Time.ReasonCode).To(Equal("MANAGER_SELECTION_AMBIGUOUS")) + Expect(d.Time.Message).NotTo(BeEmpty()) + }) + It("keeps missing links, missing versions, invalid clock and transport errors distinct", func() { + reader, _, system := softwareFixture() + d := inventory.Device{} + collectSoftware(context.Background(), reader, Document{}, system, &d, softwareEvidence) + Expect(d.Observations["firmware-inventory"].ReasonCode).To(Equal("LINK_NOT_EXPOSED")) + f := firmwareRecord(Resource{URI: "/redfish/v1/UpdateService/FirmwareInventory/os", Document: Document{"Id": "DPU_OS"}}, "", "", softwareEvidence) + Expect(f.ReasonCode).To(Equal("VERSION_NOT_EXPOSED")) + reader, root, system := softwareFixture() + reader.collections["/redfish/v1/Managers"][0].Document["DateTime"] = "invalid" + d = inventory.Device{} + collectSoftware(context.Background(), reader, root, system, &d, softwareEvidence) + Expect(d.Time.ReasonCode).To(Equal("INVALID_DATETIME")) + Expect(d.Time.DateTime).To(BeEmpty()) + for _, err := range []error{context.DeadlineExceeded, context.Canceled, &Error{Code: "RESOURCE_NOT_FOUND", Status: 404}, errors.New("secret-response")} { + state := unavailable(err) + Expect(state.ReasonCode).NotTo(BeEmpty()) + Expect(state.Message).NotTo(BeEmpty()) + Expect(state.Message).NotTo(ContainSubstring("secret-response")) + } + }) + It("returns already read members with a later collection error", func() { + first := "/redfish/v1/UpdateService/FirmwareInventory/first" + second := "/redfish/v1/UpdateService/FirmwareInventory/second" + collection := "/redfish/v1/UpdateService/FirmwareInventory" + c, closeServer := graphClient(map[string]Document{ + collection: {"Members@odata.count": 2, "Members": []any{Document{"@odata.id": first}, Document{"@odata.id": second}}}, + first: {"Id": "DPU_OS", "Version": "24.04"}, + }, nil) + defer closeServer() + records, err := c.Collection(context.Background(), collection) + Expect(Code(err)).To(Equal("RESOURCE_NOT_FOUND")) + Expect(records).To(HaveLen(1)) + Expect(records[0].URI).To(Equal(first)) + }) + It("distinguishes incomplete response bodies from response size limits", func() { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + w.WriteHeader(200) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + c := &Client{origin: server.URL, http: server.Client()} + _, err := c.Get(context.Background(), "/redfish/v1/Managers") + Expect(Code(err)).To(Equal("RESPONSE_READ_FAILED")) + }) + It("classifies an HTTP client timeout separately from other transport errors", func() { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(50 * time.Millisecond); w.WriteHeader(200) })) + defer server.Close() + client := server.Client() + client.Timeout = time.Millisecond + c := &Client{origin: server.URL, http: client} + _, err := c.Get(context.Background(), "/redfish/v1/Managers") + Expect(Code(err)).To(Equal("BMC_TIMEOUT")) + }) +}) diff --git a/pkg/dpf/releases/releases.go b/pkg/dpf/releases/releases.go new file mode 100644 index 00000000..9571fa46 --- /dev/null +++ b/pkg/dpf/releases/releases.go @@ -0,0 +1,230 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 + +// Package releases owns the embedded, release-specific DPF software bill of +// materials. Catalog entries describe software coordinates and documented +// requirements; they do not qualify hardware or authorize installation. +package releases + +import ( + "bytes" + _ "embed" + "fmt" + "io" + "slices" + "strings" + + "gopkg.in/yaml.v3" +) + +// DefaultRelease is deliberately pinned. Adding a catalog entry never selects it +// automatically; promoting the default is a separate reviewed change. +const DefaultRelease = "26.4.0" + +//go:embed releases.yaml +var catalogYAML []byte + +// Source identifies the authoritative upstream material used to curate a BOM. +// Revision is a Git commit or the digest of a downloaded release artifact. +type Source struct { + ID string `json:"id" yaml:"id"` + URL string `json:"url" yaml:"url"` + Revision string `json:"revision" yaml:"revision"` +} + +type Operator struct { + Chart string `json:"chart" yaml:"chart"` + ChartRepository string `json:"chartRepository" yaml:"chartRepository"` + Version string `json:"version" yaml:"version"` + ChartDigest string `json:"chartDigest" yaml:"chartDigest"` + Image string `json:"image" yaml:"image"` + SourceRef string `json:"sourceRef" yaml:"sourceRef"` +} + +// Component is a release coordinate, not evidence that the component is present. +// Applicability distinguishes base components from conditional and optional ones. +type Component struct { + Name string `json:"name" yaml:"name"` + Version string `json:"version" yaml:"version"` + Repository string `json:"repository" yaml:"repository"` + Applicability string `json:"applicability" yaml:"applicability"` + SourceRef string `json:"sourceRef" yaml:"sourceRef"` +} + +type Artifact struct { + Version string `json:"version" yaml:"version"` + URL string `json:"url" yaml:"url"` + Digest string `json:"digest,omitempty" yaml:"digest,omitempty"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` + SourceRef string `json:"sourceRef" yaml:"sourceRef"` +} + +// Requirement distinguishes a minimum capability version from a desired target. +// Unknown requirements have no numeric constraint and retain a reason. Scope +// identifies where a requirement applies, not where it must be measured. +type Requirement struct { + Component string `json:"component" yaml:"component"` + Scope string `json:"scope" yaml:"scope"` + State string `json:"state" yaml:"state"` + Minimum string `json:"minimum,omitempty" yaml:"minimum,omitempty"` + Target string `json:"target,omitempty" yaml:"target,omitempty"` + ReasonCode string `json:"reasonCode,omitempty" yaml:"reasonCode,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` + SourceRef string `json:"sourceRef" yaml:"sourceRef"` +} + +type HardwareBOM struct { + Family string `json:"family" yaml:"family"` + Models []string `json:"models" yaml:"models"` + BFB Artifact `json:"bfb" yaml:"bfb"` + Requirements []Requirement `json:"requirements" yaml:"requirements"` +} + +type Release struct { + Version string `json:"version" yaml:"version"` + Operator Operator `json:"operator" yaml:"operator"` + Components []Component `json:"components" yaml:"components"` + Hardware []HardwareBOM `json:"hardware" yaml:"hardware"` + Sources []Source `json:"sources" yaml:"sources"` + Limitations []string `json:"limitations" yaml:"limitations"` +} + +type catalog struct { + Releases map[string]Release `yaml:"releases"` +} + +// Resolve returns a fresh BOM with no shared mutable values. It performs no +// network access. Empty means the pinned default; callers with presence-aware +// configuration must reject a supplied empty value before calling Resolve. +func Resolve(version string) (Release, error) { + if version == "" { + version = DefaultRelease + } + c, err := load(catalogYAML) + if err != nil { + return Release{}, err + } + r, ok := c.Releases[version] + if !ok { + return Release{}, fmt.Errorf("unknown DPF release %q; catalog releases: %s", version, strings.Join(keys(c), ", ")) + } + return r, nil +} + +// SupportedReleases returns catalog keys. Presence does not assert runtime +// qualification for every component, hardware model or deployment profile. +func SupportedReleases() []string { + c, err := load(catalogYAML) + if err != nil { + return nil + } + return keys(c) +} + +func keys(c catalog) []string { + out := make([]string, 0, len(c.Releases)) + for k := range c.Releases { + out = append(out, k) + } + slices.Sort(out) + return out +} + +// RequirementsFor returns a copy of the hardware BOM for an exact documented +// model. Generic family names, unknown models and nil receivers do not match; +// callers must retain an explicit applicability-unresolved result. +func (r *Release) RequirementsFor(model string) (HardwareBOM, bool) { + if r == nil { + return HardwareBOM{}, false + } + model = strings.TrimSpace(model) + for _, h := range r.Hardware { + for _, supported := range h.Models { + if strings.EqualFold(model, supported) { + h.Models = slices.Clone(h.Models) + h.Requirements = slices.Clone(h.Requirements) + return h, true + } + } + } + return HardwareBOM{}, false +} + +func load(data []byte) (catalog, error) { + var c catalog + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&c); err != nil { + return catalog{}, fmt.Errorf("invalid embedded DPF release catalog: %w", err) + } + var trailing any + if err := dec.Decode(&trailing); err != io.EOF { + return catalog{}, fmt.Errorf("DPF release catalog must contain one document") + } + if len(c.Releases) == 0 { + return catalog{}, fmt.Errorf("DPF release catalog has no entries") + } + for key, r := range c.Releases { + if err := validate(key, r); err != nil { + return catalog{}, fmt.Errorf("invalid DPF release %q: %w", key, err) + } + } + return c, nil +} + +func validate(key string, r Release) error { + if r.Version != key || r.Operator.Version != "v"+key || r.Operator.Chart == "" || r.Operator.ChartRepository == "" || r.Operator.Image == "" || r.Operator.ChartDigest == "" { + return fmt.Errorf("missing or inconsistent release coordinates") + } + sources := map[string]bool{} + for _, s := range r.Sources { + if s.ID == "" || sources[s.ID] || !strings.HasPrefix(s.URL, "https://") || s.Revision == "" { + return fmt.Errorf("missing or duplicate authoritative source") + } + sources[s.ID] = true + } + if !sources[r.Operator.SourceRef] { + return fmt.Errorf("operator source is not declared") + } + components := map[string]bool{} + for _, c := range r.Components { + if c.Name == "" || components[c.Name] || c.Version == "" || c.Repository == "" || !sources[c.SourceRef] || !slices.Contains([]string{"base", "conditional", "optional"}, c.Applicability) { + return fmt.Errorf("invalid component coordinate or source") + } + components[c.Name] = true + } + models := map[string]bool{} + for _, h := range r.Hardware { + if h.Family == "" || len(h.Models) == 0 || h.BFB.Version == "" || !strings.HasPrefix(h.BFB.URL, "https://") || !sources[h.BFB.SourceRef] || (h.BFB.Digest == "" && (h.BFB.ReasonCode == "" || h.BFB.Message == "")) { + return fmt.Errorf("invalid hardware applicability or artifact provenance") + } + for _, model := range h.Models { + model = strings.ToLower(strings.TrimSpace(model)) + if model == "" || models[model] { + return fmt.Errorf("missing or ambiguous hardware model") + } + models[model] = true + } + seen := map[string]bool{} + for _, req := range h.Requirements { + if req.Component == "" || req.Scope == "" || seen[req.Component+"/"+req.Scope] || !sources[req.SourceRef] { + return fmt.Errorf("invalid hardware requirement or source") + } + seen[req.Component+"/"+req.Scope] = true + switch req.State { + case "known": + if req.Minimum == "" && req.Target == "" { + return fmt.Errorf("known requirement has no minimum or target") + } + case "unknown": + if req.Minimum != "" || req.Target != "" || req.ReasonCode == "" || req.Message == "" { + return fmt.Errorf("unknown requirement needs a reason and no inferred version") + } + default: + return fmt.Errorf("invalid requirement state") + } + } + } + return nil +} diff --git a/pkg/dpf/releases/releases.yaml b/pkg/dpf/releases/releases.yaml new file mode 100644 index 00000000..fe2d5950 --- /dev/null +++ b/pkg/dpf/releases/releases.yaml @@ -0,0 +1,108 @@ +# Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +# Curated software coordinates, not a Launch Kit hardware qualification catalog. +releases: + "26.4.0": + version: "26.4.0" + operator: + chart: dpf-operator + chartRepository: https://helm.ngc.nvidia.com/nvidia/doca + version: v26.4.0 + chartDigest: sha256:0f9a18a0d64958e2f1769bdb468a46ad65e0ea485dadb86cad9e6457b3cec69a + image: nvcr.io/nvidia/doca/dpf-system:v26.4.0 + sourceRef: published-chart + components: + - {name: argocd, version: v3.3.0, repository: quay.io/argoproj/argocd, applicability: base, sourceRef: platform-support} + - {name: cert-manager-controller, version: v1.19.3, repository: quay.io/jetstack/cert-manager-controller, applicability: base, sourceRef: platform-support} + - {name: flannel, version: v0.26.5, repository: docker.io/flannel/flannel, applicability: base, sourceRef: platform-support} + - {name: multus, version: v3.9.3, repository: ghcr.io/k8snetworkplumbingwg/multus-cni, applicability: base, sourceRef: platform-support} + - {name: nvidia-k8s-ipam, version: v0.3.5, repository: ghcr.io/mellanox/nvidia-k8s-ipam, applicability: base, sourceRef: platform-support} + - {name: maintenance-operator, version: v0.3.0, repository: ghcr.io/mellanox/maintenance-operator, applicability: base, sourceRef: platform-support} + - {name: network-operator, version: v26.1.0, repository: nvcr.io/nvidia/cloud-native/network-operator, applicability: base, sourceRef: platform-support} + - {name: node-feature-discovery, version: v0.18.3, repository: registry.k8s.io/nfd/node-feature-discovery, applicability: base, sourceRef: platform-support} + - {name: sriov-device-plugin, version: v3.11.0, repository: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin, applicability: base, sourceRef: platform-support} + - {name: etcd-defrag, version: v0.22.0, repository: ghcr.io/ahrtr/etcd-defrag, applicability: optional, sourceRef: platform-support} + - {name: kamaji, version: v1.34.0-25.9.3, repository: ghcr.io/nvidia/kamaji, applicability: conditional, sourceRef: platform-support} + - {name: local-path-provisioner, version: v0.0.34, repository: docker.io/rancher/local-path-provisioner, applicability: conditional, sourceRef: platform-support} + - {name: kube-state-metrics, version: v2.18.0, repository: registry.k8s.io/kube-state-metrics/kube-state-metrics, applicability: optional, sourceRef: platform-support} + - {name: node-problem-detector, version: v1.35.1, repository: registry.k8s.io/node-problem-detector/node-problem-detector, applicability: optional, sourceRef: platform-support} + - {name: prometheus, version: v2.54.1, repository: quay.io/prometheus/prometheus, applicability: optional, sourceRef: platform-support} + - {name: grafana, version: "11.1.0", repository: docker.io/grafana/grafana, applicability: optional, sourceRef: platform-support} + hardware: + - family: BlueField-3 + models: [B3210, B3220, B3240] + bfb: + version: "3.4.0-92" + url: https://content.mellanox.com/BlueField/BFBs/Ubuntu24.04/bf-bundle-3.4.0-92_26.04_ubuntu-24.04_64k_prod.bfb + reasonCode: ARTIFACT_DIGEST_NOT_RECORDED + message: The upstream release guide supplies a versioned BFB URL but no digest; immutable artifact verification remains required before provisioning. + sourceRef: zero-trust-env + requirements: + - component: nic + scope: dpu + state: known + minimum: "32.38.1002" + sourceRef: platform-support + - component: mft + scope: host + state: known + minimum: "4.33.0-169" + sourceRef: platform-support + - component: bmc + scope: dpu + state: unknown + reasonCode: RELEASE_REQUIREMENT_NOT_DOCUMENTED + message: The pinned platform requirements do not specify a BMC firmware minimum or target. The PoC capability threshold is not substituted for an official release requirement. + sourceRef: platform-support + - component: cec + scope: dpu + state: unknown + reasonCode: RELEASE_REQUIREMENT_NOT_DOCUMENTED + message: No standalone CEC requirement is declared by the pinned platform requirements; retain the observed version without inferring compatibility. + sourceRef: platform-support + - component: atf + scope: dpu + state: unknown + reasonCode: RELEASE_REQUIREMENT_NOT_DOCUMENTED + message: No standalone ATF requirement is declared by the pinned platform requirements; retain the observed version without inferring compatibility. + sourceRef: platform-support + - component: uefi + scope: dpu + state: unknown + reasonCode: RELEASE_REQUIREMENT_NOT_DOCUMENTED + message: No standalone UEFI requirement is declared by the pinned platform requirements; retain the observed version without inferring compatibility. + sourceRef: platform-support + - component: dpu-bsp + scope: dpu + state: unknown + reasonCode: RELEASE_REQUIREMENT_NOT_DOCUMENTED + message: The guide selects a BFB bundle but does not specify the BSP inventory version string; do not equate a bundle version with a BSP version. + sourceRef: zero-trust-env + - component: dpu-os + scope: dpu + state: unknown + reasonCode: RELEASE_REQUIREMENT_NOT_DOCUMENTED + message: The guide selects a BFB bundle but does not specify the DPU OS inventory version string; do not equate a bundle version with an OS version. + sourceRef: zero-trust-env + - component: golden + scope: dpu + state: unknown + reasonCode: RELEASE_REQUIREMENT_NOT_DOCUMENTED + message: No standalone golden-image requirement is declared by the pinned platform requirements; retain the observed version without inferring compatibility. + sourceRef: platform-support + sources: + - id: platform-support + url: https://github.com/NVIDIA/doca-platform/blob/8551df0d3ac4ed51a67d4a5148a7646493a30e10/docs/public/platform-support.md + revision: 8551df0d3ac4ed51a67d4a5148a7646493a30e10 + - id: zero-trust-env + url: https://github.com/NVIDIA/doca-platform/blob/8551df0d3ac4ed51a67d4a5148a7646493a30e10/docs/public/user-guides/zero-trust/use-cases/passthrough/manifests/00-env-vars/envvars.env + revision: 8551df0d3ac4ed51a67d4a5148a7646493a30e10 + - id: published-chart + url: https://helm.ngc.nvidia.com/nvidia/doca/charts/dpf-operator-v26.4.0.tgz + revision: sha256:0f9a18a0d64958e2f1769bdb468a46ad65e0ea485dadb86cad9e6457b3cec69a + limitations: + - Catalog applicability is limited to the exact listed BlueField-3 models; BlueField-4 and generic family-only observations require a separately reviewed hardware BOM. + - Software coordinates are upstream release metadata, not proof of installed versions, active firmware, runtime compatibility, or Launch Kit hardware qualification. + - Component applicability mirrors the upstream component matrix; it does not select services or authorize changing the independent host cluster. + - The published chart records build commit 50fab50af0bfa67be22e8bb4768548ad7dae57bf, separately from the public source tag commit used for documentation. + - BFB and component image bytes are not digest-locked by this catalog; provisioning must resolve and verify immutable artifacts before use. diff --git a/pkg/dpf/releases/releases_test.go b/pkg/dpf/releases/releases_test.go new file mode 100644 index 00000000..3b3d9196 --- /dev/null +++ b/pkg/dpf/releases/releases_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package releases + +import ( + "strings" + "sync" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestReleases(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "DPF release catalog") } + +var _ = Describe("release BOM resolution", func() { + It("defaults to a pinned full release and retains separately sourced coordinates", func() { + r, err := Resolve("") + Expect(err).NotTo(HaveOccurred()) + Expect(r.Version).To(Equal(DefaultRelease)) + Expect(r.Operator.Version).To(Equal("v26.4.0")) + Expect(r.Operator.Image).To(Equal("nvcr.io/nvidia/doca/dpf-system:v26.4.0")) + Expect(r.Operator.ChartDigest).To(Equal("sha256:0f9a18a0d64958e2f1769bdb468a46ad65e0ea485dadb86cad9e6457b3cec69a")) + Expect(SupportedReleases()).To(Equal([]string{"26.4.0"})) + }) + It("rejects unreviewed releases and does not interpret latest, prefixes or whitespace", func() { + for _, version := range []string{"latest", "26.4", "26.4.1", "v26.4.0", " 26.4.0", "26.4.0 "} { + _, err := Resolve(version) + Expect(err).To(MatchError(ContainSubstring("unknown DPF release"))) + } + }) + It("keeps unknown BMC requirements distinct from a documented NIC minimum", func() { + r, err := Resolve(DefaultRelease) + Expect(err).NotTo(HaveOccurred()) + h, ok := r.RequirementsFor("b3240") + Expect(ok).To(BeTrue()) + Expect(h.Family).To(Equal("BlueField-3")) + Expect(h.Requirements).To(ContainElement(And(HaveField("Component", "nic"), HaveField("Minimum", "32.38.1002"), HaveField("State", "known")))) + Expect(h.Requirements).To(ContainElement(And(HaveField("Component", "bmc"), HaveField("Minimum", ""), HaveField("Target", ""), HaveField("State", "unknown"), HaveField("ReasonCode", "RELEASE_REQUIREMENT_NOT_DOCUMENTED")))) + Expect(h.BFB.Digest).To(BeEmpty()) + Expect(h.BFB.ReasonCode).To(Equal("ARTIFACT_DIGEST_NOT_RECORDED")) + }) + It("does not apply BF3 requirements to BF4, family names, or partial model matches", func() { + r, err := Resolve(DefaultRelease) + Expect(err).NotTo(HaveOccurred()) + for _, model := range []string{"", "BlueField-3", "BlueField-4", "B4240", "B3240 extra", "XXB3240"} { + _, ok := r.RequirementsFor(model) + Expect(ok).To(BeFalse()) + } + var missing *Release + _, ok := missing.RequirementsFor("B3240") + Expect(ok).To(BeFalse()) + }) + It("isolates returned nested values from the catalog and hardware lookup", func() { + r, err := Resolve(DefaultRelease) + Expect(err).NotTo(HaveOccurred()) + h, ok := r.RequirementsFor("B3240") + Expect(ok).To(BeTrue()) + h.Models[0] = "altered" + h.Requirements[0].Minimum = "0" + Expect(r.Hardware[0].Models[0]).To(Equal("B3210")) + Expect(r.Hardware[0].Requirements[0].Minimum).To(Equal("32.38.1002")) + r.Hardware[0].Models[0] = "changed" + r.Hardware[0].Requirements[0].Minimum = "0" + r.Components[0].Version = "changed" + r.Sources[0].URL = "changed" + r.Limitations[0] = "changed" + next, err := Resolve(DefaultRelease) + Expect(err).NotTo(HaveOccurred()) + Expect(next.Hardware[0].Models[0]).To(Equal("B3210")) + Expect(next.Hardware[0].Requirements[0].Minimum).To(Equal("32.38.1002")) + Expect(next.Components[0].Version).To(Equal("v3.3.0")) + Expect(next.Sources[0].URL).To(HavePrefix("https://github.com/")) + Expect(next.Limitations[0]).NotTo(Equal("changed")) + }) + It("allows concurrent independent resolutions", func() { + var wg sync.WaitGroup + for range 10 { + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + r, err := Resolve(DefaultRelease) + Expect(err).NotTo(HaveOccurred()) + r.Hardware[0].Models[0] = "independent" + }() + } + wg.Wait() + }) + It("rejects malformed catalogs rather than hiding missing provenance", func() { + for _, data := range []string{ + "releases: {}", + string(catalogYAML) + "\n---\nreleases: {}\n", + strings.Replace(string(catalogYAML), "sourceRef: published-chart", "sourceRef: absent", 1), + strings.Replace(string(catalogYAML), "state: unknown", "state: known", 1), + strings.Replace(string(catalogYAML), "models: [B3210, B3220, B3240]", "models: [B3210, B3210]", 1), + strings.Replace(string(catalogYAML), "version: \"26.4.0\"", "version: \"26.8.0\"", 1), + strings.Replace(string(catalogYAML), "chart: dpf-operator", "unknown: true", 1), + } { + _, err := load([]byte(data)) + Expect(err).To(HaveOccurred()) + } + }) +}) diff --git a/pkg/dpf/sshprobe/collect.go b/pkg/dpf/sshprobe/collect.go index 2d1fd0e7..3b03d86a 100644 --- a/pkg/dpf/sshprobe/collect.go +++ b/pkg/dpf/sshprobe/collect.go @@ -4,6 +4,8 @@ package sshprobe import ( "context" + "errors" + "net" "path" "strconv" "strings" @@ -24,12 +26,14 @@ func (Collector) Collect(ctx context.Context, target config.Target, timeout time evidence = map[string]inventory.Evidence{} if target.SSH == nil || (target.Role != "hosts" && target.Role != "dpus") { device.ReasonCode = "SSH_CONFIGURATION_INVALID" + device.Message = "SSH discovery requires an explicit host or DPU SSH target." return device, evidence, &Error{Code: device.ReasonCode, Stage: "target"} } user, password := target.SSH.Credentials() c, err := connect(ctx, target.Endpoint, credentials{user: user, password: password, key: target.SSH.PrivateKey(), knownHosts: target.SSH.KnownHosts()}, timeout) if err != nil { device.ReasonCode = Code(err) + device.Message = Reason(err) return device, evidence, err } defer c.Close() @@ -43,7 +47,7 @@ func emptyDevice(target config.Target) inventory.Device { Qualification: "unqualified", InventoryCompleteness: "unknown", BMCAddressState: "not-evaluated", Authentication: inventory.Authentication{Method: "ssh", PasswordRead: "not-evaluated", ClientCertificateSupport: "unknown", DPFClientAuthentication: "not-evaluated"}, Firmware: []inventory.Firmware{}, EvidenceRefs: []string{}, - Time: inventory.TimeObservation{State: "unknown", ReasonCode: "NOT_EVALUATED"}, + Time: inventory.TimeObservation{State: "unknown", ReasonCode: "NOT_EVALUATED", Message: "BMC clock and NTP are not queried through OS SSH."}, Runtime: &inventory.RuntimeObservation{Observations: map[string]inventory.Observation{}}, } } @@ -54,6 +58,13 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i defer func() { if err != nil { device.ReasonCode = Code(err) + device.Message = Reason(err) + var diagnostic *Error + if errors.As(err, &diagnostic) { + if _, exists := device.Runtime.Observations[diagnostic.Stage]; !exists { + device.Runtime.Observations[diagnostic.Stage] = unavailable(err, diagnostic.Tool) + } + } } }() device.ReaderAdapter = "linux-host-ssh-v1" @@ -70,7 +81,13 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i device.EvidenceRefs = append(device.EvidenceRefs, id) return id } - read := func(stage, command string) (string, error) { return remote.Run(ctx, stage, command) } + read := func(stage, command string) (string, error) { + text, e := remote.Run(ctx, stage, command) + if e != nil { + device.Runtime.Observations[stage] = unavailable(e, commandTool(command)) + } + return text, e + } if target.Role == "dpus" { dmi, e := read("arm-dmi", privilege+"dmidecode -s system-serial-number") if e != nil { @@ -90,6 +107,13 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i device.ID = inventory.PhysicalID("dpu", device.IdentityKey) add("arm-dmi", "SerialNumber") add("arm-fru", "Product Serial") + fruFields, _ := fields(fru) + if board := fruFields["Board Product"]; board != "" && len(board) <= 256 && !strings.ContainsAny(board, "\x00\r\n") { + device.Runtime.Observations["board-product"] = observed("ipmitool", board) + add("arm-fru", "Board Product") + } else { + device.Runtime.Observations["board-product"] = missing("ipmitool") + } device.BMCAddressState = "unavailable" lan, e := read("arm-ipmi-lan", privilege+"ipmitool lan print") if e == nil { @@ -100,7 +124,7 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i add("arm-ipmi-lan", "IP Address") device.Runtime.Observations["bmcAddress"] = inventory.Observation{State: "observed", Values: []string{device.BMCEndpoint}} } else { - device.Runtime.Observations["bmcAddress"] = inventory.Observation{State: "unavailable", ReasonCode: Code(e)} + device.Runtime.Observations["bmcAddress"] = unavailable(e, "ipmitool") } } else { uuid, e := read("host-dmi", privilege+"dmidecode -s system-uuid") @@ -148,7 +172,9 @@ func collect(ctx context.Context, target config.Target, remote runner) (device i } device.Runtime.Observations["pciScope"] = inventory.Observation{State: "observed", Values: []string{"NVIDIA network PCI functions; grouped by host-local PCI slot"}} } + collectNetwork(ctx, remote, device.Runtime, add) collectRuntime(ctx, remote, device.Runtime, add) + collectHardware(ctx, remote, &device, privilege, target.Role, add) // Recheck physical identity after optional observations, so disconnects and // device changes cannot publish stale identity as a successful observation. if target.Role == "dpus" { @@ -257,7 +283,7 @@ func collectPF(ctx context.Context, remote runner, bdf string, runtime *inventor } } if err != nil { - runtime.Observations[stage] = inventory.Observation{State: "unavailable", ReasonCode: Code(err)} + runtime.Observations[stage] = unavailable(err, commandTool(p.command)) } else { runtime.Observations[stage] = inventory.Observation{State: "observed"} add(stage, p.field) @@ -296,13 +322,22 @@ func collectRuntime(ctx context.Context, remote runner, runtime *inventory.Runti for _, line := range strings.Split(text, "\n") { f := strings.Fields(line) if len(f) >= 2 && f[0] == "nameserver" { + address, _, _ := strings.Cut(f[1], "%") + if net.ParseIP(address) == nil { + err = &Error{Code: "SSH_RUNTIME_INVALID", Stage: probe.stage} + runtime.DNS = nil + break + } runtime.DNS = append(runtime.DNS, f[1]) } + if len(f) >= 2 && (f[0] == "search" || f[0] == "domain") && runtime.Network != nil { + runtime.Network.Resolver.Search = append(runtime.Network.Resolver.Search, f[1:]...) + } } } } if err != nil { - runtime.Observations[probe.stage] = inventory.Observation{State: "unavailable", ReasonCode: Code(err)} + runtime.Observations[probe.stage] = unavailable(err, commandTool(probe.command)) } else { runtime.Observations[probe.stage] = inventory.Observation{State: "observed"} add(probe.stage, probe.stage) diff --git a/pkg/dpf/sshprobe/diagnostics.go b/pkg/dpf/sshprobe/diagnostics.go new file mode 100644 index 00000000..2848a2a4 --- /dev/null +++ b/pkg/dpf/sshprobe/diagnostics.go @@ -0,0 +1,107 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "errors" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "golang.org/x/crypto/ssh" +) + +// commandFailure classifies bounded output, but never returns it. Tool diagnostics +// can include paths and credentials; only static explanations reach artifacts. +func commandFailure(err error, stage, command, output string) *Error { + code := "SSH_COMMAND_FAILED" + var status *ssh.ExitError + if errors.As(err, &status) { + switch status.ExitStatus() { + case 126: + code = "TOOL_NOT_EXECUTABLE" + case 127: + code = "TOOL_NOT_AVAILABLE" + case 66: + if stage == "dnsmasq" { + code = "PROPERTY_NOT_EXPOSED" + } + case 77: + if stage == "dnsmasq" { + code = "PERMISSION_DENIED" + } + } + } + lower := strings.ToLower(output) + switch { + case strings.Contains(lower, "command not found"), strings.Contains(lower, "not found in path"): + code = "TOOL_NOT_AVAILABLE" + case strings.Contains(lower, "permission denied"), strings.Contains(lower, "access denied"), strings.Contains(lower, "operation not permitted"), strings.Contains(lower, "a password is required"), strings.Contains(lower, "not allowed to execute"): + code = "PERMISSION_DENIED" + case strings.Contains(lower, "not supported"), strings.Contains(lower, "unsupported"), strings.Contains(lower, "unknown parameter"), strings.Contains(lower, "unrecognized option"): + code = "OPERATION_UNSUPPORTED" + } + tool := commandTool(command) + if strings.HasPrefix(command, "sudo -n ") && (strings.Contains(lower, "sudo: not found") || strings.Contains(lower, "sudo: command not found")) { + tool = "sudo" + } + return &Error{Code: code, Stage: stage, Tool: tool} +} + +func commandTool(command string) string { + f := strings.Fields(command) + if len(f) >= 3 && f[0] == "sudo" && f[1] == "-n" { + f = f[2:] + } + if len(f) == 0 { + return "" + } + return f[0] +} + +// Reason returns a safe explanation without library or remote error text. +func Reason(err error) string { + switch Code(err) { + case "TOOL_NOT_AVAILABLE": + return "Required tool is not available in the remote command environment." + case "TOOL_NOT_EXECUTABLE": + return "The remote tool exists but could not be executed." + case "PERMISSION_DENIED": + return "The configured account or noninteractive sudo policy does not permit this read." + case "OPERATION_UNSUPPORTED": + return "The installed tool or device does not support this read operation." + case "SSH_TIMEOUT_OR_CANCELLED": + return "The observation timed out or was cancelled." + case "SSH_OUTPUT_LIMIT": + return "The remote response exceeded the bounded output limit." + case "SSH_RUNTIME_INVALID": + return "The command returned missing, malformed or ambiguous observation data." + case "PROPERTY_NOT_EXPOSED": + return "The requested property is not exposed in the command response." + case "NOT_APPLICABLE": + return "This observation does not apply to this device or collection vantage." + case "SSH_HOST_KEY_REJECTED": + return "The remote SSH host key did not match the configured trust." + case "SSH_CONNECT_FAILED": + return "The configured SSH endpoint could not be reached." + case "SSH_AUTHENTICATION_OR_HANDSHAKE_FAILED": + return "SSH authentication or session negotiation failed." + default: + return "The requested read failed; no reliable observation was obtained." + } +} + +func unavailable(err error, tool string) inventory.Observation { + var e *Error + if errors.As(err, &e) && e.Tool != "" { + tool = e.Tool + } + return inventory.Observation{State: "unavailable", Values: []string{}, ReasonCode: Code(err), Message: Reason(err), Tool: tool} +} + +func missing(tool string) inventory.Observation { + return unavailable(&Error{Code: "PROPERTY_NOT_EXPOSED"}, tool) +} + +func observed(tool string, values ...string) inventory.Observation { + return inventory.Observation{State: "observed", Values: values, Tool: tool} +} diff --git a/pkg/dpf/sshprobe/hardware.go b/pkg/dpf/sshprobe/hardware.go new file mode 100644 index 00000000..76eeb09f --- /dev/null +++ b/pkg/dpf/sshprobe/hardware.go @@ -0,0 +1,236 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "context" + "regexp" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +var versionToken = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._()+-]{0,127}$`) +var bdfToken = regexp.MustCompile(`(?i)^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`) + +func collectHardware(ctx context.Context, remote runner, device *inventory.Device, privilege, role string, add func(string, string) string) { + h := &inventory.HardwareObservation{Cards: []inventory.CardHardware{}, RShim: unavailable(&Error{Code: "NOT_APPLICABLE"}, "systemctl")} + device.Runtime.Hardware = h + version, versionErr := remote.Run(ctx, "mft-version", privilege+"mlxfwmanager --version") + if versionErr == nil { + matches := regexp.MustCompile(`(?i)\bmft\s+(\d+\.\d+\.\d+(?:-\d+)?)\b`).FindAllStringSubmatch(version, -1) + if len(matches) != 1 { + versionErr = &Error{Code: "SSH_RUNTIME_INVALID", Stage: "mft-version"} + } else { + device.Runtime.Observations["mft-version"] = observed("mlxfwmanager", matches[0][1]) + add("mft-version", "MFTVersion") + } + } + if versionErr != nil { + device.Runtime.Observations["mft-version"] = unavailable(versionErr, "mlxfwmanager") + } + var cards []pciCard + if role == "hosts" { + for _, a := range device.Adapters { + cards = append(cards, pciCard{address: a.PCIAddress, model: a.Model}) + } + text, err := remote.Run(ctx, "rshim-service", "systemctl show rshim --property=LoadState --property=ActiveState") + if err == nil { + values := []string{} + for _, line := range strings.Split(text, "\n") { + key, value, ok := strings.Cut(line, "=") + if ok && (key == "LoadState" || key == "ActiveState") && versionToken.MatchString(value) { + values = append(values, key+"="+value) + } + } + if len(values) != 2 { + err = &Error{Code: "SSH_RUNTIME_INVALID", Stage: "rshim-service"} + } else { + h.RShim = observed("systemctl", values...) + add("rshim-service", "service-state") + } + } + if err != nil { + h.RShim = unavailable(err, "systemctl") + } + } else { + text, err := remote.Run(ctx, "arm-pci", "lspci -Dnn -d 15b3:") + if err == nil { + cards, err = pciCards(text) + } + if err != nil { + device.Runtime.Observations["arm-pci"] = unavailable(err, "lspci") + return + } + device.Runtime.Observations["arm-pci"] = observed("lspci") + add("arm-pci", "PCIAddress") + } + for _, card := range cards { + // Never interpolate any address other than a validated collector-owned BDF. + if !bdfToken.MatchString(card.address) { + continue + } + c := inventory.CardHardware{PCIAddress: card.address, Domain: "host-pci", Model: card.model, NVConfig: map[string]inventory.Observation{}, EvidenceRefs: []string{}, Privilege: unavailable(&Error{Code: "NOT_APPLICABLE"}, "mlxprivhost")} + if role == "hosts" { + for _, a := range device.Adapters { + if a.PCIAddress == card.address { + c.SerialNumber = a.SerialNumber + } + } + } else { + c.Domain = "arm-pci" + stage := "arm-vpd:" + card.address + text, err := remote.Run(ctx, stage, privilege+"lspci -D -vv -s "+card.address) + if err == nil { + c.SerialNumber, _, err = vpd(text) + if err == nil && c.SerialNumber == "" { + err = &Error{Code: "PROPERTY_NOT_EXPOSED", Stage: stage} + } + } + if err != nil { + device.Runtime.Observations[stage] = unavailable(err, "lspci") + } else { + device.Runtime.Observations[stage] = observed("lspci") + c.EvidenceRefs = append(c.EvidenceRefs, add(stage, "SerialNumber")) + } + } + stage := "firmware:" + card.address + text, err := remote.Run(ctx, stage, privilege+"mlxfwmanager --query -d "+card.address) + if err == nil { + c.Firmware, c.PSID, err = parseFirmware(text, card.address) + } + if err != nil { + c.Firmware = unavailable(err, "mlxfwmanager") + c.PSID = unavailable(err, "mlxfwmanager") + } else { + c.EvidenceRefs = append(c.EvidenceRefs, add(stage, "firmware-psid")) + } + if strings.Contains(strings.ToLower(card.model), "bluefield") { + for _, parameter := range []string{"INTERNAL_CPU_OFFLOAD_ENGINE", "INTERNAL_CPU_RSHIM", "NUM_OF_VFS"} { + stage = "nvconfig:" + card.address + ":" + parameter + text, err = remote.Run(ctx, stage, privilege+"mlxconfig -d "+card.address+" q "+parameter) + var value inventory.Observation + if err == nil { + value, err = parseNVConfig(text, parameter) + } + if err != nil { + value = unavailable(err, "mlxconfig") + } else { + c.EvidenceRefs = append(c.EvidenceRefs, add(stage, parameter)) + } + c.NVConfig[parameter] = value + } + if role == "dpus" { + stage = "privilege:" + card.address + text, err = remote.Run(ctx, stage, privilege+"mlxprivhost -d "+card.address+" q --full") + if err == nil { + c.Privilege, err = parsePrivilege(text) + } + if err != nil { + c.Privilege = unavailable(err, "mlxprivhost") + } else { + c.EvidenceRefs = append(c.EvidenceRefs, add(stage, "host-restrictions")) + } + } + } + h.Cards = append(h.Cards, c) + } +} + +func parseFirmware(text, bdf string) (inventory.Observation, inventory.Observation, error) { + fw, psid := missing("mlxfwmanager"), missing("mlxfwmanager") + seen := map[string]string{} + for _, line := range strings.Split(text, "\n") { + key, value, hasColon := strings.Cut(strings.TrimSpace(line), ":") + if hasColon && (key == "PCI Device Name" || key == "PSID") { + value = strings.TrimSpace(value) + if old, ok := seen[key]; ok && old != value { + return fw, psid, &Error{Code: "SSH_RUNTIME_INVALID"} + } + seen[key] = value + } + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "FW" { + if old, ok := seen["FW"]; ok && old != fields[1] { + return fw, psid, &Error{Code: "SSH_RUNTIME_INVALID"} + } + seen["FW"] = fields[1] + } + } + // An exact -d query is the binding. If a PCI address is reported, it must agree. + if address := seen["PCI Device Name"]; address != "" && !strings.EqualFold(address, bdf) && !strings.EqualFold("0000:"+address, bdf) { + return fw, psid, &Error{Code: "SSH_RUNTIME_INVALID"} + } + if v := seen["FW"]; versionToken.MatchString(v) { + fw = observed("mlxfwmanager", v) + } + if v := seen["PSID"]; versionToken.MatchString(v) { + psid = observed("mlxfwmanager", v) + } + if fw.State != "observed" && psid.State != "observed" { + return fw, psid, &Error{Code: "SSH_RUNTIME_INVALID"} + } + return fw, psid, nil +} + +func parseNVConfig(text, parameter string) (inventory.Observation, error) { + // Default mlxconfig q reports Next Boot, not necessarily the running state. + // Preserve this distinction rather than infer mode from a pending value. + next := strings.Contains(text, "Next Boot") + var value string + for _, line := range strings.Split(text, "\n") { + f := strings.Fields(line) + if len(f) > 0 && f[0] == parameter { + if len(f) != 2 || !versionToken.MatchString(f[1]) || value != "" { + return inventory.Observation{}, &Error{Code: "SSH_RUNTIME_INVALID"} + } + value = f[1] + } + } + if value == "" || !next { + return inventory.Observation{}, &Error{Code: "SSH_RUNTIME_INVALID"} + } + return observed("mlxconfig", "next-boot="+value), nil +} + +func parsePrivilege(text string) (inventory.Observation, error) { + host := "" + values := []string{} + seen := map[string]bool{} + levels := 0 + for _, line := range strings.Split(text, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), ":") + if !ok { + continue + } + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if key == "host index" { + if value == "" || strings.Trim(value, "0123456789") != "" { + return inventory.Observation{}, &Error{Code: "SSH_RUNTIME_INVALID"} + } + host = value + continue + } + if key != "level" && key != "disable_rshim" && key != "disable_tracer" && key != "disable_port_owner" && key != "disable_counter_rd" { + continue + } + id := "host[" + host + "]." + key + if host == "" || seen[id] { + return inventory.Observation{}, &Error{Code: "SSH_RUNTIME_INVALID"} + } + if key == "level" { + if value != "RESTRICTED" && value != "PRIVILEGED" { + return inventory.Observation{}, &Error{Code: "SSH_RUNTIME_INVALID"} + } + levels++ + } else if value != "TRUE" && value != "FALSE" { + return inventory.Observation{}, &Error{Code: "SSH_RUNTIME_INVALID"} + } + seen[id] = true + values = append(values, id+"="+value) + } + if levels == 0 { + return inventory.Observation{}, &Error{Code: "SSH_RUNTIME_INVALID"} + } + return observed("mlxprivhost", values...), nil +} diff --git a/pkg/dpf/sshprobe/hardware_test.go b/pkg/dpf/sshprobe/hardware_test.go new file mode 100644 index 00000000..8db9770b --- /dev/null +++ b/pkg/dpf/sshprobe/hardware_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "context" + "fmt" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("hardware observations", func() { + It("queries each host card independently and retains specific missing-tool reasons", func() { + r := hostRunner() + r.values["firmware:0000:21:00.0"] = "PCI Device Name: 0000:21:00.0\nPSID: MT_000001\nVersions: Current Available\nFW 32.1.2 N/A\n" + r.errors["firmware:0000:41:00.0"] = &Error{Code: "TOOL_NOT_AVAILABLE", Stage: "firmware:0000:41:00.0", Tool: "mlxfwmanager"} + r.values["nvconfig:0000:21:00.0:NUM_OF_VFS"] = "Configurations: Next Boot\nNUM_OF_VFS 0\n" + d, _, err := collect(context.Background(), config.Target{Role: "hosts", Endpoint: "ssh://192.0.2.11:22"}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Runtime.Hardware.Cards).To(HaveLen(2)) + a, b := d.Runtime.Hardware.Cards[0], d.Runtime.Hardware.Cards[1] + Expect(a.Firmware.Values).To(Equal([]string{"32.1.2"})) + Expect(a.PSID.Values).To(Equal([]string{"MT_000001"})) + Expect(a.SerialNumber).To(Equal(testSerial)) + Expect(a.NVConfig["NUM_OF_VFS"].Values).To(Equal([]string{"next-boot=0"})) + Expect(a.Privilege.ReasonCode).To(Equal("NOT_APPLICABLE")) + Expect(b.Firmware.State).To(Equal("unavailable")) + Expect(b.Firmware.ReasonCode).To(Equal("TOOL_NOT_AVAILABLE")) + Expect(b.Firmware.Tool).To(Equal("mlxfwmanager")) + Expect(b.Firmware.Message).To(ContainSubstring("not available")) + Expect(strings.Join(r.calls, "\n")).NotTo(ContainSubstring("mlxprivhost")) + }) + It("collects ARM-local privileges without treating every ARM-visible NIC as the DPU", func() { + r := armRunner() + r.values["mft-version"] = "mlxfwmanager, mft 4.33.0-169, built on a date" + r.values["arm-pci"] = strings.ReplaceAll(testPCI, "0000:", "0002:") + r.values["privilege:0002:21:00.0"] = "host index : 0\nlevel : RESTRICTED\ndisable_rshim : TRUE\ndisable_tracer : TRUE\ndisable_port_owner : TRUE\ndisable_counter_rd : TRUE\n" + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22", SSH: &config.SSHAccess{Sudo: true}}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Runtime.Hardware.Cards).To(HaveLen(2)) + Expect(d.Runtime.Observations["mft-version"].Values).To(Equal([]string{"4.33.0-169"})) + Expect(d.Runtime.Observations["board-product"].Values).To(Equal([]string{"B4240"})) + Expect(d.Runtime.Hardware.Cards[0].Domain).To(Equal("arm-pci")) + Expect(d.Runtime.Hardware.Cards[0].SerialNumber).To(BeEmpty()) + Expect(d.Runtime.Hardware.Cards[0].Privilege.Values).To(ContainElement("host[0].level=RESTRICTED")) + Expect(d.Runtime.Hardware.Cards[1].Privilege.ReasonCode).To(Equal("NOT_APPLICABLE")) + Expect(r.calls).To(ContainElement("privilege:0002:21:00.0 sudo -n mlxprivhost -d 0002:21:00.0 q --full")) + }) + It("rejects firmware from a different PCI function and ambiguous tool output", func() { + for _, text := range []string{"PCI Device Name: 0000:22:00.0\nFW 32.1.2 N/A", "FW 32.1.2 N/A\nFW 32.1.3 N/A", "unrecognized text"} { + _, _, err := parseFirmware(text, "0000:21:00.0") + Expect(err).To(HaveOccurred()) + } + for _, text := range []string{"Configurations: Next Boot\nNUM_OF_VFS 2\nNUM_OF_VFS 3", "Configurations: Current Next Boot\nNUM_OF_VFS 2 3", "NUM_OF_VFS 2"} { + _, err := parseNVConfig(text, "NUM_OF_VFS") + Expect(err).To(HaveOccurred()) + } + _, err := parsePrivilege("level : RESTRICTED\n") + Expect(err).To(HaveOccurred()) + }) + It("retains required-tool failures in the partial identity result", func() { + r := armRunner() + r.errors["arm-fru"] = &Error{Code: "TOOL_NOT_AVAILABLE", Stage: "arm-fru", Tool: "ipmitool"} + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22"}, r) + Expect(err).To(HaveOccurred()) + Expect(d.Runtime.Observations["arm-fru"].Message).To(ContainSubstring("not available")) + Expect(d.Runtime.Observations["arm-fru"].Tool).To(Equal("ipmitool")) + }) + It("classifies diagnostics without persisting remote text", func() { + for text, code := range map[string]string{"sudo: mlxfwmanager: command not found": "TOOL_NOT_AVAILABLE", "sudo: a password is required": "PERMISSION_DENIED", "device operation not supported": "OPERATION_UNSUPPORTED", "opaque secret-value": "SSH_COMMAND_FAILED"} { + err := commandFailure(fmt.Errorf("secret-value"), "firmware", "sudo -n mlxfwmanager --query", text) + Expect(err.Code).To(Equal(code)) + Expect(err.Tool).To(Equal("mlxfwmanager")) + Expect(Reason(err)).NotTo(ContainSubstring("secret-value")) + } + }) +}) diff --git a/pkg/dpf/sshprobe/network.go b/pkg/dpf/sshprobe/network.go new file mode 100644 index 00000000..01b1e7a8 --- /dev/null +++ b/pkg/dpf/sshprobe/network.go @@ -0,0 +1,224 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "context" + "encoding/json" + "net" + "regexp" + "sort" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" +) + +const dnsmasqRead = `sh -c 'found=0; for f in /etc/dnsmasq.conf /etc/dnsmasq.d/*.conf; do if [ -e "$f" ]; then found=1; [ -r "$f" ] || exit 77; awk '\''/^[[:space:]]*(server=|no-resolv([[:space:]]|$))/ {print}'\'' "$f" || exit; fi; done; [ "$found" -eq 1 ] || exit 66'` + +func collectNetwork(ctx context.Context, remote runner, runtime *inventory.RuntimeObservation, add func(string, string) string) { + n := &inventory.NetworkObservation{Links: []inventory.NetworkLink{}, Neighbors: []inventory.Neighbor{}, LLDP: []inventory.LLDPNeighbor{}, Resolver: inventory.ResolverObservation{Owner: "unknown", Search: []string{}, PerLink: map[string][]string{}, Domains: map[string][]string{}, DNSMasqServers: []string{}}} + runtime.Network = n + for _, p := range []struct{ stage, command string }{ + {"network-links", "ip -j -d link show"}, {"neighbors", "ip -j neighbor show"}, {"resolver-path", "readlink -f /etc/resolv.conf"}, + {"resolver-dns", "resolvectl dns"}, {"resolver-domains", "resolvectl domain"}, {"dnsmasq", dnsmasqRead}, {"lldp", "lldpctl -f keyvalue"}, + } { + text, err := remote.Run(ctx, p.stage, p.command) + if err == nil { + switch p.stage { + case "network-links": + n.Links, err = parseLinks(text) + case "neighbors": + n.Neighbors, err = parseNeighbors(text) + case "resolver-path": + value := strings.TrimSpace(text) + if !strings.HasPrefix(value, "/") || strings.ContainsAny(value, "\r\n\x00") { + err = &Error{Code: "SSH_RUNTIME_INVALID"} + break + } + n.Resolver.Path = value + switch { + case strings.Contains(value, "/systemd/resolve/"): + n.Resolver.Owner = "systemd-resolved" + case strings.Contains(value, "/NetworkManager/"): + n.Resolver.Owner = "NetworkManager" + case strings.Contains(value, "/resolvconf/"): + n.Resolver.Owner = "resolvconf" + default: + n.Resolver.Owner = "file" + } + case "resolver-dns": + n.Resolver.PerLink, err = parseResolved(text, true) + case "resolver-domains": + n.Resolver.Domains, err = parseResolved(text, false) + case "dnsmasq": + n.Resolver.DNSMasqServers, err = parseDNSMasq(text) + if err == nil { + noResolv := false + for _, line := range strings.Split(text, "\n") { + if strings.TrimSpace(line) == "no-resolv" { + noResolv = true + } + } + n.Resolver.DNSMasqNoResolv = &noResolv + } + case "lldp": + n.LLDP, err = parseLLDP(text) + } + } + if err != nil { + runtime.Observations[p.stage] = unavailable(err, commandTool(p.command)) + } else { + runtime.Observations[p.stage] = observed(commandTool(p.command)) + add(p.stage, p.stage) + } + } +} + +func parseLinks(text string) ([]inventory.NetworkLink, error) { + var rows []struct { + Name string `json:"ifname"` + MAC string `json:"address"` + Master string `json:"master"` + Info struct { + Kind string `json:"info_kind"` + } `json:"linkinfo"` + } + if err := json.Unmarshal([]byte(text), &rows); err != nil || rows == nil || len(rows) > 4096 { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + out := []inventory.NetworkLink{} + seen := map[string]bool{} + for _, r := range rows { + if r.Name == "" || seen[r.Name] { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + seen[r.Name] = true + // Non-Ethernet link-layer addresses (e.g. InfiniBand) are retained as facts. + out = append(out, inventory.NetworkLink{Name: r.Name, MAC: r.MAC, Master: r.Master, Kind: r.Info.Kind}) + } + return out, nil +} +func parseNeighbors(text string) ([]inventory.Neighbor, error) { + var rows []struct { + Address string `json:"dst"` + MAC string `json:"lladdr"` + Interface string `json:"dev"` + State []string `json:"state"` + } + if err := json.Unmarshal([]byte(text), &rows); err != nil || rows == nil || len(rows) > 8192 { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + out := []inventory.Neighbor{} + for _, r := range rows { + if net.ParseIP(r.Address) == nil || r.Interface == "" { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + if r.MAC != "" { + if mac, err := net.ParseMAC(r.MAC); err != nil || len(mac) != 6 { + continue + } + } + out = append(out, inventory.Neighbor{Address: r.Address, MAC: strings.ToLower(r.MAC), Interface: r.Interface, State: r.State}) + } + return out, nil +} + +var resolvedLine = regexp.MustCompile(`^(Global|Link [0-9]+ \(([^)]+)\)):\s*(.*)$`) + +func parseResolved(text string, addresses bool) (map[string][]string, error) { + out := map[string][]string{} + for _, line := range strings.Split(strings.TrimSpace(text), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + m := resolvedLine.FindStringSubmatch(line) + if m == nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + key := m[2] + if key == "" { + key = "global" + } + if _, exists := out[key]; exists { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + values := strings.Fields(m[3]) + for _, v := range values { + if addresses { + ip, _, _ := strings.Cut(v, "%") + if net.ParseIP(ip) == nil { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + } + } + out[key] = values + } + if len(out) == 0 { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + return out, nil +} + +var dnsmasqValue = regexp.MustCompile(`^[A-Za-z0-9._:/#%~@+-]+$`) + +func parseDNSMasq(text string) ([]string, error) { + out := []string{} + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" || line == "no-resolv" || strings.HasPrefix(line, "no-resolv ") { + continue + } + value, ok := strings.CutPrefix(line, "server=") + if !ok || !dnsmasqValue.MatchString(value) { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + out = append(out, value) + } + return out, nil +} +func parseLLDP(text string) ([]inventory.LLDPNeighbor, error) { + byKey := map[string]inventory.LLDPNeighbor{} + for _, line := range strings.Split(text, "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + key, ok = strings.CutPrefix(key, "lldp.") + if !ok { + continue + } + for _, suffix := range []string{".chassis.name", ".port.ifname", ".port.descr"} { + iface, ok := strings.CutSuffix(key, suffix) + if !ok { + continue + } + if iface == "" || len(value) > 256 || strings.ContainsAny(value, "\x00\r\n") { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + n := byKey[iface] + n.Interface = iface + if suffix == ".chassis.name" { + if n.Chassis != "" && n.Chassis != value { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + n.Chassis = value + } else if suffix == ".port.ifname" || n.Port == "" { + n.Port = value + } + byKey[iface] = n + } + } + keys := []string{} + for k := range byKey { + keys = append(keys, k) + } + sort.Strings(keys) + out := []inventory.LLDPNeighbor{} + for _, k := range keys { + out = append(out, byKey[k]) + } + if len(out) == 0 && strings.TrimSpace(text) != "" { + return nil, &Error{Code: "SSH_RUNTIME_INVALID"} + } + return out, nil +} diff --git a/pkg/dpf/sshprobe/network_test.go b/pkg/dpf/sshprobe/network_test.go new file mode 100644 index 00000000..d98f5023 --- /dev/null +++ b/pkg/dpf/sshprobe/network_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package sshprobe + +import ( + "context" + "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("network observations", func() { + It("preserves actual bridges, upstream resolvers and candidate neighbors", func() { + r := armRunner() + r.values["network-links"] = `[{"ifname":"oob_net0","address":"02:00:00:00:00:41","master":"br0"},{"ifname":"br0","linkinfo":{"info_kind":"bridge"}}]` + r.values["resolver-path"] = "/run/systemd/resolve/stub-resolv.conf\n" + r.values["resolver-dns"] = "Global: 192.0.2.53\nLink 2 (oob_net0): 2001:db8::53\n" + r.values["resolver-domains"] = "Global:\nLink 2 (oob_net0): example.test ~.\n" + r.values["dns"] = "nameserver 127.0.0.53\nsearch example.test\n" + r.values["dnsmasq"] = "server=/example.test/192.0.2.53\nno-resolv\n" + r.values["neighbors"] = `[{"dst":"192.0.2.42","dev":"oob_net0","lladdr":"02:00:00:00:00:42","state":["STALE"]},{"dst":"192.0.2.43","dev":"oob_net0","state":["FAILED"]}]` + r.values["lldp"] = "lldp.oob_net0.chassis.name=switch-a\nlldp.oob_net0.port.ifname=swp1\n" + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22"}, r) + Expect(err).NotTo(HaveOccurred()) + n := d.Runtime.Network + Expect(n.Links[0].Master).To(Equal("br0")) + Expect(n.Links[1].Kind).To(Equal("bridge")) + Expect(n.Resolver.Owner).To(Equal("systemd-resolved")) + Expect(n.Resolver.PerLink["global"]).To(Equal([]string{"192.0.2.53"})) + Expect(n.Resolver.Search).To(Equal([]string{"example.test"})) + Expect(d.Runtime.DNS).To(Equal([]string{"127.0.0.53"})) + Expect(n.Resolver.DNSMasqServers).To(Equal([]string{"/example.test/192.0.2.53"})) + Expect(n.Neighbors).To(HaveLen(2)) + Expect(n.Neighbors[1].MAC).To(BeEmpty()) + Expect(n.LLDP[0].Chassis).To(Equal("switch-a")) + Expect(strings.Join(r.calls, "\n")).NotTo(ContainSubstring("192.0.2.43")) + }) + It("does not interpret failed or malformed probes as empty successful evidence", func() { + r := armRunner() + r.values["neighbors"] = "null" + r.values["network-links"] = "bad-json" + r.errors["resolver-dns"] = &Error{Code: "TOOL_NOT_AVAILABLE", Stage: "resolver-dns", Tool: "resolvectl"} + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22"}, r) + Expect(err).NotTo(HaveOccurred()) + for _, stage := range []string{"neighbors", "network-links", "resolver-dns", "dnsmasq", "lldp"} { + Expect(d.Runtime.Observations[stage].State).To(Equal("unavailable")) + Expect(d.Runtime.Observations[stage].ReasonCode).NotTo(BeEmpty()) + Expect(d.Runtime.Observations[stage].Message).NotTo(BeEmpty()) + } + Expect(d.Runtime.Observations["resolver-dns"].Tool).To(Equal("resolvectl")) + }) + It("rejects invalid addresses and duplicate resolver records", func() { + for _, text := range []string{"Global: malformed", "Global: 192.0.2.53\nGlobal: 192.0.2.54"} { + _, err := parseResolved(text, true) + Expect(err).To(HaveOccurred()) + } + _, err := parseNeighbors(`[{"dst":"bad","dev":"eth0"}]`) + Expect(err).To(HaveOccurred()) + _, err = parseDNSMasq("server=credential with spaces") + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/pkg/dpf/sshprobe/transport.go b/pkg/dpf/sshprobe/transport.go index 115cae60..7008d978 100644 --- a/pkg/dpf/sshprobe/transport.go +++ b/pkg/dpf/sshprobe/transport.go @@ -27,6 +27,7 @@ const maxOutput = 1 << 20 type Error struct { Code string Stage string + Tool string } func (e *Error) Error() string { return e.Code + ": " + e.Stage } @@ -235,7 +236,11 @@ func (c *client) Run(ctx context.Context, stage, command string) (string, error) defer stop() session, err := c.ssh.NewSession() if err != nil { - return "", &Error{Code: "SSH_COMMAND_FAILED", Stage: stage} + code := "SSH_COMMAND_FAILED" + if ctx.Err() != nil { + code = "SSH_TIMEOUT_OR_CANCELLED" + } + return "", &Error{Code: code, Stage: stage, Tool: commandTool(command)} } defer session.Close() output := &commandOutput{} @@ -245,13 +250,13 @@ func (c *client) Run(ctx context.Context, stage, command string) (string, error) output.mu.Lock() defer output.mu.Unlock() if output.exceeded { - return "", &Error{Code: "SSH_OUTPUT_LIMIT", Stage: stage} + return "", &Error{Code: "SSH_OUTPUT_LIMIT", Stage: stage, Tool: commandTool(command)} } if ctx.Err() != nil { - return "", &Error{Code: "SSH_TIMEOUT_OR_CANCELLED", Stage: stage} + return "", &Error{Code: "SSH_TIMEOUT_OR_CANCELLED", Stage: stage, Tool: commandTool(command)} } if err != nil { - return "", &Error{Code: "SSH_COMMAND_FAILED", Stage: stage} + return "", commandFailure(err, stage, command, output.stdout.String()+"\n"+output.stderr.String()) } return output.stdout.String(), nil } diff --git a/pkg/dpf/sshprobe/transport_test.go b/pkg/dpf/sshprobe/transport_test.go index cb1563b5..1c4f335d 100644 --- a/pkg/dpf/sshprobe/transport_test.go +++ b/pkg/dpf/sshprobe/transport_test.go @@ -120,6 +120,16 @@ func (s *testServer) serve(conn net.Conn) { return case "LC_ALL=C flood": _, _ = ch.Write([]byte(strings.Repeat("x", maxOutput+1))) + case "LC_ALL=C missing": + _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{127})) + return + case "LC_ALL=C not-executable": + _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{126})) + return + case "LC_ALL=C sudo -n mlxfwmanager --query": + _, _ = ch.Stderr().Write([]byte("sh: 1: sudo: not found")) + _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{127})) + return case "LC_ALL=C fail": _, _ = ch.Stderr().Write([]byte("remote sensitive diagnostic")) _, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{7})) @@ -258,6 +268,19 @@ var _ = Describe("bounded SSH transport", func() { Expect(Code(err)).To(Equal("SSH_COMMAND_FAILED")) Expect(err.Error()).NotTo(ContainSubstring("sensitive")) }) + It("distinguishes missing executables and missing sudo without remote output", func() { + c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), server.auth(), time.Second) + Expect(err).NotTo(HaveOccurred()) + defer c.Close() + for command, code := range map[string]string{"missing": "TOOL_NOT_AVAILABLE", "not-executable": "TOOL_NOT_EXECUTABLE", "sudo -n mlxfwmanager --query": "TOOL_NOT_AVAILABLE"} { + output, err := c.Run(context.Background(), "query", command) + Expect(output).To(BeEmpty()) + Expect(Code(err)).To(Equal(code)) + if strings.HasPrefix(command, "sudo") { + Expect(err.(*Error).Tool).To(Equal("sudo")) + } + } + }) It("bounds command output", func() { c, err := connect(context.Background(), "ssh://"+server.listener.Addr().String(), server.auth(), time.Second) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/target/dpf/driver.go b/pkg/target/dpf/driver.go index fa22b15b..41c31d3b 100644 --- a/pkg/target/dpf/driver.go +++ b/pkg/target/dpf/driver.go @@ -123,7 +123,7 @@ func (d driver) Bind(inv target.Invocation) (target.Operation, error) { fmt.Fprintf(r.Stdout, "DPF discovery: %s (%s)\n", report.Discovery.Outcome, report.Discovery.Mode) for _, c := range report.Discovery.Checks { if c.State != "passed" { - fmt.Fprintf(r.Stdout, "%s %s: %s (%s)\n", c.ID, c.Subject, c.State, c.ReasonCode) + fmt.Fprintf(r.Stdout, "%s %s: %s (%s): %s\n", c.ID, c.Subject, c.State, c.ReasonCode, c.Message) } } for _, e := range report.Errors { From c25c83be9162b32c006e04153568a350bac9a2b9 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 6 Nov 2026 11:37:42 +0100 Subject: [PATCH 07/35] fix(dpf): require current evidence for software compatibility Signed-off-by: Alexander Maslennikov --- pkg/dpf/discovery/canonical_test.go | 25 +++ pkg/dpf/discovery/management.go | 33 +++- pkg/dpf/discovery/observations.go | 26 +++ pkg/dpf/discovery/reasons_test.go | 27 +++ pkg/dpf/discovery/software.go | 177 ++++++++++++++---- .../discovery/software_integration_test.go | 130 +++++++++++++ pkg/dpf/discovery/software_test.go | 93 ++++++++- pkg/dpf/sshprobe/diagnostics.go | 4 + pkg/dpf/sshprobe/hardware_test.go | 2 +- pkg/dpf/sshprobe/network.go | 31 ++- pkg/dpf/sshprobe/network_test.go | 24 +++ 11 files changed, 525 insertions(+), 47 deletions(-) create mode 100644 pkg/dpf/discovery/canonical_test.go create mode 100644 pkg/dpf/discovery/reasons_test.go create mode 100644 pkg/dpf/discovery/software_integration_test.go diff --git a/pkg/dpf/discovery/canonical_test.go b/pkg/dpf/discovery/canonical_test.go new file mode 100644 index 00000000..a0a2dd03 --- /dev/null +++ b/pkg/dpf/discovery/canonical_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Enriched observation ordering", func() { + It("canonicalizes unordered facts without mutating source facts", func() { + n1 := inventory.Neighbor{Address: "192.0.2.2", Interface: "eth1", State: []string{"REACHABLE", "PERMANENT"}} + n2 := inventory.Neighbor{Address: "192.0.2.1", Interface: "eth0", State: []string{"STALE"}} + r := inventory.RuntimeObservation{Network: &inventory.NetworkObservation{Neighbors: []inventory.Neighbor{n1, n2}, Links: []inventory.NetworkLink{{Name: "eth1"}, {Name: "eth0"}}}, Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{PCIAddress: "0000:02:00.0", Domain: "host-pci"}, {PCIAddress: "0000:01:00.0", Domain: "host-pci"}}}} + a := canonicalRuntime(r) + Expect(r.Network.Neighbors[0].Address).To(Equal("192.0.2.2")) + Expect(r.Network.Neighbors[0].State[0]).To(Equal("REACHABLE")) + Expect(r.Hardware.Cards[0].PCIAddress).To(Equal("0000:02:00.0")) + b := r + b.Network = &inventory.NetworkObservation{Neighbors: []inventory.Neighbor{n2, n1}, Links: []inventory.NetworkLink{{Name: "eth0"}, {Name: "eth1"}}} + b.Hardware = &inventory.HardwareObservation{Cards: []inventory.CardHardware{r.Hardware.Cards[1], r.Hardware.Cards[0]}} + Expect(canonicalRuntime(b)).To(Equal(a)) + }) +}) diff --git a/pkg/dpf/discovery/management.go b/pkg/dpf/discovery/management.go index df2def2f..f43257ec 100644 --- a/pkg/dpf/discovery/management.go +++ b/pkg/dpf/discovery/management.go @@ -6,7 +6,9 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" "net/url" "slices" "strings" @@ -178,7 +180,10 @@ func (m ManagementClient) Observe(ctx context.Context) (*inventory.Management, e } result.Observations = map[string]inventory.Observation{} storage, storageErr := m.Client.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{Limit: 256}) - storageFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE", Message: "The Kubernetes API inspection failed or returned incomplete evidence."} + storageFact := apiObservationFailure(storageErr) + if storageErr == nil && storage.Continue != "" { + storageFact = inventory.Observation{State: "unknown", ReasonCode: "API_COVERAGE_INCOMPLETE", Message: "StorageClass enumeration exceeded the bounded first page; not all entries were observed."} + } if storageErr == nil && storage.Continue == "" { storageFact.State = "known" storageFact.ReasonCode = "" @@ -189,10 +194,14 @@ func (m ManagementClient) Observe(ctx context.Context) (*inventory.Management, e slices.Sort(storageFact.Values) } result.Observations["storageClasses"] = storageFact - groupsFact := inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: "INSPECTION_UNAVAILABLE", Message: "The Kubernetes API inspection failed or returned incomplete evidence."} data, groupErr := m.Client.Discovery().RESTClient().Get().AbsPath("/apis").Do(ctx).Raw() var groups metav1.APIGroupList - if groupErr == nil && json.Unmarshal(data, &groups) == nil { + groupsFact := apiObservationFailure(groupErr) + decodeErr := json.Unmarshal(data, &groups) + if groupErr == nil && decodeErr != nil { + groupsFact = inventory.Observation{State: "unknown", ReasonCode: "API_RESPONSE_INVALID", Message: "The API group discovery response is not valid APIGroupList JSON."} + } + if groupErr == nil && decodeErr == nil { groupsFact.State = "known" groupsFact.ReasonCode = "" groupsFact.Message = "" @@ -212,3 +221,21 @@ func (m ManagementClient) Observe(ctx context.Context) (*inventory.Management, e func NewEvidence(at time.Time) map[string]inventory.Evidence { return map[string]inventory.Evidence{"management-identity": {Kind: "kubernetes-api", InputRef: "/management", ResourceURI: "/api/v1/namespaces/kube-system", Property: "metadata.uid", ObservedAt: at, Vantage: "executor", Authenticated: true}, "management-nodes": {Kind: "kubernetes-api", InputRef: "/management", ResourceURI: "/api/v1/nodes", ObservedAt: at, Vantage: "executor", Authenticated: true}} } + +// apiObservationFailure preserves why optional API evidence is unavailable without raw errors. +func apiObservationFailure(err error) inventory.Observation { + code, message := "API_INSPECTION_FAILED", "The Kubernetes API request failed; no complete observation was obtained." + switch { + case apierrors.IsForbidden(err): + code, message = "API_ACCESS_DENIED", "The configured Kubernetes identity is forbidden from reading this resource." + case apierrors.IsUnauthorized(err): + code, message = "API_AUTHENTICATION_FAILED", "The Kubernetes API did not accept the configured credentials." + case apierrors.IsNotFound(err): + code, message = "API_RESOURCE_NOT_FOUND", "The requested Kubernetes API resource is not available." + case errors.Is(err, context.Canceled): + code, message = "API_CANCELLED", "The Kubernetes API inspection was cancelled." + case errors.Is(err, context.DeadlineExceeded), apierrors.IsTimeout(err): + code, message = "API_TIMEOUT", "The Kubernetes API inspection timed out." + } + return inventory.Observation{State: "unknown", Values: []string{}, ReasonCode: code, Message: message} +} diff --git a/pkg/dpf/discovery/observations.go b/pkg/dpf/discovery/observations.go index 580542b0..a34a2e03 100644 --- a/pkg/dpf/discovery/observations.go +++ b/pkg/dpf/discovery/observations.go @@ -59,6 +59,32 @@ func canonicalRuntime(r inventory.RuntimeObservation) inventory.RuntimeObservati }) r.DNS = slices.Clone(r.DNS) slices.Sort(r.DNS) + if r.Network != nil { + n := *r.Network + n.Links = slices.Clone(n.Links) + slices.SortFunc(n.Links, func(a, b inventory.NetworkLink) int { return strings.Compare(a.Name, b.Name) }) + n.Neighbors = slices.Clone(n.Neighbors) + for i := range n.Neighbors { + n.Neighbors[i].State = slices.Clone(n.Neighbors[i].State) + slices.Sort(n.Neighbors[i].State) + } + slices.SortFunc(n.Neighbors, func(a, b inventory.Neighbor) int { + return cmp.Or(strings.Compare(a.Interface, b.Interface), strings.Compare(a.Address, b.Address), strings.Compare(a.MAC, b.MAC), slices.Compare(a.State, b.State)) + }) + n.LLDP = slices.Clone(n.LLDP) + slices.SortFunc(n.LLDP, func(a, b inventory.LLDPNeighbor) int { + return cmp.Or(strings.Compare(a.Interface, b.Interface), strings.Compare(a.Chassis, b.Chassis), strings.Compare(a.Port, b.Port)) + }) + r.Network = &n + } + if r.Hardware != nil { + h := *r.Hardware + h.Cards = slices.Clone(h.Cards) + slices.SortFunc(h.Cards, func(a, b inventory.CardHardware) int { + return cmp.Or(strings.Compare(a.Domain, b.Domain), strings.Compare(a.PCIAddress, b.PCIAddress)) + }) + r.Hardware = &h + } return r } diff --git a/pkg/dpf/discovery/reasons_test.go b/pkg/dpf/discovery/reasons_test.go new file mode 100644 index 00000000..b5c48c47 --- /dev/null +++ b/pkg/dpf/discovery/reasons_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "context" + "fmt" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("API diagnostic persistence", func() { + It("distinguishes authorization, absent resources, timeouts and cancellation without raw errors", func() { + resource := schema.GroupResource{Group: "storage.k8s.io", Resource: "storageclasses"} + for _, tc := range []struct { + err error + code string + }{{apierrors.NewForbidden(resource, "x", fmt.Errorf("credential-canary")), "API_ACCESS_DENIED"}, {apierrors.NewUnauthorized("credential-canary"), "API_AUTHENTICATION_FAILED"}, {apierrors.NewNotFound(resource, "x"), "API_RESOURCE_NOT_FOUND"}, {context.DeadlineExceeded, "API_TIMEOUT"}, {context.Canceled, "API_CANCELLED"}, {fmt.Errorf("credential-canary"), "API_INSPECTION_FAILED"}} { + o := apiObservationFailure(tc.err) + Expect(o.ReasonCode).To(Equal(tc.code)) + Expect(o.Message).NotTo(BeEmpty()) + Expect(o.Message).NotTo(ContainSubstring("credential-canary")) + } + }) +}) diff --git a/pkg/dpf/discovery/software.go b/pkg/dpf/discovery/software.go index 253e5d5e..2c445d22 100644 --- a/pkg/dpf/discovery/software.go +++ b/pkg/dpf/discovery/software.go @@ -3,10 +3,12 @@ package discovery import ( - "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" "slices" "strconv" "strings" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/releases" ) // SoftwareChecks reports release compatibility separately from discovery success. @@ -17,6 +19,11 @@ func SoftwareChecks(inv *inventory.Inventory) []inventory.Check { return out } for _, d := range inv.DPUs { + if !softwareIdentityCurrent(inv, d) { + out = append(out, inventory.Check{ID: "dpf.software.applicability", Subject: d.RecordKey, Vantage: "executor", RequiredPhase: "generate", State: "unknown", ReasonCode: "SOFTWARE_IDENTITY_UNAVAILABLE", Message: "Software compatibility requires a successful current observation of a unique DPU identity; inspect this device's identity observation reason.", EvidenceRefs: slices.Clone(d.EvidenceRefs)}) + continue + } + host := softwareHost(inv, d) model := d.Model if d.Runtime != nil { board := d.Runtime.Observations["board-product"] @@ -35,53 +42,32 @@ func SoftwareChecks(inv *inventory.Inventory) []inventory.Check { base.Message = "Observed model matches a documented BOM entry; hardware qualification remains separate." out = append(out, base) for _, req := range hw.Requirements { - c := inventory.Check{ID: "dpf.software." + req.Component, Subject: d.RecordKey, Vantage: "executor", RequiredPhase: "generate", State: "unknown", ReasonCode: req.ReasonCode, Message: req.Message, EvidenceRefs: slices.Clone(d.EvidenceRefs)} + // Target coordinates describe desired software, not a minimum + // readiness predicate. Unknown constraints still explain the gap. + if req.State == "known" && req.Minimum == "" { + continue + } + c := inventory.Check{ID: "dpf.software." + req.Component, Subject: d.RecordKey, Vantage: "executor", RequiredPhase: "generate", State: "unknown", ReasonCode: req.ReasonCode, Message: req.Message, EvidenceRefs: []string{}} if req.State != "known" { out = append(out, c) continue } - values := []string{} - for _, fw := range d.Firmware { - if fw.Component == req.Component && fw.State == "known" && fw.Activation == "active" { - values = append(values, fw.RawVersion) - c.EvidenceRefs = append(c.EvidenceRefs, fw.EvidenceRefs...) - } - } - if req.Component == "nic" { - for _, group := range [][]inventory.Device{inv.Hosts, []inventory.Device{d}} { - for _, device := range group { - if device.Runtime != nil && device.Runtime.Hardware != nil { - for _, card := range device.Runtime.Hardware.Cards { - if d.SerialNumber != "" && card.SerialNumber == d.SerialNumber && card.Firmware.State == "observed" { - values = append(values, card.Firmware.Values...) - c.EvidenceRefs = append(c.EvidenceRefs, card.EvidenceRefs...) - } - } - } - } - } - } - if req.Scope == "host" && req.Component == "mft" { - for _, a := range inv.Associations { - if a.DPUID == d.ID { - for _, h := range inv.Hosts { - if h.ID == a.HostID && h.Runtime != nil { - v := h.Runtime.Observations["mft-version"] - if v.State == "observed" { - values = append(values, v.Values...) - c.EvidenceRefs = append(c.EvidenceRefs, h.EvidenceRefs...) - } - } - } - } - } + values, refs, issue := softwareVersions(d, host, req) + c.EvidenceRefs = append(c.EvidenceRefs, refs...) + if issue.ReasonCode == "SOFTWARE_VERSION_CONFLICT" { + c.ReasonCode, c.Message = issue.ReasonCode, issue.Message + out = append(out, c) + continue } slices.Sort(values) values = slices.Compact(values) switch len(values) { case 0: c.ReasonCode = "SOFTWARE_VERSION_UNAVAILABLE" - c.Message = "No active or current, identity-bound version was measured for this requirement; inspect firmware/tool observation reasons." + c.Message = "No running or current, identity-bound version was measured for this requirement; inspect firmware/tool observation reasons." + if issue.ReasonCode != "" { + c.ReasonCode, c.Message = issue.ReasonCode, issue.Message + } case 1: cmp, valid := compareVersion(values[0], req.Minimum) if req.Minimum == "" || !valid { @@ -106,6 +92,121 @@ func SoftwareChecks(inv *inventory.Inventory) []inventory.Check { return out } +// A failed final identity check invalidates measurements retained for diagnosis. +// Duplicate identities must not become software-compatibility evidence either. +func softwareIdentityCurrent(inv *inventory.Inventory, d inventory.Device) bool { + if d.ObservationState != "observed" || d.ID == "" || d.SerialNumber == "" { + return false + } + count := 0 + for _, other := range inv.DPUs { + if other.ID == d.ID || other.SerialNumber == d.SerialNumber { + count++ + } + } + for _, host := range inv.Hosts { + if host.ID == d.ID { + count++ + } + } + return count == 1 +} + +// Host measurements are eligible only through a unique accepted association. +// A coincidentally matching serial elsewhere in the inventory is insufficient. +func softwareHost(inv *inventory.Inventory, d inventory.Device) *inventory.Device { + var association *inventory.Association + for i := range inv.Associations { + a := &inv.Associations[i] + if a.DPUID != d.ID { + continue + } + if association != nil || a.HostID == "" || a.HostAdapterKey == "" || a.SerialNumber != d.SerialNumber { + return nil + } + association = a + } + if association == nil { + return nil + } + var host *inventory.Device + for i := range inv.Hosts { + candidate := &inv.Hosts[i] + if candidate.ID != association.HostID { + continue + } + if host != nil || candidate.ObservationState != "observed" { + return nil + } + host = candidate + } + if host == nil { + return nil + } + for _, other := range inv.DPUs { + if other.ID == host.ID { + return nil + } + } + matches := 0 + for _, adapter := range host.Adapters { + if adapter.SerialNumber != d.SerialNumber { + continue + } + if adapter.Key != association.HostAdapterKey { + return nil + } + matches++ + } + if matches != 1 { + return nil + } + return host +} + +func softwareVersions(d inventory.Device, host *inventory.Device, req releases.Requirement) ([]string, []string, inventory.Observation) { + values, refs := []string{}, []string{} + issue := inventory.Observation{} + source := d.Observations["firmware-inventory"] + for _, fw := range d.Firmware { + if fw.Component != req.Component { + continue + } + if fw.State == "conflict" || fw.ActivationReasonCode == "ACTIVATION_CONFLICT" { + refs = append(refs, fw.EvidenceRefs...) + issue = inventory.Observation{State: "unknown", ReasonCode: "SOFTWARE_VERSION_CONFLICT", Message: "Component version or activation evidence conflicts; no alternate source can satisfy this requirement until the conflict is resolved."} + } + if source.State == "observed" && fw.State == "known" && fw.Activation == "running" { + values = append(values, fw.RawVersion) + refs = append(refs, fw.EvidenceRefs...) + } + } + if req.Component == "nic" { + for _, device := range []*inventory.Device{&d, host} { + if device == nil || device.Runtime == nil || device.Runtime.Hardware == nil { + continue + } + for _, card := range device.Runtime.Hardware.Cards { + if card.SerialNumber == d.SerialNumber && card.Firmware.State == "observed" { + values = append(values, card.Firmware.Values...) + refs = append(refs, card.EvidenceRefs...) + } + } + } + } + if req.Scope == "host" && req.Component == "mft" && host != nil && host.Runtime != nil { + version := host.Runtime.Observations["mft-version"] + if version.State == "observed" { + values = append(values, version.Values...) + refs = append(refs, host.EvidenceRefs...) + } + } + if len(values) == 0 && issue.ReasonCode == "" && source.State != "observed" && source.ReasonCode != "" && req.Scope != "host" { + issue = inventory.Observation{State: "unknown", ReasonCode: source.ReasonCode, Message: "The Redfish firmware inventory is unavailable or incomplete: " + source.Message} + } + return values, refs, issue +} + // Version grammar is deliberately narrow: numeric release components separated by // dots and an optional numeric build suffix. Vendor strings require normalization // by their specific collector before they can satisfy a minimum. diff --git a/pkg/dpf/discovery/software_integration_test.go b/pkg/dpf/discovery/software_integration_test.go new file mode 100644 index 00000000..217b5a70 --- /dev/null +++ b/pkg/dpf/discovery/software_integration_test.go @@ -0,0 +1,130 @@ +// Copyright 2026 NVIDIA CORPORATION & AFFILIATES +// SPDX-License-Identifier: Apache-2.0 +package discovery + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "time" + + "github.com/nvidia/k8s-launch-kit/pkg/dpf/config" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/inventory" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/redfish" + "github.com/nvidia/k8s-launch-kit/pkg/dpf/releases" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Redfish collector to software BOM contract", func() { + It("uses actual running activation and rejects staged, conflicting, partial and replaced observations", func() { + Expect(os.Setenv("DPF_SOFTWARE_TEST_PASSWORD", "fixture-password")).To(Succeed()) + DeferCleanup(os.Unsetenv, "DPF_SOFTWARE_TEST_PASSWORD") + dir, err := os.MkdirTemp("", "dpf-software-") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(os.RemoveAll, dir) + for _, scenario := range []struct { + name, activation, expected string + partial, replace bool + }{ + {name: "running", activation: "ActiveTargets", expected: "SOFTWARE_MINIMUM_MET"}, + {name: "staged", activation: "StagedTargets", expected: "SOFTWARE_VERSION_UNAVAILABLE"}, + {name: "conflicting activation", activation: "both", expected: "SOFTWARE_VERSION_CONFLICT"}, + {name: "partial collection", activation: "ActiveTargets", partial: true, expected: "ACCESS_DENIED"}, + {name: "identity replaced", activation: "ActiveTargets", replace: true, expected: "SOFTWARE_IDENTITY_UNAVAILABLE"}, + } { + cardReads := 0 + writes := []string{} + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes = append(writes, r.Method+" "+r.URL.Path) + } + link := func(uri string) redfish.Document { return redfish.Document{"@odata.id": uri} } + var doc redfish.Document + switch r.URL.Path { + case "/redfish/v1/": + doc = redfish.Document{"SessionService": link("/redfish/v1/SessionService"), "Systems": link("/redfish/v1/Systems"), "Chassis": link("/redfish/v1/Chassis"), "UpdateService": link("/redfish/v1/UpdateService")} + case "/redfish/v1/SessionService": + doc = redfish.Document{"Sessions": link("/redfish/v1/SessionService/Sessions")} + case "/redfish/v1/SessionService/Sessions": + w.Header().Set("X-Auth-Token", "fixture-token") + w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/owned") + w.WriteHeader(http.StatusCreated) + doc = redfish.Document{} + case "/redfish/v1/SessionService/Sessions/owned": + w.WriteHeader(http.StatusNoContent) + return + case "/redfish/v1/Systems": + doc = redfish.Document{"Members@odata.count": 1, "Members": []any{link("/redfish/v1/Systems/DPU")}} + case "/redfish/v1/Systems/DPU": + doc = redfish.Document{"UUID": "11111111-1111-4111-8111-111111111111", "Manufacturer": "NVIDIA", "Model": "BlueField-3"} + case "/redfish/v1/Chassis": + doc = redfish.Document{"Members@odata.count": 1, "Members": []any{link("/redfish/v1/Chassis/Card")}} + case "/redfish/v1/Chassis/Card": + cardReads++ + serial := "CARD-A" + if scenario.replace && cardReads > 1 { + serial = "CARD-B" + } + doc = redfish.Document{"Model": "BlueField-3", "SerialNumber": serial, "Links": redfish.Document{"ComputerSystems": []any{link("/redfish/v1/Systems/DPU")}}} + case "/redfish/v1/UpdateService": + doc = redfish.Document{"FirmwareInventory": link("/redfish/v1/UpdateService/FirmwareInventory")} + case "/redfish/v1/UpdateService/FirmwareInventory": + members := []any{link("/redfish/v1/UpdateService/FirmwareInventory/NIC")} + if scenario.partial { + members = append(members, link("/redfish/v1/UpdateService/FirmwareInventory/Denied")) + } + doc = redfish.Document{"Members@odata.count": len(members), "Members": members} + case "/redfish/v1/UpdateService/FirmwareInventory/NIC": + links := redfish.Document{scenario.activation: []any{link("/redfish/v1/Systems/DPU")}} + if scenario.activation == "both" { + links = redfish.Document{"ActiveTargets": []any{link("/redfish/v1/Systems/DPU")}, "StagedTargets": []any{link("/redfish/v1/Systems/DPU")}} + } + doc = redfish.Document{"Id": "NIC_Firmware", "Name": "NIC Firmware", "Version": "32.38.1002", "Links": links} + case "/redfish/v1/UpdateService/FirmwareInventory/Denied": + w.WriteHeader(http.StatusForbidden) + return + default: + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(doc) + })) + source := fmt.Sprintf("management:\n kubeconfigRef: {file: unused}\ncredentials:\n read: {username: reader, passwordRef: {env: DPF_SOFTWARE_TEST_PASSWORD}}\ndiscovery:\n tls: {insecureSkipVerify: true}\n dpus: [{bmcEndpoint: '%s', credentialRef: read}]\n hosts: [{bmcEndpoint: 'https://192.0.2.1', credentialRef: read}]\n", server.URL) + in, e := config.Decode([]byte(source), filepath.Join(dir, "site.yaml")) + Expect(e).NotTo(HaveOccurred()) + request, e := config.Resolve(in) + Expect(e).NotTo(HaveOccurred()) + d, _, e := (redfish.Collector{}).Collect(context.Background(), request.Targets[0], time.Second) + if scenario.replace { + Expect(redfish.Code(e)).To(Equal("IDENTITY_CHANGED")) + } else { + Expect(e).NotTo(HaveOccurred()) + } + server.Close() + Expect(writes).To(Equal([]string{"POST /redfish/v1/SessionService/Sessions", "DELETE /redfish/v1/SessionService/Sessions/owned"})) + // Exact board product is independent SSH evidence; a generic Redfish family + // is intentionally insufficient for production BOM applicability. + d.Runtime = &inventory.RuntimeObservation{Observations: map[string]inventory.Observation{"board-product": {State: "observed", Values: []string{"B3210"}}}} + release, e := releases.Resolve("") + Expect(e).NotTo(HaveOccurred()) + inv := &inventory.Inventory{Software: &release, DPUs: []inventory.Device{d}} + component := "nic" + if scenario.replace { + component = "applicability" + } + check := softwareCheck(SoftwareChecks(inv), component) + Expect(check.ReasonCode).To(Equal(scenario.expected), scenario.name) + if scenario.name == "running" { + Expect(check.State).To(Equal("passed")) + } else { + Expect(check.State).To(Equal("unknown")) + } + } + }) +}) diff --git a/pkg/dpf/discovery/software_test.go b/pkg/dpf/discovery/software_test.go index 9225fad3..1f4b1026 100644 --- a/pkg/dpf/discovery/software_test.go +++ b/pkg/dpf/discovery/software_test.go @@ -13,7 +13,7 @@ var _ = Describe("Software BOM readiness", func() { It("does not confuse inventory collection, software applicability and hardware qualification", func() { r, err := releases.Resolve("") Expect(err).NotTo(HaveOccurred()) - inv := &inventory.Inventory{Software: &r, DPUs: []inventory.Device{{RecordKey: "bf4", Model: "BlueField-4"}}} + inv := &inventory.Inventory{Software: &r, DPUs: []inventory.Device{{RecordKey: "bf4", ID: "bf4", SerialNumber: "CARD4", ObservationState: "observed", Model: "BlueField-4"}}} checks := SoftwareChecks(inv) Expect(checks).To(HaveLen(1)) Expect(checks[0].ReasonCode).To(Equal("BOM_MODEL_UNSUPPORTED")) @@ -22,7 +22,7 @@ var _ = Describe("Software BOM readiness", func() { It("requires active evidence and retains unknown requirements", func() { r, err := releases.Resolve("") Expect(err).NotTo(HaveOccurred()) - d := inventory.Device{RecordKey: "bf3", Model: "B3210", SerialNumber: "MT1234567890", Firmware: []inventory.Firmware{{Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "unknown"}}} + d := inventory.Device{RecordKey: "bf3", ID: "bf3", ObservationState: "observed", Observations: map[string]inventory.Observation{"firmware-inventory": {State: "observed"}}, Model: "B3210", SerialNumber: "MT1234567890", Firmware: []inventory.Firmware{{Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "unknown"}}} inv := &inventory.Inventory{Software: &r, DPUs: []inventory.Device{d}} find := func() inventory.Check { for _, c := range SoftwareChecks(inv) { @@ -34,11 +34,11 @@ var _ = Describe("Software BOM readiness", func() { return inventory.Check{} } Expect(find().ReasonCode).To(Equal("SOFTWARE_VERSION_UNAVAILABLE")) - inv.DPUs[0].Firmware[0].Activation = "active" + inv.DPUs[0].Firmware[0].Activation = "running" Expect(find().State).To(Equal("passed")) inv.DPUs[0].Firmware[0].RawVersion = "32.37.9999" Expect(find().State).To(Equal("failed")) - inv.DPUs[0].Firmware = append(inv.DPUs[0].Firmware, inventory.Firmware{Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "active"}) + inv.DPUs[0].Firmware = append(inv.DPUs[0].Firmware, inventory.Firmware{Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "running"}) Expect(find().ReasonCode).To(Equal("SOFTWARE_VERSION_CONFLICT")) for _, c := range SoftwareChecks(inv) { if c.State != "passed" { @@ -47,6 +47,72 @@ var _ = Describe("Software BOM readiness", func() { } } }) + It("accepts only current identity-bound source measurements", func() { + for _, scenario := range []struct { + name string + mutate func(*inventory.Inventory) + want string + }{ + {"accepted host", func(*inventory.Inventory) {}, "passed"}, + {"failed DPU", func(i *inventory.Inventory) { + i.DPUs[0].ObservationState = "unavailable" + i.DPUs[0].ReasonCode = "IDENTITY_CHANGED" + }, "identity"}, + {"missing associated adapter", func(i *inventory.Inventory) { i.Hosts[0].Adapters = nil }, "unknown"}, + {"failed host", func(i *inventory.Inventory) { i.Hosts[0].ObservationState = "unavailable" }, "unknown"}, + {"unassociated host", func(i *inventory.Inventory) { i.Associations = nil }, "unknown"}, + {"duplicate association", func(i *inventory.Inventory) { i.Associations = append(i.Associations, i.Associations[0]) }, "unknown"}, + {"duplicate DPU", func(i *inventory.Inventory) { i.DPUs = append(i.DPUs, i.DPUs[0]) }, "identity"}, + {"unrelated serial", func(i *inventory.Inventory) { i.Hosts[0].Runtime.Hardware.Cards[0].SerialNumber = "OTHER" }, "unknown"}, + } { + inv := softwareHostFixture() + scenario.mutate(inv) + checks := SoftwareChecks(inv) + if scenario.want == "identity" { + for _, c := range checks { + Expect(c.ReasonCode).To(Equal("SOFTWARE_IDENTITY_UNAVAILABLE"), scenario.name) + Expect(c.State).To(Equal("unknown")) + } + } else { + Expect(softwareCheck(checks, "nic").State).To(Equal(scenario.want), scenario.name) + } + } + }) + It("keeps explicit firmware and activation conflicts dominant over valid SSH", func() { + for _, fw := range []inventory.Firmware{ + {Component: "nic", State: "conflict", ReasonCode: "FIRMWARE_VERSION_CONFLICT"}, + {Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "unknown", ActivationReasonCode: "ACTIVATION_CONFLICT"}, + } { + inv := softwareHostFixture() + inv.DPUs[0].Firmware = []inventory.Firmware{fw} + check := softwareCheck(SoftwareChecks(inv), "nic") + Expect(check.State).To(Equal("unknown")) + Expect(check.ReasonCode).To(Equal("SOFTWARE_VERSION_CONFLICT")) + } + }) + It("retains incomplete source reasons and accepts an independent associated SSH source", func() { + inv := softwareHostFixture() + d := &inv.DPUs[0] + d.Observations = map[string]inventory.Observation{"firmware-inventory": {State: "unavailable", ReasonCode: "ACCESS_DENIED", Message: "A firmware member returned HTTP 403."}} + d.Firmware = []inventory.Firmware{{Component: "nic", State: "known", RawVersion: "32.38.1002", Activation: "running", EvidenceRefs: []string{"partial-redfish"}}} + check := softwareCheck(SoftwareChecks(inv), "nic") + Expect(check.State).To(Equal("passed")) + Expect(check.EvidenceRefs).To(ContainElement("associated-ssh")) + Expect(check.EvidenceRefs).NotTo(ContainElement("partial-redfish")) + Expect(d.Observations["firmware-inventory"].ReasonCode).To(Equal("ACCESS_DENIED")) + inv.Associations = nil + check = softwareCheck(SoftwareChecks(inv), "nic") + Expect(check.State).To(Equal("unknown")) + Expect(check.ReasonCode).To(Equal("ACCESS_DENIED")) + Expect(check.Message).To(ContainSubstring("403")) + }) + It("does not turn a desired target into a compatibility predicate", func() { + inv := softwareHostFixture() + inv.Software.Hardware[0].Requirements = append(inv.Software.Hardware[0].Requirements, releases.Requirement{Component: "desired-only", State: "known", Target: "1.2.3"}) + for _, c := range SoftwareChecks(inv) { + Expect(c.ID).NotTo(Equal("dpf.software.desired-only")) + } + }) It("compares numeric build versions without ordering vendor text", func() { c, ok := compareVersion("4.33.0-169", "4.33.0-168") Expect(ok).To(BeTrue()) @@ -62,3 +128,22 @@ var _ = Describe("Software BOM readiness", func() { } }) }) + +func softwareCheck(checks []inventory.Check, component string) inventory.Check { + for _, c := range checks { + if c.ID == "dpf.software."+component { + return c + } + } + Fail("missing software check for " + component) + return inventory.Check{} +} +func softwareHostFixture() *inventory.Inventory { + r, err := releases.Resolve("") + Expect(err).NotTo(HaveOccurred()) + return &inventory.Inventory{Software: &r, + DPUs: []inventory.Device{{RecordKey: "card", ID: "dpu-a", SerialNumber: "CARD-A", ObservationState: "observed", Model: "B3210"}}, + Hosts: []inventory.Device{{ID: "host-a", ObservationState: "observed", Adapters: []inventory.Adapter{{Key: "adapter-a", SerialNumber: "CARD-A"}}, Runtime: &inventory.RuntimeObservation{Hardware: &inventory.HardwareObservation{Cards: []inventory.CardHardware{{SerialNumber: "CARD-A", Firmware: inventory.Observation{State: "observed", Values: []string{"32.38.1002"}}, EvidenceRefs: []string{"associated-ssh"}}}}}}}, + Associations: []inventory.Association{{DPUID: "dpu-a", HostID: "host-a", SerialNumber: "CARD-A", HostAdapterKey: "adapter-a"}}, + } +} diff --git a/pkg/dpf/sshprobe/diagnostics.go b/pkg/dpf/sshprobe/diagnostics.go index 2848a2a4..9022abce 100644 --- a/pkg/dpf/sshprobe/diagnostics.go +++ b/pkg/dpf/sshprobe/diagnostics.go @@ -37,6 +37,8 @@ func commandFailure(err error, stage, command, output string) *Error { code = "TOOL_NOT_AVAILABLE" case strings.Contains(lower, "permission denied"), strings.Contains(lower, "access denied"), strings.Contains(lower, "operation not permitted"), strings.Contains(lower, "a password is required"), strings.Contains(lower, "not allowed to execute"): code = "PERMISSION_DENIED" + case strings.Contains(lower, "failed to open device"), strings.Contains(lower, "cannot open device"): + code = "DEVICE_ACCESS_FAILED" case strings.Contains(lower, "not supported"), strings.Contains(lower, "unsupported"), strings.Contains(lower, "unknown parameter"), strings.Contains(lower, "unrecognized option"): code = "OPERATION_UNSUPPORTED" } @@ -67,6 +69,8 @@ func Reason(err error) string { return "The remote tool exists but could not be executed." case "PERMISSION_DENIED": return "The configured account or noninteractive sudo policy does not permit this read." + case "DEVICE_ACCESS_FAILED": + return "The tool could not open the queried device; device access was not established." case "OPERATION_UNSUPPORTED": return "The installed tool or device does not support this read operation." case "SSH_TIMEOUT_OR_CANCELLED": diff --git a/pkg/dpf/sshprobe/hardware_test.go b/pkg/dpf/sshprobe/hardware_test.go index 8db9770b..bc9078ee 100644 --- a/pkg/dpf/sshprobe/hardware_test.go +++ b/pkg/dpf/sshprobe/hardware_test.go @@ -70,7 +70,7 @@ var _ = Describe("hardware observations", func() { Expect(d.Runtime.Observations["arm-fru"].Tool).To(Equal("ipmitool")) }) It("classifies diagnostics without persisting remote text", func() { - for text, code := range map[string]string{"sudo: mlxfwmanager: command not found": "TOOL_NOT_AVAILABLE", "sudo: a password is required": "PERMISSION_DENIED", "device operation not supported": "OPERATION_UNSUPPORTED", "opaque secret-value": "SSH_COMMAND_FAILED"} { + for text, code := range map[string]string{"sudo: mlxfwmanager: command not found": "TOOL_NOT_AVAILABLE", "sudo: a password is required": "PERMISSION_DENIED", "device operation not supported": "OPERATION_UNSUPPORTED", "opaque secret-value": "SSH_COMMAND_FAILED", "Status: Failed to open device": "DEVICE_ACCESS_FAILED"} { err := commandFailure(fmt.Errorf("secret-value"), "firmware", "sudo -n mlxfwmanager --query", text) Expect(err.Code).To(Equal(code)) Expect(err.Tool).To(Equal("mlxfwmanager")) diff --git a/pkg/dpf/sshprobe/network.go b/pkg/dpf/sshprobe/network.go index 01b1e7a8..606f7508 100644 --- a/pkg/dpf/sshprobe/network.go +++ b/pkg/dpf/sshprobe/network.go @@ -19,7 +19,7 @@ func collectNetwork(ctx context.Context, remote runner, runtime *inventory.Runti n := &inventory.NetworkObservation{Links: []inventory.NetworkLink{}, Neighbors: []inventory.Neighbor{}, LLDP: []inventory.LLDPNeighbor{}, Resolver: inventory.ResolverObservation{Owner: "unknown", Search: []string{}, PerLink: map[string][]string{}, Domains: map[string][]string{}, DNSMasqServers: []string{}}} runtime.Network = n for _, p := range []struct{ stage, command string }{ - {"network-links", "ip -j -d link show"}, {"neighbors", "ip -j neighbor show"}, {"resolver-path", "readlink -f /etc/resolv.conf"}, + {"network-links", "ip -j link show"}, {"network-link-details", "ip -j -d link show"}, {"neighbors", "ip -j neighbor show"}, {"resolver-path", "readlink -f /etc/resolv.conf"}, {"resolver-dns", "resolvectl dns"}, {"resolver-domains", "resolvectl domain"}, {"dnsmasq", dnsmasqRead}, {"lldp", "lldpctl -f keyvalue"}, } { text, err := remote.Run(ctx, p.stage, p.command) @@ -27,6 +27,12 @@ func collectNetwork(ctx context.Context, remote runner, runtime *inventory.Runti switch p.stage { case "network-links": n.Links, err = parseLinks(text) + case "network-link-details": + var details []inventory.NetworkLink + details, err = parseLinks(text) + if err == nil { + err = mergeLinkDetails(n.Links, details) + } case "neighbors": n.Neighbors, err = parseNeighbors(text) case "resolver-path": @@ -222,3 +228,26 @@ func parseLLDP(text string) ([]inventory.LLDPNeighbor, error) { } return out, nil } + +// Detailed iproute output is optional: some releases emit invalid JSON for +// VXLAN attributes. Base links remain useful without detailed link kinds. Verify +// a complete unchanged name/MAC/master projection before enriching those facts. +func mergeLinkDetails(base, details []inventory.NetworkLink) error { + if len(base) != len(details) { + return &Error{Code: "SSH_RUNTIME_INVALID", Stage: "network-link-details"} + } + byName := map[string]inventory.NetworkLink{} + for _, d := range details { + byName[d.Name] = d + } + for _, b := range base { + d, exists := byName[b.Name] + if !exists || d.MAC != b.MAC || d.Master != b.Master { + return &Error{Code: "SSH_RUNTIME_INVALID", Stage: "network-link-details"} + } + } + for i := range base { + base[i].Kind = byName[base[i].Name].Kind + } + return nil +} diff --git a/pkg/dpf/sshprobe/network_test.go b/pkg/dpf/sshprobe/network_test.go index d98f5023..154c620c 100644 --- a/pkg/dpf/sshprobe/network_test.go +++ b/pkg/dpf/sshprobe/network_test.go @@ -37,6 +37,30 @@ var _ = Describe("network observations", func() { Expect(n.LLDP[0].Chassis).To(Equal("switch-a")) Expect(strings.Join(r.calls, "\n")).NotTo(ContainSubstring("192.0.2.43")) }) + It("retains base link facts when detailed iproute VXLAN JSON is malformed", func() { + r := armRunner() + r.values["network-links"] = `[{"ifname":"oob_net0","address":"02:00:00:00:00:41","master":"br0"},{"ifname":"br0"}]` + r.values["network-link-details"] = `[{"ifname":"vxlan0","linkinfo":{"info_kind":"vxlan","info_data":{"fan_map":}}}]` + d, _, err := collect(context.Background(), config.Target{Role: "dpus", Endpoint: "ssh://192.0.2.41:22"}, r) + Expect(err).NotTo(HaveOccurred()) + Expect(d.Runtime.Network.Links).To(HaveLen(2)) + Expect(d.Runtime.Network.Links[0].MAC).To(Equal("02:00:00:00:00:41")) + Expect(d.Runtime.Network.Links[0].Master).To(Equal("br0")) + Expect(d.Runtime.Observations["network-links"].State).To(Equal("observed")) + Expect(d.Runtime.Observations["network-link-details"].ReasonCode).To(Equal("SSH_RUNTIME_INVALID")) + Expect(d.Runtime.Observations["network-link-details"].Message).NotTo(BeEmpty()) + }) + It("enriches base link kinds only when detailed observations still match", func() { + base, err := parseLinks(`[{"ifname":"br0","address":"02:00:00:00:00:01"}]`) + Expect(err).NotTo(HaveOccurred()) + details, err := parseLinks(`[{"ifname":"br0","address":"02:00:00:00:00:01","linkinfo":{"info_kind":"bridge"}}]`) + Expect(err).NotTo(HaveOccurred()) + Expect(mergeLinkDetails(base, details)).To(Succeed()) + Expect(base[0].Kind).To(Equal("bridge")) + details[0].MAC = "02:00:00:00:00:02" + Expect(mergeLinkDetails(base, details)).NotTo(Succeed()) + Expect(base[0].MAC).To(Equal("02:00:00:00:00:01")) + }) It("does not interpret failed or malformed probes as empty successful evidence", func() { r := armRunner() r.values["neighbors"] = "null" From 303716115942f0e6a1beb6f8743e56af2ccf5412 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Fri, 6 Nov 2026 11:46:06 +0100 Subject: [PATCH 08/35] docs(dpf): generate procedural discovery guides from workflow definitions Signed-off-by: Alexander Maslennikov --- .github/workflows/docs.yml | 5 + CONTRIBUTING.md | 6 + docs/llms.txt | 1 + docs/user/dpf-discovery.md | 40 +++- docs/user/dpf-workflows.md | 274 ++++++++++++++++++++++++ mkdocs.yml | 1 + openspec/README.md | 16 ++ openspec/workflows/README.md | 64 ++++++ openspec/workflows/dpf-discovery.json | 227 ++++++++++++++++++++ scripts/generate-workflow-docs.py | 135 ++++++++++++ scripts/test_generate_workflow_docs.py | 84 ++++++++ skills/k8s-launch-kit-discover/SKILL.md | 21 +- 12 files changed, 868 insertions(+), 6 deletions(-) create mode 100644 docs/user/dpf-workflows.md create mode 100644 openspec/workflows/README.md create mode 100644 openspec/workflows/dpf-discovery.json create mode 100644 scripts/generate-workflow-docs.py create mode 100644 scripts/test_generate_workflow_docs.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index aa055924..267e2f17 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,6 +45,11 @@ jobs: cache: pip cache-dependency-path: requirements-docs.txt + - name: Check generated workflow documentation + run: | + python -m unittest discover -s scripts -p 'test_generate_workflow_docs.py' -v + python scripts/generate-workflow-docs.py --check + - name: Install docs dependencies run: python -m pip install -r requirements-docs.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b27b8cea..cbf8f88f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,12 @@ Before opening a PR: - Check examples, links and any affected generated documentation. - Run `make build` and `python3 scripts/check-docs.py --binary build/l8k` to check documented flags, navigation and offline generation examples. +- For procedural workflow changes, edit the repository-owned definitions under + `openspec/workflows/` and regenerate with `python3 scripts/generate-workflow-docs.py`. + Run `python3 -m unittest discover -s scripts -p 'test_generate_workflow_docs.py'` + and `python3 scripts/generate-workflow-docs.py --check`. See the + [workflow source contract](openspec/workflows/README.md); these are documentation + inputs, not executable OpenSpec workflows or a runtime dependency. - For site changes, install `requirements-docs.txt` in an isolated Python environment and run `mkdocs build --strict`. - List documentation updates and validation in the PR description. If no diff --git a/docs/llms.txt b/docs/llms.txt index c05d09c4..f65bb405 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -19,6 +19,7 @@ This documentation covers the `l8k` CLI for deploying NVIDIA cloud-native networ - Discover cluster hardware: /k8s-launch-kit/user/discovery/ - DPF discovery preview: /k8s-launch-kit/user/dpf-discovery/ +- DPF discovery workflows: /k8s-launch-kit/user/dpf-workflows/ - Generate and review manifests: /k8s-launch-kit/advanced/generation/ - Deploy manifests: /k8s-launch-kit/advanced/deployment/ - Deploy on OpenShift: /k8s-launch-kit/user/openshift/ diff --git a/docs/user/dpf-discovery.md b/docs/user/dpf-discovery.md index 1fc7e64a..f8c27496 100644 --- a/docs/user/dpf-discovery.md +++ b/docs/user/dpf-discovery.md @@ -5,6 +5,9 @@ SPDX-License-Identifier: Apache-2.0 # DPF discovery preview +Start with the [step-by-step CLI workflows](dpf-workflows.md) for first inventory, +address-candidate lookup and refresh. This page provides the access and output reference. + `l8k discover --target dpf` collects hardware inventory from explicitly selected host/DPU Redfish or SSH endpoints. The DPF estate is independent of Host networking. Inventory mode can run before Kubernetes exists; an explicitly supplied management @@ -75,11 +78,12 @@ remote access. Lists and credentials stay in YAML; no password CLI flag exists. | Source | Values | | --- | --- | | Required site inputs | Nonempty host list; each selected endpoint and credential alias; username and password reference or SSH private-key reference | -| Defaultable | Context from a supplied kubeconfig; verified system trust; SSH port22; SSH sudo false; collection 10m; request 15s; four endpoint collectors in flight (`maxConcurrentBMCs`) | +| Defaultable | `dpf.release: 26.4.0`; context from a supplied kubeconfig; verified system trust; SSH port22; SSH sudo false; collection 10m; request 15s; four endpoint collectors in flight (`maxConcurrentBMCs`) | | Conditional | Kubeconfig and explicit DPUs for management mode; SSH known-hosts file; approved noninteractive sudo for protected reads; private Redfish CA; `systemURI` for multi-System BMCs | | Observed | Physical-card/system identities, linked host adapters, component firmware, auth/time unknowns, management nodes/version | -| Release owned | Qualified adapter completeness predicates and the probe image/architecture digests | -| Later phases | Profile/release cohort, DPU API allocation, DHCP/service networks and controller authentication | +| Release owned | Software BOM coordinates, exact-model software minima or explicit unknown requirements, and probe image/architecture digests; hardware qualification remains separate | +| Optional lookup inputs | Explicit ARM/BMC MAC selectors; optional local `dnsmasq-leases` files. Lookup returns address hints, never identity or permission to connect | +| Later phases | Deployment profile, DPU API allocation, DHCP/service networks and controller authentication | The management kubeconfig must use verified HTTPS and certificate/key or bearer credentials. File and embedded representations of the same credential/trust field are mutually @@ -180,7 +184,8 @@ no sudo password, changes no privileges and uses no remote Python helper. ARM discovery needs a booted, reachable OS with sufficient permissions. Local `ipmitool lan print` can report the BMC endpoint without Redfish. The endpoint is recorded as `observed-local-ipmi`; discovery does not connect or send credentials -to that newly observed address. Missing optional tools/data remain unknown. It +to that newly observed address. Missing optional tools/data remain unknown with a persisted `reasonCode`, safe +`message`, and `tool` where applicable; required-source failures retain their cause too. It never infers a BMC address from an ARM IP or a serial from a hostname. Host and ARM PCI addresses belong to different domains; correlation uses physical @@ -189,6 +194,33 @@ attest trusted DPU state. Source evidence, SSH trust digest and any conflicts st in the output. SSH inventory success does not establish Redfish authorization, controller mTLS enrollment or provisioning readiness. Those checks remain separate. +## Software BOM and optional address lookup + +`dpf.release` selects the embedded software BOM independently of live measurements. +Omitted/null selects the pinned `26.4.0` default; explicit empty or unknown releases +are rejected. The resolved BOM is saved under `inventory.software`, with source +references and unknown requirements. The current catalog has exact BlueField-3 +models `B3210`, `B3220` and `B3240`; generic family names and BlueField-4 do not +establish applicability. Desired BFB/component coordinates are not measured +versions. Only explicit documented minima are compatibility checks; unsupported +firmware requirements remain unknown. Model applicability and software checks do +not qualify hardware or enable DPF deployment profiles. + +SSH can report NIC firmware/PSID, MFT version, NVConfig and privilege observations +when the existing tools and permissions permit the read. Current and next-boot +configuration remain distinct. Redfish firmware keeps raw component identities, +versions, source evidence and explicit activation where exposed. A failed source, +partial firmware collection, identity change or explicit version/activation conflict +cannot satisfy a minimum through that source. An independent valid SSH observation +can still satisfy a minimum unless explicit component conflicts remain. + +`discovery.addressLookup` optionally combines named ARM/BMC MAC selectors with local +`dnsmasq-leases` snapshots and eligible neighbors already read through selected SSH +sources. Results and source/device reasons are saved under `inventory.addressLookup`. +An address candidate is unverified and never receives credentials automatically. +Follow the [candidate procedure](dpf-workflows.md#address-candidates) to prepare +lookup inputs, interpret ambiguity and explicitly add a verified endpoint. + ## Observations, reports and refresh Physical identity comes from source-specific card/system fields, keeping raw serials diff --git a/docs/user/dpf-workflows.md b/docs/user/dpf-workflows.md new file mode 100644 index 00000000..b861e944 --- /dev/null +++ b/docs/user/dpf-workflows.md @@ -0,0 +1,274 @@ + + +# DPF discovery workflows + +Use these procedures with the experimental `l8k discover --target dpf` CLI. The DPU estate is separate from Host Network Operator deployment. Start with **inventory mode**; management probe mode is not qualified in this preview. Discovery collects facts and writes a handoff. It does not create VMs, install Kubernetes or DPF, configure DPUServices, change firmware, enroll trust, or start a DPF deployment. + +For complete field and error details, see [DPF discovery reference](dpf-discovery.md). Each diagram follows the same steps as its procedure. Commands are examples to run deliberately from a machine with direct access to your selected endpoints. + +## Collect the first inventory {#initial-inventory} + +Use this workflow when you know the host access addresses. DPU ARM/BMC addresses are optional in inventory mode. Without explicit DPU endpoints, Launch Kit derives BlueField card records from host inventory and leaves their access bindings unknown. + +```mermaid +flowchart TD + n1["Choose the sources and access inputs"] + n2["Prepare credentials, trust and existing tools"] + n3["Create an inventory configuration"] + n4["Run discovery and retain its output"] + n5["Review inventory, reasons and software gaps"] + n6["Resolve the reported prerequisite or ownership issue"] + n7["Keep the handoff and decide what is next"] + n1 -->|"Sources selected"| n2 + n2 -->|"Access prepared"| n3 + n3 -->|"Configuration ready"| n4 + n4 -->|"Run finished"| n5 + n5 -->|"Complete inventory reviewed"| n7 + n5 -->|"Required failure or conflict"| n6 + n6 -->|"Retry after correction"| n2 +``` + +### 1. Choose the sources and access inputs {#initial-inventory-choose-inputs} + +Choose each source explicitly: a host operating system over SSH or its BMC over Redfish; optionally choose the DPU ARM operating system over SSH or DPU BMC over Redfish. A selected source must succeed; Launch Kit does not silently replace failed Redfish access with SSH. Host SSH can supply host-to-card identity in Zero Trust when PCI/VPD access is available. ARM SSH needs a booted, reachable DPU operating system. Neither access path proves trusted DPU state. + +| Input | What you supply or allow Launch Kit to determine | +| --- | --- | +| Always required | At least one host endpoint; the source type; a credential alias, username, and password reference or SSH key reference for each selected endpoint. Addresses and credentials are not guessed. | +| Required for SSH | A verified `knownHostsFile`; direct connectivity from the CLI machine. SSH port defaults to `22`; ambient SSH config, agents and ProxyJump are not used. | +| Conditional | An explicit DPU endpoint for direct DPU measurements; a private CA for Redfish when system trust is insufficient; `systemURI` if a BMC exposes several Systems. | +| Optional | A management kubeconfig for additional observations. Inventory needs none, but an explicitly supplied kubeconfig must work. | +| Defaultable | `dpf.release` defaults to `26.4.0`; Redfish certificate verification is enabled; SSH `sudo` defaults to `false`. Choose `discovery.probes.mode: inventory` explicitly because the default management mode is unavailable in this preview. | +| Discovered | Host/system and card identities, host-to-DPU associations, visible hardware/software/network observations, and optional local BMC address hints. | + +Do not provide product versions as measured facts. The embedded release BOM owns its software coordinates; live observations remain separate. Omitted or null `dpf.release` uses the pinned default; an empty or unknown release is rejected. + +Next: [Sources selected](#initial-inventory-prepare). + +### 2. Prepare credentials, trust and existing tools {#initial-inventory-prepare} + +Put passwords in an environment variable or a private file, and SSH private keys and known-hosts entries in files under your control. Reference them from YAML; do not put password values in command-line arguments. File references are relative to that YAML file. Verify SSH keys and Redfish CA trust through your normal provisioning process. + +For SSH inventory, ask the endpoint administrator to make the required read tools and permissions available before the run. Host identity/PCI collection uses tools such as `dmidecode` and `lspci`; DPU identity additionally uses `ipmitool`. Optional hardware checks use `mlxfwmanager`, `mlxconfig`, `mlxprivhost` and `systemctl`; network checks use existing `ip`, resolver and LLDP tools where available. Tool availability and access differ between host and ARM operating systems. + +Set `ssh.sudo: true` only where noninteractive elevated reads are authorized and configured. Launch Kit then uses `sudo -n`; it never prompts for a sudo password, installs tools, changes privileges or repairs device settings. Missing tools, denied permissions and unsupported or malformed outputs keep specific reason codes and explanations. Required identity failures stop a complete inventory; optional unknown observations remain visible for review. + +Next: [Access prepared](#initial-inventory-configure). + +### 3. Create an inventory configuration {#initial-inventory-configure} + +Save this example as `dpf-site.yaml`, replacing the documentation address and credential paths with your host values. It intentionally starts with host SSH only and requires no Kubernetes cluster. Ensure the listed files exist before running. + +```yaml +dpf: + release: 26.4.0 +credentials: + host-reader: + username: inventory + privateKeyRef: {file: ./credentials/host-key} +discovery: + hosts: + - credentialRef: host-reader + ssh: + address: 192.0.2.21:22 + knownHostsFile: ./credentials/known_hosts + sudo: true + probes: + mode: inventory +``` + +For direct DPU ARM discovery, add a separate credential alias and a `discovery.dpus` entry with the same SSH shape. For Redfish, use `bmcEndpoint` and a password credential reference instead of `ssh`; see the [complete access examples](dpf-discovery.md#inputs-and-invocation). Each endpoint has exactly one access source. Plain YAML is the CLI input, not a Kubernetes object. Do not copy previously observed `inventory` facts into a new seed as user assertions. + +Next: [Configuration ready](#initial-inventory-run). + +### 4. Run discovery and retain its output {#initial-inventory-run} + +Choose a new output path for the first run. Run from a machine that can reach every selected endpoint and read the referenced files. + +```bash +l8k discover --target dpf --user-config dpf-site.yaml \ + --save-cluster-config dpf-discovered.yaml --output json \ + > discovery.json 2> discovery.log +``` + +Record the process exit status as well as the files. `discovery.json` contains the final result; progress and diagnostics go to `discovery.log`. The command creates its own run directory next to the requested YAML. It does not send credentials to addresses found during collection. Redfish uses a temporary login session and deletes that session; SSH runs read commands on explicitly selected endpoints. + +Next: [Run finished](#initial-inventory-inspect). + +### 5. Review inventory, reasons and software gaps {#initial-inventory-inspect} + +Inspect `discovery.outcome`, `discovery.checks`, `errors`, `artifacts` and `acceptance` in the JSON report. Review saved `inventory.hosts`, `inventory.dpus`, `inventory.associations` and their evidence before using the handoff. Complete discovery writes `dpf-discovered.yaml`; partial discovery preserves an existing handoff and writes `partial.yaml` and `report.json` under `dpf-discovered.discovery//`. Use the report's artifact paths instead of guessing the run ID. + +Read each non-success observation's `reasonCode`, `message` and `tool` where present. A missing `mlxfwmanager` is an unavailable prerequisite, not proof of an incompatible NIC firmware. A denied Redfish firmware collection is different from a component property not being exposed. Conflicting versions or ambiguous activation cannot establish a software minimum. + +| Example reason | What to do | +| --- | --- | +| `TOOL_NOT_AVAILABLE` / `PERMISSION_DENIED` | Check the named endpoint tool or authorized noninteractive sudo policy; do not treat the unmeasured check as passed. | +| `ACCESS_DENIED` / `PROPERTY_NOT_EXPOSED` | Distinguish an account permission problem from a property the service does not advertise. | +| `SOFTWARE_VERSION_CONFLICT` / `SOFTWARE_IDENTITY_UNAVAILABLE` | Reconcile version/activation sources or the rejected physical identity before trusting software comparisons. | +| `SOFTWARE_BELOW_MINIMUM` | Review the release requirement and plan a separate maintenance action; discovery does not update firmware. | +| `PUBLICATION_FAILED` | Read the error explanation, resolve the local output/path problem, and rerun; only listed successfully written artifacts exist. | + +The resolved software BOM is saved under `inventory.software`, including source references and unknown requirements. Release `26.4.0` has exact BlueField-3 model entries `B3210`, `B3220` and `B3240`; generic family names and BlueField-4 are not sufficient for BOM applicability. The documented NIC/MFT minima can be evaluated when current, identity-bound measurements exist. BMC, CEC, ATF, UEFI, OS/BSP and golden-image minima remain unknown where the catalog has no supported requirement. A BFB target is a desired artifact, not an observed OS/BSP version. No site-supplied minimum can turn an unknown catalog requirement into qualified support. + +Exit zero and `discovery.outcome: complete` mean inventory collection, identity/mapping, cleanup and publication succeeded. `acceptance: not-evaluated` remains explicit. Profile planning, hardware qualification, DPF authentication, operator-created connectivity and deployment remain later work; DPF generate/deploy/validate are unavailable in this preview. + +Next: [Complete inventory reviewed](#initial-inventory-finish); [Required failure or conflict](#initial-inventory-resolve). + +### 6. Resolve the reported prerequisite or ownership issue {#initial-inventory-resolve} + +Use the specific recorded reason to correct the source configuration or ask the infrastructure owner to repair missing tools, permissions, credentials, connectivity or conflicting identity. Do not delete unknown observations or label them passed. Optional gaps may remain documented without blocking inventory; required source failures, identity ambiguity, cleanup or publication errors need resolution. If cleanup is uncertain, use the reported session/resource ownership evidence before any manual action. Rerun after correcting the cause. + +Next: [Retry after correction](#initial-inventory-prepare). + +### 7. Keep the handoff and decide what is next {#initial-inventory-finish} + +Retain the seed, discovered YAML, JSON report and stderr log together. The discovered YAML preserves configuration intent and replaces observed inventory; it is not a deployment-ready profile or a provisioning approval. If a DPU access address is missing, use [address-candidate lookup](#address-candidates). For later remeasurement of the same estate, use [refresh](#refresh-inventory). + +## Find and verify address candidates {#address-candidates} + +Use this optional inventory addition when you know an ARM or BMC management MAC but need its possible IP address. It is a locator workflow, not authentication or identity discovery. You still need the host configuration from the first workflow; this is not a standalone subnet scanner. + +```mermaid +flowchart TD + n1["Supply the management MAC and lookup evidence"] + n2["Add explicit address selectors to the seed"] + n3["Run inventory with lookup enabled"] + n4["Review candidates and unresolved lookups"] + n5["Authorize an explicit endpoint and rediscover"] + n1 -->|"MAC and source selected"| n2 + n2 -->|"Selectors configured"| n3 + n3 -->|"Candidate report available"| n4 + n4 -->|"Ownership can be independently verified"| n5 + n4 -->|"Missing or ambiguous evidence"| n1 +``` + +### 1. Supply the management MAC and lookup evidence {#address-candidates-locators} + +Obtain the exact ARM or BMC management MAC from authoritative asset records or a trustworthy observed interface. You must choose its role (`arm` or `bmc`) and a unique name. Launch Kit does not infer a management MAC from a NIC serial, data-port MAC, hostname or neighboring address. + +Optionally obtain a current dnsmasq IPv4 lease-file snapshot through your authorized DHCP administration process and place it on the CLI machine. Launch Kit reads this file locally; it does not log in to a DHCP server or query arbitrary vendor lease APIs. With no lease files, it can use suitable neighbors observed on the SSH endpoints already selected for inventory. Only `REACHABLE` or `PERMANENT` neighbors are candidates; stale entries are excluded. + +Next: [MAC and source selected](#address-candidates-configure). + +### 2. Add explicit address selectors to the seed {#address-candidates-configure} + +Merge this fragment into the existing `discovery` section of a seed without saved inventory. Replace the MACs and file path; omit `sources` if you only want eligible observed neighbors. Do not replace the required host list. + +```yaml +discovery: + addressLookup: + sources: + - file: ./inputs/dnsmasq.leases + format: dnsmasq-leases + devices: + - name: dpu-a-arm + role: arm + mac: '02:00:00:00:00:11' + - name: dpu-a-bmc + role: bmc + mac: '02:00:00:00:00:12' +``` + +The supported file contains the five dnsmasq IPv4 columns: expiry epoch, MAC, IP, hostname and client ID. An expiry of `0` means a nonexpiring record; expired records are not candidates. Each file must be readable, at most 1 MiB and at most 4096 records. A malformed file contributes no candidates and retains a source failure reason. Other DHCP formats are unsupported. + +Next: [Selectors configured](#address-candidates-run). + +### 3. Run inventory with lookup enabled {#address-candidates-run} + +Save the configured seed as `dpf-lookup.yaml` and select a new handoff path. + +```bash +l8k discover --target dpf --user-config dpf-lookup.yaml \ + --save-cluster-config dpf-candidates.yaml --output json \ + > candidates.json 2> candidates.log +``` + +Lookup happens as part of this inventory run. It reads only the configured lease files and already observed SSH neighbors. No candidate is contacted and no credential is sent to it. + +Next: [Candidate report available](#address-candidates-review). + +### 4. Review candidates and unresolved lookups {#address-candidates-review} + +Read `inventory.addressLookup.candidates` for the address, selected name/role/MAC, source and evidence reference. Read `inventory.addressLookup.observations` for each device and lease source. A single address is `ADDRESS_UNVERIFIED`; several addresses for a MAC remain ambiguous, and missing/expired/unreadable evidence keeps its reason. A candidate is not a host-to-DPU association and is never a readiness pass. + +Use your DHCP/network or asset owner to resolve multiple, missing or contradictory locators. Do not choose an address merely because it is the first result. Independent identity and endpoint trust are still required even for one candidate. + +Next: [Ownership can be independently verified](#address-candidates-bind); [Missing or ambiguous evidence](#address-candidates-locators). + +### 5. Authorize an explicit endpoint and rediscover {#address-candidates-bind} + +Verify endpoint ownership and obtain SSH known-hosts trust or Redfish CA trust and the appropriate account through your normal process. Create a new seed and add the verified endpoint under `discovery.dpus`, with its credential reference and trust policy. Use a new output when changing scope; do not edit the old observed identity to force a match. + +Run [initial inventory](#initial-inventory) for that explicit endpoint. Only its authenticated physical-card observation and unique host-card association can establish the binding. A discovered IP alone never authorizes Redfish access or proves Zero Trust attestation. + +## Refresh an existing inventory {#refresh-inventory} + +Use this workflow to remeasure the same selected estate, compare facts and preserve diagnostic history. A refresh is another live read; old successful observations do not stand in for unavailable current measurements. + +```mermaid +flowchart TD + n1["Check scope and preserve source intent"] + n2["Run an explicit in-place refresh"] + n3["Compare current facts and explain every gap"] + n4["Correct the cause and retry the same scope"] + n5["Start a new inventory for the changed estate"] + n6["Retain the refreshed handoff and reports"] + n1 -->|"Same scope ready"| n2 + n1 -->|"Scope changed"| n5 + n2 -->|"Refresh finished"| n3 + n3 -->|"Complete refresh reviewed"| n6 + n3 -->|"Partial or failed refresh"| n4 + n4 -->|"Same scope corrected"| n2 + n4 -->|"Actual scope change"| n5 +``` + +### 1. Check scope and preserve source intent {#refresh-inventory-scope} + +Start from the existing discovered YAML and keep its credential/trust references valid. Decide whether you are remeasuring the same host/DPU endpoints and trust policy. If endpoint selection, trust policy or physical identity has changed, create a fresh seed without old `inventory` and use a new output path. Investigate unexpected replacement or ownership changes before accepting them. + +Do not manually edit observed serials, associations, firmware or check outcomes. `dpf.release` selects the embedded BOM; changing the release does not upgrade software. An omitted release continues to resolve to the CLI's pinned default, so record the CLI version and review `inventory.software` when comparing runs from different binaries. Explicit release selection makes that intention visible. + +Next: [Same scope ready](#refresh-inventory-run); [Scope changed](#refresh-inventory-new-scope). + +### 2. Run an explicit in-place refresh {#refresh-inventory-run} + +Use the same YAML path for input and output and explicitly supply `--save-cluster-config`: + +```bash +l8k discover --target dpf --user-config dpf-discovered.yaml \ + --save-cluster-config dpf-discovered.yaml --output json \ + > refresh.json 2> refresh.log +``` + +The run retains `source-before.yaml`. Complete discovery replaces inventory while preserving user sections, comments and input intent. Relative file references are rebased if saving elsewhere; private files are not copied. Avoid simultaneous external edits, and use real output directories rather than symlink aliases. Credentials, trust files and configured lease files cannot be used as output destinations. + +Next: [Refresh finished](#refresh-inventory-compare). + +### 3. Compare current facts and explain every gap {#refresh-inventory-compare} + +Review `diff`, current checks, `errors`, cleanup and successfully written `artifacts` in the result. Compare physical identity, firmware, relevant hardware/mode/network observations and resolved release requirements. A failed current observation stays unknown rather than inheriting an earlier pass; semantic list ordering alone should not produce a changed fact. + +On partial collection, the previous handoff remains intact and the new partial YAML/report live under `dpf-discovered.discovery//`. Inspect the new report instead of mistaking the unchanged handoff for fresh success. Required failures need correction and another run. Optional missing tools or inaccessible properties remain explained observations; they do not make deployment readiness pass. + +Next: [Complete refresh reviewed](#refresh-inventory-retain); [Partial or failed refresh](#refresh-inventory-repair). + +### 4. Correct the cause and retry the same scope {#refresh-inventory-repair} + +Use the saved reason and affected source to repair the external prerequisite or fix the input. Preserve the unsuccessful run's report. Do not overwrite failure evidence with an old successful observation. If the cause is a real endpoint, trust or identity change, switch to a fresh seed and output. + +Next: [Same scope corrected](#refresh-inventory-run); [Actual scope change](#refresh-inventory-new-scope). + +### 5. Start a new inventory for the changed estate {#refresh-inventory-new-scope} + +Create a new seed from the desired access configuration, without copying old `inventory` facts, and select a new handoff path. Recheck endpoint ownership/trust and follow [initial inventory](#initial-inventory). Keep the previous estate's handoff and report as history. + +### 6. Retain the refreshed handoff and reports {#refresh-inventory-retain} + +Keep `source-before.yaml`, the current YAML, JSON/stdout report and stderr log. Discovery completion still means a successful inventory workflow, with deployment `acceptance: not-evaluated`. Unknown BOM constraints, hardware qualification and operator-created service checks remain visible until their later workflows are implemented and verified. diff --git a/mkdocs.yml b/mkdocs.yml index 8d0f53d8..f720ef93 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -90,6 +90,7 @@ nav: - Networking concepts: advanced/concepts.md - Deploy: - Discover cluster hardware: user/discovery.md + - DPF discovery workflows: user/dpf-workflows.md - DPF discovery preview: user/dpf-discovery.md - Generate and review manifests: advanced/generation.md - Deploy manifests: advanced/deployment.md diff --git a/openspec/README.md b/openspec/README.md index f9882f69..ab8c2091 100644 --- a/openspec/README.md +++ b/openspec/README.md @@ -87,3 +87,19 @@ promise across several boundaries. It is optional for routine conforming changes these contracts introduce no mandatory proposal/approval stage or per-flag ceremony. Keep exact flag/default/version inventories in their existing mechanical sources, operating procedures in docs/skills, and development rules in AGENTS.md. + +## Procedural workflow documentation + +[Repository-owned workflow definitions](workflows/README.md) under `workflows/` +provide the single source for the DPF CLI procedures and their flow diagrams. +`python3 scripts/generate-workflow-docs.py` renders +[DPF workflows](../docs/user/dpf-workflows.md); `--check` detects drift without +writing files. The Docs CI job runs the generator tests and drift check before +building the site. + +This is a Launch Kit documentation extension colocated with OpenSpec contracts, +**not a native executable OpenSpec workflow format**. The generator does not run +commands, validate infrastructure or enforce runtime workflow transitions. OpenSpec +structural validation remains separate; neither check proves implementation +conformance. Procedures must be reviewed against the CLI and relevant contract +scenarios when behavior changes. diff --git a/openspec/workflows/README.md b/openspec/workflows/README.md new file mode 100644 index 00000000..a7ef37e1 --- /dev/null +++ b/openspec/workflows/README.md @@ -0,0 +1,64 @@ + + +# Procedural documentation sources + +`dpf-discovery.json` owns the user steps and flow transitions for initial DPF +inventory, address-candidate lookup and refresh. Its generated page is +`docs/user/dpf-workflows.md`; edit the JSON, not the generated Markdown. + +This is a **repository-owned documentation extension**, not a native executable +OpenSpec workflow format. It does not add a runtime state machine or a discovery +plugin. Python's standard library is used only during documentation generation +and tests; the Go CLI and remote collection do not depend on Python. Commands +inside step bodies are displayed as examples and are never executed by this tool. + +## Regenerate and verify + +From the repository root: + +```sh +python3 scripts/generate-workflow-docs.py +python3 -m unittest discover -s scripts -p 'test_generate_workflow_docs.py' -v +python3 scripts/generate-workflow-docs.py --check +``` + +The Docs CI job runs the tests and `--check` before strict MkDocs. The existing +`check-docs.py` CLI example checks still run separately in code CI. The generator +is deterministic, omits timestamps, and rejects invalid metadata before writing. +`--check` reports missing/stale output with exit 1 and leaves output untouched. +`--source` and `--output` can select explicit paths for local testing. + +## Format version 1 + +The source is one JSON object with `formatVersion: 1`, `title`, `introduction`, +and a nonempty `workflows` array. An optional `$comment` carries source licensing. +Each workflow has a unique `id`, `title`, `introduction`, and nonempty `steps`. +Each step has an `id` unique within that workflow, `title`, Markdown `body`, and +`transitions`. Each transition has a destination step `to` and a nonempty `label`. +An empty transition list ends the procedure. Retry transitions may point backward. +IDs use lowercase letters, digits and hyphens and start with a letter. The first +listed step is the starting step; list steps in the order users should read them. + +The generator validates field shapes, duplicate workflow/step IDs, duplicate JSON +fields, and unknown or duplicate transition destinations. It renders the same +transitions as both Mermaid edges and next-step links. The source remains a human +procedure: no conditions are evaluated, no infrastructure is inspected, and graph +structure does not prove the commands implement the contract. Review the source, +rendered prose, examples and diagram together against the actual CLI behavior. + +## Maintaining a workflow + +1. Update applicable [OpenSpec contracts](../README.md) when the shared promise + changes, and implement/test the CLI behavior. +2. Edit the procedure's inputs, actions, failure explanations and transitions in + the JSON. Keep required site inputs separate from defaults and observations. +3. Regenerate and run generator tests, drift detection, CLI example checks and + strict MkDocs. Include the source and generated page in the same change. +4. Review runtime/hardware evidence separately. Inventory success, deployment + readiness and hardware qualification are distinct outcomes. + +The generator currently targets this one DPF procedure document. Extending it to +other guides is a future scoped documentation change, not a workflow execution API. diff --git a/openspec/workflows/dpf-discovery.json b/openspec/workflows/dpf-discovery.json new file mode 100644 index 00000000..dd322c0e --- /dev/null +++ b/openspec/workflows/dpf-discovery.json @@ -0,0 +1,227 @@ +{ + "$comment": "Copyright 2026 NVIDIA CORPORATION & AFFILIATES; SPDX-License-Identifier: Apache-2.0", + "formatVersion": 1, + "title": "DPF discovery workflows", + "introduction": "Use these procedures with the experimental `l8k discover --target dpf` CLI. The DPU estate is separate from Host Network Operator deployment. Start with **inventory mode**; management probe mode is not qualified in this preview. Discovery collects facts and writes a handoff. It does not create VMs, install Kubernetes or DPF, configure DPUServices, change firmware, enroll trust, or start a DPF deployment.\n\nFor complete field and error details, see [DPF discovery reference](dpf-discovery.md). Each diagram follows the same steps as its procedure. Commands are examples to run deliberately from a machine with direct access to your selected endpoints.", + "workflows": [ + { + "id": "initial-inventory", + "title": "Collect the first inventory", + "introduction": "Use this workflow when you know the host access addresses. DPU ARM/BMC addresses are optional in inventory mode. Without explicit DPU endpoints, Launch Kit derives BlueField card records from host inventory and leaves their access bindings unknown.", + "steps": [ + { + "id": "choose-inputs", + "title": "Choose the sources and access inputs", + "body": "Choose each source explicitly: a host operating system over SSH or its BMC over Redfish; optionally choose the DPU ARM operating system over SSH or DPU BMC over Redfish. A selected source must succeed; Launch Kit does not silently replace failed Redfish access with SSH. Host SSH can supply host-to-card identity in Zero Trust when PCI/VPD access is available. ARM SSH needs a booted, reachable DPU operating system. Neither access path proves trusted DPU state.\n\n| Input | What you supply or allow Launch Kit to determine |\n| --- | --- |\n| Always required | At least one host endpoint; the source type; a credential alias, username, and password reference or SSH key reference for each selected endpoint. Addresses and credentials are not guessed. |\n| Required for SSH | A verified `knownHostsFile`; direct connectivity from the CLI machine. SSH port defaults to `22`; ambient SSH config, agents and ProxyJump are not used. |\n| Conditional | An explicit DPU endpoint for direct DPU measurements; a private CA for Redfish when system trust is insufficient; `systemURI` if a BMC exposes several Systems. |\n| Optional | A management kubeconfig for additional observations. Inventory needs none, but an explicitly supplied kubeconfig must work. |\n| Defaultable | `dpf.release` defaults to `26.4.0`; Redfish certificate verification is enabled; SSH `sudo` defaults to `false`. Choose `discovery.probes.mode: inventory` explicitly because the default management mode is unavailable in this preview. |\n| Discovered | Host/system and card identities, host-to-DPU associations, visible hardware/software/network observations, and optional local BMC address hints. |\n\nDo not provide product versions as measured facts. The embedded release BOM owns its software coordinates; live observations remain separate. Omitted or null `dpf.release` uses the pinned default; an empty or unknown release is rejected.", + "transitions": [ + { + "to": "prepare", + "label": "Sources selected" + } + ] + }, + { + "id": "prepare", + "title": "Prepare credentials, trust and existing tools", + "body": "Put passwords in an environment variable or a private file, and SSH private keys and known-hosts entries in files under your control. Reference them from YAML; do not put password values in command-line arguments. File references are relative to that YAML file. Verify SSH keys and Redfish CA trust through your normal provisioning process.\n\nFor SSH inventory, ask the endpoint administrator to make the required read tools and permissions available before the run. Host identity/PCI collection uses tools such as `dmidecode` and `lspci`; DPU identity additionally uses `ipmitool`. Optional hardware checks use `mlxfwmanager`, `mlxconfig`, `mlxprivhost` and `systemctl`; network checks use existing `ip`, resolver and LLDP tools where available. Tool availability and access differ between host and ARM operating systems.\n\nSet `ssh.sudo: true` only where noninteractive elevated reads are authorized and configured. Launch Kit then uses `sudo -n`; it never prompts for a sudo password, installs tools, changes privileges or repairs device settings. Missing tools, denied permissions and unsupported or malformed outputs keep specific reason codes and explanations. Required identity failures stop a complete inventory; optional unknown observations remain visible for review.", + "transitions": [ + { + "to": "configure", + "label": "Access prepared" + } + ] + }, + { + "id": "configure", + "title": "Create an inventory configuration", + "body": "Save this example as `dpf-site.yaml`, replacing the documentation address and credential paths with your host values. It intentionally starts with host SSH only and requires no Kubernetes cluster. Ensure the listed files exist before running.\n\n```yaml\ndpf:\n release: 26.4.0\ncredentials:\n host-reader:\n username: inventory\n privateKeyRef: {file: ./credentials/host-key}\ndiscovery:\n hosts:\n - credentialRef: host-reader\n ssh:\n address: 192.0.2.21:22\n knownHostsFile: ./credentials/known_hosts\n sudo: true\n probes:\n mode: inventory\n```\n\nFor direct DPU ARM discovery, add a separate credential alias and a `discovery.dpus` entry with the same SSH shape. For Redfish, use `bmcEndpoint` and a password credential reference instead of `ssh`; see the [complete access examples](dpf-discovery.md#inputs-and-invocation). Each endpoint has exactly one access source. Plain YAML is the CLI input, not a Kubernetes object. Do not copy previously observed `inventory` facts into a new seed as user assertions.", + "transitions": [ + { + "to": "run", + "label": "Configuration ready" + } + ] + }, + { + "id": "run", + "title": "Run discovery and retain its output", + "body": "Choose a new output path for the first run. Run from a machine that can reach every selected endpoint and read the referenced files.\n\n```bash\nl8k discover --target dpf --user-config dpf-site.yaml \\\n --save-cluster-config dpf-discovered.yaml --output json \\\n > discovery.json 2> discovery.log\n```\n\nRecord the process exit status as well as the files. `discovery.json` contains the final result; progress and diagnostics go to `discovery.log`. The command creates its own run directory next to the requested YAML. It does not send credentials to addresses found during collection. Redfish uses a temporary login session and deletes that session; SSH runs read commands on explicitly selected endpoints.", + "transitions": [ + { + "to": "inspect", + "label": "Run finished" + } + ] + }, + { + "id": "inspect", + "title": "Review inventory, reasons and software gaps", + "body": "Inspect `discovery.outcome`, `discovery.checks`, `errors`, `artifacts` and `acceptance` in the JSON report. Review saved `inventory.hosts`, `inventory.dpus`, `inventory.associations` and their evidence before using the handoff. Complete discovery writes `dpf-discovered.yaml`; partial discovery preserves an existing handoff and writes `partial.yaml` and `report.json` under `dpf-discovered.discovery//`. Use the report's artifact paths instead of guessing the run ID.\n\nRead each non-success observation's `reasonCode`, `message` and `tool` where present. A missing `mlxfwmanager` is an unavailable prerequisite, not proof of an incompatible NIC firmware. A denied Redfish firmware collection is different from a component property not being exposed. Conflicting versions or ambiguous activation cannot establish a software minimum.\n\n| Example reason | What to do |\n| --- | --- |\n| `TOOL_NOT_AVAILABLE` / `PERMISSION_DENIED` | Check the named endpoint tool or authorized noninteractive sudo policy; do not treat the unmeasured check as passed. |\n| `ACCESS_DENIED` / `PROPERTY_NOT_EXPOSED` | Distinguish an account permission problem from a property the service does not advertise. |\n| `SOFTWARE_VERSION_CONFLICT` / `SOFTWARE_IDENTITY_UNAVAILABLE` | Reconcile version/activation sources or the rejected physical identity before trusting software comparisons. |\n| `SOFTWARE_BELOW_MINIMUM` | Review the release requirement and plan a separate maintenance action; discovery does not update firmware. |\n| `PUBLICATION_FAILED` | Read the error explanation, resolve the local output/path problem, and rerun; only listed successfully written artifacts exist. |\n\nThe resolved software BOM is saved under `inventory.software`, including source references and unknown requirements. Release `26.4.0` has exact BlueField-3 model entries `B3210`, `B3220` and `B3240`; generic family names and BlueField-4 are not sufficient for BOM applicability. The documented NIC/MFT minima can be evaluated when current, identity-bound measurements exist. BMC, CEC, ATF, UEFI, OS/BSP and golden-image minima remain unknown where the catalog has no supported requirement. A BFB target is a desired artifact, not an observed OS/BSP version. No site-supplied minimum can turn an unknown catalog requirement into qualified support.\n\nExit zero and `discovery.outcome: complete` mean inventory collection, identity/mapping, cleanup and publication succeeded. `acceptance: not-evaluated` remains explicit. Profile planning, hardware qualification, DPF authentication, operator-created connectivity and deployment remain later work; DPF generate/deploy/validate are unavailable in this preview.", + "transitions": [ + { + "to": "finish", + "label": "Complete inventory reviewed" + }, + { + "to": "resolve", + "label": "Required failure or conflict" + } + ] + }, + { + "id": "resolve", + "title": "Resolve the reported prerequisite or ownership issue", + "body": "Use the specific recorded reason to correct the source configuration or ask the infrastructure owner to repair missing tools, permissions, credentials, connectivity or conflicting identity. Do not delete unknown observations or label them passed. Optional gaps may remain documented without blocking inventory; required source failures, identity ambiguity, cleanup or publication errors need resolution. If cleanup is uncertain, use the reported session/resource ownership evidence before any manual action. Rerun after correcting the cause.", + "transitions": [ + { + "to": "prepare", + "label": "Retry after correction" + } + ] + }, + { + "id": "finish", + "title": "Keep the handoff and decide what is next", + "body": "Retain the seed, discovered YAML, JSON report and stderr log together. The discovered YAML preserves configuration intent and replaces observed inventory; it is not a deployment-ready profile or a provisioning approval. If a DPU access address is missing, use [address-candidate lookup](#address-candidates). For later remeasurement of the same estate, use [refresh](#refresh-inventory).", + "transitions": [] + } + ] + }, + { + "id": "address-candidates", + "title": "Find and verify address candidates", + "introduction": "Use this optional inventory addition when you know an ARM or BMC management MAC but need its possible IP address. It is a locator workflow, not authentication or identity discovery. You still need the host configuration from the first workflow; this is not a standalone subnet scanner.", + "steps": [ + { + "id": "locators", + "title": "Supply the management MAC and lookup evidence", + "body": "Obtain the exact ARM or BMC management MAC from authoritative asset records or a trustworthy observed interface. You must choose its role (`arm` or `bmc`) and a unique name. Launch Kit does not infer a management MAC from a NIC serial, data-port MAC, hostname or neighboring address.\n\nOptionally obtain a current dnsmasq IPv4 lease-file snapshot through your authorized DHCP administration process and place it on the CLI machine. Launch Kit reads this file locally; it does not log in to a DHCP server or query arbitrary vendor lease APIs. With no lease files, it can use suitable neighbors observed on the SSH endpoints already selected for inventory. Only `REACHABLE` or `PERMANENT` neighbors are candidates; stale entries are excluded.", + "transitions": [ + { + "to": "configure", + "label": "MAC and source selected" + } + ] + }, + { + "id": "configure", + "title": "Add explicit address selectors to the seed", + "body": "Merge this fragment into the existing `discovery` section of a seed without saved inventory. Replace the MACs and file path; omit `sources` if you only want eligible observed neighbors. Do not replace the required host list.\n\n```yaml\ndiscovery:\n addressLookup:\n sources:\n - file: ./inputs/dnsmasq.leases\n format: dnsmasq-leases\n devices:\n - name: dpu-a-arm\n role: arm\n mac: '02:00:00:00:00:11'\n - name: dpu-a-bmc\n role: bmc\n mac: '02:00:00:00:00:12'\n```\n\nThe supported file contains the five dnsmasq IPv4 columns: expiry epoch, MAC, IP, hostname and client ID. An expiry of `0` means a nonexpiring record; expired records are not candidates. Each file must be readable, at most 1 MiB and at most 4096 records. A malformed file contributes no candidates and retains a source failure reason. Other DHCP formats are unsupported.", + "transitions": [ + { + "to": "run", + "label": "Selectors configured" + } + ] + }, + { + "id": "run", + "title": "Run inventory with lookup enabled", + "body": "Save the configured seed as `dpf-lookup.yaml` and select a new handoff path.\n\n```bash\nl8k discover --target dpf --user-config dpf-lookup.yaml \\\n --save-cluster-config dpf-candidates.yaml --output json \\\n > candidates.json 2> candidates.log\n```\n\nLookup happens as part of this inventory run. It reads only the configured lease files and already observed SSH neighbors. No candidate is contacted and no credential is sent to it.", + "transitions": [ + { + "to": "review", + "label": "Candidate report available" + } + ] + }, + { + "id": "review", + "title": "Review candidates and unresolved lookups", + "body": "Read `inventory.addressLookup.candidates` for the address, selected name/role/MAC, source and evidence reference. Read `inventory.addressLookup.observations` for each device and lease source. A single address is `ADDRESS_UNVERIFIED`; several addresses for a MAC remain ambiguous, and missing/expired/unreadable evidence keeps its reason. A candidate is not a host-to-DPU association and is never a readiness pass.\n\nUse your DHCP/network or asset owner to resolve multiple, missing or contradictory locators. Do not choose an address merely because it is the first result. Independent identity and endpoint trust are still required even for one candidate.", + "transitions": [ + { + "to": "bind", + "label": "Ownership can be independently verified" + }, + { + "to": "locators", + "label": "Missing or ambiguous evidence" + } + ] + }, + { + "id": "bind", + "title": "Authorize an explicit endpoint and rediscover", + "body": "Verify endpoint ownership and obtain SSH known-hosts trust or Redfish CA trust and the appropriate account through your normal process. Create a new seed and add the verified endpoint under `discovery.dpus`, with its credential reference and trust policy. Use a new output when changing scope; do not edit the old observed identity to force a match.\n\nRun [initial inventory](#initial-inventory) for that explicit endpoint. Only its authenticated physical-card observation and unique host-card association can establish the binding. A discovered IP alone never authorizes Redfish access or proves Zero Trust attestation.", + "transitions": [] + } + ] + }, + { + "id": "refresh-inventory", + "title": "Refresh an existing inventory", + "introduction": "Use this workflow to remeasure the same selected estate, compare facts and preserve diagnostic history. A refresh is another live read; old successful observations do not stand in for unavailable current measurements.", + "steps": [ + { + "id": "scope", + "title": "Check scope and preserve source intent", + "body": "Start from the existing discovered YAML and keep its credential/trust references valid. Decide whether you are remeasuring the same host/DPU endpoints and trust policy. If endpoint selection, trust policy or physical identity has changed, create a fresh seed without old `inventory` and use a new output path. Investigate unexpected replacement or ownership changes before accepting them.\n\nDo not manually edit observed serials, associations, firmware or check outcomes. `dpf.release` selects the embedded BOM; changing the release does not upgrade software. An omitted release continues to resolve to the CLI's pinned default, so record the CLI version and review `inventory.software` when comparing runs from different binaries. Explicit release selection makes that intention visible.", + "transitions": [ + { + "to": "run", + "label": "Same scope ready" + }, + { + "to": "new-scope", + "label": "Scope changed" + } + ] + }, + { + "id": "run", + "title": "Run an explicit in-place refresh", + "body": "Use the same YAML path for input and output and explicitly supply `--save-cluster-config`:\n\n```bash\nl8k discover --target dpf --user-config dpf-discovered.yaml \\\n --save-cluster-config dpf-discovered.yaml --output json \\\n > refresh.json 2> refresh.log\n```\n\nThe run retains `source-before.yaml`. Complete discovery replaces inventory while preserving user sections, comments and input intent. Relative file references are rebased if saving elsewhere; private files are not copied. Avoid simultaneous external edits, and use real output directories rather than symlink aliases. Credentials, trust files and configured lease files cannot be used as output destinations.", + "transitions": [ + { + "to": "compare", + "label": "Refresh finished" + } + ] + }, + { + "id": "compare", + "title": "Compare current facts and explain every gap", + "body": "Review `diff`, current checks, `errors`, cleanup and successfully written `artifacts` in the result. Compare physical identity, firmware, relevant hardware/mode/network observations and resolved release requirements. A failed current observation stays unknown rather than inheriting an earlier pass; semantic list ordering alone should not produce a changed fact.\n\nOn partial collection, the previous handoff remains intact and the new partial YAML/report live under `dpf-discovered.discovery//`. Inspect the new report instead of mistaking the unchanged handoff for fresh success. Required failures need correction and another run. Optional missing tools or inaccessible properties remain explained observations; they do not make deployment readiness pass.", + "transitions": [ + { + "to": "retain", + "label": "Complete refresh reviewed" + }, + { + "to": "repair", + "label": "Partial or failed refresh" + } + ] + }, + { + "id": "repair", + "title": "Correct the cause and retry the same scope", + "body": "Use the saved reason and affected source to repair the external prerequisite or fix the input. Preserve the unsuccessful run's report. Do not overwrite failure evidence with an old successful observation. If the cause is a real endpoint, trust or identity change, switch to a fresh seed and output.", + "transitions": [ + { + "to": "run", + "label": "Same scope corrected" + }, + { + "to": "new-scope", + "label": "Actual scope change" + } + ] + }, + { + "id": "new-scope", + "title": "Start a new inventory for the changed estate", + "body": "Create a new seed from the desired access configuration, without copying old `inventory` facts, and select a new handoff path. Recheck endpoint ownership/trust and follow [initial inventory](#initial-inventory). Keep the previous estate's handoff and report as history.", + "transitions": [] + }, + { + "id": "retain", + "title": "Retain the refreshed handoff and reports", + "body": "Keep `source-before.yaml`, the current YAML, JSON/stdout report and stderr log. Discovery completion still means a successful inventory workflow, with deployment `acceptance: not-evaluated`. Unknown BOM constraints, hardware qualification and operator-created service checks remain visible until their later workflows are implemented and verified.", + "transitions": [] + } + ] + } + ] +} diff --git a/scripts/generate-workflow-docs.py b/scripts/generate-workflow-docs.py new file mode 100644 index 00000000..df1e4831 --- /dev/null +++ b/scripts/generate-workflow-docs.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +"""Render repository-owned OpenSpec procedure metadata; never execute commands.""" + +import argparse +import html +import json +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "openspec/workflows/dpf-discovery.json" +OUTPUT = ROOT / "docs/user/dpf-workflows.md" +ID = re.compile(r"^[a-z][a-z0-9-]*$") + + +def text(value, where): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{where} must be nonempty text") + + +def keys(value, required, optional, where): + if not isinstance(value, dict): + raise ValueError(f"{where} must be an object") + if set(value) - required - optional or required - set(value): + raise ValueError(f"{where} has unknown or missing fields") + + +def validate(doc): + keys(doc, {"formatVersion", "title", "introduction", "workflows"}, {"$comment"}, "document") + if type(doc["formatVersion"]) is not int or doc["formatVersion"] != 1: + raise ValueError("unsupported formatVersion") + text(doc["title"], "title") + text(doc["introduction"], "introduction") + if not isinstance(doc["workflows"], list) or not doc["workflows"]: + raise ValueError("workflows must be a nonempty list") + workflow_ids = set() + for workflow in doc["workflows"]: + keys(workflow, {"id", "title", "introduction", "steps"}, set(), "workflow") + identifier = workflow["id"] + if not isinstance(identifier, str) or not ID.fullmatch(identifier) or identifier in workflow_ids: + raise ValueError("invalid or duplicate workflow ID") + workflow_ids.add(identifier) + text(workflow["title"], "workflow title") + text(workflow["introduction"], "workflow introduction") + if not isinstance(workflow["steps"], list) or not workflow["steps"]: + raise ValueError("steps must be a nonempty list") + step_ids = set() + for step in workflow["steps"]: + keys(step, {"id", "title", "body", "transitions"}, set(), "step") + identifier = step["id"] + if not isinstance(identifier, str) or not ID.fullmatch(identifier) or identifier in step_ids: + raise ValueError("invalid or duplicate step ID") + step_ids.add(identifier) + text(step["title"], "step title") + text(step["body"], "step body") + if not isinstance(step["transitions"], list): + raise ValueError("transitions must be a list") + for step in workflow["steps"]: + destinations = set() + for transition in step["transitions"]: + keys(transition, {"to", "label"}, set(), "transition") + if not isinstance(transition["to"], str) or transition["to"] not in step_ids: + raise ValueError("transition points to an unknown step") + text(transition["label"], "transition label") + if transition["to"] in destinations: + raise ValueError("duplicate transition destination") + destinations.add(transition["to"]) + + +def diagram_text(value): + # Titles/labels are data, not Mermaid markup. In particular, newlines and + # quotes cannot terminate a diagram node or inject another graph statement. + return html.escape(" ".join(value.split()), quote=True) + + +def render(doc): + validate(doc) + lines = ["", "", f"# {doc['title']}", "", doc["introduction"].strip(), ""] + for workflow in doc["workflows"]: + lines += [f"## {workflow['title']} {{#{workflow['id']}}}", "", + workflow["introduction"].strip(), "", "```mermaid", "flowchart TD"] + # Sequential node identifiers avoid Mermaid keyword collisions (e.g. end). + nodes = {step["id"]: f"n{i}" for i, step in enumerate(workflow["steps"], 1)} + for step in workflow["steps"]: + lines.append(f' {nodes[step["id"]]}["{diagram_text(step["title"])}"]') + for step in workflow["steps"]: + for transition in step["transitions"]: + lines.append(f' {nodes[step["id"]]} -->|"{diagram_text(transition["label"])}"| {nodes[transition["to"]]}') + lines += ["```", ""] + for number, step in enumerate(workflow["steps"], 1): + lines += [f"### {number}. {step['title']} {{#{workflow['id']}-{step['id']}}}", + "", step["body"].strip(), ""] + if step["transitions"]: + links = [f"[{t['label']}](#{workflow['id']}-{t['to']})" for t in step["transitions"]] + lines += ["Next: " + "; ".join(links) + ".", ""] + return "\n".join(lines) + + +def unique_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON field: {key}") + result[key] = value + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail on drift without writing output") + parser.add_argument("--source", type=Path, default=SOURCE) + parser.add_argument("--output", type=Path, default=OUTPUT) + args = parser.parse_args(argv) + try: + content = render(json.loads(args.source.read_text(encoding="utf-8"), object_pairs_hook=unique_keys)) + if args.check: + if not args.output.is_file() or args.output.read_text(encoding="utf-8") != content: + raise ValueError("workflow documentation is missing or stale; run scripts/generate-workflow-docs.py") + else: + args.output.write_text(content, encoding="utf-8") + except (OSError, ValueError) as error: + print(f"Workflow documentation: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_generate_workflow_docs.py b/scripts/test_generate_workflow_docs.py new file mode 100644 index 00000000..d33e61a8 --- /dev/null +++ b/scripts/test_generate_workflow_docs.py @@ -0,0 +1,84 @@ +# Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +"""Documentation generation contracts; no CLI or infrastructure is invoked.""" + +import copy +from contextlib import redirect_stderr +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest + +SPEC = importlib.util.spec_from_file_location("workflow_docs", Path(__file__).with_name("generate-workflow-docs.py")) +generator = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(generator) + + +class WorkflowDocsTests(unittest.TestCase): + def setUp(self): + self.doc = {"formatVersion": 1, "title": "Procedures", "introduction": "Read first.", "workflows": [ + {"id": "inventory", "title": "Inventory", "introduction": "Collect facts.", "steps": [ + {"id": "run", "title": "Run", "body": "```sh\nprintf 'example only'\n```", "transitions": [{"to": "inspect", "label": "complete"}]}, + {"id": "inspect", "title": "Inspect", "body": "Read the report.", "transitions": []}]}]} + + def test_deterministic_procedure_and_graph(self): + original = copy.deepcopy(self.doc) + rendered = generator.render(self.doc) + self.assertEqual(rendered, generator.render(self.doc)) + self.assertEqual(self.doc, original) + self.assertIn('n1 -->|"complete"| n2', rendered) + self.assertIn("[complete](#inventory-inspect)", rendered) + self.assertIn("printf 'example only'", rendered) + self.assertIn("### 2. Inspect", rendered) + + def test_rejects_duplicate_ids_and_unknown_destinations(self): + for change in ( + lambda d: d["workflows"].append(copy.deepcopy(d["workflows"][0])), + lambda d: d["workflows"][0]["steps"].append(copy.deepcopy(d["workflows"][0]["steps"][0])), + lambda d: d["workflows"][0]["steps"][0]["transitions"][0].update(to="absent"), + ): + doc = copy.deepcopy(self.doc) + change(doc) + with self.assertRaises(ValueError): + generator.render(doc) + + def test_rejects_malformed_shapes_and_unknown_fields(self): + for change in ( + lambda d: d.update(formatVersion=True), + lambda d: d.update(workflows=[]), + lambda d: d.update(command="must not execute"), + lambda d: d["workflows"][0].update(steps=[]), + lambda d: d["workflows"][0]["steps"][0].update(transitions="bad"), + ): + doc = copy.deepcopy(self.doc) + change(doc) + with self.assertRaises(ValueError): + generator.render(doc) + with self.assertRaises(ValueError): + json.loads('{"id":"one", "id":"two"}', object_pairs_hook=generator.unique_keys) + + def test_escapes_diagram_labels(self): + self.doc["workflows"][0]["steps"][0]["title"] = 'Read "report"\n