diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 88760ceb9d..bf8cd2e33f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1362,6 +1362,25 @@ jobs: - name: Run Protolint run: protolint lint -config_path=.protolint.yaml crates/rpc/proto/ + migration-police: + permissions: + contents: read + needs: + - changes + if: ${{ contains(github.ref, 'pull-request/') }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Check new migration timestamps + run: | + git fetch --no-tags origin main:refs/remotes/origin/main + bash scripts/check-migration-filenames.sh --base origin/main + proto-breaking-changes: name: Proto Breaking Changes Check runs-on: ubuntu-latest @@ -2059,6 +2078,7 @@ jobs: - build-release-artifacts-arm-host - security-secret-scan - lint-police + - migration-police - check-rest-core-proto-sync - build-machine-a-tron - build-mat-k8s-controller diff --git a/AGENTS.md b/AGENTS.md index ecf2db1aa0..4c7517f8a5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -169,6 +169,11 @@ verification expectations. See [`STYLE_GUIDE.md`](STYLE_GUIDE.md) for detailed Rust coding conventions. Make sure to review it to ensure changes meet the expected style of the codebase. +Name new Core database migrations with the fully populated +`YYYYMMDDhhmmss_description.sql` format described in +[`STYLE_GUIDE.md`](STYLE_GUIDE.md#database-migrations). The `migration-police` +CI job checks only newly added migrations, so existing filenames remain accepted. + ### Documentation Give every fenced code block a language identifier. Use `bash` or `sh` for diff --git a/Cargo.lock b/Cargo.lock index a798547c73..71284542b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,7 +199,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -210,7 +210,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1588,6 +1588,8 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "reqwest 0.13.4", + "reqwest-middleware", + "reqwest-tracing", "rustls-pemfile", "rustls-pki-types", "serde", @@ -1599,6 +1601,7 @@ dependencies = [ "tonic", "tower", "tower-http 0.7.0", + "trace-propagation", "tracing", "tracing-log", "tracing-opentelemetry 0.33.0", @@ -2675,8 +2678,10 @@ dependencies = [ "async-trait", "carbide-api-db", "carbide-api-model", + "carbide-credential-rotation", "carbide-health-metrics", "carbide-rack", + "carbide-redfish", "carbide-secrets", "carbide-utils", "carbide-uuid", @@ -3720,7 +3725,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4928,7 +4933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6332,7 +6337,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6834,7 +6839,7 @@ dependencies = [ [[package]] name = "libredfish" version = "0.0.0" -source = "git+https://github.com/NVIDIA/libredfish.git?tag=v0.45.1#7d6a335f7ebe1e489c574f075a9adc1ce171a1bf" +source = "git+https://github.com/NVIDIA/libredfish.git?tag=v0.46.1#cee87be0dd8742d0bab9162f8419c780aaad0e3e" dependencies = [ "chrono", "clap", @@ -7688,7 +7693,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9974,7 +9979,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10042,7 +10047,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10760,7 +10765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -11362,7 +11367,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -12874,7 +12879,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6794e36afa..96f524b0b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ repository = "https://github.com/NVIDIA/infra-controller" [workspace.dependencies] clap = { version = "4", features = ["derive", "env"] } -libredfish = { git = "https://github.com/NVIDIA/libredfish.git", tag = "v0.45.1" } +libredfish = { git = "https://github.com/NVIDIA/libredfish.git", tag = "v0.46.1" } librms = { git = "https://github.com/NVIDIA/nv-rms-client.git", tag = "v0.10.0" } ansi-to-html = "0.2.2" diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md index c5ce25834c..a5be6305a9 100644 --- a/STYLE_GUIDE.md +++ b/STYLE_GUIDE.md @@ -342,6 +342,14 @@ your interface `async` just so you can use the tokio Mutex. That way callers can async themselves. Async work should generally be traceable to some I/O or timer that needs to be used, otherwise code should typically be synchronous. +## Database migrations + +Name new Core database migration files with a fully populated 14-digit timestamp: +`YYYYMMDDhhmmss_description.sql`. Use the actual hour, minute, and second values instead of a +trailing `0000` minute-and-second placeholder so independently authored migrations are less likely +to collide. Existing migration filenames remain unchanged, and migrations already on `main` are +immutable. + ## Database transactions Transactions should be used to group write operations together such that they can be rolled back on failure. But do diff --git a/crates/admin-cli/src/credential/force_bmc/args.rs b/crates/admin-cli/src/credential/force_bmc/args.rs index 04bc751734..7c1422d249 100644 --- a/crates/admin-cli/src/credential/force_bmc/args.rs +++ b/crates/admin-cli/src/credential/force_bmc/args.rs @@ -31,7 +31,10 @@ Force an immediate credential rotation by machine ID: Force a switch BMC by switch ID: $ nico-admin-cli credential force-bmc set --id sw100nt038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg -Force it by BMC MAC instead (machine or switch): +Force a power shelf BMC (PMC) by power shelf ID: + $ nico-admin-cli credential force-bmc set --id ps100ht038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg + +Force it by BMC MAC instead (machine, switch, or power shelf): $ nico-admin-cli credential force-bmc set --bmc-mac 00:11:22:33:44:55 Clear a pending force-converge request: @@ -40,13 +43,11 @@ Clear a pending force-converge request: ")] pub enum Args { #[clap( - about = "Request an immediate BMC credential rotation. Machines and switches are \ - supported; power shelf IDs are accepted but not yet supported." + about = "Request an immediate BMC credential rotation of a machine, switch, or power shelf." )] Set(ForceSet), #[clap( - about = "Clear a pending BMC force-converge request. Machines and switches are supported; \ - power shelf IDs are accepted but not yet supported." + about = "Clear a pending BMC force-converge request for a machine, switch, or power shelf." )] Clear(ForceClear), } @@ -61,7 +62,10 @@ Force-converge a machine BMC now by machine ID: Force-converge a switch BMC now by switch ID: $ nico-admin-cli credential force-bmc set --id sw100nt038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg -Force-converge a BMC now by BMC MAC (machine or switch): +Force-converge a power shelf BMC (PMC) now by power shelf ID: + $ nico-admin-cli credential force-bmc set --id ps100ht038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg + +Force-converge a BMC now by BMC MAC (machine, switch, or power shelf): $ nico-admin-cli credential force-bmc set --bmc-mac 00:11:22:33:44:55 ")] @@ -71,14 +75,13 @@ pub struct ForceSet { long, required_unless_present_any = ["bmc_mac"], help = "ID of the machine, DPU, switch, or power shelf that owns the BMC. \ - Power shelf IDs are allowed for forward compatibility but are not yet supported. \ Provide this or --bmc-mac." )] pub id: Option, #[clap( long, - help = "MAC of the BMC to target (machine or switch). Provide this \ + help = "MAC of the BMC to target (machine, switch, or power shelf). Provide this \ or --id; if an id is also given they must identify the same device." )] pub bmc_mac: Option, @@ -104,6 +107,9 @@ Clear a pending force-converge request by machine ID: Clear a pending force-converge request by switch ID: $ nico-admin-cli credential force-bmc clear --id sw100nt038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg +Clear a pending force-converge request by power shelf ID: + $ nico-admin-cli credential force-bmc clear --id ps100ht038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg + Clear a pending force-converge request by BMC MAC: $ nico-admin-cli credential force-bmc clear --bmc-mac 00:11:22:33:44:55 @@ -114,8 +120,7 @@ pub struct ForceClear { long, required_unless_present_any = ["bmc_mac"], help = "Machine, DPU, switch, or power shelf ID whose pending BMC force-converge request \ - should be cleared. Power shelf IDs are allowed for forward compatibility but are \ - not yet supported. Provide this or --bmc-mac." + should be cleared. Provide this or --bmc-mac." )] pub id: Option, @@ -171,9 +176,11 @@ mod tests { } #[test] - fn power_shelf_limit_is_documented() { + fn power_shelf_is_a_documented_target() { let help = ForceSet::command().render_long_help().to_string(); - assert!(help.contains("Power shelf IDs are allowed for forward compatibility")); - assert!(help.contains("not yet supported")); + assert!( + help.contains("power shelf"), + "force-bmc help should document power shelf as a supported target" + ); } } diff --git a/crates/admin-cli/src/rpc.rs b/crates/admin-cli/src/rpc.rs index e9a0e3e869..9ba333ac9b 100644 --- a/crates/admin-cli/src/rpc.rs +++ b/crates/admin-cli/src/rpc.rs @@ -1238,6 +1238,7 @@ impl ApiClient { .create_vpc(VpcCreationRequest { vni: None, routing_profile_type: None, + routing_profile_overrides: None, tenant_organization_id: "devenv_test_org".to_string(), tenant_keyset_id: None, network_virtualization_type: Some( @@ -1296,6 +1297,7 @@ impl ApiClient { .create_vpc(VpcCreationRequest { vni: None, routing_profile_type: None, + routing_profile_overrides: None, tenant_organization_id: "devenv_test_org".to_string(), tenant_keyset_id: None, network_virtualization_type: Some(VpcVirtualizationType::Flat.into()), diff --git a/crates/admin-cli/src/vpc/create/args.rs b/crates/admin-cli/src/vpc/create/args.rs index 4b27155fc1..45729b991d 100644 --- a/crates/admin-cli/src/vpc/create/args.rs +++ b/crates/admin-cli/src/vpc/create/args.rs @@ -75,6 +75,7 @@ impl From for forge::VpcCreationRequest { default_nvlink_logical_partition_id: None, vni: None, routing_profile_type: None, + routing_profile_overrides: None, } } } diff --git a/crates/admin-cli/src/vpc/show/cmd.rs b/crates/admin-cli/src/vpc/show/cmd.rs index 52721bf4c8..ca44c5f0e3 100644 --- a/crates/admin-cli/src/vpc/show/cmd.rs +++ b/crates/admin-cli/src/vpc/show/cmd.rs @@ -111,6 +111,7 @@ fn vpc_config(vpc: &forgerpc::Vpc) -> forgerpc::VpcConfig { default_nvlink_logical_partition_id: vpc.default_nvlink_logical_partition_id, vni: vpc.vni, routing_profile_type: vpc.routing_profile_type.clone(), + routing_profile_overrides: None, } } } @@ -184,6 +185,19 @@ pub fn convert_vpc_to_nice_format(vpc: &forgerpc::Vpc) -> CarbideCliResult CarbideCliResult CarbideCliResult` | — | `hardware` | Component manager for NvLink switches and power shelves. | -| `vpcs` | `Option>` | — | `networking` | VPCs to create at startup. Use the `CreateVpc` gRPC to create them later instead. | +| `vpcs` | `Option>` | — | `networking` | VPCs to create at startup (see [VpcDefinition](#vpcdefinition)). Use the `CreateVpc` gRPC to create them later instead. | | `allow_bmc_basic_auth_fallback` | `bool` | `false` | `security` | When `true`, `GetBmcCredentials` may return `UsernamePassword` credentials for BMCs whose Redfish ServiceRoot does not expose `SessionService`. When `false`, such BMCs surface a `NoSessionService` error and no basic-auth fallback is performed. | | `rack_validation_config` | `RackValidationConfig` | *(default)* | `hardware` | Rack-level validation: multi-node partition tests after firmware upgrade and maintenance to verify rack health (see [RackValidationConfig](#rackvalidationconfig)). | | `oem_manager_profiles` | `BiosProfileVendor` | `{}` | `machines` | Vendor-specific iDRAC/BMC manager attributes applied during machine setup, before BMC lockdown. Keyed by vendor → model → profile → attribute name; targets the manager OEM attributes endpoint (e.g. Dell `DellAttributes`), as opposed to `bios_profiles` which targets BIOS settings. Model names are normalized to lowercase with underscores (e.g. `"PowerEdge R760"` → `"poweredge_r760"`). | @@ -516,15 +516,29 @@ client-certificate authentication is not used. | Field | Type | Default | Description | |-------|------|---------|-------------| -| `route_target_imports` | `Vec` | `[]` | Route targets imported into DPU VRFs for VPC routes. | -| `route_targets_on_exports` | `Vec` | `[]` | Route targets added to routes exported by the DPU. | -| `internal` | `bool` | `false` | Whether the profile uses internal VNI allocation. | -| `leak_default_route_from_underlay` | `bool` | `false` | Leak the default route from the underlay/default VRF into tenant VRFs. | -| `leak_tenant_host_routes_to_underlay` | `bool` | `false` | Leak tenant host routes into the underlay/default VRF. | -| `tenant_leak_communities_accepted` | `bool` | `false` | Honor route-leak communities sent by the tenant host OS. | -| `accepted_leaks_from_underlay` | `Vec` | `[]` | Specific underlay/default VRF prefixes allowed to leak into tenant VRFs. Routing only; does not affect ACLs. | -| `allowed_anycast_prefixes` | `Vec` | `[]` | IPv4 or IPv6 prefixes that tenant hosts are allowed to announce to the DPU as anycast routes. | -| `access_tier` | `u32` | `0` | Routing profile access tier. Lower values grant broader access. | +| `route_target_imports` | `Option>` | — (effective `[]`) | Route targets imported into DPU VRFs for VPC routes. | +| `route_targets_on_exports` | `Option>` | — (effective `[]`) | Route targets added to routes exported by the DPU. | +| `internal` | `Option` | — (effective `false`) | Whether the profile uses internal VNI allocation. This property cannot be overridden on a VPC. | +| `leak_default_route_from_underlay` | `Option` | — (effective `false`) | Leak the default route from the underlay/default VRF into tenant VRFs. | +| `leak_tenant_host_routes_to_underlay` | `Option` | — (effective `false`) | Leak tenant host routes into the underlay/default VRF. | +| `tenant_leak_communities_accepted` | `Option` | — (effective `false`) | Honor route-leak communities sent by the tenant host OS. | +| `accepted_leaks_from_underlay` | `Option>` | — (effective `[]`) | Specific underlay/default VRF prefixes allowed to leak into tenant VRFs. Routing only; does not affect ACLs. | +| `allowed_anycast_prefixes` | `Option>` | — (effective `[]`) | IPv4 or IPv6 prefixes that tenant hosts are allowed to announce to the DPU as anycast routes. | +| `access_tier` | `Option` | — (effective `0`) | Routing profile access tier. Lower values grant broader access. This property cannot be overridden on a VPC. | + +Unset properties retain presence information so a VPC's inline +`routing_profile_overrides` can inherit them. After the named profile and VPC +override are combined, properties still unset use the effective defaults above. + +### `VpcDefinition` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `organization_id` | `Option` | — | Tenant organization that owns the seeded VPC. | +| `network_virtualization_type` | `VpcVirtualizationType` | **required** | Data plane used by the VPC. | +| `routing_profile_type` | `Option` | — | Named FNN routing profile recorded on the seeded VPC. | +| `routing_profile_overrides` | `Option` | — | Unsupported for seeded VPCs. Any configured value causes startup to fail; inline overrides are accepted only by VPC creation requests. | +| `vni` | `Option` | — | Desired VNI; when absent, one is allocated. | ### `PrefixFilterPolicyEntry` diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index f4035a566c..de9bba0c75 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -15,6 +15,7 @@ * limitations under the License. */ +use std::borrow::Cow; use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -62,9 +63,13 @@ use model::network_security_group::NetworkSecurityGroupRule; use model::network_segment::NetworkDefinition; use model::resource_pool::define::ResourcePoolDef; use model::tenant::identity_config::SigningAlgorithm; +use model::vpc::VpcConfig; +pub use model::vpc::{PrefixFilterPolicyEntry, RouteTargetConfig}; use regex::Regex; use serde::{Deserialize, Deserializer, Serialize}; +use crate::CarbideError; + pub(crate) const DEFAULT_DPU_NUM_OF_VFS: u32 = 16; pub(crate) const MAX_DPU_NUM_OF_VFS: u32 = 126; @@ -1930,17 +1935,6 @@ pub struct SpdmConfig { pub nras_config: Option, } -/// A BGP route target used in FNN VRF import/export policies. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub struct RouteTargetConfig { - /// Autonomous System Number component of the route target. - #[serde(default)] - pub asn: u32, - /// Virtual Network Identifier component of the route target. - #[serde(default)] - pub vni: u32, -} - /// Fabric Nearest Neighbor (FNN) configuration for L3 VNI-based overlay networking. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct FnnConfig { @@ -1970,35 +1964,37 @@ pub struct FnnConfig { pub use_vpc_vrf_loopback: bool, } +/// A named routing-profile definition whose unset properties use effective +/// defaults unless a VPC supplies an inline override. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Default)] pub struct FnnRoutingProfileConfig { /// These are used for import policies to import routes /// that match these targets. #[serde(default)] - pub route_target_imports: Vec, + pub route_target_imports: Option>, /// These are used for tagging routes exported by the DPU #[serde(default)] - pub route_targets_on_exports: Vec, + pub route_targets_on_exports: Option>, /// Is this an internal or external tenant/VPC profile #[serde(default)] - pub internal: bool, + pub internal: Option, /// Should DPUs leak the default route from the /// underlay into the tenant VRF? #[serde(default)] - pub leak_default_route_from_underlay: bool, + pub leak_default_route_from_underlay: Option, /// Should DPUs leak the routes for the host IPs into /// into the underlay? #[serde(default)] - pub leak_tenant_host_routes_to_underlay: bool, + pub leak_tenant_host_routes_to_underlay: Option, /// Are route-leak communities sent by the host OS honored by the DPU for allowing /// routes advertised by the host OS to be leaked into the underlay? #[serde(default)] - pub tenant_leak_communities_accepted: bool, + pub tenant_leak_communities_accepted: Option, /// An explicit/granular list of prefixes that should /// be allowed to leak from the default VRF into the tenant @@ -2007,12 +2003,12 @@ pub struct FnnRoutingProfileConfig { /// These are purely for routing purposes and will not have any /// impact on ACLs. #[serde(default)] - pub accepted_leaks_from_underlay: Vec, + pub accepted_leaks_from_underlay: Option>, /// Prefixes that tenant hosts are allowed to announce /// to the DPU as anycast routes. #[serde(default)] - pub allowed_anycast_prefixes: Vec, + pub allowed_anycast_prefixes: Option>, /// Currently controls which profiles a tenant can use /// when creating VPCs. Lower value means broader access. @@ -2024,17 +2020,83 @@ pub struct FnnRoutingProfileConfig { /// - A tenant with ADMIN could create ADMIN VPCs and INTERNAL VPCs. /// - A tenant with INTERNAL could only create INTERNAL VPCs. #[serde(default)] - pub access_tier: u32, + pub access_tier: Option, +} + +impl FnnConfig { + /// Resolves the named routing profile and applies properties set on the VPC. + pub(crate) fn resolve_vpc_routing_profile( + &self, + vpc: &VpcConfig, + ) -> Result, CarbideError> { + let profile_type = + vpc.routing_profile_type + .as_ref() + .ok_or_else(|| CarbideError::Internal { + message: "tenant routing profile type not found in VPC record".to_string(), + })?; + let base_profile = + self.routing_profiles + .get(profile_type) + .ok_or_else(|| CarbideError::NotFoundError { + kind: "routing_profile_type", + id: profile_type.to_string(), + })?; + + // Apply properties explicitly set on the VPC over the named base profile. + let Some(overrides) = vpc.routing_profile_overrides.as_ref() else { + return Ok(Cow::Borrowed(base_profile)); + }; + + Ok(Cow::Owned(FnnRoutingProfileConfig { + route_target_imports: overrides + .route_target_imports + .clone() + .or_else(|| base_profile.route_target_imports.clone()), + route_targets_on_exports: overrides + .route_targets_on_exports + .clone() + .or_else(|| base_profile.route_targets_on_exports.clone()), + // VPCs must inherit the base profile's allocation and access controls. + internal: base_profile.internal, + leak_default_route_from_underlay: overrides + .leak_default_route_from_underlay + .or(base_profile.leak_default_route_from_underlay), + leak_tenant_host_routes_to_underlay: overrides + .leak_tenant_host_routes_to_underlay + .or(base_profile.leak_tenant_host_routes_to_underlay), + tenant_leak_communities_accepted: overrides + .tenant_leak_communities_accepted + .or(base_profile.tenant_leak_communities_accepted), + accepted_leaks_from_underlay: overrides + .accepted_leaks_from_underlay + .clone() + .or_else(|| base_profile.accepted_leaks_from_underlay.clone()), + allowed_anycast_prefixes: overrides + .allowed_anycast_prefixes + .clone() + .or_else(|| base_profile.allowed_anycast_prefixes.clone()), + access_tier: base_profile.access_tier, + })) + } } impl From<&FnnRoutingProfileConfig> for rpc::forge::RoutingProfile { fn from(profile: &FnnRoutingProfileConfig) -> Self { Self { - tenant_leak_communities_accepted: profile.tenant_leak_communities_accepted, - leak_default_route_from_underlay: profile.leak_default_route_from_underlay, - leak_tenant_host_routes_to_underlay: profile.leak_tenant_host_routes_to_underlay, + tenant_leak_communities_accepted: profile + .tenant_leak_communities_accepted + .unwrap_or_default(), + leak_default_route_from_underlay: profile + .leak_default_route_from_underlay + .unwrap_or_default(), + leak_tenant_host_routes_to_underlay: profile + .leak_tenant_host_routes_to_underlay + .unwrap_or_default(), accepted_leaks_from_underlay: profile .accepted_leaks_from_underlay + .as_deref() + .unwrap_or_default() .iter() .map(|entry| rpc::forge::PrefixFilterPolicyEntry { prefix: entry.prefix.to_string(), @@ -2042,6 +2104,8 @@ impl From<&FnnRoutingProfileConfig> for rpc::forge::RoutingProfile { .collect(), allowed_anycast_prefixes: profile .allowed_anycast_prefixes + .as_deref() + .unwrap_or_default() .iter() .map(|entry| rpc::forge::PrefixFilterPolicyEntry { prefix: entry.prefix.to_string(), @@ -2049,6 +2113,8 @@ impl From<&FnnRoutingProfileConfig> for rpc::forge::RoutingProfile { .collect(), route_target_imports: profile .route_target_imports + .as_deref() + .unwrap_or_default() .iter() .map(|route_target| rpc::common::RouteTarget { asn: route_target.asn, @@ -2057,6 +2123,8 @@ impl From<&FnnRoutingProfileConfig> for rpc::forge::RoutingProfile { .collect(), route_targets_on_exports: profile .route_targets_on_exports + .as_deref() + .unwrap_or_default() .iter() .map(|route_target| rpc::common::RouteTarget { asn: route_target.asn, @@ -2067,13 +2135,22 @@ impl From<&FnnRoutingProfileConfig> for rpc::forge::RoutingProfile { } } -/// Entries used for prefix-list policies on the DPUS. -/// Default behavior is max-len lte 32 -/// We can change that with additional fields on this struct -/// if necessary in the future. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub struct PrefixFilterPolicyEntry { - pub prefix: IpNetwork, +impl From<&FnnRoutingProfileConfig> for rpc::forge::VpcEffectiveRoutingProfile { + fn from(profile: &FnnRoutingProfileConfig) -> Self { + let routing_profile = rpc::forge::RoutingProfile::from(profile); + Self { + route_target_imports: routing_profile.route_target_imports, + route_targets_on_exports: routing_profile.route_targets_on_exports, + leak_default_route_from_underlay: routing_profile.leak_default_route_from_underlay, + leak_tenant_host_routes_to_underlay: routing_profile + .leak_tenant_host_routes_to_underlay, + tenant_leak_communities_accepted: routing_profile.tenant_leak_communities_accepted, + accepted_leaks_from_underlay: routing_profile.accepted_leaks_from_underlay, + allowed_anycast_prefixes: routing_profile.allowed_anycast_prefixes, + internal: profile.internal.unwrap_or_default(), + access_tier: profile.access_tier.unwrap_or_default(), + } + } } /// FNN configuration specific to the admin network. @@ -3480,12 +3557,206 @@ mod tests { use model::expected_machine::HostDpuPolicy; use model::network_segment::NetworkDefinitionSegmentType; use model::resource_pool; + use model::vpc::VpcRoutingProfileOverrides; use super::*; use crate::test_support::network_segment::FIXTURE_TENANT_ORG_ID; const TEST_DATA_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/src/cfg/test_data"); + fn vpc_config( + routing_profile_type: Option<&str>, + routing_profile_overrides: Option, + ) -> VpcConfig { + VpcConfig { + tenant_organization_id: "test-tenant".to_string(), + tenant_keyset_id: None, + network_virtualization_type: VpcVirtualizationType::Fnn, + network_security_group_id: None, + default_nvlink_logical_partition_id: None, + vni: None, + routing_profile_type: routing_profile_type.map(str::to_string), + routing_profile_overrides, + } + } + + /// Verifies existing routing-profile TOML values deserialize unchanged + /// after the fields become presence-aware. + #[test] + fn fnn_routing_profile_options_accept_existing_toml_syntax() { + let profile: FnnRoutingProfileConfig = Figment::new() + .merge(Toml::string( + r#" + route_target_imports = [{ asn = 64512, vni = 10 }] + route_targets_on_exports = [] + internal = true + leak_default_route_from_underlay = false + leak_tenant_host_routes_to_underlay = true + tenant_leak_communities_accepted = false + accepted_leaks_from_underlay = [{ prefix = "10.0.0.0/8" }] + allowed_anycast_prefixes = [] + access_tier = 2 + "#, + )) + .extract() + .expect("existing routing-profile syntax must remain valid"); + + assert_eq!( + profile, + FnnRoutingProfileConfig { + route_target_imports: Some(vec![RouteTargetConfig { + asn: 64512, + vni: 10, + }]), + route_targets_on_exports: Some(vec![]), + internal: Some(true), + leak_default_route_from_underlay: Some(false), + leak_tenant_host_routes_to_underlay: Some(true), + tenant_leak_communities_accepted: Some(false), + accepted_leaks_from_underlay: Some(vec![PrefixFilterPolicyEntry { + prefix: "10.0.0.0/8".parse().expect("valid test prefix"), + }]), + allowed_anycast_prefixes: Some(vec![]), + access_tier: Some(2), + } + ); + } + + /// Verifies seed-time VPC TOML preserves unsupported inline overrides + /// so startup validation can reject them instead of silently ignoring them. + #[test] + fn vpc_definition_preserves_routing_profile_overrides_for_seed_validation() { + // Parse a seeded VPC with representative unsupported override values. + let config: InitialObjectsConfig = Figment::new() + .merge(Toml::string( + r#" + [vpcs.inline-profile] + organization_id = "inline-profile-test" + network_virtualization_type = "fnn" + routing_profile_type = "BASE" + + [vpcs.inline-profile.routing_profile_overrides] + route_target_imports = [] + leak_default_route_from_underlay = false + allowed_anycast_prefixes = [{ prefix = "192.0.2.0/24" }] + "#, + )) + .extract() + .expect("seed validation must receive configured routing-profile overrides"); + let definition = config + .vpcs + .as_ref() + .expect("configured VPCs") + .get("inline-profile") + .expect("inline-profile VPC"); + + // Explicit empty and false values remain visible to startup validation. + assert_eq!( + definition, + &VpcDefinition { + organization_id: Some("inline-profile-test".to_string()), + network_virtualization_type: VpcVirtualizationType::Fnn, + routing_profile_type: Some("BASE".to_string()), + routing_profile_overrides: Some(VpcRoutingProfileOverrides { + route_target_imports: Some(vec![]), + leak_default_route_from_underlay: Some(false), + allowed_anycast_prefixes: Some(vec![PrefixFilterPolicyEntry { + prefix: "192.0.2.0/24".parse().expect("valid test prefix"), + }]), + ..Default::default() + }), + vni: None, + } + ); + } + + /// Verifies VPC properties override only present fields while `internal` + /// and `access_tier` remain owned by the base profile. + #[test] + fn vpc_routing_profile_overrides_are_presence_aware() { + // Build a complete base and an override containing explicit default values. + let inherited_export = RouteTargetConfig { asn: 1, vni: 2 }; + let inherited_anycast = PrefixFilterPolicyEntry { + prefix: "192.0.2.0/24".parse().expect("valid test prefix"), + }; + let base = FnnRoutingProfileConfig { + route_target_imports: Some(vec![RouteTargetConfig { asn: 3, vni: 4 }]), + route_targets_on_exports: Some(vec![inherited_export.clone()]), + internal: Some(true), + leak_default_route_from_underlay: Some(true), + leak_tenant_host_routes_to_underlay: Some(true), + tenant_leak_communities_accepted: Some(true), + accepted_leaks_from_underlay: Some(vec![PrefixFilterPolicyEntry { + prefix: "198.51.100.0/24".parse().expect("valid test prefix"), + }]), + allowed_anycast_prefixes: Some(vec![inherited_anycast.clone()]), + access_tier: Some(2), + }; + let overrides = VpcRoutingProfileOverrides { + route_target_imports: Some(vec![]), + leak_default_route_from_underlay: Some(false), + tenant_leak_communities_accepted: Some(false), + accepted_leaks_from_underlay: Some(vec![]), + ..Default::default() + }; + let fnn = FnnConfig { + admin_vpc: None, + common_internal_route_target: None, + additional_route_target_imports: vec![], + routing_profiles: HashMap::from([("BASE".to_string(), base)]), + use_vpc_vrf_loopback: false, + }; + let vpc = vpc_config(Some("BASE"), Some(overrides)); + + // Explicit empty and false values override; absent values inherit. + assert_eq!( + fnn.resolve_vpc_routing_profile(&vpc).unwrap().as_ref(), + &FnnRoutingProfileConfig { + route_target_imports: Some(vec![]), + route_targets_on_exports: Some(vec![inherited_export]), + internal: Some(true), + leak_default_route_from_underlay: Some(false), + leak_tenant_host_routes_to_underlay: Some(true), + tenant_leak_communities_accepted: Some(false), + accepted_leaks_from_underlay: Some(vec![]), + allowed_anycast_prefixes: Some(vec![inherited_anycast]), + access_tier: Some(2), + } + ); + } + + #[test] + fn vpc_routing_profile_resolution_reports_consistent_errors() { + let fnn = FnnConfig { + admin_vpc: None, + common_internal_route_target: None, + additional_route_target_imports: vec![], + routing_profiles: HashMap::new(), + use_vpc_vrf_loopback: false, + }; + + check_values( + [ + Check { + scenario: "routing profile type absent from VPC", + input: None, + expect: "internal error: tenant routing profile type not found in VPC record" + .to_string(), + }, + Check { + scenario: "named routing profile absent from FNN config", + input: Some("MISSING"), + expect: "routing_profile_type not found: MISSING".to_string(), + }, + ], + |profile_type| { + fnn.resolve_vpc_routing_profile(&vpc_config(profile_type, None)) + .unwrap_err() + .to_string() + }, + ); + } + #[test] fn deny_prefixes_accept_both_address_families() { let config: CarbideConfig = Figment::new() @@ -5416,6 +5687,7 @@ firmware_url = "https://firmware.example.com/fw-b.bin" organization_id: Some(FIXTURE_TENANT_ORG_ID.to_string()), network_virtualization_type: VpcVirtualizationType::Flat, routing_profile_type: None, + routing_profile_overrides: None, vni: None, } ); diff --git a/crates/api-core/src/db_init.rs b/crates/api-core/src/db_init.rs index 48e82afc4f..8cb51dd9eb 100644 --- a/crates/api-core/src/db_init.rs +++ b/crates/api-core/src/db_init.rs @@ -195,11 +195,31 @@ pub async fn create_initial_networks( Ok(()) } +pub(crate) fn validate_initial_vpcs( + vpcs: &HashMap, +) -> Result<(), model::ConfigValidationError> { + for (name, definition) in vpcs { + // Inline overrides are supported only by runtime VPC creation requests. + if definition.routing_profile_overrides.is_some() { + return Err( + model::ConfigValidationError::InitialVpcRoutingProfileOverridesUnsupported { + name: name.clone(), + }, + ); + } + } + Ok(()) +} + pub async fn create_initial_vpcs( db_pool: &Pool, vpcs: &HashMap, vni_pool: &ResourcePool, ) -> Result<(), CarbideError> { + // Retain validation at the mutation boundary as defense in depth. Startup + // also validates during SeedData resolution, before any reconciliation. + validate_initial_vpcs(vpcs).map_err(CarbideError::InvalidConfiguration)?; + let mut txn = Transaction::begin(db_pool).await?; for (name, def) in vpcs { if db::vpc::find_by_name(&mut txn, name) @@ -248,11 +268,12 @@ pub async fn create_initial_vpcs( }, network_security_group_id: None, routing_profile_type: def.routing_profile_type.clone(), + routing_profile_overrides: def.routing_profile_overrides.clone(), vni: Some(vni), }; // Validation - if def.routing_profile_type.is_some() { + if def.routing_profile_type.is_some() || def.routing_profile_overrides.is_some() { def.network_virtualization_type .ensure_supports_routing_profiles() .map_err(CarbideError::from)?; @@ -539,6 +560,7 @@ pub(crate) async fn create_admin_vpc( // For consistency, but admin routing profile is defined in-line in the // FNN config. routing_profile_type: None, // It's purely informational. Admin profile is pulled from an inline-config and not tied to a name or ID. + routing_profile_overrides: None, network_security_group_id: None, network_virtualization_type: carbide_network::virtualization::VpcVirtualizationType::Fnn, metadata: Metadata { diff --git a/crates/api-core/src/dpf_services.rs b/crates/api-core/src/dpf_services.rs index deb9155d65..e3342b2af6 100644 --- a/crates/api-core/src/dpf_services.rs +++ b/crates/api-core/src/dpf_services.rs @@ -26,7 +26,9 @@ use carbide_dpf::types::{ DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_NAME, DOCA_XPLANE_SERVICE_NAME, DPU_AGENT_SERVICE_NAME, DTS_SERVICE_NAME, FMDS_SERVICE_NAME, OTEL_COLLECTOR_SERVICE_NAME, }; -use carbide_dpf::{ServiceDefinition, ServiceInterface, ServiceNAD, ServiceNADResourceType}; +use carbide_dpf::{ + IntOrString, ServiceDefinition, ServiceInterface, ServiceNAD, ServiceNADResourceType, +}; use crate::cfg::file::{ DpfBootstrapCaObjectKind, DpfDpuAgentBootstrapCa, DpfExtraService, @@ -45,6 +47,10 @@ pub const DEFAULT_DOCA_IMAGE_REGISTRY: &str = "nvcr.io/nvidia/doca"; /// Default Carbide container image registry prefix. pub const DEFAULT_CARBIDE_IMAGE_REGISTRY: &str = "nvcr.io/0837451325059433/carbide-dev"; +/// Astra Service Helm and Image Registries +pub const DOCA_WEAVE_CHART_REPO_URL: &str = "oci://harbor.mellanox.com/cloud-orchestration-dev/dpf"; +pub const DOCA_WEAVE_IMAGE_REGISTRY: &str = "nvcr.io/nvstaging/doca"; + /// HBN service Definitions pub const DOCA_HBN_SERVICE_HELM_NAME: &str = "doca-hbn"; pub const DOCA_HBN_SERVICE_HELM_VERSION: &str = "3.4.0"; @@ -78,16 +84,16 @@ pub const OTEL_COLLECTOR_SERVICE_HELM_NAME: &str = "nico-otelcol"; pub const OTEL_COLLECTOR_SERVICE_IMAGE_NAME: &str = "otelcol-contrib"; /// Weave DHCP agent service definitions. -pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_HELM_NAME: &str = "doca-weave-dhcp-agent"; -pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_HELM_VERSION: &str = "1.0"; -pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_NAME: &str = "doca_weave_dhcp_agent"; -pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_TAG: &str = "3.2.1-doca3.2.1"; +pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_HELM_NAME: &str = "dpf-weave"; +pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_HELM_VERSION: &str = "v26.5.0-1f8f4e1e"; +pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_NAME: &str = "weave-system"; +pub const DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_TAG: &str = "v26.5.0-f2c9f7c4-nightly"; /// Weave flow (ovs) controller service definitions. -pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_HELM_NAME: &str = "doca-weave-flow-controller"; -pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_HELM_VERSION: &str = "1.0"; -pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_NAME: &str = "doca_weave_flow_controller"; -pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_TAG: &str = "3.2.1-doca3.2.1"; +pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_HELM_NAME: &str = "dpf-weave"; +pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_HELM_VERSION: &str = "v26.5.0-1f8f4e1e"; +pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_NAME: &str = "weave-system"; +pub const DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_TAG: &str = "v26.5.0-f2c9f7c4-nightly"; /// Xplane service definitions. pub const DOCA_XPLANE_SERVICE_HELM_NAME: &str = "doca-xplane"; @@ -237,11 +243,11 @@ pub(crate) fn default_otelcol_service() -> DpfServiceConfig { pub(crate) fn default_doca_weave_dhcp_agent_service() -> DpfServiceConfig { DpfServiceConfig { name: DOCA_WEAVE_DHCP_AGENT_SERVICE_NAME.to_string(), - helm_repo_url: DEFAULT_DOCA_HELM_REGISTRY.to_string(), + helm_repo_url: DOCA_WEAVE_CHART_REPO_URL.to_string(), helm_chart: DOCA_WEAVE_DHCP_AGENT_SERVICE_HELM_NAME.to_string(), helm_version: DOCA_WEAVE_DHCP_AGENT_SERVICE_HELM_VERSION.to_string(), docker_repo_url: format!( - "{DEFAULT_DOCA_IMAGE_REGISTRY}/{DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_NAME}" + "{DOCA_WEAVE_IMAGE_REGISTRY}/{DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_NAME}" ), docker_image_tag: DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_TAG.to_string(), docker_image_pull_secret: None, @@ -251,11 +257,11 @@ pub(crate) fn default_doca_weave_dhcp_agent_service() -> DpfServiceConfig { pub(crate) fn default_doca_weave_flow_controller_service() -> DpfServiceConfig { DpfServiceConfig { name: DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_NAME.to_string(), - helm_repo_url: DEFAULT_DOCA_HELM_REGISTRY.to_string(), + helm_repo_url: DOCA_WEAVE_CHART_REPO_URL.to_string(), helm_chart: DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_HELM_NAME.to_string(), helm_version: DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_HELM_VERSION.to_string(), docker_repo_url: format!( - "{DEFAULT_DOCA_IMAGE_REGISTRY}/{DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_NAME}" + "{DOCA_WEAVE_IMAGE_REGISTRY}/{DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_NAME}" ), docker_image_tag: DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_TAG.to_string(), docker_image_pull_secret: None, @@ -534,14 +540,33 @@ pub fn otelcol_service(cfg: &DpfServiceConfig) -> ServiceDefinition { pub fn doca_weave_dhcp_agent_service(cfg: &DpfServiceConfig) -> ServiceDefinition { let mut helm_values = serde_json::json!({ - "image": { - "repository": cfg.docker_repo_url, - "tag": cfg.docker_image_tag, + "weaveDHCPAgent": { + "containers": { + "weaveDHCPAgent": { + "image": { + "repository": cfg.docker_repo_url, + "tag": cfg.docker_image_tag, + } + } + } } }); apply_image_pull_secrets(&mut helm_values, cfg); ServiceDefinition { helm_values: Some(helm_values), + config_values: Some(serde_json::json!({ + "weaveDHCPAgent": { + "enabled": true, + "dhcpNetworks": { + "createNADs": true, + "networks": weave_dhcp_agent_networks(), + } + } + })), + service_daemon_set_resources: Some(BTreeMap::from([( + "nvidia.com/bf_sf".to_string(), + IntOrString::Int(WEAVE_DHCP_AGENT_NETWORKS.len() as i32), + )])), ..ServiceDefinition::new( &cfg.name, &cfg.helm_repo_url, @@ -551,16 +576,77 @@ pub fn doca_weave_dhcp_agent_service(cfg: &DpfServiceConfig) -> ServiceDefinitio } } +const WEAVE_DHCP_AGENT_NETWORKS: &[&str] = &[ + "r0swpln0", "r1swpln0", "r0swpln1", "r1swpln1", "r2swpln0", "r3swpln0", "r2swpln1", "r3swpln1", +]; + +fn weave_dhcp_agent_networks() -> Vec { + WEAVE_DHCP_AGENT_NETWORKS + .iter() + .map(|port| { + serde_json::json!({ + "name": format!("dhcp-{port}"), + "bridge": format!("br-dhcp-{port}"), + "resourceName": "nvidia.com/bf_sf", + "interfaceName": port, + }) + }) + .collect() +} + +/// PCI address of each BF4 Astra uplink paired with the switch port it carries. +/// The underlay, overlay, and bridge names are all derived from the port name. +const WEAVE_FLOW_CONTROLLER_UNDERLAY_PORTS: &[(&str, &str)] = &[ + ("0005:06:00.0", "r0swpln0"), + ("0005:03:00.0", "r1swpln0"), + ("0000:06:00.0", "r2swpln0"), + ("0000:03:00.0", "r3swpln0"), + ("0004:03:00.0", "r0swpln1"), + ("0004:06:00.0", "r1swpln1"), + ("0001:03:00.0", "r2swpln1"), + ("0001:06:00.0", "r3swpln1"), +]; + +fn weave_flow_controller_underlay_interfaces() -> Vec { + WEAVE_FLOW_CONTROLLER_UNDERLAY_PORTS + .iter() + .map(|(pci_address, port)| { + serde_json::json!({ + "pciAddress": pci_address, + "underlayInterface": format!("brcx-{port}"), + "overlayDHCPInterface": port, + "dhcpBridgeName": format!("br-dhcp-{port}"), + "dropBridgeName": format!("br-drop-{port}"), + }) + }) + .collect() +} + pub fn doca_weave_flow_controller_service(cfg: &DpfServiceConfig) -> ServiceDefinition { let mut helm_values = serde_json::json!({ - "image": { - "repository": cfg.docker_repo_url, - "tag": cfg.docker_image_tag, + "weaveFlowController": { + "containers": { + "weaveFlowController": { + "image": { + "repository": cfg.docker_repo_url, + "tag": cfg.docker_image_tag, + } + } + } } }); apply_image_pull_secrets(&mut helm_values, cfg); ServiceDefinition { helm_values: Some(helm_values), + config_values: Some(serde_json::json!({ + "weaveFlowController": { + "enabled": true, + "underlayConfigMapData": { + "nicIDType": "mac", + "interfaces": weave_flow_controller_underlay_interfaces(), + } + } + })), ..ServiceDefinition::new( &cfg.name, &cfg.helm_repo_url, @@ -620,9 +706,9 @@ pub fn mandatory_services( #[cfg(test)] mod tests { - use carbide_dpf::build_service_interface; use carbide_dpf::sdk::build_dpu_interfaces_vec; use carbide_dpf::types::DpuServiceInterfaceTemplateType; + use carbide_dpf::{build_service_configuration, build_service_interface}; use carbide_test_support::value_scenarios; use url::Url; @@ -793,6 +879,88 @@ mod tests { ); } + // ---- weave services ---- + + #[test] + fn weave_dhcp_agent_service_emits_networks_and_sf_resources() { + let svc = doca_weave_dhcp_agent_service(&default_doca_weave_dhcp_agent_service()); + let helm_values = svc.helm_values.as_ref().expect("helm_values must be set"); + assert_eq!( + helm_values["weaveDHCPAgent"]["containers"]["weaveDHCPAgent"]["image"]["repository"], + format!("{DOCA_WEAVE_IMAGE_REGISTRY}/{DOCA_WEAVE_DHCP_AGENT_SERVICE_IMAGE_NAME}") + ); + assert!( + helm_values.get("image").is_none(), + "DHCP-agent image must not use a top-level image key" + ); + + let configuration = + build_service_configuration(&svc, TEST_NS, "bf4astra", &BTreeMap::new()); + let service_configuration = configuration + .spec + .service_configuration + .expect("serviceConfiguration must be set"); + let values = service_configuration + .helm_chart + .expect("helmChart must be set") + .values + .expect("helmChart values must be set"); + + assert_eq!(values["weaveDHCPAgent"]["enabled"], true); + assert_eq!(values["weaveDHCPAgent"]["dhcpNetworks"]["createNADs"], true); + assert_eq!( + values["weaveDHCPAgent"]["dhcpNetworks"]["networks"], + serde_json::Value::Array(weave_dhcp_agent_networks()) + ); + + let resources = service_configuration + .service_daemon_set + .expect("serviceDaemonSet must be set") + .resources + .expect("serviceDaemonSet resources must be set"); + assert_eq!( + resources.get("nvidia.com/bf_sf"), + Some(&IntOrString::Int(8)) + ); + } + + #[test] + fn weave_flow_controller_service_emits_underlay_config_values() { + let svc = doca_weave_flow_controller_service(&default_doca_weave_flow_controller_service()); + let helm_values = svc.helm_values.expect("helm_values must be set"); + assert_eq!( + helm_values["weaveFlowController"]["containers"]["weaveFlowController"]["image"]["repository"], + format!("{DOCA_WEAVE_IMAGE_REGISTRY}/{DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_IMAGE_NAME}") + ); + assert!( + helm_values.get("image").is_none(), + "flow-controller image must not use a top-level image key" + ); + + let config = svc.config_values.expect("config_values must be set"); + assert_eq!( + config["weaveFlowController"]["enabled"], + serde_json::json!(true) + ); + assert_eq!( + config["weaveFlowController"]["underlayConfigMapData"]["nicIDType"], + "mac" + ); + + let interfaces = config["weaveFlowController"]["underlayConfigMapData"]["interfaces"] + .as_array() + .expect("interfaces must be an array"); + assert_eq!( + interfaces.len(), + WEAVE_FLOW_CONTROLLER_UNDERLAY_PORTS.len(), + "every Astra uplink must be present in underlayConfigMapData.interfaces" + ); + assert_eq!( + interfaces.as_slice(), + weave_flow_controller_underlay_interfaces().as_slice() + ); + } + // ---- dpu_service_interfaces ---- #[test] diff --git a/crates/api-core/src/ethernet_virtualization.rs b/crates/api-core/src/ethernet_virtualization.rs index da6cef560d..ef803dafa1 100644 --- a/crates/api-core/src/ethernet_virtualization.rs +++ b/crates/api-core/src/ethernet_virtualization.rs @@ -117,6 +117,8 @@ fn validate_interface_routing_profile( for prefix in &interface_profile.allowed_anycast_prefixes { if !vpc_profile .allowed_anycast_prefixes + .as_deref() + .unwrap_or_default() .iter() .any(|allowed| prefix_contains(allowed.prefix, *prefix)) { @@ -167,23 +169,9 @@ pub(crate) async fn validate_instance_interface_routing_profiles( "instance interface routing_profile requires FNN routing profiles".to_string(), ) })?; - let profile_type = - vpc.config - .routing_profile_type - .as_ref() - .ok_or_else(|| CarbideError::Internal { - message: "tenant routing profile type not found in VPC record".to_string(), - })?; - let vpc_profile = - fnn.routing_profiles - .get(profile_type) - .ok_or_else(|| CarbideError::NotFoundError { - kind: "routing_profile_type", - id: profile_type.to_string(), - })?; - - // The interface profile must be a subset of the operator profile. - validate_interface_routing_profile(vpc_profile, interface_profile)?; + // The interface profile must narrow the fully resolved VPC profile. + let vpc_profile = fnn.resolve_vpc_routing_profile(&vpc.config)?; + validate_interface_routing_profile(vpc_profile.as_ref(), interface_profile)?; } Ok(()) @@ -656,48 +644,37 @@ pub async fn tenant_network( let vpc_vni = vpc.as_ref().and_then(|v| v.status.vni).unwrap_or_default() as u32; // Resolve the routing profile from the VPC attached to this interface. - let (vpc_routing_profile, interface_routing_profile) = - match (vpc.as_ref(), fnn_config) { - (Some(vpc), Some(fnn)) - if vpc.config.network_virtualization_type == VpcVirtualizationType::Fnn => - { - let profile_type = vpc.config.routing_profile_type.as_ref().ok_or_else(|| { - CarbideError::Internal { - message: "tenant routing profile type not found in VPC record".to_string(), - } - })?; - let profile = fnn.routing_profiles.get(profile_type).ok_or_else(|| { - CarbideError::NotFoundError { - kind: "routing_profile_type", - id: profile_type.to_string(), - } - })?; - - ( - Some(rpc::RoutingProfile::from(profile)), - iface - .routing_profile - .as_ref() - .map(rpc::FlatInterfaceRoutingProfile::from), - ) - } - (Some(vpc), None) - if vpc.config.network_virtualization_type == VpcVirtualizationType::Fnn => - { - return Err(CarbideError::Internal { - message: "FNN VPC found but no FNN config found".to_string(), - } - .into()); - } - _ if iface.routing_profile.is_some() => { - return Err(CarbideError::InvalidArgument( - "instance interface routing_profile is only supported for FNN VPC interfaces" - .to_string(), - ) - .into()); + let (vpc_routing_profile, interface_routing_profile) = match (vpc.as_ref(), fnn_config) { + (Some(vpc), Some(fnn)) + if vpc.config.network_virtualization_type == VpcVirtualizationType::Fnn => + { + let profile = fnn.resolve_vpc_routing_profile(&vpc.config)?; + + ( + Some(rpc::RoutingProfile::from(profile.as_ref())), + iface + .routing_profile + .as_ref() + .map(rpc::FlatInterfaceRoutingProfile::from), + ) + } + (Some(vpc), None) + if vpc.config.network_virtualization_type == VpcVirtualizationType::Fnn => + { + return Err(CarbideError::Internal { + message: "FNN VPC found but no FNN config found".to_string(), } - _ => (None, None), - }; + .into()); + } + _ if iface.routing_profile.is_some() => { + return Err(CarbideError::InvalidArgument( + "instance interface routing_profile is only supported for FNN VPC interfaces" + .to_string(), + ) + .into()); + } + _ => (None, None), + }; let rpc_ft: rpc::InterfaceFunctionType = iface.function_id.function_type().into(); let (svi_ip, svi_ip_v6) = ds.svi_ips(network_virtualization_type, is_l2_segment)?; @@ -842,13 +819,15 @@ mod test { /// Returns a test FNN routing profile with the provided allowed anycast prefixes. fn routing_profile_with_anycast(prefixes: &[&str]) -> FnnRoutingProfileConfig { FnnRoutingProfileConfig { - allowed_anycast_prefixes: prefixes - .iter() - .map(|prefix| crate::cfg::file::PrefixFilterPolicyEntry { - prefix: prefix.parse().unwrap(), - }) - .collect(), - leak_default_route_from_underlay: true, + allowed_anycast_prefixes: Some( + prefixes + .iter() + .map(|prefix| crate::cfg::file::PrefixFilterPolicyEntry { + prefix: prefix.parse().unwrap(), + }) + .collect(), + ), + leak_default_route_from_underlay: Some(true), ..Default::default() } } diff --git a/crates/api-core/src/handlers/bmc_credential_rotation.rs b/crates/api-core/src/handlers/bmc_credential_rotation.rs index dd305bebfa..3ec614c2a3 100644 --- a/crates/api-core/src/handlers/bmc_credential_rotation.rs +++ b/crates/api-core/src/handlers/bmc_credential_rotation.rs @@ -27,11 +27,11 @@ use crate::api::{Api, log_machine_id, log_request_data}; /// Operator force-converge escape hatch: record (or clear) a request to /// immediately rotate a device's BMC credentials, bypassing the passive /// site-wide gate and the device's backoff quarantine. The target BMC is -/// addressed by the owning device's id (machine or switch), its BMC MAC, or a -/// combination (see [`resolve_target`]); the flag is written on that device's -/// row. The owning device's state controller consumes the request on its next -/// sweep; this handler only writes the flag (it performs no Redfish work -/// itself). +/// addressed by the owning device's id (a machine, switch, or power shelf), its +/// BMC MAC, or a combination (see [`resolve_target`]); the flag is written on +/// that device's row. The owning device's state controller consumes the request +/// on its next sweep; this handler only writes the flag (it performs no Redfish +/// work itself). pub(crate) async fn trigger_bmc_credential_rotation( api: &Api, request: Request, @@ -39,11 +39,10 @@ pub(crate) async fn trigger_bmc_credential_rotation( log_request_data(&request); let req = request.into_inner(); let mode = req.mode(); - let device_id = reject_unsupported_device_id(req.device_id)?; let mut txn = api.txn_begin().await?; - let target = resolve_target(&mut txn, device_id, req.bmc_mac).await?; + let target = resolve_target(&mut txn, req.device_id, req.bmc_mac).await?; match mode { Mode::Set => match target { @@ -53,11 +52,8 @@ pub(crate) async fn trigger_bmc_credential_rotation( DeviceId::Switch(id) => { db::switch::set_bmc_credential_rotation_requested(&mut txn, id).await?; } - DeviceId::PowerShelf(_) => { - return Err(CarbideError::InvalidArgument( - "power shelf BMC credential rotation is not yet supported".to_string(), - ) - .into()); + DeviceId::PowerShelf(id) => { + db::power_shelf::set_bmc_credential_rotation_requested(&mut txn, id).await?; } }, Mode::Clear => match target { @@ -67,11 +63,8 @@ pub(crate) async fn trigger_bmc_credential_rotation( DeviceId::Switch(id) => { db::switch::clear_bmc_credential_rotation_requested(&mut txn, id).await?; } - DeviceId::PowerShelf(_) => { - return Err(CarbideError::InvalidArgument( - "power shelf BMC credential rotation is not yet supported".to_string(), - ) - .into()); + DeviceId::PowerShelf(id) => { + db::power_shelf::clear_bmc_credential_rotation_requested(&mut txn, id).await?; } }, // An omitted `mode` decodes as `Unspecified`; reject it rather than let @@ -88,24 +81,12 @@ pub(crate) async fn trigger_bmc_credential_rotation( Ok(Response::new(())) } -fn reject_unsupported_device_id( - device_id: Option, -) -> Result, CarbideError> { - match device_id { - Some(DeviceId::PowerShelf(_)) => Err(CarbideError::InvalidArgument( - "power shelf BMC credential rotation is not yet supported".to_string(), - )), - device_id => Ok(device_id), - } -} - /// Resolve the device that owns the target BMC from an operator request that -/// carries a `device_id`, a BMC MAC, or both. A supported device has exactly one -/// BMC, so any single identifier uniquely names it. When a MAC is supplied -/// alongside a `device_id` they must agree, which lets an operator double-check -/// that a MAC pulled from an alert really is the BMC of the device they mean. -/// Power shelf IDs are part of the shared type for forward compatibility but -/// are rejected until power shelf BMC rotation is implemented. +/// carries a `device_id` (a machine, switch, or power shelf), a BMC MAC, or +/// both. A device has exactly one BMC, so any single identifier uniquely names +/// it. When a MAC is supplied alongside a `device_id` they must resolve to the +/// same device, which lets an operator double-check that a MAC pulled from an +/// alert really is the BMC of the device they mean. async fn resolve_target( txn: &mut PgConnection, device_id: Option, @@ -119,54 +100,32 @@ async fn resolve_target( }) .transpose()?; - // A MAC uniquely names one BMC device; resolve which device kind owns it. - let mac_target = match bmc_mac { - Some(mac) => Some(resolve_mac_owner(txn, mac).await?), + // A MAC uniquely names one BMC device; resolve which device kind owns it, + // keeping the parsed MAC alongside its owner for cross-check error messages. + let mac_address_and_target = match bmc_mac { + Some(mac) => Some((mac, resolve_mac_owner(txn, mac).await?)), None => None, }; - let target = match (device_id, mac_target) { - // Explicit machine id, optionally cross-checked against the MAC's owner. - (Some(DeviceId::Machine(machine_id)), None) => DeviceId::Machine(machine_id), - (Some(DeviceId::Machine(machine_id)), Some(DeviceId::Machine(mac_machine_id))) => { - if machine_id != mac_machine_id { - return Err(CarbideError::InvalidArgument(format!( - "bmc {} belongs to machine {mac_machine_id}, not the requested machine {machine_id}", - bmc_mac.expect("a mac target implies a parsed mac") - ))); - } - DeviceId::Machine(machine_id) - } - (Some(DeviceId::Machine(machine_id)), Some(DeviceId::Switch(switch_id))) => { - return Err(CarbideError::InvalidArgument(format!( - "bmc {} belongs to switch {switch_id}, not the requested machine {machine_id}", - bmc_mac.expect("a mac target implies a parsed mac") - ))); - } - // Explicit switch id, optionally cross-checked against the MAC's owner. - (Some(DeviceId::Switch(switch_id)), None) => DeviceId::Switch(switch_id), - (Some(DeviceId::Switch(switch_id)), Some(DeviceId::Switch(mac_switch_id))) => { - if switch_id != mac_switch_id { + let target = match (device_id, mac_address_and_target) { + // Both supplied: the explicit id and the MAC's owner must be the same + // device, so a mismatched cross-check is rejected. + (Some(id_target), Some((mac, mac_target))) => { + if id_target != mac_target { return Err(CarbideError::InvalidArgument(format!( - "bmc {} belongs to switch {mac_switch_id}, not the requested switch {switch_id}", - bmc_mac.expect("a mac target implies a parsed mac") + "bmc {} belongs to {} {}, not the requested {} {}", + mac, + mac_target.kind(), + mac_target, + id_target.kind(), + id_target, ))); } - DeviceId::Switch(switch_id) - } - (Some(DeviceId::Switch(switch_id)), Some(DeviceId::Machine(machine_id))) => { - return Err(CarbideError::InvalidArgument(format!( - "bmc {} belongs to machine {machine_id}, not the requested switch {switch_id}", - bmc_mac.expect("a mac target implies a parsed mac") - ))); + id_target } + (Some(id_target), None) => id_target, // MAC only: the owner the MAC resolved to. - (None, Some(target)) => target, - (Some(DeviceId::PowerShelf(_)), _) | (_, Some(DeviceId::PowerShelf(_))) => { - return Err(CarbideError::InvalidArgument( - "power shelf BMC credential rotation is not yet supported".to_string(), - )); - } + (None, Some((_mac, mac_target))) => mac_target, (None, None) => { return Err(CarbideError::InvalidArgument( "one of device_id or bmc_mac must be provided".to_string(), @@ -181,8 +140,9 @@ async fn resolve_target( } /// Resolve which device kind owns a BMC MAC. A physical BMC MAC lives on exactly -/// one interface row, keyed to a machine *or* a switch, so try the machine -/// resolver first (its `machine_id`-keyed BMC interface) then the switch one. +/// one interface row, keyed to a machine, a switch, *or* a power shelf, so try +/// the machine resolver first (its `machine_id`-keyed BMC interface), then the +/// switch one, then the power shelf one. async fn resolve_mac_owner( txn: &mut PgConnection, mac: MacAddress, @@ -193,6 +153,9 @@ async fn resolve_mac_owner( if let Some(switch_id) = db::switch::find_switch_id_by_bmc_mac(txn, mac).await? { return Ok(DeviceId::Switch(switch_id)); } + if let Some(power_shelf) = db::power_shelf::find_by_bmc_mac_address(txn, mac).await? { + return Ok(DeviceId::PowerShelf(power_shelf.id)); + } Err(CarbideError::NotFoundError { kind: "BMC", id: mac.to_string(), @@ -201,37 +164,111 @@ async fn resolve_mac_owner( #[cfg(test)] mod tests { - use std::str::FromStr; - - use carbide_test_support::Outcome::*; - use carbide_test_support::scenarios; - use carbide_uuid::machine::MachineId; - use carbide_uuid::power_shelf::PowerShelfId; - use carbide_uuid::switch::SwitchId; + use carbide_uuid::power_shelf::{PowerShelfId, PowerShelfIdSource, PowerShelfType}; + use model::power_shelf::{NewPowerShelf, PowerShelfConfig}; use super::*; - const MACHINE_ID: &str = "fm100ht038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg"; - const SWITCH_ID: &str = "sw100nt038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg"; - const POWER_SHELF_ID: &str = "ps100ht038bg3qsho433vkg684heguv282qaggmrsh2ugn1qk096n2c6hcg"; - - #[test] - fn rejects_only_power_shelf_device_ids() { - let machine_id = MachineId::from_str(MACHINE_ID).unwrap(); - let switch_id = SwitchId::from_str(SWITCH_ID).unwrap(); - let power_shelf_id = PowerShelfId::from_str(POWER_SHELF_ID).unwrap(); - - scenarios!( - run = |device_id| reject_unsupported_device_id(device_id).map_err(drop); - "supported targets" { - Some(DeviceId::Machine(machine_id)) => Yields(Some(DeviceId::Machine(machine_id))), - Some(DeviceId::Switch(switch_id)) => Yields(Some(DeviceId::Switch(switch_id))), - None => Yields(None), - } + fn mac(last: u8) -> MacAddress { + MacAddress::new([0x02, 0, 0, 0, 0, last]) + } - "future target" { - Some(DeviceId::PowerShelf(power_shelf_id)) => Fails, - } + /// Insert a power shelf carrying `bmc_mac` on its row so `resolve_mac_owner` + /// resolves the MAC to it. `power_shelves.bmc_mac_address` has a foreign key + /// into `expected_power_shelves`, so seed that row first. Returns the shelf's + /// id. + async fn seed_power_shelf( + conn: &mut PgConnection, + seed: u8, + bmc_mac: MacAddress, + ) -> PowerShelfId { + sqlx::query( + "INSERT INTO expected_power_shelves \ + (serial_number, bmc_mac_address, bmc_username, bmc_password) \ + VALUES ($1, $2::macaddr, 'admin', 'pw')", + ) + .bind(format!("resolve-target-sn-{seed}")) + .bind(bmc_mac) + .execute(&mut *conn) + .await + .expect("seeding the expected_power_shelves row should succeed"); + + let id = PowerShelfId::new( + PowerShelfIdSource::ProductBoardChassisSerial, + [seed; 32], + PowerShelfType::Rack, + ); + let new_power_shelf = NewPowerShelf { + id, + config: PowerShelfConfig { + name: format!("resolve-target-shelf-{seed}"), + capacity: Some(100), + voltage: Some(240), + }, + bmc_mac_address: Some(bmc_mac), + metadata: None, + rack_id: None, + }; + db::power_shelf::create(conn, &new_power_shelf) + .await + .expect("seeding a power shelf should succeed"); + id + } + + /// A power shelf device id resolves to the owning power shelf, whether + /// addressed by id alone, by its BMC MAC alone, or by both when they agree. + #[crate::sqlx_test] + async fn resolve_target_resolves_a_power_shelf_by_id_and_mac(pool: sqlx::PgPool) { + let mut conn = pool.acquire().await.unwrap(); + let shelf_mac = mac(1); + let shelf_id = seed_power_shelf(&mut conn, 1, shelf_mac).await; + + // Id only: no MAC lookup, returns the addressed shelf. + let by_id = resolve_target(&mut conn, Some(DeviceId::PowerShelf(shelf_id)), None) + .await + .expect("a power shelf id alone should resolve"); + assert_eq!(by_id, DeviceId::PowerShelf(shelf_id)); + + // MAC only: the owner the MAC resolves to. + let by_mac = resolve_target(&mut conn, None, Some(shelf_mac.to_string())) + .await + .expect("a power shelf BMC MAC alone should resolve to its shelf"); + assert_eq!(by_mac, DeviceId::PowerShelf(shelf_id)); + + // Id + agreeing MAC: the cross-check passes. + let by_both = resolve_target( + &mut conn, + Some(DeviceId::PowerShelf(shelf_id)), + Some(shelf_mac.to_string()), + ) + .await + .expect("a power shelf id with its own BMC MAC should resolve"); + assert_eq!(by_both, DeviceId::PowerShelf(shelf_id)); + } + + /// A power shelf id cross-checked against a BMC MAC that belongs to a + /// *different* device is rejected rather than silently trusting either. + #[crate::sqlx_test] + async fn resolve_target_rejects_a_power_shelf_id_that_disagrees_with_the_mac( + pool: sqlx::PgPool, + ) { + let mut conn = pool.acquire().await.unwrap(); + let shelf_a = seed_power_shelf(&mut conn, 1, mac(1)).await; + let mac_b = mac(2); + let _shelf_b = seed_power_shelf(&mut conn, 2, mac_b).await; + + // Request shelf A but hand over shelf B's BMC MAC: the id and the MAC's + // owner disagree, so the request is an invalid argument. + let err = resolve_target( + &mut conn, + Some(DeviceId::PowerShelf(shelf_a)), + Some(mac_b.to_string()), + ) + .await + .expect_err("a power shelf id that disagrees with the MAC owner must be rejected"); + assert!( + matches!(err, CarbideError::InvalidArgument(_)), + "expected InvalidArgument, got {err:?}" ); } } diff --git a/crates/api-core/src/handlers/finder.rs b/crates/api-core/src/handlers/finder.rs index de2bd66923..f7b9fcb899 100644 --- a/crates/api-core/src/handlers/finder.rs +++ b/crates/api-core/src/handlers/finder.rs @@ -29,6 +29,7 @@ use carbide_uuid::network::NetworkSegmentId; use carbide_uuid::vpc::VpcId; use db::{DatabaseError, ObjectColumnFilter, instance, network_segment, vpc}; use model::allocation_type::AllocationType; +use model::machine_interface::InterfaceType; use model::network_segment::NetworkSegmentSearchConfig; use model::resource_pool::ResourcePoolEntryState; use model::route_server::RouteServerSourceType; @@ -36,19 +37,24 @@ use model::route_server::RouteServerSourceType; use crate::CarbideError; use crate::api::Api; -/// Returns true when this machine-interface address should be labeled as operator/static BMC -/// (`IpTypeStaticBmcIp`): either explicitly static allocation, or an address on the synthetic -/// `static-assignments` segment used for external IPs outside Carbide-managed prefixes. -fn machine_interface_address_is_operator_static( +/// Returns true when this machine-interface address should be labeled as an operator/static BMC +/// (`IpTypeStaticBmcIp`). +/// +/// Static allocation metadata and the synthetic `static-assignments` segment do not identify the +/// endpoint role, so either condition must be paired with `InterfaceType::Bmc`. +fn machine_interface_address_is_static_bmc( segment_name: &str, allocation_type: AllocationType, + interface_type: InterfaceType, ) -> bool { - allocation_type == AllocationType::Static - || segment_name == network_segment::STATIC_ASSIGNMENTS_SEGMENT_NAME + interface_type == InterfaceType::Bmc + && (allocation_type == AllocationType::Static + || segment_name == network_segment::STATIC_ASSIGNMENTS_SEGMENT_NAME) } /// Resolves an IP to zero or more typed matches (BMC, instance, static BMC, etc.). Static BMC -/// classification for `machine_interface_addresses` uses [`machine_interface_address_is_operator_static`]. +/// classification for `machine_interface_addresses` uses +/// [`machine_interface_address_is_static_bmc`]. pub(crate) async fn find_ip_address( api: &Api, request: tonic::Request, @@ -288,14 +294,17 @@ async fn search( }) } - // machine_interface_addresses: classify operator/static BMC as StaticBmcIp (see - // machine_interface_address_is_operator_static). + // machine_interface_addresses: classify operator/static BMC as StaticBmcIp while + // retaining static Data addresses as MachineAddress. MachineAddresses => { let out = db::machine_interface_address::find_by_address(db, addr).await?; match out { Some(e) => { - let is_static_bmc = - machine_interface_address_is_operator_static(&e.name, e.allocation_type); + let is_static_bmc = machine_interface_address_is_static_bmc( + &e.name, + e.allocation_type, + e.interface_type, + ); let (ip_type, type_label) = if is_static_bmc { (rpc::IpType::StaticBmcIp, "static BMC IP") @@ -303,12 +312,16 @@ async fn search( (rpc::IpType::MachineAddress, "machine address") }; - let message = match e.machine_id.as_ref() { - Some(machine_id) => format!( + let message = match (e.machine_id.as_ref(), e.switch_id.as_ref()) { + (Some(machine_id), _) => format!( "{ip} is a {type_label} on machine {} (interface {}) on network segment {} of type {}", machine_id, e.id, e.name, e.network_segment_type, ), - None => format!( + (None, Some(switch_id)) => format!( + "{ip} is a {type_label} on switch {} (interface {}) on network segment {} of type {}", + switch_id, e.id, e.name, e.network_segment_type, + ), + (None, None) => format!( "{ip} is a {type_label} on interface {} on network segment {} of type {}. It is not attached to a machine.", e.id, e.name, e.network_segment_type, ), @@ -333,9 +346,10 @@ async fn search( let is_static = db::machine_interface_address::find_by_address(db, addr) .await? .is_some_and(|row| { - machine_interface_address_is_operator_static( + machine_interface_address_is_static_bmc( &row.name, row.allocation_type, + row.interface_type, ) }); @@ -359,6 +373,7 @@ async fn search( None => None, } } + ExploredEndpoint => { let out = db::explored_endpoints::find_by_ips(db, vec![addr]).await?; out.first().map(|ee| rpc::IpAddressMatch { @@ -566,3 +581,69 @@ async fn by_mac( Ok(None) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn static_bmc_classification_requires_bmc_interface_type() { + let static_assignments = network_segment::STATIC_ASSIGNMENTS_SEGMENT_NAME; + let cases = [ + ( + "static BMC allocation", + "underlay", + AllocationType::Static, + InterfaceType::Bmc, + true, + ), + ( + "BMC on static assignments", + static_assignments, + AllocationType::Dhcp, + InterfaceType::Bmc, + true, + ), + ( + "dynamic BMC", + "underlay", + AllocationType::Dhcp, + InterfaceType::Bmc, + false, + ), + ( + "static Data allocation", + "underlay", + AllocationType::Static, + InterfaceType::Data, + false, + ), + ( + "Data on static assignments", + static_assignments, + AllocationType::Dhcp, + InterfaceType::Data, + false, + ), + ( + "dynamic Data", + "underlay", + AllocationType::Dhcp, + InterfaceType::Data, + false, + ), + ]; + + for (name, segment_name, allocation_type, interface_type, expected) in cases { + assert_eq!( + machine_interface_address_is_static_bmc( + segment_name, + allocation_type, + interface_type, + ), + expected, + "{name}", + ); + } + } +} diff --git a/crates/api-core/src/handlers/vpc.rs b/crates/api-core/src/handlers/vpc.rs index e0958301dc..b4b11f2bd4 100644 --- a/crates/api-core/src/handlers/vpc.rs +++ b/crates/api-core/src/handlers/vpc.rs @@ -26,7 +26,8 @@ use db::{self, ObjectColumnFilter, network_security_group}; use model::resource_pool; use model::tenant::{InvalidTenantOrg, Tenant}; use model::vpc::{ - NewVpc, UpdateVpc, UpdateVpcVirtualization, VpcStatus, VpcVirtualizationTypeCapabilities, + NewVpc, UpdateVpc, UpdateVpcVirtualization, VpcRoutingProfileOverrides, VpcStatus, + VpcVirtualizationTypeCapabilities, }; use sqlx::PgConnection; use tonic::{Request, Response, Status}; @@ -107,25 +108,29 @@ pub(crate) async fn create( Some(v) => vpc_virtualization_type_try_from_rpc(v).map_err(CarbideError::from)?, }; - if vpc_creation_request.routing_profile_type.is_some() { + if vpc_creation_request.routing_profile_type.is_some() + || vpc_creation_request.routing_profile_overrides.is_some() + { requested_virtualization_type .ensure_supports_routing_profiles() .map_err(CarbideError::from)?; } + let requested_profile_type = vpc_creation_request.routing_profile_type.clone(); + let mut new_vpc = NewVpc::try_from(request.into_inner())?; + let ResolvedVpcRouting { - profile_type: requested_profile_type, + profile_type: resolved_profile_type, internal, } = resolve_vpc_routing( requested_virtualization_type, - vpc_creation_request.routing_profile_type.as_deref(), + requested_profile_type.as_deref(), + new_vpc.routing_profile_overrides.as_ref(), tenant.as_ref(), api.runtime_config.fnn.as_ref(), - &vpc_creation_request.tenant_organization_id, + &new_vpc.tenant_organization_id, )?; - let mut new_vpc = NewVpc::try_from(request.into_inner())?; - let vni = Some( allocate_vpc_vni( api, @@ -137,11 +142,11 @@ pub(crate) async fn create( .await?, ); - new_vpc.routing_profile_type = requested_profile_type; + new_vpc.routing_profile_type = resolved_profile_type; let vpc = db::vpc::persist(new_vpc, VpcStatus { vni }, &mut txn).await?; - let rpc_out: rpc::Vpc = vpc.into(); + let rpc_out = vpc_to_rpc(vpc, api.runtime_config.fnn.as_ref()); txn.commit().await?; @@ -219,7 +224,7 @@ pub(crate) async fn update( txn.commit().await?; Ok(Response::new(rpc::VpcUpdateResult { - vpc: Some(vpc.into()), + vpc: Some(vpc_to_rpc(vpc, api.runtime_config.fnn.as_ref())), })) } @@ -319,7 +324,7 @@ pub(crate) async fn delete( } .into()); }; - profile.internal + profile.internal.unwrap_or_default() } }; @@ -387,13 +392,45 @@ pub(crate) async fn find_by_ids( let result = db_vpcs .map(|vpc| rpc::VpcList { - vpcs: vpc.into_iter().map(rpc::Vpc::from).collect(), + vpcs: vpc + .into_iter() + .map(|vpc| vpc_to_rpc(vpc, api.runtime_config.fnn.as_ref())) + .collect(), }) .map(Response::new)?; Ok(result) } +/// Converts a persisted VPC to RPC and populates its runtime-derived effective routing profile. +/// +/// The effective profile is omitted when routing profiles are unsupported, FNN is disabled, or +/// the named runtime profile cannot be resolved. +fn vpc_to_rpc(vpc: model::vpc::Vpc, fnn_config: Option<&FnnConfig>) -> rpc::Vpc { + let effective_routing_profile = if vpc + .config + .network_virtualization_type + .supports_routing_profiles() + { + fnn_config.and_then(|fnn| { + // A persisted VPC can outlive its named runtime profile. In that + // case the status remains available without an effective profile. + fnn.resolve_vpc_routing_profile(&vpc.config) + .ok() + .map(|profile| rpc::VpcEffectiveRoutingProfile::from(profile.as_ref())) + }) + } else { + None + }; + + let mut rpc_vpc = rpc::Vpc::from(vpc); + rpc_vpc + .status + .get_or_insert_default() + .effective_routing_profile = effective_routing_profile; + rpc_vpc +} + /// Allocate a value from the vpc vni resource pool. /// /// If the pool exists but is empty or has en error, return that. @@ -524,9 +561,9 @@ impl Default for ResolvedVpcRouting { /// Resolves the routing-profile and `internal` flag for a VPC create /// request from (1) the VPC's virtualization type's capabilities, -/// (2) the request's `routing_profile_type`, (3) the tenant's -/// `routing_profile_type`, and (4) the site's FNN config. Surfaces -/// any contradictions as [`CarbideError`]. +/// (2) the request's `routing_profile_type` and inline overrides, +/// (3) the tenant's `routing_profile_type`, and (4) the site's FNN +/// config. Surfaces any contradictions as [`CarbideError`]. /// /// This exists as a function so that resolution rules can be /// more easily unit-tested directly, vs. as part of a wider @@ -534,6 +571,7 @@ impl Default for ResolvedVpcRouting { pub(crate) fn resolve_vpc_routing( virt_type: VpcVirtualizationType, requested_profile_type: Option<&str>, + vpc_profile_overrides: Option<&VpcRoutingProfileOverrides>, tenant: Option<&Tenant>, fnn_config: Option<&FnnConfig>, organization_id: &str, @@ -556,36 +594,46 @@ pub(crate) fn resolve_vpc_routing( // Falling back to a default. With FNN disabled, assume // internal (legacy/pre-FNN behavior); with FNN enabled, // external must be assumed. - (None, None) => Ok(ResolvedVpcRouting { + (None, None) if vpc_profile_overrides.is_none() => Ok(ResolvedVpcRouting { profile_type: None, internal: fnn_config.is_none(), }), - // Request asks for a routing profile but no tenant context - // exists to validate it against -- reject. - (Some(_), None) => Err(CarbideError::FailedPrecondition(format!( - "VPC routing-profile type requested but no tenant or routing profile-type found for organization id `{organization_id}`" + // A requested profile type or inline profile needs tenant context + // to establish and authorize its named base profile. + (_, None) => Err(CarbideError::FailedPrecondition(format!( + "VPC routing profile type or overrides requested but no tenant or routing profile type found for organization id `{organization_id}`" ))), // Tenant has a routing profile; resolve the request against it. (request_profile_type, Some(tenant_profile_type)) => { - match (fnn_config, request_profile_type) { - // FNN disabled but the request named a profile -- reject. - (None, Some(_)) => Err(CarbideError::FailedPrecondition( - "FNN configuration required to request routing-profile for VPCs".to_string(), - )), + match fnn_config { + // Explicit profile properties require FNN configuration. + None if request_profile_type.is_some() || vpc_profile_overrides.is_some() => { + Err(CarbideError::FailedPrecondition( + "FNN configuration required to request routing-profile for VPCs" + .to_string(), + )) + } // FNN disabled with no explicit request: inherit the // tenant's profile name; force `internal=true` (legacy // pre-FNN behavior). - (None, None) => Ok(ResolvedVpcRouting { + None => Ok(ResolvedVpcRouting { profile_type: Some(tenant_profile_type.to_owned()), internal: true, }), - // FNN enabled with no explicit request: inherit the - // tenant's profile name and its `internal` flag. - (Some(fnn), None) => { + // Resolve the selected base and enforce its tenant access + // boundary. Inline VPC properties cannot change access tiers. + Some(fnn) => { + let profile_type = request_profile_type.unwrap_or(tenant_profile_type); + let base_profile = fnn.routing_profiles.get(profile_type).ok_or_else(|| { + CarbideError::NotFoundError { + kind: "routing_profile", + id: profile_type.to_owned(), + } + })?; let tenant_profile = fnn.routing_profiles .get(tenant_profile_type) @@ -593,32 +641,9 @@ pub(crate) fn resolve_vpc_routing( kind: "routing_profile", id: tenant_profile_type.to_owned(), })?; - Ok(ResolvedVpcRouting { - profile_type: Some(tenant_profile_type.to_owned()), - internal: tenant_profile.internal, - }) - } - - // FNN enabled and the request named a profile: use the - // request's profile, but check that its access tier - // isn't broader than the tenant's. Higher tier value = - // more restricted; lower = broader. - (Some(fnn), Some(profile_type)) => { - let routing_profile = - fnn.routing_profiles.get(profile_type).ok_or_else(|| { - CarbideError::NotFoundError { - kind: "routing_profile", - id: profile_type.to_owned(), - } - })?; - let tenant_profile = - fnn.routing_profiles - .get(tenant_profile_type) - .ok_or_else(|| CarbideError::NotFoundError { - kind: "routing_profile", - id: tenant_profile_type.to_owned(), - })?; - if routing_profile.access_tier < tenant_profile.access_tier { + if base_profile.access_tier.unwrap_or_default() + < tenant_profile.access_tier.unwrap_or_default() + { return Err(CarbideError::FailedPrecondition( "requested VPC routing-profile access tier is broader than associated tenant routing-profile access tier" .to_string(), @@ -626,7 +651,7 @@ pub(crate) fn resolve_vpc_routing( } Ok(ResolvedVpcRouting { profile_type: Some(profile_type.to_owned()), - internal: routing_profile.internal, + internal: base_profile.internal.unwrap_or_default(), }) } } @@ -638,6 +663,8 @@ pub(crate) fn resolve_vpc_routing( mod tests { use std::collections::HashMap; + use carbide_test_support::Outcome::{FailsWith, Yields}; + use carbide_test_support::scenarios; use config_version::ConfigVersion; use model::metadata::Metadata; @@ -668,186 +695,198 @@ mod tests { fn profile(internal: bool, access_tier: u32) -> FnnRoutingProfileConfig { FnnRoutingProfileConfig { - internal, - access_tier, + internal: Some(internal), + access_tier: Some(access_tier), ..Default::default() } } - #[test] - fn flat_short_circuits_regardless_of_inputs() { - let resolved = resolve_vpc_routing( - VpcVirtualizationType::Flat, - Some("EXTERNAL"), - Some(&tenant_with_profile(Some("INTERNAL"))), - Some(&fnn_with_profiles(&[ - ("EXTERNAL", profile(false, 2)), - ("INTERNAL", profile(true, 1)), - ])), - "test-org", - ) - .expect("Flat must short-circuit cleanly"); - assert_eq!(resolved.profile_type, None); - assert!(resolved.internal); - } - - #[test] - fn no_request_no_tenant_no_fnn_defaults_to_internal() { - let resolved = - resolve_vpc_routing(VpcVirtualizationType::Fnn, None, None, None, "test-org") - .expect("no-request no-tenant no-fnn is the legacy pre-FNN default"); - assert_eq!(resolved.profile_type, None); - assert!( - resolved.internal, - "FNN disabled means we default to internal" - ); + struct RoutingResolutionInput { + network_virtualization_type: VpcVirtualizationType, + requested_profile_type: Option<&'static str>, + routing_profile_overrides: Option, + tenant: Option, + fnn_config: Option, } + type RoutingResolution = (Option, bool); - #[test] - fn no_request_no_tenant_with_fnn_defaults_to_external() { - let fnn = fnn_with_profiles(&[]); - let resolved = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - None, - None, - Some(&fnn), - "test-org", - ) - .expect("no-request no-tenant with-fnn must succeed"); - assert_eq!(resolved.profile_type, None); - assert!( - !resolved.internal, - "FNN enabled means we default to external" - ); + #[derive(Debug, PartialEq, Eq)] + enum RoutingResolutionFailure { + FailedPrecondition, + NotFound(&'static str), + Unexpected(String), } - #[test] - fn request_but_no_tenant_is_rejected() { - let err = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - Some("EXTERNAL"), - None, - None, + fn resolve_routing_case( + input: RoutingResolutionInput, + ) -> Result { + resolve_vpc_routing( + input.network_virtualization_type, + input.requested_profile_type, + input.routing_profile_overrides.as_ref(), + input.tenant.as_ref(), + input.fnn_config.as_ref(), "test-org", ) - .expect_err("request without tenant must be rejected"); - assert!(matches!(err, CarbideError::FailedPrecondition(_))); - } - - #[test] - fn fnn_disabled_with_request_is_rejected() { - let tenant = tenant_with_profile(Some("INTERNAL")); - let err = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - Some("EXTERNAL"), - Some(&tenant), - None, - "test-org", - ) - .expect_err("FNN-disabled + explicit request must be rejected"); - assert!(matches!(err, CarbideError::FailedPrecondition(_))); + .map(|resolved| (resolved.profile_type, resolved.internal)) + .map_err(|error| match error { + CarbideError::FailedPrecondition(_) => RoutingResolutionFailure::FailedPrecondition, + CarbideError::NotFoundError { kind, .. } => RoutingResolutionFailure::NotFound(kind), + error => RoutingResolutionFailure::Unexpected(error.to_string()), + }) } #[test] - fn fnn_disabled_no_request_inherits_tenant_profile() { - let tenant = tenant_with_profile(Some("INTERNAL")); - let resolved = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - None, - Some(&tenant), - None, - "test-org", - ) - .expect("FNN-disabled + tenant profile must inherit"); - assert_eq!(resolved.profile_type.as_deref(), Some("INTERNAL")); - assert!( - resolved.internal, - "legacy pre-FNN behavior forces internal=true" - ); - } + fn resolve_vpc_routing_scenarios() { + use RoutingResolutionFailure::{FailedPrecondition, NotFound}; + use VpcVirtualizationType::{Flat, Fnn}; + + scenarios!(resolve_routing_case: + "VPC types without routing-profile support short-circuit" { + // Flat VPCs ignore all routing-profile inputs and retain the internal allocation + // default because they have no NICo-managed routing profile. + RoutingResolutionInput { + network_virtualization_type: Flat, + requested_profile_type: Some("EXTERNAL"), + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("INTERNAL"))), + fnn_config: Some(fnn_with_profiles(&[ + ("EXTERNAL", profile(false, 2)), + ("INTERNAL", profile(true, 1)), + ])), + } => Yields((None, true)), + } - #[test] - fn fnn_enabled_no_request_inherits_tenant_profile_internal_flag() { - let tenant = tenant_with_profile(Some("EXTERNAL")); - let fnn = fnn_with_profiles(&[("EXTERNAL", profile(false, 2))]); - let resolved = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - None, - Some(&tenant), - Some(&fnn), - "test-org", - ) - .expect("FNN-enabled + tenant profile must inherit name + internal flag"); - assert_eq!(resolved.profile_type.as_deref(), Some("EXTERNAL")); - assert!(!resolved.internal); - } + "missing tenant context" { + // Without FNN or explicit routing properties, preserve the legacy internal + // allocation default even when no tenant record is available. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: None, + tenant: None, + fnn_config: None, + } => Yields((None, true)), + // Enabling FNN changes the no-profile allocation default to external. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: None, + tenant: None, + fnn_config: Some(fnn_with_profiles(&[])), + } => Yields((None, false)), + // A named profile request needs tenant context to authorize its access tier. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: Some("EXTERNAL"), + routing_profile_overrides: None, + tenant: None, + fnn_config: None, + } => FailsWith(FailedPrecondition), + // Inline properties need tenant context to select and authorize their base + // profile, even when no profile name is requested explicitly. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: Some(VpcRoutingProfileOverrides::default()), + tenant: None, + fnn_config: None, + } => FailsWith(FailedPrecondition), + } - #[test] - fn fnn_enabled_request_overrides_when_access_tier_permits() { - // tenant tier 0 (broad); request tier 2 (narrower) -- allowed - let tenant = tenant_with_profile(Some("ADMIN")); - let fnn = - fnn_with_profiles(&[("ADMIN", profile(true, 0)), ("EXTERNAL", profile(false, 2))]); - let resolved = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - Some("EXTERNAL"), - Some(&tenant), - Some(&fnn), - "test-org", - ) - .expect("narrower request than tenant access tier must succeed"); - assert_eq!(resolved.profile_type.as_deref(), Some("EXTERNAL")); - assert!(!resolved.internal); - } + "FNN configuration disabled" { + // An explicit profile cannot be resolved when its FNN definition is unavailable. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: Some("EXTERNAL"), + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("INTERNAL"))), + fnn_config: None, + } => FailsWith(FailedPrecondition), + // Inline properties cannot be applied without an FNN base-profile definition. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: Some(VpcRoutingProfileOverrides::default()), + tenant: Some(tenant_with_profile(Some("INTERNAL"))), + fnn_config: None, + } => FailsWith(FailedPrecondition), + // With no explicit VPC routing properties, preserve pre-FNN behavior by + // inheriting the tenant's stored profile name and forcing internal allocation. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("INTERNAL"))), + fnn_config: None, + } => Yields((Some("INTERNAL".to_string()), true)), + } - #[test] - fn fnn_enabled_request_broader_than_tenant_is_rejected() { - // tenant tier 2 (narrow); request tier 0 (broader) -- rejected - let tenant = tenant_with_profile(Some("EXTERNAL")); - let fnn = - fnn_with_profiles(&[("EXTERNAL", profile(false, 2)), ("ADMIN", profile(true, 0))]); - let err = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - Some("ADMIN"), - Some(&tenant), - Some(&fnn), - "test-org", - ) - .expect_err("broader request than tenant access tier must be rejected"); - assert!(matches!(err, CarbideError::FailedPrecondition(_))); - } + "tenant profile inheritance" { + // With FNN enabled and no explicit request, inherit both the tenant profile name + // and its internal-allocation policy. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("EXTERNAL"))), + fnn_config: Some(fnn_with_profiles(&[("EXTERNAL", profile(false, 2))])), + } => Yields((Some("EXTERNAL".to_string()), false)), + // Inline properties overlay the tenant's base profile but cannot change its + // protected internal-allocation policy. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: Some(VpcRoutingProfileOverrides::default()), + tenant: Some(tenant_with_profile(Some("INTERNAL"))), + fnn_config: Some(fnn_with_profiles(&[("INTERNAL", profile(true, 1))])), + } => Yields((Some("INTERNAL".to_string()), true)), + } - #[test] - fn unknown_requested_profile_yields_not_found() { - let tenant = tenant_with_profile(Some("EXTERNAL")); - let fnn = fnn_with_profiles(&[("EXTERNAL", profile(false, 2))]); - let err = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - Some("DOES_NOT_EXIST"), - Some(&tenant), - Some(&fnn), - "test-org", - ) - .expect_err("request naming an undefined routing profile must error"); - assert!( - matches!(err, CarbideError::NotFoundError { kind, .. } if kind == "routing_profile") - ); - } + "requested profile access tier" { + // An ADMIN tenant may select a narrower EXTERNAL routing profile. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: Some("EXTERNAL"), + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("ADMIN"))), + fnn_config: Some(fnn_with_profiles(&[ + ("ADMIN", profile(true, 0)), + ("EXTERNAL", profile(false, 2)), + ])), + } => Yields((Some("EXTERNAL".to_string()), false)), + // An EXTERNAL tenant may not broaden its access by selecting ADMIN. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: Some("ADMIN"), + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("EXTERNAL"))), + fnn_config: Some(fnn_with_profiles(&[ + ("EXTERNAL", profile(false, 2)), + ("ADMIN", profile(true, 0)), + ])), + } => FailsWith(FailedPrecondition), + } - #[test] - fn unknown_tenant_profile_yields_not_found() { - let tenant = tenant_with_profile(Some("UNDEFINED")); - let fnn = fnn_with_profiles(&[("EXTERNAL", profile(false, 2))]); - let err = resolve_vpc_routing( - VpcVirtualizationType::Fnn, - None, - Some(&tenant), - Some(&fnn), - "test-org", - ) - .expect_err("tenant naming an undefined routing profile must error"); - assert!( - matches!(err, CarbideError::NotFoundError { kind, .. } if kind == "routing_profile") + "unknown named profile" { + // Reject an explicit profile name that has no runtime FNN definition. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: Some("DOES_NOT_EXIST"), + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("EXTERNAL"))), + fnn_config: Some(fnn_with_profiles(&[("EXTERNAL", profile(false, 2))])), + } => FailsWith(NotFound("routing_profile")), + // Reject an unresolved tenant base profile even when the VPC does not request a + // different profile explicitly. + RoutingResolutionInput { + network_virtualization_type: Fnn, + requested_profile_type: None, + routing_profile_overrides: None, + tenant: Some(tenant_with_profile(Some("UNDEFINED"))), + fnn_config: Some(fnn_with_profiles(&[("EXTERNAL", profile(false, 2))])), + } => FailsWith(NotFound("routing_profile")), + } ); } } diff --git a/crates/api-core/src/instance/mod.rs b/crates/api-core/src/instance/mod.rs index bcb653646f..18c1a0e9c6 100644 --- a/crates/api-core/src/instance/mod.rs +++ b/crates/api-core/src/instance/mod.rs @@ -1324,6 +1324,27 @@ pub async fn allocate_instance( .ok_or_else(|| CarbideError::internal("instance allocation returned no result".to_string())) } +fn not_allocatable_error(machine_id: MachineId, reason: NotAllocatableReason) -> CarbideError { + match reason { + NotAllocatableReason::InvalidState(state) => CarbideError::InvalidArgument(format!( + "could not create instance on machine {machine_id} given machine state {state:?}" + )), + NotAllocatableReason::PendingInstanceCreation => CarbideError::InvalidArgument(format!( + "could not create instance on machine {machine_id}. machine is already used by another instance creation request", + )), + NotAllocatableReason::PendingBootConfiguration => { + CarbideError::FailedPrecondition(format!( + "machine {machine_id} has a pending boot configuration; retry after it has been applied" + )) + } + NotAllocatableReason::NoDpuSnapshots => { + CarbideError::internal(format!("machine {machine_id} has no DPU. cannot allocate")) + } + NotAllocatableReason::MaintenanceMode => CarbideError::MaintenanceMode, + NotAllocatableReason::HealthAlert(_) => CarbideError::UnhealthyHost, + } +} + /// Allocates multiple instances in a single transaction. /// Rolls back entirely if any allocation fails. /// @@ -1517,26 +1538,20 @@ pub async fn batch_allocate_instances( })?; if let Err(e) = mh_snapshot.is_usable_as_instance(request.allow_unhealthy_machine) { - tracing::error!( - %machine_id, - error = %e, - "Host can not be used as instance due to reason", - ); - return Err(match e { - NotAllocatableReason::InvalidState(s) => CarbideError::InvalidArgument(format!( - "could not create instance on machine {machine_id} given machine state {s:?}" - )), - NotAllocatableReason::PendingInstanceCreation => { - CarbideError::InvalidArgument(format!( - "could not create instance on machine {machine_id}. machine is already used by another instance creation request", - )) - } - NotAllocatableReason::NoDpuSnapshots => CarbideError::internal(format!( - "machine {machine_id} has no DPU. cannot allocate" - )), - NotAllocatableReason::MaintenanceMode => CarbideError::MaintenanceMode, - NotAllocatableReason::HealthAlert(_) => CarbideError::UnhealthyHost, - }); + if matches!(&e, NotAllocatableReason::PendingBootConfiguration) { + tracing::info!( + %machine_id, + error = %e, + "Host can not be used as instance due to reason", + ); + } else { + tracing::error!( + %machine_id, + error = %e, + "Host can not be used as instance due to reason", + ); + } + return Err(not_allocatable_error(machine_id, e)); } if mh_snapshot.host_snapshot.config.dpf.used_for_ingestion @@ -2335,6 +2350,25 @@ mod tests { }, ); } + + #[test] + fn pending_boot_configuration_has_a_safe_allocation_error() { + let machine_id = "fm100htes3rn1npvbtm5qd57dkilaag7ljugl1llmm7rfuq1ov50i0rpl30" + .parse() + .unwrap(); + + assert!(matches!( + not_allocatable_error( + machine_id, + NotAllocatableReason::PendingBootConfiguration, + ), + CarbideError::FailedPrecondition(message) + if message + == format!( + "machine {machine_id} has a pending boot configuration; retry after it has been applied" + ) + )); + } } #[cfg(test)] diff --git a/crates/api-core/src/ipxe.rs b/crates/api-core/src/ipxe.rs index 69dffb0bc3..7e4130ef1a 100644 --- a/crates/api-core/src/ipxe.rs +++ b/crates/api-core/src/ipxe.rs @@ -469,6 +469,7 @@ exit || let pxe_script = match &machine.current_state() { ManagedHostState::Ready | ManagedHostState::HostInit { .. } + | ManagedHostState::BootConfiguring { .. } | ManagedHostState::BomValidating { .. } | ManagedHostState::Measuring { measuring_state: MeasuringState::WaitingForMeasurements, diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index c128bec1b6..51f5f47338 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -804,6 +804,8 @@ impl<'a> SeedData<'a> { carbide_config, false, )?; + db_init::validate_initial_vpcs(&initial_vpcs)?; + let initial_pools = Self::merge_objects( initial_objects.and_then(|io| io.pools.as_ref()), carbide_config.pools.as_ref(), @@ -1501,6 +1503,9 @@ async fn initialize_and_start_controllers<'a>( rack_firmware_reprovisioning_enabled: carbide_config .power_shelf_state_controller .rack_firmware_reprovisioning_enabled, + redfish_client_pool: shared_redfish_pool.clone(), + bmc_rotation_gate: carbide_credential_rotation::BmcRotationGate::new(), + bmc_rotation_enabled: carbide_config.bmc_rotation_enabled, } .into(), ) @@ -1870,6 +1875,7 @@ mod tests { organization_id: None, network_virtualization_type, routing_profile_type: None, + routing_profile_overrides: None, vni: None, } } @@ -1981,6 +1987,9 @@ mod tests { ConflictingNetwork, ConflictingVpc, InvalidNetwork, + InvalidVpcOverrides { + source: SeedSource, + }, NoOptionalObjects, MissingPools, } @@ -1996,10 +2005,23 @@ mod tests { enum ResolveFailure { Conflict(String), InvalidNetwork, + InvalidVpcOverrides, MissingPools, Unexpected(String), } + fn classify_config_validation_error(error: &model::ConfigValidationError) -> ResolveFailure { + match error { + model::ConfigValidationError::InitialVpcRoutingProfileOverridesUnsupported { + .. + } => ResolveFailure::InvalidVpcOverrides, + // The only other configuration validation performed by + // `SeedData::resolve` is `NetworkDefinition::validate`. + model::ConfigValidationError::InvalidValue(_) => ResolveFailure::InvalidNetwork, + _ => ResolveFailure::Unexpected(error.to_string()), + } + } + fn names(entries: &[&str]) -> BTreeSet { entries.iter().map(|name| name.to_string()).collect() } @@ -2093,6 +2115,29 @@ mod tests { ResolveInput::InvalidNetwork => { config.networks = Some(seed_map(&[("test-network", network_definition(9214))])); } + ResolveInput::InvalidVpcOverrides { + source: SeedSource::InitialObjects, + } => { + initial_objects.vpcs = Some(seed_map(&[( + "test-vpc", + VpcDefinition { + routing_profile_overrides: Some(Default::default()), + ..vpc_definition(VpcVirtualizationType::Fnn) + }, + )])); + use_initial_objects = true; + } + ResolveInput::InvalidVpcOverrides { + source: SeedSource::LegacyConfig, + } => { + config.vpcs = Some(seed_map(&[( + "test-vpc", + VpcDefinition { + routing_profile_overrides: Some(Default::default()), + ..vpc_definition(VpcVirtualizationType::Fnn) + }, + )])); + } ResolveInput::NoOptionalObjects => {} ResolveInput::MissingPools => { config.pools = None; @@ -2112,11 +2157,8 @@ mod tests { { return Err(ResolveFailure::MissingPools); } - if error - .downcast_ref::() - .is_some() - { - return Err(ResolveFailure::InvalidNetwork); + if let Some(error) = error.downcast_ref::() { + return Err(classify_config_validation_error(error)); } let message = error.to_string(); @@ -2396,6 +2438,20 @@ attributes = { attribute1 = "site", additional_attribute3 = "site" } input: ResolveInput::InvalidNetwork, expect: FailsWith(ResolveFailure::InvalidNetwork), }, + Case { + scenario: "initial-objects VPC overrides fail during resolution", + input: ResolveInput::InvalidVpcOverrides { + source: SeedSource::InitialObjects, + }, + expect: FailsWith(ResolveFailure::InvalidVpcOverrides), + }, + Case { + scenario: "legacy VPC overrides fail during resolution", + input: ResolveInput::InvalidVpcOverrides { + source: SeedSource::LegacyConfig, + }, + expect: FailsWith(ResolveFailure::InvalidVpcOverrides), + }, Case { scenario: "resource pool definitions remain required", input: ResolveInput::MissingPools, @@ -2405,4 +2461,14 @@ attributes = { attribute1 = "site", additional_attribute3 = "site" } resolve_seed_data, ); } + + #[test] + fn seed_resolution_preserves_unexpected_config_validation_errors() { + let error = + model::ConfigValidationError::DuplicateTenantKeysetId("duplicate-keyset".to_string()); + assert_eq!( + classify_config_validation_error(&error), + ResolveFailure::Unexpected(error.to_string()) + ); + } } diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index f99bdd69d1..ea8d0144f5 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -218,29 +218,17 @@ impl TestEnvOverrides { ( "EXTERNAL".to_string(), crate::cfg::file::FnnRoutingProfileConfig { - access_tier: 2, - internal: false, - route_target_imports: vec![], - route_targets_on_exports: vec![], - leak_default_route_from_underlay: false, - leak_tenant_host_routes_to_underlay: false, - tenant_leak_communities_accepted: false, - accepted_leaks_from_underlay: vec![], - allowed_anycast_prefixes: vec![], + access_tier: Some(2), + internal: Some(false), + ..Default::default() }, ), ( "INTERNAL".to_string(), crate::cfg::file::FnnRoutingProfileConfig { - access_tier: 1, - internal: true, - route_target_imports: vec![], - route_targets_on_exports: vec![], - leak_default_route_from_underlay: false, - leak_tenant_host_routes_to_underlay: false, - tenant_leak_communities_accepted: false, - accepted_leaks_from_underlay: vec![], - allowed_anycast_prefixes: vec![], + access_tier: Some(1), + internal: Some(true), + ..Default::default() }, ), ]), @@ -424,6 +412,7 @@ impl TestEnv { ManagedHostState::HostInit { machine_state: mc } } ManagedHostState::Ready => state.clone(), + ManagedHostState::BootConfiguring { .. } => state.clone(), ManagedHostState::Maintenance { .. } => state.clone(), ManagedHostState::Assigned { .. } => state.clone(), ManagedHostState::WaitingForCleanup { .. } => state.clone(), @@ -1662,6 +1651,9 @@ pub async fn create_test_env_with_overrides( credential_manager: credential_manager.clone(), per_object_metrics_registry: per_object_metrics_registry.clone(), rack_firmware_reprovisioning_enabled: false, + redfish_client_pool: redfish_sim.clone(), + bmc_rotation_gate: carbide_credential_rotation::BmcRotationGate::new(), + bmc_rotation_enabled: false, } .into(), ) diff --git a/crates/api-core/src/tests/finder.rs b/crates/api-core/src/tests/finder.rs index c64cf4bb50..7e6c43e6db 100644 --- a/crates/api-core/src/tests/finder.rs +++ b/crates/api-core/src/tests/finder.rs @@ -147,10 +147,34 @@ async fn test_inner(ip: &str, ip_type: IpType, env: &TestEnv, caller: &str) { assert!(!res.matches.is_empty(), "{caller} not found"); // In integration testing DHCP relay is in a network segment, // so we get multiple matches. Wouldn't happen in live. - for m in res.matches { - if m.ip_type == ip_type as i32 { - return; // success + if let Some(ip_match) = res + .matches + .into_iter() + .find(|ip_match| ip_match.ip_type == ip_type as i32) + { + match ip_type { + IpType::MachineAddress => { + assert!( + ip_match.message.contains("machine address"), + "{caller} used incorrect Data-interface wording: {}", + ip_match.message, + ); + assert!( + !ip_match.message.contains("BMC"), + "{caller} described a Data interface as BMC: {}", + ip_match.message, + ); + } + IpType::StaticBmcIp => { + assert!( + ip_match.message.contains("static BMC IP"), + "{caller} used incorrect BMC-interface wording: {}", + ip_match.message, + ); + } + _ => {} } + return; } panic!("{caller} did not have correct IPType"); } @@ -423,7 +447,7 @@ async fn test_identify_serial(db_pool: sqlx::PgPool) -> Result<(), eyre::Report> } /// `FindIpAddress` returns `IpTypeStaticBmcIp` when the address is a static/operator BMC -/// allocation (`AllocationType::Static` after `preallocate_machine_interface` on underlay). +/// allocation. #[crate::sqlx_test] async fn test_static_bmc_ip_finder(db_pool: sqlx::PgPool) -> Result<(), eyre::Report> { use std::net::IpAddr; @@ -434,9 +458,14 @@ async fn test_static_bmc_ip_finder(db_pool: sqlx::PgPool) -> Result<(), eyre::Re let bmc_mac = "AA:BB:CC:DD:EE:99".parse().unwrap(); let mut txn = db_pool.begin().await.unwrap(); - db::machine_interface::preallocate_machine_interface(txn.as_mut(), bmc_mac, static_ip, None) - .await - .expect("preallocate static BMC interface"); + db::machine_interface::preallocate_bmc_machine_interface( + txn.as_mut(), + bmc_mac, + static_ip, + None, + ) + .await + .expect("preallocate static BMC interface"); txn.commit().await.unwrap(); // Query the IP via finder @@ -452,15 +481,107 @@ async fn test_static_bmc_ip_finder(db_pool: sqlx::PgPool) -> Result<(), eyre::Re assert!(!res.matches.is_empty(), "Should find at least one match"); - // Verify it's classified as StaticBmcIp - let has_static_bmc_ip = res + let static_bmc_match = res .matches - .iter() - .any(|m| m.ip_type == IpType::StaticBmcIp as i32); + .into_iter() + .find(|ip_match| ip_match.ip_type == IpType::StaticBmcIp as i32) + .expect("static BMC IP should be classified as IpTypeStaticBmcIp"); + assert!( + static_bmc_match.message.contains("static BMC IP"), + "static BMC wording should match the interface type: {}", + static_bmc_match.message, + ); + + Ok(()) +} + +/// A static NVOS/Data address remains a MachineAddress even when it lives on the synthetic +/// `static-assignments` segment. When the interface is associated with a switch, finder output +/// identifies that switch. +#[crate::sqlx_test] +async fn test_static_data_ip_finder(db_pool: sqlx::PgPool) -> Result<(), eyre::Report> { + use std::net::IpAddr; + + use carbide_uuid::switch::SwitchId; + use model::machine_interface_address::MachineInterfaceAssociation; + use model::switch::{NewSwitch, SwitchConfig}; + + let env = create_test_env(db_pool.clone()).await; + let static_ip: IpAddr = "10.86.241.49".parse().unwrap(); + let nvos_mac = "AA:BB:CC:DD:EE:98".parse().unwrap(); + let switch_id = SwitchId::from(uuid::Uuid::new_v4()); + + let mut txn = db_pool.begin().await.unwrap(); + db::machine_interface::preallocate_machine_interface(txn.as_mut(), nvos_mac, static_ip, None) + .await + .expect("preallocate static NVOS Data interface"); + let interface = db::machine_interface::find_by_mac_address(txn.as_mut(), nvos_mac) + .await + .expect("find static NVOS Data interface") + .into_iter() + .next() + .expect("static NVOS Data interface should exist"); + db::switch::create( + txn.as_mut(), + &NewSwitch { + id: switch_id, + config: SwitchConfig { + name: "gb-nvl-136-switch02".to_string(), + enable_nmxc: false, + fabric_manager_config: None, + }, + bmc_mac_address: None, + metadata: None, + rack_id: None, + slot_number: None, + tray_index: None, + }, + ) + .await + .expect("create associated switch"); + db::machine_interface::associate_interface_with_machine( + &interface.id, + MachineInterfaceAssociation::Switch(switch_id), + txn.as_mut(), + ) + .await + .expect("associate NVOS Data interface with switch"); + txn.commit().await.unwrap(); + + let req = rpc::forge::FindIpAddressRequest { + ip: static_ip.to_string(), + }; + let res = env + .api + .find_ip_address(tonic::Request::new(req)) + .await + .expect("find_ip_address should succeed") + .into_inner(); assert!( - has_static_bmc_ip, - "Static IP should be classified as IpTypeStaticBmcIp" + res.matches + .iter() + .all(|ip_match| ip_match.ip_type != IpType::StaticBmcIp as i32), + "static NVOS Data IP must never be classified as StaticBmcIp: {:?}", + res.matches, + ); + let machine_address_match = res + .matches + .into_iter() + .find(|ip_match| ip_match.ip_type == IpType::MachineAddress as i32) + .expect("static NVOS Data IP should remain a MachineAddress"); + assert!( + machine_address_match.owner_id.is_none(), + "static NVOS Data address must not expose a switch ID as a machine owner", + ); + assert!( + machine_address_match.message.contains("machine address") + && machine_address_match + .message + .contains(&switch_id.to_string()) + && !machine_address_match.message.contains("BMC"), + "static NVOS Data wording should identify the switch without calling it BMC: {}", + machine_address_match.message, ); Ok(()) diff --git a/crates/api-core/src/tests/instance_allocate.rs b/crates/api-core/src/tests/instance_allocate.rs index dbc21ded25..95d5c5a73f 100644 --- a/crates/api-core/src/tests/instance_allocate.rs +++ b/crates/api-core/src/tests/instance_allocate.rs @@ -230,11 +230,11 @@ async fn test_allocate_instance_rejects_interface_anycast_prefix_outside_vpc_pro routing_profiles: HashMap::from([( profile_type.to_string(), FnnRoutingProfileConfig { - internal: true, - access_tier: 0, - allowed_anycast_prefixes: vec![PrefixFilterPolicyEntry { + internal: Some(true), + access_tier: Some(0), + allowed_anycast_prefixes: Some(vec![PrefixFilterPolicyEntry { prefix: "192.0.2.0/24".parse().unwrap(), - }], + }]), ..Default::default() }, )]), diff --git a/crates/api-core/src/tests/instance_config_update.rs b/crates/api-core/src/tests/instance_config_update.rs index a3c2f68adb..8d005b6430 100644 --- a/crates/api-core/src/tests/instance_config_update.rs +++ b/crates/api-core/src/tests/instance_config_update.rs @@ -867,11 +867,11 @@ async fn test_update_instance_config_rejects_interface_anycast_prefix_outside_vp routing_profiles: HashMap::from([( profile_type.to_string(), FnnRoutingProfileConfig { - internal: true, - access_tier: 0, - allowed_anycast_prefixes: vec![PrefixFilterPolicyEntry { + internal: Some(true), + access_tier: Some(0), + allowed_anycast_prefixes: Some(vec![PrefixFilterPolicyEntry { prefix: "192.0.2.0/24".parse().unwrap(), - }], + }]), ..Default::default() }, )]), diff --git a/crates/api-core/src/tests/ipxe.rs b/crates/api-core/src/tests/ipxe.rs index d000e6d24e..7d610fc719 100644 --- a/crates/api-core/src/tests/ipxe.rs +++ b/crates/api-core/src/tests/ipxe.rs @@ -22,12 +22,15 @@ use chrono::Utc; use common::api_fixtures::{ TestEnv, TestEnvOverrides, create_test_env, create_test_env_with_overrides, get_config, }; +use config_version::ConfigVersion; use db::{self}; use futures_util::FutureExt; use mac_address::MacAddress; use model::machine::{ CleanupContext, DpuInitState, HostReprovisionState, MachineState, ManagedHostState, + ReadyBootConfigState, SetBootOrderInfo, SetBootOrderState, }; +use model::machine_boot_interface::MachineBootInterfaceTarget; use model::test_support::ManagedHostConfig; use rpc::forge::CloudInitInstructionsRequest; use rpc::forge::forge_server::Forge; @@ -341,6 +344,35 @@ async fn test_pxe_host(pool: sqlx::PgPool) { .await; assert!(instructions.pxe_script.contains("x86_64/scout.efi")); + move_machine_to_needed_state( + host_id, + &ManagedHostState::BootConfiguring { + desired_version: ConfigVersion::new(7), + desired_boot_interface: MachineBootInterfaceTarget::MacOnly( + "02:00:00:00:00:01".parse().unwrap(), + ), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::SetBootOrder { + set_boot_order_info: SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: 0, + }, + }, + }, + &env.pool, + ) + .await; + + let instructions = get_pxe_instructions( + &env, + host_interface_id, + rpc::forge::MachineArchitecture::X86, + None, + ) + .await; + assert!(instructions.pxe_script.contains("x86_64/scout.efi")); + move_machine_to_needed_state( host_id, &ManagedHostState::HostReprovision { diff --git a/crates/api-core/src/tests/machine_network.rs b/crates/api-core/src/tests/machine_network.rs index 6b23d89a01..7b5029581a 100644 --- a/crates/api-core/src/tests/machine_network.rs +++ b/crates/api-core/src/tests/machine_network.rs @@ -353,20 +353,24 @@ async fn test_managed_host_network_config_includes_routing_profile_prefix_lists( routing_profiles: HashMap::from([( profile_type.to_string(), FnnRoutingProfileConfig { - internal: true, - access_tier: 0, - accepted_leaks_from_underlay: expected_leaks - .iter() - .map(|prefix| PrefixFilterPolicyEntry { - prefix: prefix.parse().unwrap(), - }) - .collect(), - allowed_anycast_prefixes: expected_allowed_anycast_prefixes - .iter() - .map(|prefix| PrefixFilterPolicyEntry { - prefix: prefix.parse().unwrap(), - }) - .collect(), + internal: Some(true), + access_tier: Some(0), + accepted_leaks_from_underlay: Some( + expected_leaks + .iter() + .map(|prefix| PrefixFilterPolicyEntry { + prefix: prefix.parse().unwrap(), + }) + .collect(), + ), + allowed_anycast_prefixes: Some( + expected_allowed_anycast_prefixes + .iter() + .map(|prefix| PrefixFilterPolicyEntry { + prefix: prefix.parse().unwrap(), + }) + .collect(), + ), ..Default::default() }, )]), @@ -469,7 +473,7 @@ async fn test_managed_host_network_config_narrows_interface_anycast_prefixes(poo vni: 123, }; - // Configure an FNN routing profile with anycast prefixes broad enough for the interface. + // Configure inherited properties and a base prefix that the VPC will replace. let env = api_fixtures::create_test_env_with_overrides( pool, TestEnvOverrides::default().with_fnn_config(Some(FnnConfig { @@ -479,16 +483,13 @@ async fn test_managed_host_network_config_narrows_interface_anycast_prefixes(poo routing_profiles: HashMap::from([( profile_type.to_string(), FnnRoutingProfileConfig { - internal: true, - access_tier: 0, - leak_default_route_from_underlay: true, - route_target_imports: vec![inherited_import.clone()], - allowed_anycast_prefixes: vpc_allowed_anycast_prefixes - .iter() - .map(|prefix| PrefixFilterPolicyEntry { - prefix: prefix.parse().unwrap(), - }) - .collect(), + internal: Some(true), + access_tier: Some(0), + leak_default_route_from_underlay: Some(true), + route_target_imports: Some(vec![inherited_import.clone()]), + allowed_anycast_prefixes: Some(vec![PrefixFilterPolicyEntry { + prefix: "203.0.113.0/24".parse().unwrap(), + }]), ..Default::default() }, )]), @@ -497,7 +498,7 @@ async fn test_managed_host_network_config_narrows_interface_anycast_prefixes(poo ) .await; - // Create a tenant and FNN VPC using that VPC-level routing profile. + // Override the VPC anycast list while inheriting the other base properties. let tenant = env .api .create_tenant(tonic::Request::new(rpc::forge::CreateTenantRequest { @@ -524,11 +525,22 @@ async fn test_managed_host_network_config_narrows_interface_anycast_prefixes(poo }) .network_virtualization_type(rpc::forge::VpcVirtualizationType::Fnn as i32) .routing_profile_type(profile_type.to_string()) + .routing_profile_overrides(rpc::forge::VpcRoutingProfileOverrides { + allowed_anycast_prefixes: Some(rpc::forge::PrefixFilterPolicyEntries { + values: vpc_allowed_anycast_prefixes + .iter() + .map(|prefix| rpc::forge::PrefixFilterPolicyEntry { + prefix: prefix.to_string(), + }) + .collect(), + }), + ..Default::default() + }) .rpc(), ) .await; - // Allocate an instance with a per-interface anycast prefix subset. + // Allocate an instance with a subset of the effective VPC override. let mut network_config = common::api_fixtures::instance::single_interface_network_config(segment_id); network_config.interfaces[0].routing_profile = @@ -617,20 +629,20 @@ async fn test_managed_host_network_config_includes_per_vpc_routing_profiles(pool ( "INTERNAL".to_string(), FnnRoutingProfileConfig { - internal: true, - access_tier: 1, - leak_default_route_from_underlay: true, - route_target_imports: vec![internal_import.clone()], + internal: Some(true), + access_tier: Some(1), + leak_default_route_from_underlay: Some(true), + route_target_imports: Some(vec![internal_import.clone()]), ..Default::default() }, ), ( "EXTERNAL".to_string(), FnnRoutingProfileConfig { - internal: false, - access_tier: 2, - leak_tenant_host_routes_to_underlay: true, - route_targets_on_exports: vec![external_export.clone()], + internal: Some(false), + access_tier: Some(2), + leak_tenant_host_routes_to_underlay: Some(true), + route_targets_on_exports: Some(vec![external_export.clone()]), ..Default::default() }, ), @@ -941,11 +953,11 @@ async fn test_managed_host_network_config_omits_admin_fnn_vrf_loopback_by_defaul enabled: true, vpc_vni: Some(10000), routing_profile: FnnRoutingProfileConfig { - leak_default_route_from_underlay: true, - route_target_imports: vec![RouteTargetConfig { + leak_default_route_from_underlay: Some(true), + route_target_imports: Some(vec![RouteTargetConfig { asn: 64512, vni: 10000, - }], + }]), ..Default::default() }, }); diff --git a/crates/api-core/src/tests/machine_states.rs b/crates/api-core/src/tests/machine_states.rs index ff7df726fe..141fdde1a6 100644 --- a/crates/api-core/src/tests/machine_states.rs +++ b/crates/api-core/src/tests/machine_states.rs @@ -30,7 +30,7 @@ use carbide_site_explorer::config::SiteExplorerConfig; use carbide_utils::arch::CpuArchitecture; use carbide_uuid::machine::MachineId; use carbide_uuid::machine_validation::MachineValidationId; -use chrono::Duration; +use chrono::{Duration, Utc}; use common::api_fixtures::dpu::{ create_dpu_machine, create_dpu_machine_in_waiting_for_network_install, }; @@ -57,12 +57,15 @@ use model::hardware_info::TpmEkCertificate; use model::machine::health_override::HARDWARE_HEALTH_OVERRIDE_PREFIX; use model::machine::machine_search_config::MachineSearchConfig; use model::machine::{ - BiosConfigInfo, BiosConfigState, CleanupContext, CleanupState, DpuDiscoveringState, - DpuInitState, DpuReprovisionStates, FailureCause, FailureDetails, FailureSource, - HostPlatformConfigurationState, InstallDpuOsState, InstanceState, LockdownMode, MachineState, - MachineValidatingState, ManagedHostState, MeasuringState, PowerState, SetBootOrderInfo, - SetBootOrderState, SetSecureBootState, SpdmMeasuringState, StateMachineArea, ValidationState, + BiosConfigInfo, BiosConfigState, BomValidating, BomValidatingContext, CleanupContext, + CleanupState, DpuDiscoveringState, DpuInitState, DpuReprovisionStates, FailureCause, + FailureDetails, FailureSource, HostPlatformConfigurationState, InstallDpuOsState, + InstanceState, LockdownMode, MachineState, MachineValidatingState, MachineValidationContext, + ManagedHostState, MeasuringState, PowerState, ReadyBootConfigState, + ReadyBootConfigTerminalFailure, SetBootOrderInfo, SetBootOrderState, SetSecureBootState, + SpdmMeasuringState, StateMachineArea, ValidationState, }; +use model::machine_boot_interface::MachineBootInterfaceTarget; use model::machine_validation::MachineValidationState; use model::network_segment::NetworkSegmentType; use model::site_explorer::{EndpointExplorationReport, ExploredDpu, ExploredManagedHost}; @@ -2949,6 +2952,855 @@ async fn host_inband_nic_mac(env: &TestEnv, host_id: MachineId) -> MacAddress { .mac_address } +/// Replaces the fixture host's boot target with distinct operator intent. +/// +/// Site Explorer initializes the durable target during ingestion. Using +/// `machine_desired_boot_interface::set` with a different MAC exercises the +/// normal intent path and guarantees a new version without force-reconcile. +async fn set_pending_boot_interface( + env: &TestEnv, + mh: &TestManagedHost, +) -> config_version::Versioned { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + let current = host + .config + .desired_boot_interface + .as_ref() + .expect("Site Explorer should initialize a durable boot target"); + let first_candidate = MacAddress::new([0x02, 0, 0, 0, 0xfe, 0x01]); + let second_candidate = MacAddress::new([0x02, 0, 0, 0, 0xfe, 0x02]); + let replacement_mac = if current.value.mac_address() == first_candidate { + second_candidate + } else { + first_candidate + }; + let replacement = MachineBootInterfaceTarget::MacOnly(replacement_mac); + + let pending = db::machine_desired_boot_interface::set(txn.as_mut(), &host.id, &replacement) + .await + .expect("setting distinct boot-interface intent should persist a new version"); + assert_ne!( + pending.version, current.version, + "distinct intent must create a pending desired version" + ); + txn.commit().await.unwrap(); + pending +} + +/// Replaces the fixture's DMI vendor so controller vendor branches can be +/// exercised after ordinary ingestion has completed. +async fn set_host_hardware_vendor(env: &TestEnv, mh: &TestManagedHost, vendor: &str) { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + let mut hardware_info = host + .status + .hardware_info + .expect("fixture host should have hardware information"); + hardware_info + .dmi_data + .as_mut() + .expect("fixture host should have DMI information") + .sys_vendor = vendor.to_string(); + db::machine_topology::set_topology_update_needed(txn.as_mut(), &host.id, true) + .await + .unwrap(); + db::machine_topology::create_or_update(txn.as_mut(), &host.id, &hardware_info) + .await + .unwrap(); + txn.commit().await.unwrap(); +} + +/// A zero-DPU host skips the DPU reachability wait. Disable still polls before +/// platform configuration, while Enable preserves the established direct +/// validation path even when reported lockdown status is stale. +#[crate::sqlx_test] +async fn test_zero_dpu_lockdown_wait_preserves_mode_transition(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let polling = ManagedHostState::HostInit { + machine_state: MachineState::WaitingForLockdown { + lockdown_info: model::machine::LockdownInfo { + state: model::machine::LockdownState::PollingLockdownStatus, + mode: LockdownMode::Disable, + }, + }, + }; + + set_host_controller_state_stuck_in( + &env, + mh.host().id, + &ManagedHostState::HostInit { + machine_state: MachineState::WaitingForLockdown { + lockdown_info: model::machine::LockdownInfo { + state: model::machine::LockdownState::TimeWaitForDPUDown, + mode: LockdownMode::Disable, + }, + }, + }, + 0, + ) + .await; + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!(host.current_state(), &polling); + drop(txn); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.current_state(), + &ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 0 }, + }, + ); + drop(txn); + + set_host_controller_state_stuck_in( + &env, + mh.host().id, + &ManagedHostState::HostInit { + machine_state: MachineState::WaitingForLockdown { + lockdown_info: model::machine::LockdownInfo { + state: model::machine::LockdownState::TimeWaitForDPUDown, + mode: LockdownMode::Enable, + }, + }, + }, + 0, + ) + .await; + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.current_state(), + &ManagedHostState::BomValidating { + bom_validating_state: BomValidating::MatchingSku(BomValidatingContext { + machine_validation_context: Some(MachineValidationContext::Discovery), + ..BomValidatingContext::default() + }), + }, + ); +} + +/// Discovery completion must not publish a transient Ready state while a +/// desired boot-interface version still needs controller convergence. +#[crate::sqlx_test] +async fn test_discovered_host_with_pending_boot_config_enters_convergence(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mut host_config = ManagedHostConfig::zero_dpu(); + host_config.vendor = Some(bmc_vendor::BMCVendor::Supermicro); + let mh = create_managed_host_with_config(&env, host_config).await; + set_host_hardware_vendor(&env, &mh, "Supermicro").await; + let pending = set_pending_boot_interface(&env, &mh).await; + + set_host_controller_state_stuck_in( + &env, + mh.host().id, + &ManagedHostState::HostInit { + machine_state: MachineState::WaitingForLockdown { + lockdown_info: model::machine::LockdownInfo { + state: model::machine::LockdownState::PollingLockdownStatus, + mode: LockdownMode::Enable, + }, + }, + }, + 0, + ) + .await; + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + let lockdown_checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.pending_boot_interface_config_version(), + Some(pending.version), + "a locked Supermicro read must not publish boot verification" + ); + drop(txn); + assert!( + !env.redfish_sim + .actions_since(&lockdown_checkpoint) + .all_hosts() + .iter() + .any(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. })), + "HostInit must defer Supermicro verification until the unlocked Ready flow" + ); + + set_host_controller_state_stuck_in( + &env, + mh.host().id, + &ManagedHostState::HostInit { + machine_state: MachineState::Discovered { + skip_reboot_wait: true, + }, + }, + 0, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.current_state(), + &ManagedHostState::BootConfiguring { + desired_version: pending.version, + desired_boot_interface: pending.value, + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::Prepare, + }, + ); +} + +/// Supermicro boot-order reads can be stale under lockdown. Ready must create +/// its exact verification while unlocked, then retain that proof while +/// restoring lockdown instead of trusting a contradictory locked read. +#[crate::sqlx_test] +async fn test_supermicro_ready_boot_config_uses_unlocked_verification(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mut host_config = ManagedHostConfig::zero_dpu(); + host_config.vendor = Some(bmc_vendor::BMCVendor::Supermicro); + let mh = create_managed_host_with_config(&env, host_config).await; + set_host_hardware_vendor(&env, &mh, "Supermicro").await; + let pending = set_pending_boot_interface(&env, &mh).await; + let expected_mac = pending.value.mac_address().to_string(); + let locking = ManagedHostState::BootConfiguring { + desired_version: pending.version, + desired_boot_interface: pending.value.clone(), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + }; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + let convergence_checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + env.run_machine_state_controller_iteration_until_state_matches(&mh.id, 10, locking.clone()) + .await; + + let history = mh.host().parsed_history(None).await; + assert!( + history.iter().any(|state| matches!( + state, + ManagedHostState::BootConfiguring { + desired_version, + boot_config_state: ReadyBootConfigState::UnlockHost { .. }, + .. + } if *desired_version == pending.version + )), + "Supermicro convergence must pass through the unlock choreography: {history:#?}" + ); + let convergence_actions = env + .redfish_sim + .actions_since(&convergence_checkpoint) + .all_hosts(); + assert!( + convergence_actions.iter().any(|action| matches!( + action, + RedfishSimAction::Power(libredfish::SystemPowerControl::ForceRestart) + )), + "Supermicro must reboot after lockdown is disabled: {convergence_actions:?}" + ); + assert!( + convergence_actions.iter().any(|action| matches!( + action, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if boot_interface_mac == &expected_mac + )), + "Supermicro must verify the exact target before relocking: {convergence_actions:?}" + ); + + // Model the stale locked view reported by Supermicro after the durable + // unlocked verification has advanced the controller to LockHost. + env.redfish_sim.set_is_boot_order_setup(false); + let lock_checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!(host.current_state(), &ManagedHostState::Ready); + assert_eq!(host.pending_boot_interface_config_version(), None); + assert_eq!( + host.status + .boot_interface_status_observation + .as_ref() + .map(|observation| observation.config_version), + Some(pending.version), + ); + drop(txn); + + let lock_actions = env.redfish_sim.actions_since(&lock_checkpoint).all_hosts(); + assert!( + !lock_actions + .iter() + .any(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. })), + "LockHost must not replace the durable unlocked proof with a stale locked read: {lock_actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|state| *state == libredfish::EnabledDisabled::Enabled), + "the target must be verified only after lockdown is restored" + ); +} + +/// A pending desired version on an unassigned Ready host is fully owned by the +/// state controller: it persists each convergence phase, repairs Redfish, +/// restores lockdown, and only then records the exact desired version verified. +#[crate::sqlx_test] +async fn test_ready_converges_pending_desired_boot_interface(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let pending = set_pending_boot_interface(&env, &mh).await; + let expected_mac = pending.value.mac_address().to_string(); + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + let checkpoint = env.redfish_sim.timepoint(); + let reconciliation_started_at = Utc::now(); + + env.run_machine_state_controller_iteration().await; + + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.current_state(), + &ManagedHostState::BootConfiguring { + desired_version: pending.version, + desired_boot_interface: pending.value.clone(), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::Prepare, + }, + "Ready must persist the captured target before Redfish work" + ); + assert_ne!( + host.status + .boot_interface_status_observation + .as_ref() + .map(|observation| observation.config_version), + Some(pending.version), + "entering BootConfiguring is not itself a Redfish verification" + ); + } + + env.run_machine_state_controller_iteration_until_state_matches( + &mh.id, + 20, + ManagedHostState::Ready, + ) + .await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + let persisted_desired = host + .config + .desired_boot_interface + .as_ref() + .expect("convergence must retain operator intent"); + assert_eq!(persisted_desired.version, pending.version); + assert_eq!(persisted_desired.value, pending.value); + let observation = host + .status + .boot_interface_status_observation + .as_ref() + .expect("successful convergence should record a Redfish observation"); + assert_eq!( + observation.config_version, pending.version, + "only the exact desired version that was checked may be marked verified" + ); + assert!( + observation.observed_at >= reconciliation_started_at, + "verification must come from this convergence pass" + ); + assert!( + !observation.assumed, + "controller verification must replace any migration-assumed baseline" + ); + assert_eq!( + host.pending_boot_interface_config_version(), + None, + "the verified host should no longer have pending boot intent" + ); + drop(txn); + + let history = mh.host().parsed_history(None).await; + let prepare_index = history + .iter() + .position(|state| { + matches!( + state, + ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface, + boot_config_state: ReadyBootConfigState::Prepare, + .. + } if *desired_version == pending.version + && desired_boot_interface == &pending.value + ) + }) + .expect("Ready should persist BootConfiguring/Prepare"); + let set_order_index = history + .iter() + .position(|state| { + matches!( + state, + ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface, + boot_config_state: ReadyBootConfigState::SetBootOrder { .. }, + .. + } if *desired_version == pending.version + && desired_boot_interface == &pending.value + ) + }) + .expect("boot-order drift should persist its remediation substate"); + let lock_index = history + .iter() + .position(|state| { + matches!( + state, + ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + .. + } if *desired_version == pending.version + && desired_boot_interface == &pending.value + ) + }) + .expect("successful repair should persist LockHost before verification"); + assert!( + prepare_index < set_order_index && set_order_index < lock_index, + "persisted convergence phases should retain their causal order: {history:#?}" + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions.iter().any(|action| matches!( + action, + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } + if boot_interface_mac == &expected_mac + )), + "the controller should repair the desired target's boot order: {actions:?}" + ); + assert!( + actions.iter().any(|action| matches!( + action, + RedfishSimAction::Power(libredfish::SystemPowerControl::ForceRestart) + )), + "boot-order remediation should reboot to apply the change: {actions:?}" + ); + assert!( + actions + .iter() + .filter(|action| matches!( + action, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if boot_interface_mac == &expected_mac + )) + .count() + >= 2, + "the controller should observe the desired target before repair and again at final verification: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|state| *state == libredfish::EnabledDisabled::Enabled), + "the BMC must be locked before the desired version is published as verified" + ); +} + +/// A correct ordinary host proves its target before opening lockdown and must +/// not issue any boot mutation or reboot. +#[crate::sqlx_test] +async fn test_ready_boot_config_skips_unlock_when_already_correct(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let pending = set_pending_boot_interface(&env, &mh).await; + let expected_mac = pending.value.mac_address().to_string(); + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + let checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + env.run_machine_state_controller_iteration_until_state_matches( + &mh.id, + 5, + ManagedHostState::Ready, + ) + .await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + let observation = host + .status + .boot_interface_status_observation + .as_ref() + .expect("the already-correct target should be verified"); + assert_eq!(observation.config_version, pending.version); + assert!(!observation.assumed); + drop(txn); + + let history = mh.host().parsed_history(None).await; + assert!(history.iter().any(|state| matches!( + state, + ManagedHostState::BootConfiguring { + desired_version, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + .. + } if *desired_version == pending.version + ))); + assert!( + !history.iter().any(|state| matches!( + state, + ManagedHostState::BootConfiguring { + desired_version, + boot_config_state: + ReadyBootConfigState::UnlockHost { .. } + | ReadyBootConfigState::ConfigureBios { .. } + | ReadyBootConfigState::SetBootOrder { .. }, + .. + } if *desired_version == pending.version + )), + "an already-correct ordinary host should not be opened or mutated: {history:#?}" + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .filter(|action| matches!( + action, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if boot_interface_mac == &expected_mac + )) + .count() + >= 2, + "the target should be observed before cleanup and at final verification: {actions:?}" + ); + assert!( + !actions.iter().any(|action| matches!( + action, + RedfishSimAction::MachineSetup { .. } + | RedfishSimAction::SetBootOrderDpuFirst { .. } + | RedfishSimAction::Power(_) + )), + "the correct fast path must remain observation-only: {actions:?}" + ); +} + +/// Hosts whose lifecycle profile intentionally leaves lockdown disabled still +/// take the already-correct fast path. LockHost has no policy restoration to +/// perform for them, but it still re-observes the exact target before marking +/// the desired version verified. +#[crate::sqlx_test] +async fn test_ready_boot_config_disable_lockdown_skips_unneeded_cleanup(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mut expected_machine_data = ExpectedMachineData::default(); + expected_machine_data + .host_lifecycle_profile + .disable_lockdown = Some(true); + let mh = create_managed_host_with_config( + &env, + ManagedHostConfig::zero_dpu().with_expected_machine_data(expected_machine_data), + ) + .await; + let pending = set_pending_boot_interface(&env, &mh).await; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + + env.run_machine_state_controller_iteration().await; + env.run_machine_state_controller_iteration().await; + let client_calls_after_preflight = env.redfish_sim.create_client_calls().len(); + + env.run_machine_state_controller_iteration_until_state_matches( + &mh.id, + 2, + ManagedHostState::Ready, + ) + .await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.status + .boot_interface_status_observation + .as_ref() + .map(|observation| observation.config_version), + Some(pending.version), + ); + drop(txn); + + assert_eq!( + env.redfish_sim.create_client_calls().len(), + client_calls_after_preflight + 1, + "LockHost should reconnect only for the final exact-target observation" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|state| *state == libredfish::EnabledDisabled::Enabled), + "an already-correct host should not have lockdown changed" + ); + let history = mh.host().parsed_history(None).await; + assert!( + !history.iter().any(|state| matches!( + state, + ManagedHostState::BootConfiguring { + boot_config_state: ReadyBootConfigState::UnlockHost { .. }, + .. + } + )), + "an already-correct host must not enter the unlock choreography: {history:#?}" + ); +} + +/// A machine-level failure is more fundamental than boot convergence. Once it +/// has been durably captured on a host whose profile has no lockdown cleanup +/// barrier, it must remain reportable even without Redfish access. +#[crate::sqlx_test] +async fn test_ready_boot_config_machine_failure_does_not_wait_for_redfish(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mut expected_machine_data = ExpectedMachineData::default(); + expected_machine_data + .host_lifecycle_profile + .disable_lockdown = Some(true); + let mh = create_managed_host_with_config( + &env, + ManagedHostConfig::zero_dpu().with_expected_machine_data(expected_machine_data), + ) + .await; + let pending = set_pending_boot_interface(&env, &mh).await; + let details = FailureDetails { + cause: FailureCause::Discovery { + err: "host BMC became unavailable".to_string(), + }, + failed_at: Utc::now(), + source: FailureSource::StateMachine, + }; + let locking = ManagedHostState::BootConfiguring { + desired_version: pending.version, + desired_boot_interface: pending.value, + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: Some(ReadyBootConfigTerminalFailure::Machine { + machine_id: mh.host().id, + details: details.clone(), + }), + }, + }; + + set_host_controller_state_stuck_in(&env, mh.host().id, &locking, 0).await; + let client_calls_before_failure = env.redfish_sim.create_client_calls().len(); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.current_state(), + &ManagedHostState::Failed { + details, + machine_id: mh.host().id, + retry_count: 0, + } + ); + assert_eq!( + env.redfish_sim.create_client_calls().len(), + client_calls_before_failure, + "machine failure must transition without opening a Redfish client" + ); +} + +/// `LockHost` is a durable restart boundary. The resumed invocation must +/// restore lockdown before directly re-reading the captured target. +#[crate::sqlx_test] +async fn test_ready_boot_config_lock_host_is_restart_safe(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let pending = set_pending_boot_interface(&env, &mh).await; + let expected_mac = pending.value.mac_address().to_string(); + let locking = ManagedHostState::BootConfiguring { + desired_version: pending.version, + desired_boot_interface: pending.value.clone(), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + }; + + set_host_controller_state_stuck_in(&env, mh.host().id, &locking, 0).await; + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + let checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!(host.current_state(), &ManagedHostState::Ready); + let observation = host + .status + .boot_interface_status_observation + .as_ref() + .expect("resumed LockHost should publish a direct verification"); + assert_eq!(observation.config_version, pending.version); + assert!(!observation.assumed); + drop(txn); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions.iter().any(|action| matches!( + action, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if boot_interface_mac == &expected_mac + )), + "resumed LockHost must directly observe its captured target after lockdown: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|state| *state == libredfish::EnabledDisabled::Enabled), + "resumed LockHost must restore lockdown before verification" + ); +} + +/// LockHost is a fail-closed cleanup boundary. Losing the BMC interface mapping +/// must keep the host parked there instead of propagating an error that could +/// let later failure handling bypass lockdown restoration. +#[crate::sqlx_test] +async fn test_ready_boot_config_lock_host_waits_for_redfish_access(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let pending = set_pending_boot_interface(&env, &mh).await; + let locking = ManagedHostState::BootConfiguring { + desired_version: pending.version, + desired_boot_interface: pending.value, + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + }; + + set_host_controller_state_stuck_in(&env, mh.host().id, &locking, 0).await; + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + let bmc_ip = host + .bmc_addr() + .expect("fixture host has a BMC address") + .ip(); + sqlx::query("DELETE FROM machine_interface_addresses WHERE address = $1::inet") + .bind(bmc_ip) + .execute(txn.as_mut()) + .await + .unwrap(); + txn.commit().await.unwrap(); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!( + host.current_state(), + &locking, + "cleanup must wait in LockHost until Redfish access is restored" + ); + assert_eq!( + host.pending_boot_interface_config_version(), + Some(pending.version), + "a connectivity failure must not publish verification" + ); +} + +/// A successful Redfish write is not sufficient evidence that lockdown was +/// actually restored. Keep the desired version pending in LockHost until a +/// direct status read confirms the policy. +#[crate::sqlx_test] +async fn test_ready_boot_config_waits_for_observed_lockdown_before_verifying(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let pending = set_pending_boot_interface(&env, &mh).await; + let locking = ManagedHostState::BootConfiguring { + desired_version: pending.version, + desired_boot_interface: pending.value.clone(), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + }; + + set_host_controller_state_stuck_in(&env, mh.host().id, &locking, 0).await; + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + env.redfish_sim.set_lockdown_bmc_applies(false); + let checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert_eq!(host.current_state(), &locking); + assert_eq!( + host.pending_boot_interface_config_version(), + Some(pending.version), + "an unobserved lockdown write must not publish verification" + ); + drop(txn); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + !actions + .iter() + .any(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. })), + "final boot inspection must wait until lockdown is observed: {actions:?}" + ); +} + /// Asserts that, within the recorded boot-order actions, `machine_setup` ran /// before the `set_boot_order_dpu_first` reorder, and that BOTH targeted /// `expected_mac` -- the resolved boot NIC. This is the ordering the recovery diff --git a/crates/api-core/src/tests/network_segment.rs b/crates/api-core/src/tests/network_segment.rs index 0e42d1d15f..712e345216 100644 --- a/crates/api-core/src/tests/network_segment.rs +++ b/crates/api-core/src/tests/network_segment.rs @@ -95,6 +95,7 @@ async fn create_stretchable_segment_for_svi_test_with_vpc_type( }, network_security_group_id: None, routing_profile_type: None, + routing_profile_overrides: None, vni: None, }, VpcStatus { vni: None }, @@ -712,6 +713,7 @@ pub async fn test_create_initial_vpc_and_attached_network( organization_id: Some(FIXTURE_TENANT_ORG_ID.to_string()), network_virtualization_type: VpcVirtualizationType::Flat, routing_profile_type: None, + routing_profile_overrides: None, vni: None, }, )]); @@ -913,6 +915,7 @@ async fn initial_vpc_allocation_failures_preserve_errors_and_emit( organization_id: Some(FIXTURE_TENANT_ORG_ID.to_string()), network_virtualization_type: VpcVirtualizationType::Flat, routing_profile_type: None, + routing_profile_overrides: None, vni: failure.requested_vni(), }, )]); diff --git a/crates/api-core/src/tests/power_shelf_state_controller/bmc_rotation.rs b/crates/api-core/src/tests/power_shelf_state_controller/bmc_rotation.rs new file mode 100644 index 0000000000..7d8b13a78e --- /dev/null +++ b/crates/api-core/src/tests/power_shelf_state_controller/bmc_rotation.rs @@ -0,0 +1,608 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! End-to-end coverage for power-shelf-controller BMC (PMC) credential rotation +//! (REQ-2): with the site-wide flag enabled, a staged target drives a Ready +//! power shelf through `PowerShelfControllerState::RotatingBmc` and back to +//! Ready, converging the device and persisting the rotated per-device secret. +//! Mirrors the switch-controller integration test +//! `ready_switch_converges_bmc_to_site_target`. + +use std::sync::Arc; +use std::time::Duration; + +use carbide_credential_rotation::BmcRotationGate; +use carbide_power_shelf_controller::context::PowerShelfStateHandlerServices; +use carbide_power_shelf_controller::handler::PowerShelfStateHandler; +use carbide_power_shelf_controller::io::PowerShelfStateControllerIO; +use carbide_secrets::credentials::{ + BmcCredentialType, CredentialKey, CredentialReader, CredentialWriter, Credentials, +}; +use carbide_uuid::machine::MachineInterfaceId; +use carbide_uuid::network::NetworkSegmentId; +use carbide_uuid::power_shelf::PowerShelfId; +use chrono::Utc; +use db::credential_rotation::{ + CredentialRotationType, device_rotation_status, increment_rotate_attempt, + record_device_converged, set_next_target_version, +}; +use db::power_shelf as db_power_shelf; +use mac_address::MacAddress; +use model::allocation_type::AllocationType; +use model::power_shelf::{PowerShelf, PowerShelfControllerState}; +use state_controller::config::IterationConfig; +use state_controller::controller::StateController; +use tokio_util::sync::CancellationToken; + +use super::fixtures::power_shelf::set_power_shelf_controller_state; +use crate::tests::common; +use crate::tests::common::api_fixtures::{TestEnv, create_test_env}; + +type TestResult = Result>; + +const BMC: CredentialRotationType = CredentialRotationType::Bmc; + +fn per_device_key(mac: MacAddress) -> CredentialKey { + CredentialKey::BmcCredentials { + credential_type: BmcCredentialType::BmcRoot { + bmc_mac_address: mac, + }, + } +} + +fn rotate_to_key(version: u32) -> CredentialKey { + CredentialKey::BmcCredentials { + credential_type: BmcCredentialType::site_wide_root(version), + } +} + +fn creds(username: &str, password: &str) -> Credentials { + Credentials::UsernamePassword { + username: username.to_string(), + password: password.to_string(), + } +} + +/// Services with the passive BMC-rotation gate toggled by `bmc_rotation_enabled` +/// and no component manager, so a Ready power shelf idles on power-state polling +/// and reaches the BMC gate. A fresh [`BmcRotationGate`] refreshes its cached +/// aggregate live on first use each iteration. +fn power_shelf_services( + env: &TestEnv, + pool: &sqlx::PgPool, + bmc_rotation_enabled: bool, +) -> PowerShelfStateHandlerServices { + PowerShelfStateHandlerServices { + db_pool: pool.clone(), + component_manager: None, + credential_manager: env.test_credential_manager.clone(), + per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled: false, + redfish_client_pool: env.redfish_sim.clone(), + bmc_rotation_gate: BmcRotationGate::new(), + bmc_rotation_enabled, + } +} + +/// Run a single power-shelf-controller iteration with a fresh controller (and a +/// fresh rotation gate), mirroring `run_switch_controller_with_services`. +async fn run_power_shelf_controller_with_services( + pool: sqlx::PgPool, + work_lock_manager_handle: db::work_lock_manager::WorkLockManagerHandle, + services: PowerShelfStateHandlerServices, +) { + let cancel_token = CancellationToken::new(); + let mut controller = StateController::::builder() + .iteration_config(IterationConfig { + iteration_time: Duration::from_millis(50), + processor_dispatch_interval: Duration::from_millis(10), + ..Default::default() + }) + .database(pool, work_lock_manager_handle) + .processor_id(uuid::Uuid::new_v4().to_string()) + .services(services.into()) + .state_handler(Arc::new(PowerShelfStateHandler::default())) + .build_for_manual_iterations(cancel_token) + .unwrap(); + controller.run_single_iteration().await; +} + +/// Link a `Bmc` machine_interface (with a MAC and IP) back to the power shelf so +/// the shelf load query resolves `bmc_info`, giving the controller an +/// addressable PMC endpoint to rotate. Returns the PMC MAC. +async fn seed_pmc_endpoint( + pool: &sqlx::PgPool, + power_shelf_id: PowerShelfId, +) -> TestResult { + let mut txn = pool.begin().await?; + + let segment_id: NetworkSegmentId = sqlx::query_scalar( + "INSERT INTO network_segments (name, version, network_segment_type) + VALUES ($1, 'V1-T0', 'tenant') RETURNING id", + ) + .bind(format!("pmc-{power_shelf_id}")) + .fetch_one(txn.as_mut()) + .await?; + + let pmc_mac = "02:00:00:00:0b:01"; + let bmc_interface_id: MachineInterfaceId = sqlx::query_scalar( + "INSERT INTO machine_interfaces + (power_shelf_id, association_type, segment_id, mac_address, + primary_interface, hostname, interface_type) + VALUES ($1, 'PowerShelf', $2, $3::macaddr, false, 'pmc', 'Bmc') + RETURNING id", + ) + .bind(power_shelf_id) + .bind(segment_id) + .bind(pmc_mac) + .fetch_one(txn.as_mut()) + .await?; + + db::machine_interface_address::insert( + txn.as_mut(), + bmc_interface_id, + "10.30.40.50".parse()?, + AllocationType::Dhcp, + ) + .await?; + + txn.commit().await?; + + Ok(pmc_mac.parse()?) +} + +async fn load_power_shelf(pool: &sqlx::PgPool, id: &PowerShelfId) -> TestResult { + let mut conn = pool.acquire().await?; + Ok(db_power_shelf::find_by_id(&mut conn, id) + .await? + .expect("power shelf should exist")) +} + +/// Stage a PMC that lags a freshly published site-wide target v1: seed the PMC's +/// "old" per-device secret, record it converged at the v0 baseline, advance the +/// target to v1, and write the rotate-to secret `RotateCredential` would have +/// staged. After this the device's rotation row lags the target, so the passive +/// gate (or a force request) will drive a rotation that converges it to "new". +async fn stage_lagging_pmc(env: &TestEnv, pool: &sqlx::PgPool, pmc_mac: MacAddress) -> TestResult { + env.redfish_sim.seed_user("root", "old"); + env.test_credential_manager + .set_credentials(&per_device_key(pmc_mac), &creds("root", "old")) + .await + .expect("staging the per-device secret should succeed"); + { + let mut conn = pool.acquire().await?; + record_device_converged(&mut conn, pmc_mac, BMC).await?; + set_next_target_version(&mut conn, BMC, 0, serde_json::json!({})) + .await? + .expect("target must advance from version 0"); + } + env.test_credential_manager + .set_credentials(&rotate_to_key(1), &creds("root", "new")) + .await + .expect("staging the rotate-to secret should succeed"); + Ok(()) +} + +/// Move a power shelf directly to `Ready` so the BMC-rotation gate is the only +/// pending work when the controller next sweeps it. +async fn move_to_ready(pool: &sqlx::PgPool, power_shelf_id: &PowerShelfId) -> TestResult { + let mut txn = pool.begin().await?; + set_power_shelf_controller_state( + txn.as_mut(), + power_shelf_id, + PowerShelfControllerState::Ready, + ) + .await?; + txn.commit().await?; + Ok(()) +} + +/// A Ready power shelf whose PMC lags a freshly staged site-wide target rotates +/// on its own once the feature flag is on: the entry guard promotes it to +/// `RotatingBmc`, the rotation converges the device and rewrites the per-device +/// secret, and the power shelf returns to Ready. +#[crate::sqlx_test] +async fn ready_power_shelf_converges_pmc_to_site_target(pool: sqlx::PgPool) -> TestResult { + let env = create_test_env(pool.clone()).await; + + let power_shelf_id = common::api_fixtures::site_explorer::new_power_shelf( + &env, + Some("BMC Rotation Test Power Shelf".to_string()), + Some(5000), + Some(240), + Some("Data Center A, Rack 1".to_string()), + ) + .await?; + let pmc_mac = seed_pmc_endpoint(&pool, power_shelf_id).await?; + + // Move the power shelf to Ready so the BMC-rotation gate is the only pending + // work. + { + let mut txn = pool.begin().await?; + set_power_shelf_controller_state( + txn.as_mut(), + &power_shelf_id, + PowerShelfControllerState::Ready, + ) + .await?; + txn.commit().await?; + } + + // The PMC currently holds the per-device "old" secret. + env.redfish_sim.seed_user("root", "old"); + env.test_credential_manager + .set_credentials(&per_device_key(pmc_mac), &creds("root", "old")) + .await + .expect("staging the per-device secret should succeed"); + + // Stage a site-wide rotation to version 1: record the device converged at + // the v0 baseline, advance the target, and write the rotate-to secret that + // `RotateCredential` would have staged. + { + let mut conn = pool.acquire().await?; + record_device_converged(&mut conn, pmc_mac, BMC).await?; + set_next_target_version(&mut conn, BMC, 0, serde_json::json!({})) + .await? + .expect("target must advance from version 0"); + } + env.test_credential_manager + .set_credentials(&rotate_to_key(1), &creds("root", "new")) + .await + .expect("staging the rotate-to secret should succeed"); + + // The device lags the staged target before the controller runs. + { + let mut conn = pool.acquire().await?; + let status = device_rotation_status(&mut conn, BMC, pmc_mac) + .await? + .expect("device rotation row should exist"); + assert!( + !status.converged, + "device should lag the staged target before rotation" + ); + } + + // Iteration 1: Ready observes the lag and enters RotatingBmc. + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, true), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::RotatingBmc { .. } + ), + "expected RotatingBmc after the entry guard fires, got {:?}", + power_shelf.controller_state.value, + ); + + // Iteration 2: the rotation converges the device and returns to Ready. + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, true), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::Ready + ), + "expected Ready once rotation settles, got {:?}", + power_shelf.controller_state.value, + ); + + // The device is converged at the target, and the per-device secret is the + // rotated value. + { + let mut conn = pool.acquire().await?; + let status = device_rotation_status(&mut conn, BMC, pmc_mac) + .await? + .expect("device rotation row should exist"); + assert!( + status.converged, + "device should be converged after rotation" + ); + assert_eq!( + status.current_version, + Some(1), + "device should be recorded at target version 1" + ); + } + let persisted = env + .test_credential_manager + .get_credentials(&per_device_key(pmc_mac)) + .await + .expect("reading the per-device secret should succeed") + .expect("per-device secret should still be set"); + assert_eq!( + persisted, + creds("root", "new"), + "per-device secret should be rotated to the new password" + ); + + Ok(()) +} + +/// A failing PMC rotation must not trap the power shelf in `RotatingBmc`: the +/// engine quarantines the device (backoff) and the handler returns to `Ready`, +/// and because the passive gate skips quarantined devices the shelf then stays +/// in `Ready` rather than hot-looping Ready -> RotatingBmc every sweep. The +/// bounded transient-retry budget is unit-tested separately on `advance`; this +/// covers the device-fault (quarantine) arm end-to-end. +#[crate::sqlx_test] +async fn failing_pmc_rotation_returns_to_ready_and_quarantines(pool: sqlx::PgPool) -> TestResult { + let env = create_test_env(pool.clone()).await; + + let power_shelf_id = common::api_fixtures::site_explorer::new_power_shelf( + &env, + Some("BMC Rotation Failure Test Power Shelf".to_string()), + Some(5000), + Some(240), + Some("Data Center A, Rack 1".to_string()), + ) + .await?; + let pmc_mac = seed_pmc_endpoint(&pool, power_shelf_id).await?; + + { + let mut txn = pool.begin().await?; + set_power_shelf_controller_state( + txn.as_mut(), + &power_shelf_id, + PowerShelfControllerState::Ready, + ) + .await?; + txn.commit().await?; + } + + env.redfish_sim.seed_user("root", "old"); + env.test_credential_manager + .set_credentials(&per_device_key(pmc_mac), &creds("root", "old")) + .await + .expect("staging the per-device secret should succeed"); + + // Stage a lagging target to version 1 but deliberately do NOT stage the + // rotate-to secret, so the rotation attempt fails and the engine quarantines + // the device (the `rotate_bmc_quarantines_when_rotate_to_secret_is_not_staged` + // arm) rather than converging. + { + let mut conn = pool.acquire().await?; + record_device_converged(&mut conn, pmc_mac, BMC).await?; + set_next_target_version(&mut conn, BMC, 0, serde_json::json!({})) + .await? + .expect("target must advance from version 0"); + } + + // Iteration 1: Ready observes the lag and enters RotatingBmc. + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, true), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::RotatingBmc { .. } + ), + "expected RotatingBmc after the entry guard fires, got {:?}", + power_shelf.controller_state.value, + ); + + // Iteration 2: the rotation attempt fails; the engine quarantines the device + // and the handler settles back to Ready (RotatingBmc is not terminal). + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, true), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::Ready + ), + "expected Ready after a failed rotation settles, got {:?}", + power_shelf.controller_state.value, + ); + { + let mut conn = pool.acquire().await?; + let status = device_rotation_status(&mut conn, BMC, pmc_mac) + .await? + .expect("device rotation row should exist"); + assert!(!status.converged, "a failed rotation must not converge"); + assert!( + status.quarantined, + "a failed rotation must quarantine the device with backoff" + ); + assert!( + status.rotate_attempts >= 1, + "a failed rotation must record at least one attempt, got {}", + status.rotate_attempts + ); + } + + // Further sweeps must NOT re-enter RotatingBmc: the passive gate skips the + // quarantined device, so the shelf idles in Ready until backoff elapses + // instead of hot-looping Ready -> RotatingBmc -> Ready forever. + for _ in 0..3 { + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, true), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::Ready + ), + "a quarantined shelf must stay in Ready, got {:?}", + power_shelf.controller_state.value, + ); + } + + Ok(()) +} + +/// With the site-wide feature flag off (the production default), a Ready power +/// shelf whose PMC lags the staged target must NOT rotate on its own: the +/// passive gate is the fleet kill-switch, so the shelf stays in `Ready`. Mirrors +/// the machine-controller `feature_flag_off_suppresses_passive_rotation`. +#[crate::sqlx_test] +async fn feature_flag_off_suppresses_pmc_rotation(pool: sqlx::PgPool) -> TestResult { + let env = create_test_env(pool.clone()).await; + + let power_shelf_id = common::api_fixtures::site_explorer::new_power_shelf( + &env, + Some("BMC Rotation Kill-Switch Test Power Shelf".to_string()), + Some(5000), + Some(240), + Some("Data Center A, Rack 1".to_string()), + ) + .await?; + let pmc_mac = seed_pmc_endpoint(&pool, power_shelf_id).await?; + move_to_ready(&pool, &power_shelf_id).await?; + stage_lagging_pmc(&env, &pool, pmc_mac).await?; + + // A full sweep with the flag OFF must leave the lagging shelf in Ready. + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, false), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::Ready + ), + "expected Ready to be preserved while the feature flag is off, got {:?}", + power_shelf.controller_state.value, + ); + { + let mut conn = pool.acquire().await?; + let status = device_rotation_status(&mut conn, BMC, pmc_mac) + .await? + .expect("device rotation row should exist"); + assert!( + !status.converged, + "the PMC must remain unrotated while the feature flag is off" + ); + } + + Ok(()) +} + +/// The operator force-converge escape hatch overrides both the site-wide flag +/// (off here) and the device's active backoff quarantine: the targeted PMC is +/// rotated on the next sweep and the one-shot request is cleared afterward. +/// Mirrors the machine-controller `force_request_converges_quarantined_bmc_when_disabled`. +#[crate::sqlx_test] +async fn force_request_converges_quarantined_pmc_when_disabled(pool: sqlx::PgPool) -> TestResult { + let env = create_test_env(pool.clone()).await; + + let power_shelf_id = common::api_fixtures::site_explorer::new_power_shelf( + &env, + Some("BMC Rotation Force Test Power Shelf".to_string()), + Some(5000), + Some(240), + Some("Data Center A, Rack 1".to_string()), + ) + .await?; + let pmc_mac = seed_pmc_endpoint(&pool, power_shelf_id).await?; + move_to_ready(&pool, &power_shelf_id).await?; + stage_lagging_pmc(&env, &pool, pmc_mac).await?; + + // Quarantine the device (so the passive gate would skip it even if enabled) + // and record the operator's force-converge request on the shelf row. + { + let mut conn = pool.acquire().await?; + increment_rotate_attempt( + &mut conn, + pmc_mac, + BMC, + "seed backoff", + Utc::now() + chrono::Duration::seconds(3600), + ) + .await?; + db_power_shelf::set_bmc_credential_rotation_requested(&mut conn, power_shelf_id).await?; + } + + // Iteration 1: the force request drives entry into RotatingBmc despite the + // disabled site-wide flag. + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, false), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::RotatingBmc { .. } + ), + "expected RotatingBmc from the force request, got {:?}", + power_shelf.controller_state.value, + ); + + // Iteration 2: the forced tick bypasses backoff, converges the device, and + // returns to Ready. + run_power_shelf_controller_with_services( + pool.clone(), + env.api.work_lock_manager_handle.clone(), + power_shelf_services(&env, &pool, false), + ) + .await; + let power_shelf = load_power_shelf(&pool, &power_shelf_id).await?; + assert!( + matches!( + power_shelf.controller_state.value, + PowerShelfControllerState::Ready + ), + "expected Ready once the forced rotation settles, got {:?}", + power_shelf.controller_state.value, + ); + + // The device converged despite its quarantine, and the one-shot request was + // cleared so the shelf does not re-enter RotatingBmc on the next sweep. + { + let mut conn = pool.acquire().await?; + let status = device_rotation_status(&mut conn, BMC, pmc_mac) + .await? + .expect("device rotation row should exist"); + assert!( + status.converged, + "the forced rotation must converge the PMC despite its quarantine" + ); + } + assert!( + !power_shelf.bmc_credential_rotation_requested, + "the one-shot force request must be cleared after a settled forced rotation" + ); + + Ok(()) +} diff --git a/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs b/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs index 53f60df299..9591d5f274 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs @@ -67,6 +67,9 @@ async fn services( credential_manager: Arc::new(TestCredentialManager::default()), per_object_metrics_registry: env.per_object_metrics_registry(), rack_firmware_reprovisioning_enabled: false, + redfish_client_pool: env.redfish_sim.clone(), + bmc_rotation_gate: carbide_credential_rotation::BmcRotationGate::new(), + bmc_rotation_enabled: false, } } diff --git a/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs b/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs index 89ecc541bc..fd7bf2b731 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs @@ -145,6 +145,9 @@ fn services_with_component_manager( })), per_object_metrics_registry: env.per_object_metrics_registry(), rack_firmware_reprovisioning_enabled: false, + redfish_client_pool: env.redfish_sim.clone(), + bmc_rotation_gate: carbide_credential_rotation::BmcRotationGate::new(), + bmc_rotation_enabled: false, } } diff --git a/crates/api-core/src/tests/power_shelf_state_controller/mod.rs b/crates/api-core/src/tests/power_shelf_state_controller/mod.rs index 26ecbf79c8..dea8550533 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/mod.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/mod.rs @@ -28,6 +28,7 @@ use tokio_util::sync::CancellationToken; use crate::tests::common; use crate::tests::common::api_fixtures::create_test_env; +mod bmc_rotation; mod error_state; mod fixtures; mod maintenance; @@ -73,6 +74,9 @@ async fn test_power_shelf_deletion_with_state_controller( credential_manager: credential_manager.clone(), per_object_metrics_registry: env.per_object_metrics_registry(), rack_firmware_reprovisioning_enabled: false, + redfish_client_pool: env.redfish_sim.clone(), + bmc_rotation_gate: carbide_credential_rotation::BmcRotationGate::new(), + bmc_rotation_enabled: false, } .into(), ) diff --git a/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs b/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs index 4568f054e9..006f84c621 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs @@ -55,6 +55,9 @@ fn services( credential_manager: Arc::new(TestCredentialManager::default()), per_object_metrics_registry: env.per_object_metrics_registry(), rack_firmware_reprovisioning_enabled, + redfish_client_pool: env.redfish_sim.clone(), + bmc_rotation_gate: carbide_credential_rotation::BmcRotationGate::new(), + bmc_rotation_enabled: false, } } diff --git a/crates/api-core/src/tests/vpc.rs b/crates/api-core/src/tests/vpc.rs index f91612ea20..90c414abc2 100644 --- a/crates/api-core/src/tests/vpc.rs +++ b/crates/api-core/src/tests/vpc.rs @@ -18,13 +18,15 @@ use std::collections::HashMap; use std::ops::DerefMut; use carbide_network::virtualization::VpcVirtualizationType; +use carbide_test_support::Outcome::FailsWith; +use carbide_test_support::{Case, check_cases_async}; use carbide_uuid::vpc::VpcId; use common::api_fixtures::{create_test_env, populate_network_security_groups}; use config_version::ConfigVersion; use db::vpc::{self}; use db::{self, ObjectColumnFilter}; use model::metadata::Metadata; -use model::vpc::{UpdateVpc, UpdateVpcVirtualization}; +use model::vpc::{UpdateVpc, UpdateVpcVirtualization, VpcDefinition, VpcRoutingProfileOverrides}; use rpc::forge::forge_server::Forge; use crate::test_support::network_segment::FIXTURE_TENANT_ORG_ID; @@ -108,7 +110,7 @@ async fn create_vpc_for_tenant_without_profile( .await .unwrap_err() .message() - .contains("no tenant or routing profile-type found") + .contains("no tenant or routing profile type found") ); // Try to request a VPC with a routing profile when the tenant has no routing profile type @@ -127,7 +129,7 @@ async fn create_vpc_for_tenant_without_profile( .await .unwrap_err() .message() - .contains("no tenant or routing profile-type found") + .contains("no tenant or routing profile type found") ); Ok(()) @@ -151,29 +153,17 @@ async fn create_vpc(pool: sqlx::PgPool) -> Result<(), Box ( "INTERNAL".to_string(), crate::cfg::file::FnnRoutingProfileConfig { - internal: true, - route_target_imports: vec![], - route_targets_on_exports: vec![], - leak_default_route_from_underlay: false, - leak_tenant_host_routes_to_underlay: false, - tenant_leak_communities_accepted: false, - access_tier: 1, - accepted_leaks_from_underlay: vec![], - allowed_anycast_prefixes: vec![], + internal: Some(true), + access_tier: Some(1), + ..Default::default() }, ), ( "PRIVILEGED_INTERNAL".to_string(), crate::cfg::file::FnnRoutingProfileConfig { - internal: true, - route_target_imports: vec![], - route_targets_on_exports: vec![], - leak_default_route_from_underlay: false, - leak_tenant_host_routes_to_underlay: false, - tenant_leak_communities_accepted: false, - access_tier: 0, - accepted_leaks_from_underlay: vec![], - allowed_anycast_prefixes: vec![], + internal: Some(true), + access_tier: Some(0), + ..Default::default() }, ), ]), @@ -585,7 +575,7 @@ async fn create_vpc(pool: sqlx::PgPool) -> Result<(), Box } #[crate::sqlx_test] -async fn create_vpc_without_fnn_rejects_explicit_routing_profile( +async fn create_vpc_without_fnn_rejects_routing_profile_fields( pool: sqlx::PgPool, ) -> Result<(), Box> { let env = create_test_env_with_overrides( @@ -614,27 +604,322 @@ async fn create_vpc_without_fnn_rejects_explicit_routing_profile( txn.commit().await?; }; - // Requesting a VPC routing profile on a non-FNN VPC type (default - // is ETV) should fail early at the API gate. The REST API enforces - // this upstream; carbide-core enforces it as defense-in-depth via - // `ensure_supports_routing_profiles`. - assert!( - env.api - .create_vpc( - VpcCreationRequest::builder(&tenant_organization_id) + // Requesting either VPC routing-profile field on a non-FNN VPC type + // (default is ETV) should fail early at the API gate. The REST API + // enforces this upstream; carbide-core enforces it as defense-in-depth + // via `ensure_supports_routing_profiles`. + check_cases_async( + [ + Case { + scenario: "routing_profile_type", + input: VpcCreationRequest::builder(&tenant_organization_id) .metadata(rpc::forge::Metadata { name: "Forge".to_string(), ..Default::default() }) .routing_profile_type("PRIVILEGED_INTERNAL".to_string()) .tonic_request(), - ) - .await - .unwrap_err() - .message() - .contains("do not support routing profiles") + expect: FailsWith(true), + }, + Case { + scenario: "routing_profile_overrides", + input: VpcCreationRequest::builder(&tenant_organization_id) + .metadata(rpc::forge::Metadata { + name: "Forge".to_string(), + ..Default::default() + }) + .routing_profile_overrides(rpc::forge::VpcRoutingProfileOverrides { + leak_default_route_from_underlay: Some(false), + ..Default::default() + }) + .tonic_request(), + expect: FailsWith(true), + }, + ], + |request| { + let api = env.api.clone(); + async move { + api.create_vpc(request) + .await + .map(drop) + .map_err(|error| { + error.message().contains( + "`routing_profile_type` and `routing_profile_overrides` fields are FNN-only", + ) + }) + } + }, + ) + .await; + + Ok(()) +} + +/// Verifies inline routing-profile values survive persistence and the API +/// returns their current effective profile on create and find. +#[crate::sqlx_test] +async fn create_vpc_persists_inline_routing_profile_overrides( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let profile_type = "INLINE_PROFILE_TEST"; + + // Configure the named profile that provides the VPC's protected base. + let env = create_test_env_with_overrides( + pool, + TestEnvOverrides::default().with_fnn_config(Some(crate::cfg::file::FnnConfig { + admin_vpc: None, + common_internal_route_target: None, + additional_route_target_imports: vec![], + routing_profiles: HashMap::from([( + profile_type.to_string(), + crate::cfg::file::FnnRoutingProfileConfig { + route_target_imports: Some(vec![crate::cfg::file::RouteTargetConfig { + asn: 65000, + vni: 100, + }]), + internal: Some(true), + leak_default_route_from_underlay: Some(true), + allowed_anycast_prefixes: Some(vec![ + crate::cfg::file::PrefixFilterPolicyEntry { + prefix: "198.51.100.0/24".parse()?, + }, + ]), + access_tier: Some(1), + ..Default::default() + }, + )]), + use_vpc_vrf_loopback: false, + })), + ) + .await; + let tenant = env + .api + .create_tenant(tonic::Request::new(rpc::forge::CreateTenantRequest { + organization_id: "inline-profile-test".to_string(), + routing_profile_type: Some(profile_type.to_string()), + metadata: Some(rpc::forge::Metadata { + name: "inline-profile-test".to_string(), + ..Default::default() + }), + })) + .await? + .into_inner() + .tenant + .expect("created tenant"); + let routing_profile_overrides = rpc::forge::VpcRoutingProfileOverrides { + route_target_imports: Some(rpc::common::RouteTargets { values: vec![] }), + leak_default_route_from_underlay: Some(false), + allowed_anycast_prefixes: Some(rpc::forge::PrefixFilterPolicyEntries { + values: vec![rpc::forge::PrefixFilterPolicyEntry { + prefix: "192.0.2.0/24".to_string(), + }], + }), + ..Default::default() + }; + let expected_effective_profile = rpc::forge::VpcEffectiveRoutingProfile { + route_target_imports: vec![], + leak_default_route_from_underlay: false, + allowed_anycast_prefixes: vec![rpc::forge::PrefixFilterPolicyEntry { + prefix: "192.0.2.0/24".to_string(), + }], + internal: true, + access_tier: 1, + ..Default::default() + }; + + // Create the VPC and verify the API echoes the presence-aware override + // alongside the profile resolved from the current API configuration. + let created = env + .api + .create_vpc(tonic::Request::new( + VpcCreationRequest::builder(&tenant.organization_id) + .metadata(rpc::forge::Metadata { + name: "inline profile vpc".to_string(), + ..Default::default() + }) + .network_virtualization_type(rpc::forge::VpcVirtualizationType::Fnn as i32) + .routing_profile_type(profile_type.to_string()) + .routing_profile_overrides(routing_profile_overrides.clone()) + .rpc(), + )) + .await? + .into_inner(); + assert_eq!( + forge_vpc_config(&created).routing_profile_overrides, + Some(routing_profile_overrides.clone()) + ); + assert_eq!( + created + .status + .as_ref() + .and_then(|status| status.effective_routing_profile.as_ref()), + Some(&expected_effective_profile) ); + // Read through the public find API to prove the override was persisted. + let found = env + .api + .find_vpcs_by_ids(tonic::Request::new(rpc::forge::VpcsByIdsRequest { + vpc_ids: vec![created.id.expect("created VPC ID")], + })) + .await? + .into_inner() + .vpcs + .pop() + .expect("persisted VPC"); + assert_eq!( + forge_vpc_config(&found).routing_profile_overrides, + Some(routing_profile_overrides) + ); + assert_eq!( + found + .status + .as_ref() + .and_then(|status| status.effective_routing_profile.as_ref()), + Some(&expected_effective_profile) + ); + + // Updating unrelated VPC configuration must return the same current + // effective profile as create and find. + let updated = env + .api + .update_vpc( + VpcUpdateRequest::builder() + .set_id(found.id) + .metadata(rpc::forge::Metadata { + name: "updated inline profile vpc".to_string(), + ..Default::default() + }) + .tonic_request(), + ) + .await? + .into_inner() + .vpc + .expect("updated VPC"); + assert_eq!( + updated + .status + .as_ref() + .and_then(|status| status.effective_routing_profile.as_ref()), + Some(&expected_effective_profile) + ); + + // The virtualization update API currently permits this transition even + // though callers are instructed not to use it. A non-FNN VPC must not + // report an effective routing profile retained from its former FNN state. + let updated_id = updated.id.expect("updated VPC ID"); + env.api + .update_vpc_virtualization(tonic::Request::new( + rpc::forge::VpcUpdateVirtualizationRequest { + id: Some(updated_id), + if_version_match: None, + network_virtualization_type: Some(rpc::forge::VpcVirtualizationType::Flat as i32), + }, + )) + .await?; + let transitioned = env + .api + .find_vpcs_by_ids(tonic::Request::new(rpc::forge::VpcsByIdsRequest { + vpc_ids: vec![updated_id], + })) + .await? + .into_inner() + .vpcs + .pop() + .expect("transitioned VPC"); + assert!( + transitioned + .status + .as_ref() + .and_then(|status| status.effective_routing_profile.as_ref()) + .is_none() + ); + + Ok(()) +} + +/// Verifies seeded inline overrides are rejected even for existing VPCs +/// because idempotent seed handling must not hide unsupported configuration. +#[crate::sqlx_test] +async fn initial_vpc_inline_overrides_are_rejected_before_existing_vpc_check( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env_with_overrides(pool, TestEnvOverrides::no_network_segments()).await; + let vpc_name = "seeded-inline-profile"; + + // Seed a valid VPC so the rejected definition exercises the existing-VPC path. + let initial_vpcs = HashMap::from([( + vpc_name.to_string(), + VpcDefinition { + organization_id: Some(FIXTURE_TENANT_ORG_ID.to_string()), + network_virtualization_type: VpcVirtualizationType::Fnn, + routing_profile_type: Some("BASE".to_string()), + routing_profile_overrides: None, + vni: None, + }, + )]); + db_init::create_initial_vpcs( + &env.pool, + &initial_vpcs, + env.common_pools.ethernet.pool_vpc_vni.as_ref(), + ) + .await?; + + // Re-submit the same name with an override and a valid-looking named base. + let vpcs = HashMap::from([( + vpc_name.to_string(), + VpcDefinition { + organization_id: Some(FIXTURE_TENANT_ORG_ID.to_string()), + network_virtualization_type: VpcVirtualizationType::Fnn, + routing_profile_type: Some("BASE".to_string()), + routing_profile_overrides: Some(VpcRoutingProfileOverrides { + leak_default_route_from_underlay: Some(true), + ..Default::default() + }), + vni: None, + }, + )]); + + // Reject the unsupported override before the existing-VPC short-circuit. + let error = db_init::create_initial_vpcs( + &env.pool, + &vpcs, + env.common_pools.ethernet.pool_vpc_vni.as_ref(), + ) + .await + .expect_err("seeded inline overrides must always be rejected"); + assert!( + error + .to_string() + .contains("cannot define `routing_profile_overrides`"), + "unexpected error: {error}" + ); + + // Read through the public API and verify the original VPC remains unchanged. + let found_ids = env + .api + .find_vpc_ids(tonic::Request::new(rpc::forge::VpcSearchFilter { + name: Some(vpc_name.to_string()), + tenant_org_id: None, + label: None, + })) + .await? + .into_inner() + .vpc_ids; + assert_eq!(found_ids.len(), 1); + + let found = env + .api + .find_vpcs_by_ids(tonic::Request::new(rpc::forge::VpcsByIdsRequest { + vpc_ids: found_ids, + })) + .await? + .into_inner() + .vpcs + .pop() + .expect("persisted VPC"); + assert!(forge_vpc_config(&found).routing_profile_overrides.is_none()); + Ok(()) } diff --git a/crates/api-core/tests/integration/compute_allocation.rs b/crates/api-core/tests/integration/compute_allocation.rs index 92165f48e0..a6153599f1 100644 --- a/crates/api-core/tests/integration/compute_allocation.rs +++ b/crates/api-core/tests/integration/compute_allocation.rs @@ -28,7 +28,6 @@ use carbide_uuid::instance_type::InstanceTypeId; use carbide_uuid::machine::MachineId; use carbide_uuid::network::NetworkSegmentId; use model::instance_type::InstanceTypeMachineCapabilityFilter; -use model::machine::ManagedHostState; use model::machine::capabilities::MachineCapabilityType; use model::metadata::Metadata as DbMetadata; use model::test_support::ManagedHostConfig; @@ -181,7 +180,7 @@ async fn create_managed_host(env: &TestEnv) -> TestManagedHost { .await .0; mh.host.discover_primary_iface(env.admin_segment).await; - mh.advance_state(ManagedHostState::Ready).await; + mh.advance_to_converged_ready().await; TestManagedHost { id: mh.host.id } } diff --git a/crates/api-core/tests/integration/forge_agent_control.rs b/crates/api-core/tests/integration/forge_agent_control.rs index eea6bdc0c6..f1bac09798 100644 --- a/crates/api-core/tests/integration/forge_agent_control.rs +++ b/crates/api-core/tests/integration/forge_agent_control.rs @@ -383,7 +383,7 @@ async fn waiting_for_measurements_returns_measure(pool: PgPool) { #[sqlx_test] async fn ready_returns_noop(pool: PgPool) { let TestContext { env: _env, mh, .. } = init(pool).await; - mh.advance_state(ManagedHostState::Ready).await; + mh.advance_to_converged_ready().await; let response = mh.host.forge_agent_control().await; assert!(matches!(response.action, Some(Action::Noop(_)))); diff --git a/crates/api-db/migrations/20260722120000_bmc_suppressions.sql b/crates/api-db/migrations/20260722120001_bmc_suppressions.sql similarity index 100% rename from crates/api-db/migrations/20260722120000_bmc_suppressions.sql rename to crates/api-db/migrations/20260722120001_bmc_suppressions.sql diff --git a/crates/api-db/migrations/20260730120000_machine_boot_interface_status.sql b/crates/api-db/migrations/20260730120000_machine_boot_interface_status.sql new file mode 100644 index 0000000000..c921d15d3b --- /dev/null +++ b/crates/api-db/migrations/20260730120000_machine_boot_interface_status.sql @@ -0,0 +1,34 @@ +-- Track the desired boot-interface generation that machine-controller +-- currently treats as converged. The status is intentionally retained +-- when the desired target changes: comparing its version with desired_version +-- makes the new work pending without discarding the last convergence status. +ALTER TABLE machine_boot_interfaces + ADD COLUMN verified_version varchar(64), + ADD COLUMN observed_at timestamp with time zone, + ADD COLUMN assumed boolean NOT NULL DEFAULT false, + -- Existing rows satisfy this constraint through the unset-status branch. + -- Add it normally so this single migration remains atomic without implying + -- that validation releases the transaction's table lock. + ADD CONSTRAINT machine_boot_interfaces_status_consistent + CHECK ( + ( + verified_version IS NULL + AND observed_at IS NULL + AND NOT assumed + ) + OR ( + verified_version IS NOT NULL + AND observed_at IS NOT NULL + ) + ); + +-- Avoid scheduling a fleet-wide boot reconfiguration when this status first +-- rolls out. These rows predate status tracking, so record the baseline +-- explicitly as assumed rather than presenting it as a Redfish observation. +UPDATE machine_boot_interfaces AS boot_interface +SET verified_version = desired_version, + observed_at = CURRENT_TIMESTAMP, + assumed = true +FROM machines AS machine +WHERE machine.id = boot_interface.machine_id + AND machine.controller_state->>'state' IN ('ready', 'assigned'); diff --git a/crates/api-db/migrations/20260730120001_power_shelf_bmc_credential_rotation_requested.sql b/crates/api-db/migrations/20260730120001_power_shelf_bmc_credential_rotation_requested.sql new file mode 100644 index 0000000000..431b25d9d0 --- /dev/null +++ b/crates/api-db/migrations/20260730120001_power_shelf_bmc_credential_rotation_requested.sql @@ -0,0 +1,12 @@ +-- Add bmc_credential_rotation_requested column to power_shelves table. +-- bmc_credential_rotation_requested: an operator "force-converge this PMC now" +-- escape hatch (REQ-2), the power-shelf analogue of +-- machines.bmc_credential_rotation_requested and +-- switches.bmc_credential_rotation_requested. When true, the power-shelf state +-- controller enters RotatingBmc and force-converges the power shelf BMC (PMC), +-- bypassing the passive site-wide gate and the device's backoff quarantine. A +-- power shelf has exactly one BMC, so the flag's presence on the row names the +-- target device. + +ALTER TABLE power_shelves + ADD COLUMN bmc_credential_rotation_requested BOOLEAN NOT NULL DEFAULT false; diff --git a/crates/api-db/migrations/20260731123242_vpc_routing_profile_overrides.sql b/crates/api-db/migrations/20260731123242_vpc_routing_profile_overrides.sql new file mode 100644 index 0000000000..7a181a5c5b --- /dev/null +++ b/crates/api-db/migrations/20260731123242_vpc_routing_profile_overrides.sql @@ -0,0 +1,4 @@ +-- Presence-aware overlay on the VPC's named routing profile. NULL and omitted +-- properties inherit from the base; internal and access_tier are never stored. +ALTER TABLE vpcs + ADD COLUMN routing_profile_overrides jsonb; diff --git a/crates/api-db/src/machine.rs b/crates/api-db/src/machine.rs index ec3cd1aeb1..a3f8585b60 100644 --- a/crates/api-db/src/machine.rs +++ b/crates/api-db/src/machine.rs @@ -48,10 +48,10 @@ use model::machine::nvlink::MachineNvLinkStatusObservation; use model::machine::spx::MachineSpxStatusObservation; use model::machine::upgrade_policy::AgentUpgradePolicy; use model::machine::{ - Dpf, DpuInfo, DpuInfoStatusObservation, DpuOsOperationalState, DpuRepresentorStatus, - FailureDetails, HostProfile, Machine, MachineInterfaceSnapshot, MachineLastRebootRequested, - MachineLastRebootRequestedMode, MachineMaintenanceOperation, MachineValidationContext, - ManagedHostState, ReprovisionRequest, UpgradeDecision, + CURRENT_STATE_MODEL_VERSION, Dpf, DpuInfo, DpuInfoStatusObservation, DpuOsOperationalState, + DpuRepresentorStatus, FailureDetails, HostProfile, Machine, MachineInterfaceSnapshot, + MachineLastRebootRequested, MachineLastRebootRequestedMode, MachineMaintenanceOperation, + MachineValidationContext, ManagedHostState, ReprovisionRequest, UpgradeDecision, }; use model::machine_interface_address::MachineInterfaceAssociation; use model::metadata::Metadata; @@ -134,7 +134,15 @@ pub async fn get_or_create( // Host and DPU machines are created in same `discover_machine` call. Update same // state in both machines. let state = ManagedHostState::Created; - let machine = create(txn, common_pools, stable_machine_id, state, None, 2).await?; + let machine = create( + txn, + common_pools, + stable_machine_id, + state, + None, + CURRENT_STATE_MODEL_VERSION, + ) + .await?; crate::machine_interface::associate_interface_with_machine( &interface.id, MachineInterfaceAssociation::Machine(machine.id), diff --git a/crates/api-db/src/machine_desired_boot_interface.rs b/crates/api-db/src/machine_desired_boot_interface.rs index 4a9e0a084e..4cd1b96143 100644 --- a/crates/api-db/src/machine_desired_boot_interface.rs +++ b/crates/api-db/src/machine_desired_boot_interface.rs @@ -16,6 +16,7 @@ */ use carbide_uuid::machine::{MachineId, MachineType}; +use chrono::{DateTime, Utc}; use config_version::{ConfigVersion, Versioned}; use mac_address::MacAddress; use model::machine_boot_interface::{ @@ -32,6 +33,7 @@ struct DesiredBootInterfaceRow { desired_mac_address: Option, desired_interface_id: Option, desired_version: Option, + rollout_baseline_eligible: bool, } impl DesiredBootInterfaceRow { @@ -104,7 +106,11 @@ async fn load( machine.version AS machine_version, boot_interface.desired_mac_address, boot_interface.desired_interface_id, - boot_interface.desired_version + boot_interface.desired_version, + COALESCE( + machine.controller_state->>'state' IN ('ready', 'assigned'), + false + ) AS rollout_baseline_eligible FROM machines machine LEFT JOIN machine_boot_interfaces boot_interface ON boot_interface.machine_id = machine.id @@ -131,7 +137,11 @@ async fn load_for_update( machine.version AS machine_version, boot_interface.desired_mac_address, boot_interface.desired_interface_id, - boot_interface.desired_version + boot_interface.desired_version, + COALESCE( + machine.controller_state->>'state' IN ('ready', 'assigned'), + false + ) AS rollout_baseline_eligible FROM machines machine LEFT JOIN machine_boot_interfaces boot_interface ON boot_interface.machine_id = machine.id @@ -222,13 +232,26 @@ fn next_version(expected_version: Option) -> ConfigVersion { .unwrap_or_else(ConfigVersion::initial) } +#[derive(Clone, Copy)] +enum VerificationPolicy { + Pending, + AssumeVerified, + CarryCurrentForward, +} + async fn update( txn: &mut PgConnection, machine_id: &MachineId, current_machine_version: ConfigVersion, expected_version: Option, target: &MachineBootInterfaceTarget, + verification_policy: VerificationPolicy, ) -> DatabaseResult> { + let (assume_verified, carry_current_forward) = match verification_policy { + VerificationPolicy::Pending => (false, false), + VerificationPolicy::AssumeVerified => (true, false), + VerificationPolicy::CarryCurrentForward => (false, true), + }; let desired_version = next_version(expected_version); let machine_version = current_machine_version.increment(); let updated: Option = if let Some(expected_version) = expected_version { @@ -236,7 +259,11 @@ async fn update( UPDATE machine_boot_interfaces SET desired_mac_address = $1, desired_interface_id = $2, - desired_version = $3 + desired_version = $3, + verified_version = CASE + WHEN $6 AND verified_version = $5 THEN $3 + ELSE verified_version + END WHERE machine_id = $4 AND desired_version = $5 RETURNING machine_id @@ -247,6 +274,7 @@ async fn update( .bind(desired_version) .bind(machine_id) .bind(expected_version) + .bind(carry_current_forward) .fetch_optional(&mut *txn) .await .map_err(|error| DatabaseError::query(query, error))? @@ -256,9 +284,20 @@ async fn update( machine_id, desired_mac_address, desired_interface_id, - desired_version + desired_version, + verified_version, + observed_at, + assumed + ) + VALUES ( + $1, + $2, + $3, + $4, + CASE WHEN $5 THEN $4 END, + CASE WHEN $5 THEN CURRENT_TIMESTAMP END, + $5 ) - VALUES ($1, $2, $3, $4) ON CONFLICT (machine_id) DO NOTHING RETURNING machine_id "#; @@ -267,6 +306,7 @@ async fn update( .bind(target.mac_address()) .bind(target.interface_id()) .bind(desired_version) + .bind(assume_verified) .fetch_optional(&mut *txn) .await .map_err(|error| DatabaseError::query(query, error))? @@ -337,6 +377,7 @@ pub async fn try_set( current_machine_version, expected_version, target, + VerificationPolicy::Pending, ) .await? .is_some()) @@ -371,6 +412,7 @@ pub async fn set( current_machine_version, expected_version, target, + VerificationPolicy::Pending, ) .await? else { @@ -389,6 +431,12 @@ pub async fn set( /// `initialize_if_unset` stores Site Explorer's initial target without /// replacing a target that is already present. +/// +/// A stable host already in `Ready` or `Assigned` receives an explicitly +/// assumed compatibility observation. This covers a missing row during a +/// mixed-component rollout or later repair without scheduling fleet-wide +/// remediation. Normal lifecycle initialization happens on a predicted host or +/// in `HostInit`, so newly provisioned hosts remain pending real verification. pub async fn initialize_if_unset( txn: &mut PgConnection, machine_id: &MachineId, @@ -399,11 +447,25 @@ pub async fn initialize_if_unset( let row = load_for_update(txn, machine_id).await?; let current_machine_version = row.machine_version; + let assume_verified = machine_id.machine_type().is_host() && row.rollout_baseline_eligible; + let verification_policy = if assume_verified { + VerificationPolicy::AssumeVerified + } else { + VerificationPolicy::Pending + }; if let Some(current) = row.decode(machine_id)? { return Ok(current); } - let Some(version) = update(txn, machine_id, current_machine_version, None, target).await? + let Some(version) = update( + txn, + machine_id, + current_machine_version, + None, + target, + verification_policy, + ) + .await? else { return Err(DatabaseError::Internal { message: format!( @@ -421,6 +483,8 @@ pub async fn initialize_if_unset( /// `enrich_interface_id` adds a Redfish id to a matching MAC-only target. /// /// Once a pair is stored, later observations do not replace its id. +/// Enrichment advances a current status because it strengthens the same +/// physical target's identity; stale or pending status remains unchanged. pub async fn enrich_interface_id( txn: &mut PgConnection, machine_id: &MachineId, @@ -458,6 +522,7 @@ pub async fn enrich_interface_id( current_machine_version, expected_version, &target, + VerificationPolicy::CarryCurrentForward, ) .await? else { @@ -474,15 +539,60 @@ pub async fn enrich_interface_id( })) } +/// Records a Redfish observation only if the desired boot-interface version +/// still matches the version the caller observed. +/// +/// A `false` return means the desired target was removed or replaced before +/// the observation could be committed. The caller must not treat that newer +/// target as verified. +pub async fn mark_verified( + txn: &mut PgConnection, + machine_id: &MachineId, + expected_desired_version: ConfigVersion, + observed_at: DateTime, +) -> Result { + validate_machine_id(machine_id)?; + + // Desired-target writers lock the parent machine row before touching this + // child row. Preserve that order so a concurrent operator write cannot + // deadlock verification against the state-controller transition, which + // also updates the parent before commit. + load_for_update(txn, machine_id).await?; + + let query = r#" + UPDATE machine_boot_interfaces + SET verified_version = desired_version, + observed_at = $1, + assumed = false + WHERE machine_id = $2 + AND desired_version = $3 + RETURNING machine_id + "#; + let updated: Option = sqlx::query_scalar(query) + .bind(observed_at) + .bind(machine_id) + .bind(expected_desired_version) + .fetch_optional(txn) + .await + .map_err(|error| DatabaseError::query(query, error))?; + + Ok(updated.is_some()) +} + #[cfg(test)] mod tests { use carbide_uuid::machine::{MachineIdSource, MachineType}; + use model::machine::machine_search_config::MachineSearchConfig; + use model::machine::{InstanceState, MachineState, ManagedHostState}; use sqlx::PgPool; + use sqlx::types::Json; use super::*; const MIGRATION: &str = include_str!("../migrations/20260728120000_machine_boot_interfaces.sql"); + const STATUS_MIGRATION: &str = + include_str!("../migrations/20260730120000_machine_boot_interface_status.sql"); fn machine_id(machine_type: MachineType, marker: u8) -> MachineId { let mut hardware_id = [0u8; 32]; @@ -500,7 +610,10 @@ mod tests { ) -> Result { let query = r#" INSERT INTO machines (id, dpf) - VALUES ($1, '{}'::jsonb) + VALUES ( + $1, + '{"enabled": false, "used_for_ingestion": false}'::jsonb + ) RETURNING version "#; sqlx::query_scalar(query) @@ -525,6 +638,33 @@ mod tests { .await } + async fn set_controller_state( + txn: &mut PgConnection, + machine_id: &MachineId, + state: ManagedHostState, + ) -> Result<(), sqlx::Error> { + sqlx::query("UPDATE machines SET controller_state = $1 WHERE id = $2") + .bind(Json(state)) + .bind(machine_id) + .execute(txn) + .await?; + Ok(()) + } + + async fn status_observation( + txn: &mut PgConnection, + machine_id: &MachineId, + ) -> Result<(Option, Option>, bool), sqlx::Error> { + sqlx::query_as( + "SELECT verified_version, observed_at, assumed + FROM machine_boot_interfaces + WHERE machine_id = $1", + ) + .bind(machine_id) + .fetch_one(txn) + .await + } + fn assert_target( actual: &Versioned, expected: &MachineBootInterfaceTarget, @@ -562,6 +702,234 @@ mod tests { Ok(()) } + #[crate::sqlx_test] + async fn rollout_baseline_only_applies_to_existing_stable_ready_or_assigned_hosts( + pool: PgPool, + ) -> Result<(), Box> { + let mut txn = pool.begin().await?; + let cases = [ + ( + machine_id(MachineType::Host, 30), + ManagedHostState::Ready, + true, + ), + ( + machine_id(MachineType::Host, 31), + ManagedHostState::Assigned { + instance_state: InstanceState::Init, + }, + true, + ), + ( + machine_id(MachineType::Host, 32), + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 0 }, + }, + false, + ), + ( + machine_id(MachineType::PredictedHost, 33), + ManagedHostState::Ready, + false, + ), + ]; + + for (index, (machine_id, state, expect_assumed)) in cases.into_iter().enumerate() { + seed_machine(txn.as_mut(), &machine_id).await?; + set_controller_state(txn.as_mut(), &machine_id, state).await?; + let target = + MachineBootInterfaceTarget::MacOnly(MacAddress::new([2, 0, 0, 0, 3, index as u8])); + let initialized = initialize_if_unset(txn.as_mut(), &machine_id, &target).await?; + let (verified_version, observed_at, assumed) = + status_observation(txn.as_mut(), &machine_id).await?; + + if expect_assumed { + assert_eq!(verified_version, Some(initialized.version)); + assert!(observed_at.is_some()); + assert!(assumed); + } else { + assert_eq!(verified_version, None); + assert_eq!(observed_at, None); + assert!(!assumed); + } + } + + Ok(()) + } + + #[crate::sqlx_test] + async fn verification_is_a_cas_and_is_exposed_in_machine_snapshots( + pool: PgPool, + ) -> Result<(), Box> { + let mut txn = pool.begin().await?; + let machine_id = machine_id(MachineType::Host, 34); + seed_machine(txn.as_mut(), &machine_id).await?; + let target = MachineBootInterfaceTarget::MacOnly(MacAddress::new([2, 0, 0, 0, 3, 4])); + let initialized = initialize_if_unset(txn.as_mut(), &machine_id, &target).await?; + let observed_at = + DateTime::from_timestamp(1_722_000_000, 123_000_000).expect("fixture timestamp"); + + assert!( + !mark_verified( + txn.as_mut(), + &machine_id, + ConfigVersion::invalid(), + observed_at, + ) + .await? + ); + assert_eq!( + status_observation(txn.as_mut(), &machine_id).await?, + (None, None, false) + ); + + assert!(mark_verified(txn.as_mut(), &machine_id, initialized.version, observed_at,).await?); + assert_eq!( + status_observation(txn.as_mut(), &machine_id).await?, + (Some(initialized.version), Some(observed_at), false) + ); + + let machine = + crate::machine::find_one(txn.as_mut(), &machine_id, MachineSearchConfig::default()) + .await? + .expect("machine snapshot"); + let observation = machine + .status + .boot_interface_status_observation + .expect("boot interface status observation"); + assert_eq!(observation.config_version, initialized.version); + assert_eq!(observation.observed_at, observed_at); + assert!(!observation.assumed); + + let replacement = MachineBootInterfaceTarget::MacOnly(MacAddress::new([2, 0, 0, 0, 3, 5])); + let updated = set(txn.as_mut(), &machine_id, &replacement).await?; + assert_ne!(updated.version, initialized.version); + assert!(!mark_verified(txn.as_mut(), &machine_id, initialized.version, Utc::now(),).await?); + assert_eq!( + status_observation(txn.as_mut(), &machine_id).await?, + (Some(initialized.version), Some(observed_at), false), + "changing the target keeps the last factual observation but makes its version stale", + ); + + Ok(()) + } + + #[crate::sqlx_test] + #[allow(txn_held_across_await)] // Intentionally hold a row lock while exercising concurrency. + async fn concurrent_set_and_mark_verified_use_parent_first_lock_order( + pool: PgPool, + ) -> Result<(), Box> { + let machine_id = machine_id(MachineType::Host, 35); + let initial_target = + MachineBootInterfaceTarget::MacOnly(MacAddress::new([2, 0, 0, 0, 3, 6])); + let replacement = MachineBootInterfaceTarget::MacOnly(MacAddress::new([2, 0, 0, 0, 3, 7])); + + let mut setup_txn = pool.begin().await?; + seed_machine(setup_txn.as_mut(), &machine_id).await?; + let initial = initialize_if_unset(setup_txn.as_mut(), &machine_id, &initial_target).await?; + setup_txn.commit().await?; + + // Hold the parent lock as `set` does. `mark_verified` must wait here, + // before it can lock the child intent row. + let mut setter_txn = pool.begin().await?; + let setter_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(setter_txn.as_mut()) + .await?; + load_for_update(setter_txn.as_mut(), &machine_id).await?; + + let (verification_pid_tx, verification_pid_rx) = tokio::sync::oneshot::channel(); + let verification_pool = pool.clone(); + let mut verification_task = tokio::spawn(async move { + let mut txn = verification_pool + .begin() + .await + .map_err(|error| error.to_string())?; + let verification_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(txn.as_mut()) + .await + .map_err(|error| error.to_string())?; + verification_pid_tx + .send(verification_pid) + .map_err(|_| "could not signal verification backend pid".to_string())?; + + let marked = mark_verified(txn.as_mut(), &machine_id, initial.version, Utc::now()) + .await + .map_err(|error| error.to_string())?; + + // The real state-controller transaction updates the parent state + // after recording verification. This remains safe because + // mark_verified already owns the parent lock. + sqlx::query("UPDATE machines SET updated = updated WHERE id = $1") + .bind(machine_id) + .execute(txn.as_mut()) + .await + .map_err(|error| error.to_string())?; + txn.commit().await.map_err(|error| error.to_string())?; + + Ok::(marked) + }); + + let verification_pid = verification_pid_rx + .await + .map_err(|error| std::io::Error::other(error.to_string()))?; + let wait_for_parent_lock = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let blocked_by_setter: bool = + sqlx::query_scalar("SELECT $1 = ANY(pg_blocking_pids($2))") + .bind(setter_pid) + .bind(verification_pid) + .fetch_one(&pool) + .await?; + if blocked_by_setter { + return Ok::<(), sqlx::Error>(()); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await; + match wait_for_parent_lock { + Ok(result) => result?, + Err(_) => { + verification_task.abort(); + let _ = verification_task.await; + return Err(std::io::Error::other( + "verification did not wait on the setter's parent lock", + ) + .into()); + } + } + + let updated = set(setter_txn.as_mut(), &machine_id, &replacement).await?; + setter_txn.commit().await?; + + let marked = + match tokio::time::timeout(std::time::Duration::from_secs(5), &mut verification_task) + .await + { + Ok(result) => result + .map_err(|error| std::io::Error::other(error.to_string()))? + .map_err(std::io::Error::other)?, + Err(_) => { + verification_task.abort(); + let _ = verification_task.await; + return Err(std::io::Error::other("set and verification deadlocked").into()); + } + }; + + assert!(!marked, "a superseded desired version must not be verified"); + let desired = get(&pool, &machine_id).await?.expect("replacement target"); + assert_target(&desired, &replacement); + assert_eq!(desired.version, updated.version); + let mut conn = pool.acquire().await?; + assert_eq!( + status_observation(conn.as_mut(), &machine_id).await?, + (None, None, false), + "stale verification must not stamp the replacement version", + ); + + Ok(()) + } + #[crate::sqlx_test] async fn try_set_uses_cas_without_bumping_noops( pool: PgPool, @@ -693,6 +1061,10 @@ mod tests { let mac_address = MacAddress::new([2, 0, 0, 0, 0, 5]); let target = MachineBootInterfaceTarget::MacOnly(mac_address); let initialized = initialize_if_unset(txn.as_mut(), &machine_id, &target).await?; + assert_eq!( + status_observation(txn.as_mut(), &machine_id).await?, + (None, None, false), + ); let mismatch = enrich_interface_id( txn.as_mut(), @@ -722,6 +1094,11 @@ mod tests { enriched.version.version_nr(), initialized.version.version_nr() + 1 ); + assert_eq!( + status_observation(txn.as_mut(), &machine_id).await?, + (None, None, false), + "enrichment must not make an unverified target look converged", + ); let unchanged = enrich_interface_id(txn.as_mut(), &machine_id, mac_address, "replacement") .await? @@ -737,6 +1114,51 @@ mod tests { Ok(()) } + #[crate::sqlx_test] + async fn enrichment_carries_current_verification_without_hiding_target_changes( + pool: PgPool, + ) -> Result<(), Box> { + let mut txn = pool.begin().await?; + let machine_id = machine_id(MachineType::Host, 36); + seed_machine(txn.as_mut(), &machine_id).await?; + let mac_address = MacAddress::new([2, 0, 0, 0, 3, 8]); + let initialized = initialize_if_unset( + txn.as_mut(), + &machine_id, + &MachineBootInterfaceTarget::MacOnly(mac_address), + ) + .await?; + let observed_at = + DateTime::from_timestamp(1_722_000_100, 123_000_000).expect("fixture timestamp"); + assert!(mark_verified(txn.as_mut(), &machine_id, initialized.version, observed_at,).await?); + + let enriched = + enrich_interface_id(txn.as_mut(), &machine_id, mac_address, "NIC.Slot.8-1-1") + .await? + .expect("enriched target"); + assert_eq!( + enriched.version.version_nr(), + initialized.version.version_nr() + 1, + "the enriched value remains a distinct immutable desired generation", + ); + assert_eq!( + status_observation(txn.as_mut(), &machine_id).await?, + (Some(enriched.version), Some(observed_at), false), + "adding an id for the same MAC must not schedule redundant convergence", + ); + + let replacement = MachineBootInterfaceTarget::MacOnly(MacAddress::new([2, 0, 0, 0, 3, 9])); + let replaced = set(txn.as_mut(), &machine_id, &replacement).await?; + assert_ne!(replaced.version, enriched.version); + assert_eq!( + status_observation(txn.as_mut(), &machine_id).await?, + (Some(enriched.version), Some(observed_at), false), + "a different target must retain only the stale factual observation", + ); + + Ok(()) + } + #[crate::sqlx_test] async fn desired_targets_reject_dpu_ids( pool: PgPool, @@ -771,6 +1193,16 @@ mod tests { enrich_interface_id(txn.as_mut(), &machine_id, mac_address, "id").await, Err(DatabaseError::InvalidArgument(_)) )); + assert!(matches!( + mark_verified( + txn.as_mut(), + &machine_id, + ConfigVersion::initial(), + Utc::now(), + ) + .await, + Err(DatabaseError::InvalidArgument(_)) + )); Ok(()) } @@ -913,4 +1345,98 @@ mod tests { Ok(()) } + + #[crate::sqlx_test] + async fn status_migration_scopes_rollout_baseline_and_constrains_status( + pool: PgPool, + ) -> Result<(), Box> { + sqlx::query("DROP TABLE machine_boot_interfaces") + .execute(&pool) + .await?; + + let mut txn = pool.begin().await?; + let existing_host = machine_id(MachineType::Host, 40); + let in_flight_host = machine_id(MachineType::Host, 42); + seed_machine(txn.as_mut(), &existing_host).await?; + seed_machine(txn.as_mut(), &in_flight_host).await?; + set_controller_state(txn.as_mut(), &existing_host, ManagedHostState::Ready).await?; + set_controller_state( + txn.as_mut(), + &in_flight_host, + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 0 }, + }, + ) + .await?; + txn.commit().await?; + + sqlx::raw_sql(MIGRATION).execute(&pool).await?; + let existing_version = ConfigVersion::initial(); + for (machine_id, mac_address) in [ + (existing_host, MacAddress::new([2, 0, 0, 0, 4, 0])), + (in_flight_host, MacAddress::new([2, 0, 0, 0, 4, 2])), + ] { + sqlx::query( + "INSERT INTO machine_boot_interfaces ( + machine_id, + desired_mac_address, + desired_version + ) + VALUES ($1, $2, $3)", + ) + .bind(machine_id) + .bind(mac_address) + .bind(existing_version) + .execute(&pool) + .await?; + } + + sqlx::raw_sql(STATUS_MIGRATION).execute(&pool).await?; + let (verified_version, observed_at, assumed): ( + Option, + Option>, + bool, + ) = sqlx::query_as( + "SELECT verified_version, observed_at, assumed + FROM machine_boot_interfaces + WHERE machine_id = $1", + ) + .bind(existing_host) + .fetch_one(&pool) + .await?; + assert_eq!(verified_version, Some(existing_version)); + assert!(observed_at.is_some()); + assert!(assumed); + + let in_flight_status: (Option, Option>, bool) = + sqlx::query_as( + "SELECT verified_version, observed_at, assumed + FROM machine_boot_interfaces + WHERE machine_id = $1", + ) + .bind(in_flight_host) + .fetch_one(&pool) + .await?; + assert_eq!(in_flight_status, (None, None, false)); + + for inconsistent_update in [ + "UPDATE machine_boot_interfaces + SET observed_at = CURRENT_TIMESTAMP + WHERE machine_id = $1", + "UPDATE machine_boot_interfaces + SET assumed = true + WHERE machine_id = $1", + ] { + let result = sqlx::query(inconsistent_update) + .bind(in_flight_host) + .execute(&pool) + .await; + assert!( + result.is_err(), + "the status consistency constraint should reject {inconsistent_update:?}", + ); + } + + Ok(()) + } } diff --git a/crates/api-db/src/machine_interface_address.rs b/crates/api-db/src/machine_interface_address.rs index 23f50558f1..48cac96373 100644 --- a/crates/api-db/src/machine_interface_address.rs +++ b/crates/api-db/src/machine_interface_address.rs @@ -19,8 +19,10 @@ use std::net::IpAddr; use carbide_network::ip::{IdentifyAddressFamily, IpAddressFamily}; use carbide_uuid::machine::{MachineId, MachineInterfaceId}; use carbide_uuid::network::NetworkSegmentId; +use carbide_uuid::switch::SwitchId; use mac_address::MacAddress; use model::allocation_type::{AllocationType, AssignStaticResult}; +use model::machine_interface::InterfaceType; use model::network_segment::NetworkSegmentType; use sqlx::{FromRow, PgConnection}; @@ -65,16 +67,17 @@ pub async fn find_ipv4_for_interface( .map_err(|e| DatabaseError::query(query, e)) } -/// Looks up which machine interface owns an IP, with segment metadata and **allocation type**. +/// Looks up which machine interface owns an IP, with association, segment, role, and allocation +/// metadata. /// -/// `allocation_type` is used by the IP finder to classify operator static assignments -/// (`AllocationType::Static` or addresses on the `static-assignments` segment) as -/// `IpTypeStaticBmcIp` where appropriate. +/// The IP finder uses `interface_type` together with `allocation_type` and segment metadata to +/// distinguish static BMC addresses from static Data addresses. pub async fn find_by_address( txn: impl DbReader<'_>, address: IpAddr, ) -> Result, DatabaseError> { - let query = "SELECT mi.id, mi.machine_id, ns.name, ns.network_segment_type, mia.allocation_type + let query = "SELECT mi.id, mi.machine_id, mi.switch_id, mi.interface_type, + ns.name, ns.network_segment_type, mia.allocation_type FROM machine_interface_addresses mia INNER JOIN machine_interfaces mi ON mi.id = mia.interface_id INNER JOIN network_segments ns ON ns.id = mi.segment_id @@ -291,12 +294,14 @@ pub async fn has_address_for_family( .map_err(|e| DatabaseError::query(query, e)) } -/// Row shape for [`find_by_address`]: interface identity, owning segment, and how the address was -/// assigned (DHCP vs static / operator-configured). +/// Row shape for [`find_by_address`]: interface identity, association, role, owning segment, and +/// how the address was assigned (DHCP vs static / operator-configured). #[derive(Debug, FromRow)] pub struct MachineInterfaceSearchResult { pub id: MachineInterfaceId, pub machine_id: Option, + pub switch_id: Option, + pub interface_type: InterfaceType, pub name: String, pub network_segment_type: NetworkSegmentType, pub allocation_type: AllocationType, diff --git a/crates/api-db/src/migrations/mod.rs b/crates/api-db/src/migrations/mod.rs index a31ee059c8..89d27fa6e2 100644 --- a/crates/api-db/src/migrations/mod.rs +++ b/crates/api-db/src/migrations/mod.rs @@ -199,6 +199,27 @@ async fn load_and_validate_history( mod tests { use super::*; + #[test] + fn migration_versions_are_unique() { + let mut versions = HashSet::new(); + + for migration in std::iter::once(&MIGRATION_LAYOUT.legacy) + .chain( + MIGRATION_LAYOUT + .epochs + .iter() + .flat_map(|epoch| [&epoch.squash, &epoch.post_squash]), + ) + .flat_map(|migrator| migrator.iter()) + { + assert!( + versions.insert(migration.version), + "duplicate migration version {}", + migration.version + ); + } + } + #[test] fn epochs_are_ordered_and_point_to_their_squash_migration() { let mut previous = None; diff --git a/crates/api-db/src/power_shelf.rs b/crates/api-db/src/power_shelf.rs index e0b904bd15..8c87191171 100644 --- a/crates/api-db/src/power_shelf.rs +++ b/crates/api-db/src/power_shelf.rs @@ -124,6 +124,7 @@ pub async fn create( status: None, deleted: None, bmc_mac_address: new_power_shelf.bmc_mac_address, + bmc_credential_rotation_requested: false, bmc_info: None, controller_state: Versioned { value: state, @@ -351,6 +352,52 @@ pub async fn clear_power_shelf_maintenance_requested( Ok(()) } +/// Record an operator force-converge request against a power shelf's BMC (PMC) +/// (REQ-2). The power-shelf state controller consumes it on its next sweep. +/// Mirrors [`crate::switch::set_bmc_credential_rotation_requested`]. +pub async fn set_bmc_credential_rotation_requested( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, +) -> DatabaseResult<()> { + let query = "UPDATE power_shelves SET bmc_credential_rotation_requested = true WHERE id = $1 RETURNING id"; + sqlx::query_scalar::<_, PowerShelfId>(query) + .bind(power_shelf_id) + .fetch_one(txn) + .await + .map_err(|e| match e { + // `RETURNING id` yields no row for an unknown power shelf; surface a + // clean not-found rather than a generic wrapped error. + sqlx::Error::RowNotFound => DatabaseError::NotFoundError { + kind: "power_shelf", + id: power_shelf_id.to_string(), + }, + e => DatabaseError::new("power_shelf::set_bmc_credential_rotation_requested", e), + })?; + Ok(()) +} + +/// Clear a power shelf's force-converge request (REQ-2), committed with the +/// return to `Ready` once a forced tick settles. Mirrors +/// [`crate::switch::clear_bmc_credential_rotation_requested`]. +pub async fn clear_bmc_credential_rotation_requested( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, +) -> DatabaseResult<()> { + let query = "UPDATE power_shelves SET bmc_credential_rotation_requested = false WHERE id = $1 RETURNING id"; + sqlx::query_scalar::<_, PowerShelfId>(query) + .bind(power_shelf_id) + .fetch_one(txn) + .await + .map_err(|e| match e { + sqlx::Error::RowNotFound => DatabaseError::NotFoundError { + kind: "power_shelf", + id: power_shelf_id.to_string(), + }, + e => DatabaseError::new("power_shelf::clear_bmc_credential_rotation_requested", e), + })?; + Ok(()) +} + /// Sets `power_shelf_reprovisioning_requested` so the Ready handler can enter /// `ReProvisioning` when rack-firmware reprovisioning is enabled. /// diff --git a/crates/api-db/src/sql/machine_snapshots.sql.template b/crates/api-db/src/sql/machine_snapshots.sql.template index 49a2d0fb8c..1c116335bc 100644 --- a/crates/api-db/src/sql/machine_snapshots.sql.template +++ b/crates/api-db/src/sql/machine_snapshots.sql.template @@ -9,6 +9,9 @@ m.*, boot_interface.desired_mac_address AS desired_boot_interface_mac, boot_interface.desired_interface_id AS desired_boot_interface_id, boot_interface.desired_version AS desired_boot_interface_version, +boot_interface.verified_version AS boot_interface_verified_version, +boot_interface.observed_at AS boot_interface_observed_at, +COALESCE(boot_interface.assumed, false) AS boot_interface_observation_assumed, sku.device_type as hw_sku_device_type, COALESCE(i.json, '[]') AS interfaces, COALESCE(t.json, '[]') AS topology, diff --git a/crates/api-db/src/sql/managed_hosts.sql.template b/crates/api-db/src/sql/managed_hosts.sql.template index 96421adb93..52f00df197 100644 --- a/crates/api-db/src/sql/managed_hosts.sql.template +++ b/crates/api-db/src/sql/managed_hosts.sql.template @@ -26,6 +26,9 @@ SELECT m.* FROM ( boot_interface.desired_mac_address AS desired_boot_interface_mac, boot_interface.desired_interface_id AS desired_boot_interface_id, boot_interface.desired_version AS desired_boot_interface_version, + boot_interface.verified_version AS boot_interface_verified_version, + boot_interface.observed_at AS boot_interface_observed_at, + COALESCE(boot_interface.assumed, false) AS boot_interface_observation_assumed, sku.device_type as hw_sku_device_type, COALESCE(i.json, '[]') AS interfaces, COALESCE(t.json, '[]') AS topology, @@ -131,6 +134,9 @@ SELECT m.* FROM ( boot_interface.desired_mac_address AS desired_boot_interface_mac, boot_interface.desired_interface_id AS desired_boot_interface_id, boot_interface.desired_version AS desired_boot_interface_version, + boot_interface.verified_version AS boot_interface_verified_version, + boot_interface.observed_at AS boot_interface_observed_at, + COALESCE(boot_interface.assumed, false) AS boot_interface_observation_assumed, COALESCE(i.json, '[]') AS interfaces, COALESCE(t.json, '[]') AS topology, COALESCE(bmc.json, t.bmc_info - 'ip' - 'mac', '{}'::jsonb) AS bmc_info diff --git a/crates/api-db/src/vpc.rs b/crates/api-db/src/vpc.rs index d609b2df07..fde54c4dce 100644 --- a/crates/api-db/src/vpc.rs +++ b/crates/api-db/src/vpc.rs @@ -67,7 +67,8 @@ pub async fn persist( let query = "INSERT INTO vpcs (id, name, organization_id, network_security_group_id, version, network_virtualization_type, description, - labels, routing_profile_type, vni, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *"; + labels, routing_profile_type, routing_profile_overrides, vni, status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING *"; sqlx::query_as(query) .bind(value.id) .bind(&value.metadata.name) @@ -78,6 +79,7 @@ pub async fn persist( .bind(&value.metadata.description) .bind(sqlx::types::Json(&value.metadata.labels)) .bind(value.routing_profile_type) + .bind(value.routing_profile_overrides.map(sqlx::types::Json)) .bind(value.vni) .bind(sqlx::types::Json(&status)) .fetch_one(txn) diff --git a/crates/api-model/src/lib.rs b/crates/api-model/src/lib.rs index 17a19558d8..aa931e53c4 100644 --- a/crates/api-model/src/lib.rs +++ b/crates/api-model/src/lib.rs @@ -116,6 +116,12 @@ pub enum ConfigValidationError { #[error("invalid value: {0}")] InvalidValue(String), + #[error( + "initial VPC `{name}` cannot define `routing_profile_overrides`; inline routing-profile \ + overrides are only supported by VPC creation requests" + )] + InitialVpcRoutingProfileOverridesUnsupported { name: String }, + #[error("found unknown segments")] UnknownSegments, diff --git a/crates/api-model/src/machine/json.rs b/crates/api-model/src/machine/json.rs index b405b43066..acfe16aa94 100644 --- a/crates/api-model/src/machine/json.rs +++ b/crates/api-model/src/machine/json.rs @@ -41,7 +41,7 @@ use crate::machine::{ ManagedHostState, ReprovisionRequest, UpgradeDecision, }; use crate::machine_boot_interface::{ - MachineBootInterfaceTarget, canonical_redfish_boot_interface_id, + BootInterfaceStatusObservation, MachineBootInterfaceTarget, canonical_redfish_boot_interface_id, }; use crate::metadata::Metadata; use crate::power_manager::PowerOptions; @@ -109,6 +109,10 @@ pub struct MachineSnapshotPgJson { pub desired_boot_interface_mac: Option, pub desired_boot_interface_id: Option, pub desired_boot_interface_version: Option, + pub boot_interface_verified_version: Option, + pub boot_interface_observed_at: Option>, + #[serde(default)] + pub boot_interface_observation_assumed: bool, pub hw_sku_status: Option, #[serde(default)] // Power options are valid only for host, not for DPUs. pub power_options: Option, @@ -171,6 +175,37 @@ fn decode_desired_boot_interface( } } +fn decode_boot_interface_status_observation( + config_version: Option, + observed_at: Option>, + assumed: bool, +) -> sqlx::Result> { + match (config_version, observed_at, assumed) { + (None, None, false) => Ok(None), + (Some(config_version), Some(observed_at), assumed) => { + let config_version = + config_version + .parse() + .map_err(|error| sqlx::Error::ColumnDecode { + index: "boot_interface_verified_version".to_string(), + source: Box::new(error), + })?; + Ok(Some(BootInterfaceStatusObservation { + config_version, + observed_at, + assumed, + })) + } + _ => Err(sqlx::Error::ColumnDecode { + index: "boot_interface_(verified_version,observed_at,assumed)".to_string(), + source: Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "boot interface verified version and observation time must both be set or both be null, and assumed requires an observation", + )), + }), + } +} + impl TryFrom for Machine { type Error = sqlx::Error; @@ -196,6 +231,11 @@ impl TryFrom for Machine { value.desired_boot_interface_id, value.desired_boot_interface_version, )?; + let boot_interface_status_observation = decode_boot_interface_status_observation( + value.boot_interface_verified_version, + value.boot_interface_observed_at, + value.boot_interface_observation_assumed, + )?; let version: ConfigVersion = value @@ -261,6 +301,7 @@ impl TryFrom for Machine { }, status: MachineStatus { interfaces: value.interfaces, + boot_interface_status_observation, hardware_info, bmc_info: value.bmc_info, last_reboot_time: value.last_reboot_time, @@ -332,6 +373,13 @@ mod tests { }, } + #[derive(Debug)] + struct ObservationInput { + config_version: Option, + observed_at: Option>, + assumed: bool, + } + fn summarize(value: Option>) -> Decoded { match value { None => Decoded::Unset, @@ -464,4 +512,96 @@ mod tests { }, ); } + + #[test] + fn boot_interface_status_columns_decode_atomically() { + let observed_at = DateTime::from_timestamp(1_722_000_000, 123_000_000) + .expect("fixture timestamp is valid"); + let version = ConfigVersion::new(7); + let config_version = version.version_string(); + + check_cases( + [ + Case { + scenario: "no observation", + input: ObservationInput { + config_version: None, + observed_at: None, + assumed: false, + }, + expect: Yields(None), + }, + Case { + scenario: "Redfish observation", + input: ObservationInput { + config_version: Some(config_version.clone()), + observed_at: Some(observed_at), + assumed: false, + }, + expect: Yields(Some(BootInterfaceStatusObservation { + config_version: version, + observed_at, + assumed: false, + })), + }, + Case { + scenario: "rollout baseline", + input: ObservationInput { + config_version: Some(config_version.clone()), + observed_at: Some(observed_at), + assumed: true, + }, + expect: Yields(Some(BootInterfaceStatusObservation { + config_version: version, + observed_at, + assumed: true, + })), + }, + Case { + scenario: "version without time", + input: ObservationInput { + config_version: Some(config_version), + observed_at: None, + assumed: false, + }, + expect: Fails, + }, + Case { + scenario: "time without version", + input: ObservationInput { + config_version: None, + observed_at: Some(observed_at), + assumed: false, + }, + expect: Fails, + }, + Case { + scenario: "assumed without observation", + input: ObservationInput { + config_version: None, + observed_at: None, + assumed: true, + }, + expect: Fails, + }, + Case { + scenario: "malformed version", + input: ObservationInput { + config_version: Some("not-a-version".to_string()), + observed_at: Some(observed_at), + assumed: false, + }, + expect: Fails, + }, + ], + |ObservationInput { + config_version, + observed_at, + assumed, + }| { + decode_boot_interface_status_observation(config_version, observed_at, assumed) + .map_err(drop) + }, + ); + } } diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index d3660262c1..da4b5a07ce 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -42,7 +42,7 @@ use super::StateSla; use super::instance::snapshot::InstanceSnapshot; use super::instance::status::extension_service::InstanceExtensionServiceStatusObservation; use super::instance::status::network::InstanceNetworkStatusObservation; -use super::machine_boot_interface::MachineBootInterface; +use super::machine_boot_interface::{MachineBootInterface, MachineBootInterfaceTarget}; use super::metadata::Metadata; use crate::controller_outcome::PersistentStateHandlerOutcome; use crate::dpa_interface::DpaInterface; @@ -120,7 +120,14 @@ fn default_true() -> bool { } // This should be updated on each new model introduction -pub const CURRENT_STATE_MODEL_VERSION: i16 = 2; +pub const CURRENT_STATE_MODEL_VERSION: i16 = 3; + +fn pending_boot_interface_config_version( + desired_version: Option, + verified_version: Option, +) -> Option { + desired_version.filter(|desired_version| Some(*desired_version) != verified_version) +} /// Represents the current state of `Machine` #[derive(Debug, Clone)] @@ -228,6 +235,8 @@ pub enum NotAllocatableReason { "the machine has a pending instance creation request, that has not yet been processed by the state handler" )] PendingInstanceCreation, + #[error("the machine has a pending boot configuration")] + PendingBootConfiguration, #[error("there are no dpu_snapshots, but associated_dpu_machine_ids is non-empty")] NoDpuSnapshots, #[error("the machine is in maintenance mode")] @@ -462,6 +471,7 @@ impl ManagedHostStateSnapshot { /// - the Machine has not yet been target of an instance creation request /// - no health alerts which classification `PreventAllocations` to be set /// - the machine not to be in Maintenance Mode + /// - the desired boot-interface generation to have a matching observation pub fn is_usable_as_instance(&self, allow_unhealthy: bool) -> Result<(), NotAllocatableReason> { // TODO: allow other states than Ready when allow_unhealthy=true. Will require changes to state machine (see Matthias). if !matches!(self.managed_state, ManagedHostState::Ready) { @@ -477,6 +487,18 @@ impl ManagedHostStateSnapshot { return Err(NotAllocatableReason::PendingInstanceCreation); } + // A desired boot-interface update and instance allocation can race + // before machine-controller has persisted BootConfiguring. Keep the + // host unavailable as soon as the desired version lacks a matching + // convergence status. + if self + .host_snapshot + .pending_boot_interface_config_version() + .is_some() + { + return Err(NotAllocatableReason::PendingBootConfiguration); + } + if self.dpu_snapshots.is_empty() && !self.host_snapshot.associated_dpu_machine_ids().is_empty() { @@ -939,6 +961,25 @@ impl Machine { self.state.version } + /// Returns the desired boot-interface version whose persisted convergence + /// status is not current, if any. + /// + /// Comparing versions keeps the Ready-state decision DB-only. Redfish is + /// queried only after the controller has persisted + /// [`ManagedHostState::BootConfiguring`]. + pub fn pending_boot_interface_config_version(&self) -> Option { + pending_boot_interface_config_version( + self.config + .desired_boot_interface + .as_ref() + .map(|desired| desired.version), + self.status + .boot_interface_status_observation + .as_ref() + .map(|observation| observation.config_version), + ) + } + /// Latest health report received from forge-dpu-agent. pub fn dpu_agent_health_report(&self) -> Option<&HealthReport> { self.health_reports @@ -1166,6 +1207,23 @@ pub enum ManagedHostState { /// Host is Ready for instance creation. Ready, + /// An unassigned Ready host is converging its Redfish boot configuration + /// to the desired boot interface persisted on the machine. + /// + /// The desired target and version are captured when the repair starts. + /// The controller checks that version before issuing new Redfish writes, + /// uses the captured target while work is in flight, and records it + /// verified only when the version is still current after final observation. + BootConfiguring { + desired_version: ConfigVersion, + desired_boot_interface: MachineBootInterfaceTarget, + /// Number of complete reconciliation passes retried because the final + /// Redfish observation drifted after lockdown was restored. + #[serde(default)] + post_lock_verification_retry_count: u32, + boot_config_state: ReadyBootConfigState, + }, + /// Host is executing an operator-requested maintenance operation. Maintenance { operation: MachineMaintenanceOperation, @@ -1309,6 +1367,74 @@ pub enum MachineValidatingState { validation_id: MachineValidationId, }, } + +/// `ReadyBootConfigTerminalFailure` defers a terminal condition until Ready +/// boot convergence restores lockdown. +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum ReadyBootConfigTerminalFailure { + /// The boot-config convergence flow could not complete automatically. + Convergence { failure: String }, + /// An independent host or DPU failure appeared while lockdown was open. + /// Preserve its original attribution while routing through LockHost. + Machine { + machine_id: MachineId, + details: FailureDetails, + }, +} + +/// `ReadyBootConfigState` persists progress while an unassigned Ready host +/// converges its desired Redfish boot configuration. +/// +/// BIOS and boot-order job details reuse the same model types as HostInit, +/// assigned platform configuration, and validation so controller restarts +/// retain vendor job IDs, recovery substates, and retry budgets. +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] +#[serde(tag = "state", rename_all = "lowercase")] +pub enum ReadyBootConfigState { + /// Observe the target, then inspect lockdown only when a repair may write. + Prepare, + /// Disable lockdown, including any vendor-specific reboot and wait. + UnlockHost { + #[serde(default)] + unlock_host_state: UnlockHostState, + }, + /// Observe BIOS and boot order and select the smallest required repair. + CheckHostConfig, + /// Run `machine_setup` for the desired boot interface. + ConfigureBios { + #[serde(default)] + retry_count: u32, + }, + /// Wait for the vendor BIOS configuration job returned by `machine_setup`. + WaitingForBiosJob { bios_config_info: BiosConfigInfo }, + /// Verify that the BIOS configuration has been applied. + PollingBiosSetup { + #[serde(default)] + retry_count: u32, + }, + /// Set, apply, and verify boot order. + SetBootOrder { + set_boot_order_info: SetBootOrderInfo, + }, + /// Restore the configured lockdown policy before either conditionally + /// marking the desired boot-interface version verified or surfacing a + /// terminal convergence failure. + LockHost { + /// Failure deferred until lockdown has been restored. Absent on the + /// successful convergence path. + #[serde(default, skip_serializing_if = "Option::is_none")] + terminal_failure: Option, + }, + /// Automated convergence could not complete safely after lockdown was + /// restored. The host remains unavailable until an operator changes its + /// desired boot interface, starting a fresh pass from + /// [`ReadyBootConfigState::Prepare`], or successfully completes a + /// maintenance operation, which returns the host to + /// [`ManagedHostState::Ready`]. + Failed { failure: String }, +} + #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] #[serde(tag = "validation_type", rename_all = "lowercase")] pub enum ValidationState { @@ -2335,6 +2461,23 @@ impl Display for SpdmMeasuringState { } } +impl Display for ReadyBootConfigState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let name = match self { + Self::Prepare => "Prepare", + Self::UnlockHost { .. } => "UnlockHost", + Self::CheckHostConfig => "CheckHostConfig", + Self::ConfigureBios { .. } => "ConfigureBios", + Self::WaitingForBiosJob { .. } => "WaitingForBiosJob", + Self::PollingBiosSetup { .. } => "PollingBiosSetup", + Self::SetBootOrder { .. } => "SetBootOrder", + Self::LockHost { .. } => "LockHost", + Self::Failed { .. } => "Failed", + }; + f.write_str(name) + } +} + impl Display for ManagedHostState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -2363,6 +2506,11 @@ impl Display for ManagedHostState { write!(f, "HostInitializing/{machine_state}") } ManagedHostState::Ready => write!(f, "Ready"), + ManagedHostState::BootConfiguring { + boot_config_state, .. + } => { + write!(f, "BootConfiguring/{boot_config_state}") + } ManagedHostState::Maintenance { operation } => { write!(f, "Maintenance({operation:?})") } @@ -2459,6 +2607,11 @@ impl ManagedHostState { format!("HostInitializing/{machine_state}") } ManagedHostState::Ready => "Ready".to_string(), + ManagedHostState::BootConfiguring { + boot_config_state, .. + } => { + format!("BootConfiguring/{boot_config_state}") + } ManagedHostState::Maintenance { operation } => { format!("Maintenance({operation:?})") } @@ -2662,6 +2815,13 @@ pub fn state_sla( _ => StateSla::with_sla(slas::HOST_INIT, time_in_state), }, ManagedHostState::Ready => StateSla::no_sla(), + ManagedHostState::BootConfiguring { + boot_config_state: ReadyBootConfigState::Failed { .. }, + .. + } => StateSla::with_sla(std::time::Duration::ZERO, time_in_state), + ManagedHostState::BootConfiguring { .. } => { + StateSla::with_sla(slas::BOOT_CONFIGURING, time_in_state) + } ManagedHostState::Maintenance { .. } => { StateSla::with_sla(slas::MAINTENANCE, time_in_state) } @@ -3095,7 +3255,7 @@ mod tests { use std::str::FromStr; use carbide_test_support::Outcome::*; - use carbide_test_support::{Check, check_values, scenarios}; + use carbide_test_support::{Check, check_values, scenarios, value_scenarios}; use super::*; use crate::test_support::machine_snapshot::{ @@ -3133,6 +3293,151 @@ mod tests { reprovision_requested: false, }; + #[test] + fn pending_boot_interface_version_requires_matching_verification() { + let desired = ConfigVersion::new(7); + let stale = ConfigVersion::new(6); + + value_scenarios!( + run = |(desired_version, verified_version)| { + pending_boot_interface_config_version(desired_version, verified_version) + }; + "no desired target needs no verification" { + (None, None) => None, + } + "an unobserved desired target needs verification" { + (Some(desired), None) => Some(desired), + } + "a stale observation needs verification" { + (Some(desired), Some(stale)) => Some(desired), + } + "a matching observation is converged" { + (Some(desired), Some(desired)) => None, + } + ); + } + + #[test] + fn ready_boot_config_defaults_survive_persisted_state_loading() { + scenarios!( + run = |json| serde_json::from_str::(json).map_err(drop); + "unlock starts by disabling lockdown" { + r#"{"state":"unlockhost"}"# => Yields(ReadyBootConfigState::UnlockHost { + unlock_host_state: UnlockHostState::DisableLockdown, + }), + } + + "BIOS setup starts with no retries" { + r#"{"state":"configurebios"}"# => Yields(ReadyBootConfigState::ConfigureBios { + retry_count: 0, + }), + } + + "BIOS verification starts with no retries" { + r#"{"state":"pollingbiossetup"}"# => Yields( + ReadyBootConfigState::PollingBiosSetup { retry_count: 0 }, + ), + } + + "lockdown restoration defaults to the success path" { + r#"{"state":"lockhost"}"# => Yields(ReadyBootConfigState::LockHost { + terminal_failure: None, + }), + } + ); + } + + #[test] + fn ready_boot_config_terminal_outcomes_round_trip() { + let machine_id = + MachineId::from_str("fm100ds7blqjsadm2uuh3qqbf1h7k8pmf47um6v9uckrg7l03po8mhqgvng") + .unwrap(); + let failure_details = FailureDetails { + cause: FailureCause::BiosSetupFailed { + err: "BIOS job retries exhausted".to_string(), + }, + failed_at: DateTime::::UNIX_EPOCH, + source: FailureSource::StateMachine, + }; + + check_values( + [ + Check { + scenario: "convergence failure waits for lockdown", + input: ReadyBootConfigState::LockHost { + terminal_failure: Some(ReadyBootConfigTerminalFailure::Convergence { + failure: "BIOS job retries exhausted".to_string(), + }), + }, + expect: true, + }, + Check { + scenario: "independent machine failure keeps its attribution", + input: ReadyBootConfigState::LockHost { + terminal_failure: Some(ReadyBootConfigTerminalFailure::Machine { + machine_id, + details: failure_details, + }), + }, + expect: true, + }, + Check { + scenario: "terminal convergence failure persists", + input: ReadyBootConfigState::Failed { + failure: "BIOS job retries exhausted".to_string(), + }, + expect: true, + }, + ], + |state| { + serde_json::from_str::( + &serde_json::to_string(&state).unwrap(), + ) + .unwrap() + == state + }, + ); + } + + #[test] + fn ready_host_with_unverified_boot_interface_is_not_allocatable() { + let mut snapshot = managed_host_state_snapshot(); + let desired_version = ConfigVersion::new(7); + let desired_boot_interface = + MachineBootInterfaceTarget::MacOnly(MacAddress::new([1, 2, 3, 4, 5, 6])); + snapshot.host_snapshot.config.desired_boot_interface = + Some(Versioned::new(desired_boot_interface, desired_version)); + snapshot + .host_snapshot + .status + .boot_interface_status_observation = None; + + assert_eq!( + snapshot.is_usable_as_instance(false), + Err(NotAllocatableReason::PendingBootConfiguration) + ); + } + + #[test] + fn boot_configuring_state_has_stable_state_strings() { + let state = ManagedHostState::BootConfiguring { + desired_version: ConfigVersion::new(7), + desired_boot_interface: MachineBootInterfaceTarget::MacOnly(MacAddress::new([ + 1, 2, 3, 4, 5, 6, + ])), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::Failed { + failure: "payload must not enter state labels".to_string(), + }, + }; + let dpu_id = + MachineId::from_str("fm100ds7blqjsadm2uuh3qqbf1h7k8pmf47um6v9uckrg7l03po8mhqgvng") + .unwrap(); + + assert_eq!(state.to_string(), "BootConfiguring/Failed"); + assert_eq!(state.dpu_state_string(&dpu_id), "BootConfiguring/Failed"); + } + #[test] fn machine_bmc_vendor_delegates_to_hardware_info() { let mut with_hardware_info = host_machine(); @@ -3874,6 +4179,32 @@ mod tests { input: stale(ManagedHostState::Ready), expect: (None, false), }, + Check { + scenario: "active boot configuration uses the convergence SLA", + input: stale(ManagedHostState::BootConfiguring { + desired_version: ConfigVersion::initial(), + desired_boot_interface: MachineBootInterfaceTarget::MacOnly( + MacAddress::new([1, 2, 3, 4, 5, 6]), + ), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::Prepare, + }), + expect: (seconds(5_400), true), + }, + Check { + scenario: "terminal boot configuration immediately breaches its SLA", + input: stale(ManagedHostState::BootConfiguring { + desired_version: ConfigVersion::initial(), + desired_boot_interface: MachineBootInterfaceTarget::MacOnly( + MacAddress::new([1, 2, 3, 4, 5, 6]), + ), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::Failed { + failure: "BIOS job retries exhausted".to_string(), + }, + }), + expect: (seconds(0), true), + }, Check { scenario: "maintenance uses the maintenance SLA", input: stale(ManagedHostState::Maintenance { diff --git a/crates/api-model/src/machine/slas.rs b/crates/api-model/src/machine/slas.rs index 74f55c500a..097796da75 100644 --- a/crates/api-model/src/machine/slas.rs +++ b/crates/api-model/src/machine/slas.rs @@ -29,6 +29,10 @@ pub const DPUINIT_NOTINIT: Duration = Duration::from_secs(30 * 60); // EnableIpmiOverLan WaitingForPlatformConfiguration PollingBiosSetup UefiSetup Discovered Lockdown PollingLockdownStatus MachineValidating pub const HOST_INIT: Duration = Duration::from_secs(30 * 60); +// Ready-state boot-interface convergence may include vendor BIOS jobs, +// controlled reboots, and final observation after lockdown is restored. +pub const BOOT_CONFIGURING: Duration = Duration::from_secs(90 * 60); + pub const WAITING_FOR_CLEANUP: Duration = Duration::from_secs(30 * 60); pub const CREATED: Duration = Duration::from_secs(30 * 60); diff --git a/crates/api-model/src/machine/status.rs b/crates/api-model/src/machine/status.rs index 91ae96b2df..86e43f5640 100644 --- a/crates/api-model/src/machine/status.rs +++ b/crates/api-model/src/machine/status.rs @@ -22,6 +22,7 @@ use crate::machine::infiniband::MachineInfinibandStatusObservation; use crate::machine::nvlink::MachineNvLinkStatusObservation; use crate::machine::spx::MachineSpxStatusObservation; use crate::machine::{FailureDetails, MachineInterfaceSnapshot, MachineLastRebootRequested}; +use crate::machine_boot_interface::BootInterfaceStatusObservation; use crate::power_manager::PowerOptions; use crate::sku::SkuStatus; @@ -31,6 +32,8 @@ use crate::sku::SkuStatus; #[derive(Debug, Clone)] pub struct MachineStatus { pub interfaces: Vec, + /// Latest persisted convergence status for the desired boot interface. + pub boot_interface_status_observation: Option, pub hardware_info: Option, pub bmc_info: BmcInfo, pub last_reboot_time: Option>, diff --git a/crates/api-model/src/machine_boot_interface.rs b/crates/api-model/src/machine_boot_interface.rs index c8383b0a8b..bc3e2d3fd5 100644 --- a/crates/api-model/src/machine_boot_interface.rs +++ b/crates/api-model/src/machine_boot_interface.rs @@ -15,6 +15,8 @@ * limitations under the License. */ use carbide_utils::none_if_empty::NoneIfEmpty; +use chrono::{DateTime, Utc}; +use config_version::ConfigVersion; use mac_address::MacAddress; use serde::{Deserialize, Serialize}; @@ -98,6 +100,21 @@ pub enum MachineBootInterfaceTarget { MacOnly(MacAddress), } +/// Status for the desired boot-interface generation currently treated as converged. +/// +/// `assumed` is true for the compatibility baseline used when an already-stable +/// host has no persisted row, including during mixed-component rollout. Real +/// Redfish verification always records it as false. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BootInterfaceStatusObservation { + /// Desired boot-interface configuration version this status applies to. + pub config_version: ConfigVersion, + /// Time this status was recorded. + pub observed_at: DateTime, + /// Whether this is a compatibility baseline rather than a Redfish observation. + pub assumed: bool, +} + impl MachineBootInterfaceTarget { /// Builds the strongest usable target from an endpoint record. /// diff --git a/crates/api-model/src/power_shelf/mod.rs b/crates/api-model/src/power_shelf/mod.rs index b20cee1770..fc88756e80 100644 --- a/crates/api-model/src/power_shelf/mod.rs +++ b/crates/api-model/src/power_shelf/mod.rs @@ -74,6 +74,14 @@ pub struct PowerShelf { pub bmc_mac_address: Option, + /// Operator "force-converge this power shelf PMC now" request (REQ-2). When + /// `true`, the power-shelf state controller enters `RotatingBmc` and + /// force-converges the PMC on its next sweep, bypassing the passive + /// site-wide gate and the device's backoff quarantine. A power shelf has + /// exactly one BMC, so the flag's presence on the row names the target + /// device. + pub bmc_credential_rotation_requested: bool, + /// BMC/PMC endpoint (MAC/IP + machine-interface id) resolved from the `Bmc` /// machine_interface linked back to this power shelf. Populated by the /// standard power-shelf load query, so every consumer (handlers, state @@ -144,6 +152,9 @@ impl<'r> FromRow<'r, PgRow> for PowerShelf { status: status.map(|s| s.0), deleted: row.try_get("deleted")?, bmc_mac_address: row.try_get("bmc_mac_address").ok().flatten(), + bmc_credential_rotation_requested: row + .try_get("bmc_credential_rotation_requested") + .unwrap_or(false), bmc_info, controller_state: Versioned { value: controller_state.0, @@ -235,6 +246,15 @@ pub enum PowerShelfControllerState { /// The PowerShelf is ready for use. Ready, + /// The PowerShelf's BMC (PMC) credential is being converged to the staged + /// site-wide rotation target (REQ-2), entered from `Ready` at lowest + /// precedence. The shared engine owns crash-safety and per-device backoff, + /// so this state carries only a retry budget for transient handler failures. + RotatingBmc { + #[serde(default)] + retry_count: u32, + }, + Maintenance { operation: PowerShelfMaintenanceOperation, }, @@ -270,6 +290,10 @@ pub fn state_sla(state: &PowerShelfControllerState, state_version: &ConfigVersio time_in_state, ), PowerShelfControllerState::Ready => StateSla::no_sla(), + PowerShelfControllerState::RotatingBmc { .. } => StateSla::with_sla( + std::time::Duration::from_secs(slas::ROTATING_BMC), + time_in_state, + ), PowerShelfControllerState::Maintenance { .. } => StateSla::with_sla( std::time::Duration::from_secs(slas::MAINTENANCE), time_in_state, @@ -342,6 +366,13 @@ mod tests { )), } + "rotatingbmc carries its retry count" { + PowerShelfControllerState::RotatingBmc { retry_count: 4 } => Yields(( + r#"{"state":"rotatingbmc","retry_count":4}"#.to_string(), + PowerShelfControllerState::RotatingBmc { retry_count: 4 }, + )), + } + "error with cause" { PowerShelfControllerState::Error { cause: "cause goes here".to_string(), @@ -487,6 +518,18 @@ mod tests { r#"{"state":"ready"}"# => Yields(PowerShelfControllerState::Ready), } + "rotatingbmc round-trips its retry count" { + r#"{"state":"rotatingbmc","retry_count":4}"# => Yields(PowerShelfControllerState::RotatingBmc { + retry_count: 4, + }), + } + + "rotatingbmc absent retry_count defaults to 0" { + r#"{"state":"rotatingbmc"}"# => Yields(PowerShelfControllerState::RotatingBmc { + retry_count: 0, + }), + } + "deleting tag" { r#"{"state":"deleting"}"# => Yields(PowerShelfControllerState::Deleting), } @@ -747,6 +790,10 @@ mod tests { } => (secs(slas::REPROVISIONING), true), } + "rotatingbmc has the rotating-bmc SLA" { + PowerShelfControllerState::RotatingBmc { retry_count: 0 } => (secs(slas::ROTATING_BMC), true), + } + "ready carries no SLA" { PowerShelfControllerState::Ready => (None, false), } diff --git a/crates/api-model/src/power_shelf/slas.rs b/crates/api-model/src/power_shelf/slas.rs index 215516b577..991b8db49b 100644 --- a/crates/api-model/src/power_shelf/slas.rs +++ b/crates/api-model/src/power_shelf/slas.rs @@ -36,5 +36,10 @@ pub const DELETING: u64 = 300; // 5 minutes /// SLA for PowerShelf maintenance (PowerOn / PowerOff) in seconds pub const MAINTENANCE: u64 = 300; // 5 minutes +/// SLA for PowerShelf BMC (PMC) credential rotation in seconds. Generous enough +/// to absorb a slow BMC plus the rotation engine's short per-device backoff +/// without tripping the SLA on the first retry. +pub const ROTATING_BMC: u64 = 15 * 60; // 15 minutes + /// SLA for PowerShelf rack-level reprovisioning (firmware wait) in seconds pub const REPROVISIONING: u64 = 3600; // 1 hour diff --git a/crates/api-model/src/test_support/machine_snapshot.rs b/crates/api-model/src/test_support/machine_snapshot.rs index a8b213fe4f..1169b437ba 100644 --- a/crates/api-model/src/test_support/machine_snapshot.rs +++ b/crates/api-model/src/test_support/machine_snapshot.rs @@ -45,8 +45,8 @@ use crate::machine::json::MachineSnapshotPgJson; use crate::machine::network::{MachineNetworkStatusObservation, ManagedHostNetworkConfig}; use crate::machine::topology::{DiscoveryData, MachineTopology, TopologyData}; use crate::machine::{ - Dpf, FailureCause, FailureDetails, FailureSource, HostProfile, Machine, - MachineInterfaceSnapshot, MachineLastRebootRequested, MachineLastRebootRequestedMode, + CURRENT_STATE_MODEL_VERSION, Dpf, FailureCause, FailureDetails, FailureSource, HostProfile, + Machine, MachineInterfaceSnapshot, MachineLastRebootRequested, MachineLastRebootRequestedMode, ManagedHostState, ManagedHostStateSnapshot, UpgradeDecision, }; use crate::machine_interface::InterfaceType; @@ -457,7 +457,7 @@ pub fn machine_snapshot_pg_json(machine_id: MachineId) -> MachineSnapshotPgJson asn: Some(4_200_000_042), controller_state_outcome: None, current_machine_validation_id: None, - machine_state_model_version: 2, + machine_state_model_version: i32::from(CURRENT_STATE_MODEL_VERSION), instance_type_id: Some(uuid::Uuid::from_u128(0x4001).into()), interfaces, topology: vec![MachineTopology { @@ -485,6 +485,9 @@ pub fn machine_snapshot_pg_json(machine_id: MachineId) -> MachineSnapshotPgJson desired_boot_interface_mac: None, desired_boot_interface_id: None, desired_boot_interface_version: None, + boot_interface_verified_version: None, + boot_interface_observed_at: None, + boot_interface_observation_assumed: false, hw_sku_status: None, power_options: None, hw_sku_device_type: Some("compute".to_string()), diff --git a/crates/api-model/src/vpc/capability.rs b/crates/api-model/src/vpc/capability.rs index 649fc39f19..0ec116824c 100644 --- a/crates/api-model/src/vpc/capability.rs +++ b/crates/api-model/src/vpc/capability.rs @@ -280,7 +280,8 @@ pub enum VpcCapabilityError { Ipv6Unsupported { vpc_type: VpcVirtualizationType }, #[error( - "{vpc_type} VPCs do not support routing profiles; the `routing_profile_type` field is FNN-only" + "{vpc_type} VPCs do not support routing profiles; the `routing_profile_type` and \ + `routing_profile_overrides` fields are FNN-only" )] RoutingProfilesUnsupported { vpc_type: VpcVirtualizationType }, diff --git a/crates/api-model/src/vpc/mod.rs b/crates/api-model/src/vpc/mod.rs index 4dbb3b754b..2eace0363e 100644 --- a/crates/api-model/src/vpc/mod.rs +++ b/crates/api-model/src/vpc/mod.rs @@ -15,6 +15,7 @@ * limitations under the License. */ pub mod capability; +pub mod routing_profile; use std::collections::HashMap; use std::net::IpAddr; @@ -31,6 +32,7 @@ use carbide_uuid::vpc::VpcId; use carbide_uuid::vpc_peering::VpcPeeringId; use chrono::{DateTime, Utc}; use config_version::ConfigVersion; +pub use routing_profile::{PrefixFilterPolicyEntry, RouteTargetConfig, VpcRoutingProfileOverrides}; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgRow; use sqlx::{FromRow, Row}; @@ -46,6 +48,7 @@ pub struct VpcConfig { pub default_nvlink_logical_partition_id: Option, pub vni: Option, pub routing_profile_type: Option, + pub routing_profile_overrides: Option, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -71,6 +74,7 @@ pub struct VpcDefinition { pub organization_id: Option, pub network_virtualization_type: VpcVirtualizationType, pub routing_profile_type: Option, + pub routing_profile_overrides: Option, pub vni: Option, } @@ -89,6 +93,7 @@ pub struct NewVpc { pub metadata: Metadata, pub network_security_group_id: Option, pub routing_profile_type: Option, + pub routing_profile_overrides: Option, pub vni: Option, } @@ -131,6 +136,11 @@ impl<'r> sqlx::FromRow<'r, PgRow> for Vpc { network_security_group_id: row.try_get("network_security_group_id")?, network_virtualization_type: row.try_get("network_virtualization_type")?, routing_profile_type: row.try_get("routing_profile_type")?, + routing_profile_overrides: row + .try_get::>, _>( + "routing_profile_overrides", + )? + .map(|profile| profile.0), vni: row.try_get("vni")?, default_nvlink_logical_partition_id: None, }, diff --git a/crates/api-model/src/vpc/routing_profile.rs b/crates/api-model/src/vpc/routing_profile.rs new file mode 100644 index 0000000000..c83829d4eb --- /dev/null +++ b/crates/api-model/src/vpc/routing_profile.rs @@ -0,0 +1,55 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use ipnetwork::IpNetwork; +use serde::{Deserialize, Serialize}; + +/// A BGP route target used in FNN VRF import/export policies. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct RouteTargetConfig { + /// Autonomous System Number component of the route target. + #[serde(default)] + pub asn: u32, + + /// Virtual Network Identifier component of the route target. + #[serde(default)] + pub vni: u32, +} + +/// An entry used by a DPU prefix-list policy. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct PrefixFilterPolicyEntry { + /// Prefix matched by the policy. + pub prefix: IpNetwork, +} + +/// Routing-profile values set directly on a VPC. +/// +/// Each present value overrides the corresponding property from the VPC's +/// named routing profile. `internal` and `access_tier` are intentionally absent +/// because VPCs cannot override the base profile's allocation and access +/// controls. +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct VpcRoutingProfileOverrides { + pub route_target_imports: Option>, + pub route_targets_on_exports: Option>, + pub leak_default_route_from_underlay: Option, + pub leak_tenant_host_routes_to_underlay: Option, + pub tenant_leak_communities_accepted: Option, + pub accepted_leaks_from_underlay: Option>, + pub allowed_anycast_prefixes: Option>, +} diff --git a/crates/api-web/src/tests/env.rs b/crates/api-web/src/tests/env.rs index 30fa7bdd8e..2a1d5ab585 100644 --- a/crates/api-web/src/tests/env.rs +++ b/crates/api-web/src/tests/env.rs @@ -21,7 +21,6 @@ use carbide_api_core::test_support::fixture_config::{ use carbide_test_harness::dns::TestDomain; use carbide_test_harness::network::segment::TestNetworkSegment; use carbide_test_harness::prelude::*; -use model::machine::ManagedHostState; use model::test_support::ManagedHostConfig; pub struct TestEnv { @@ -84,7 +83,7 @@ impl TestEnv { host.report_dpu_network_status().await; host.insert_empty_host_health_report("test-harness-health") .await; - host.advance_state(ManagedHostState::Ready).await; + host.advance_to_converged_ready().await; (host, build_data) } } diff --git a/crates/api-web/src/vpc.rs b/crates/api-web/src/vpc.rs index 1b5c4c8179..53acc0357d 100644 --- a/crates/api-web/src/vpc.rs +++ b/crates/api-web/src/vpc.rs @@ -57,6 +57,7 @@ fn vpc_config(vpc: &forgerpc::Vpc) -> forgerpc::VpcConfig { default_nvlink_logical_partition_id: vpc.default_nvlink_logical_partition_id, vni: vpc.vni, routing_profile_type: vpc.routing_profile_type.clone(), + routing_profile_overrides: None, } } } @@ -187,6 +188,10 @@ struct VpcDetail { tenant_keyset_id: String, network_virtualization_type: String, routing_profile_type: String, + has_routing_profile_overrides: bool, + routing_profile_overrides: String, + has_effective_routing_profile: bool, + effective_routing_profile: String, vni: String, metadata_detail: super::MetadataDetail, peerings: Vec, @@ -195,6 +200,15 @@ struct VpcDetail { impl From for VpcDetail { fn from(vpc: forgerpc::Vpc) -> Self { let config = vpc_config(&vpc); + let routing_profile_overrides = config + .routing_profile_overrides + .as_ref() + .and_then(|overrides| serde_json::to_string_pretty(overrides).ok()); + let effective_routing_profile = vpc + .status + .as_ref() + .and_then(|status| status.effective_routing_profile.as_ref()) + .and_then(|profile| serde_json::to_string_pretty(profile).ok()); Self { network_virtualization_type: format!( "{:?}", @@ -204,6 +218,10 @@ impl From for VpcDetail { tenant_organization_id: config.tenant_organization_id, tenant_keyset_id: config.tenant_keyset_id.unwrap_or_default(), routing_profile_type: config.routing_profile_type.unwrap_or("None".to_string()), + has_routing_profile_overrides: routing_profile_overrides.is_some(), + routing_profile_overrides: routing_profile_overrides.unwrap_or_default(), + has_effective_routing_profile: effective_routing_profile.is_some(), + effective_routing_profile: effective_routing_profile.unwrap_or_default(), vni: vpc_allocated_vni(&vpc) .map(|vni| vni.to_string()) .unwrap_or_default(), @@ -344,3 +362,60 @@ async fn fetch_vpc_peerings(state: Arc, vpc_id_string: String) -> VecVPC Detail VNI{{ vni }} +

Routing Profile Overrides

+{% if has_routing_profile_overrides %} +
{{ routing_profile_overrides }}
+{% else %} +

None

+{% endif %} + +

Current Effective Routing Profile

+{% if has_effective_routing_profile %} +
{{ effective_routing_profile }}
+{% else %} +

Not applicable or unavailable from the current API configuration.

+{% endif %} +

Metadata

{{ metadata_detail|safe }} diff --git a/crates/bmc-mock/src/redfish/computer_system.rs b/crates/bmc-mock/src/redfish/computer_system.rs index 6c5c29ad54..17fb78a1d3 100644 --- a/crates/bmc-mock/src/redfish/computer_system.rs +++ b/crates/bmc-mock/src/redfish/computer_system.rs @@ -16,6 +16,7 @@ */ use std::borrow::Cow; +use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -59,6 +60,25 @@ pub fn reset_target(system_id: &str) -> String { ) } +/// Return the HPE iLO boot settings resource used for persistent boot ordering. +fn hpe_boot_resource(system_id: &str) -> redfish::Resource<'static> { + redfish::Resource { + odata_id: Cow::Owned(format!( + "/redfish/v1/Systems/{system_id}/Bios/oem/hpe/boot/" + )), + odata_type: Cow::Borrowed("#HpeServerBootSettings.v2_0_0.HpeServerBootSettings"), + id: Cow::Borrowed("boot"), + name: Cow::Borrowed("Boot Settings"), + } +} + +/// HPE iLO settings payload for its persistent boot order. +#[derive(serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +struct HpeBootSettingsPatch { + persistent_boot_config_order: Vec, +} + pub fn add_routes(r: Router, bmc_vendor: redfish::oem::BmcVendor) -> Router { const SYSTEM_ID: &str = "{system_id}"; const ETH_ID: &str = "{eth_id}"; @@ -67,7 +87,8 @@ pub fn add_routes(r: Router, bmc_vendor: redfish::oem::BmcVendor) -> R const LOG_ENTRY_ID: &str = "{log_entry_id}"; const PROCESSOR_ID: &str = "{processor_id}"; let bios = redfish::bios::resource(SYSTEM_ID); - r.route(&collection().odata_id, get(get_system_collection)) + let routes = r + .route(&collection().odata_id, get(get_system_collection)) .route( &resource(SYSTEM_ID).odata_id, get(get_system).patch(patch_system), @@ -97,6 +118,11 @@ pub fn add_routes(r: Router, bmc_vendor: redfish::oem::BmcVendor) -> R &redfish::boot_option::resource(SYSTEM_ID, BOOT_OPTION_ID).odata_id, get(get_boot_option), ) + .route( + &bmc_vendor + .make_settings_odata_id(&redfish::boot_option::resource(SYSTEM_ID, BOOT_OPTION_ID)), + patch(patch_boot_option_settings), + ) .route(&bios.odata_id, get(get_bios).patch(patch_bios_settings)) .route( &redfish::log_service::system_collection(SYSTEM_ID).odata_id, @@ -141,7 +167,20 @@ pub fn add_routes(r: Router, bmc_vendor: redfish::oem::BmcVendor) -> R .route( &redfish::bios::change_password_target(&bios), post(change_bios_password_action), + ); + if matches!(bmc_vendor, redfish::oem::BmcVendor::Hpe) { + let hpe_boot = hpe_boot_resource(SYSTEM_ID); + // CombinedServer normalizes requests by removing trailing slashes before + // routing them, while the Redfish resource still advertises canonical + // trailing-slash OData identifiers. + let hpe_boot_path = hpe_boot.odata_id.trim_end_matches('/'); + routes.route(hpe_boot_path, get(get_hpe_boot)).route( + &format!("{hpe_boot_path}/settings"), + patch(patch_hpe_boot_settings), ) + } else { + routes + } } pub struct SingleSystemConfig { @@ -183,6 +222,10 @@ pub struct SingleSystemState { config: SingleSystemConfig, virtual_media: Option, boot_order_override: Mutex>>, + // HPE iLO uses OEM structured boot strings here, not the BootOption IDs + // exposed by the standard ComputerSystem BootOrder property. + hpe_boot_order_override: Mutex>>, + boot_option_overrides: Mutex>, boot_source_override: Mutex, secure_boot_enabled: Arc, bios_overrides: Arc>, @@ -268,6 +311,8 @@ impl SingleSystemState { config, virtual_media, boot_order_override: Mutex::new(None), + hpe_boot_order_override: Mutex::new(None), + boot_option_overrides: Mutex::new(HashMap::new()), boot_source_override: Mutex::new(BootSourceOverride::default()), secure_boot_enabled: Arc::new(AtomicBool::new(false)), bios_overrides: Arc::new(Mutex::new(serde_json::json!({}))), @@ -297,6 +342,32 @@ impl SingleSystemState { .find(|v| v.id == option_id) } + fn boot_option(&self, option_id: &str) -> Option { + let option = self.find_boot_option(option_id)?; + let overrides = self.boot_option_overrides.lock().expect("mutex poisoned"); + Some( + option.to_json().patch( + overrides + .get(option_id) + .cloned() + .unwrap_or_else(|| json!({})), + ), + ) + } + + fn patch_boot_option(&self, option_id: &str, patch_request: serde_json::Value) -> bool { + if self.find_boot_option(option_id).is_none() { + return false; + } + + let mut overrides = self.boot_option_overrides.lock().expect("mutex poisoned"); + let current = overrides + .entry(option_id.to_string()) + .or_insert_with(|| json!({})); + *current = current.clone().patch(patch_request); + true + } + fn set_boot_order_override(&self, boot_order: Vec) { *self.boot_order_override.lock().unwrap() = Some(boot_order); } @@ -305,6 +376,33 @@ impl SingleSystemState { self.boot_order_override.lock().unwrap().clone() } + /// Return the HPE OEM persistent order without changing standard BootOrder state. + fn hpe_boot_order(&self) -> Vec { + self.hpe_boot_order_override + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| { + self.config + .boot_options + .iter() + .flatten() + .map(|option| { + let prefix = match option.kind { + BootOptionKind::Disk => "HD", + BootOptionKind::Network => "NIC", + }; + format!("{prefix}.BootOption.{}", option.boot_reference()) + }) + .collect() + }) + } + + /// Persist an HPE OEM boot order independently from standard BootOrder state. + fn set_hpe_boot_order(&self, boot_order: Vec) { + *self.hpe_boot_order_override.lock().unwrap() = Some(boot_order); + } + pub(crate) fn virtual_media(&self) -> Option<&redfish::virtual_media::VirtualMediaState> { self.virtual_media.as_ref() } @@ -750,11 +848,54 @@ async fn get_boot_option( state .system_state .find(&system_id) - .and_then(|system_state| system_state.find_boot_option(&boot_option_id)) - .map(|boot_option| boot_option.to_json().into_ok_response()) + .and_then(|system_state| system_state.boot_option(&boot_option_id)) + .map(JsonExt::into_ok_response) .unwrap_or_else(http::not_found) } +async fn patch_boot_option_settings( + State(state): State, + Path((system_id, boot_option_id)): Path<(String, String)>, + Json(patch_request): Json, +) -> Response { + let Some(system_state) = state.system_state.find(&system_id) else { + return http::not_found(); + }; + if !system_state.patch_boot_option(&boot_option_id, patch_request) { + return http::not_found(); + } + json!({}).into_ok_response() +} + +/// Return the HPE iLO persistent boot-order resource. +async fn get_hpe_boot(State(state): State, Path(system_id): Path) -> Response { + let Some(system_state) = state.system_state.find(&system_id) else { + return http::not_found(); + }; + let boot_order = system_state.hpe_boot_order(); + hpe_boot_resource(&system_id) + .json_patch() + .patch(json!({ + "BootSources": [], + "DefaultBootOrder": ["PcieSlotNic", "PcieSlotStorage"], + "PersistentBootConfigOrder": boot_order, + })) + .into_ok_response() +} + +/// Apply the HPE iLO persistent boot order staged through its settings resource. +async fn patch_hpe_boot_settings( + State(state): State, + Path(system_id): Path, + Json(request): Json, +) -> Response { + let Some(system_state) = state.system_state.find(&system_id) else { + return http::not_found(); + }; + system_state.set_hpe_boot_order(request.persistent_boot_config_order); + json!({}).into_ok_response() +} + async fn get_log_services_collection( State(state): State, Path(system_id): Path, @@ -1092,3 +1233,71 @@ impl SystemBuilder { self.value } } + +#[cfg(test)] +mod tests { + use axum::Router; + use axum::body::{Body, to_bytes}; + use axum::http::header::CONTENT_TYPE; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + use tower_http::normalize_path::NormalizePathLayer; + + use super::*; + use crate::test_support::{NoopCallbacks, host_info}; + use crate::{HardwareType, MachineRouterOptions, machine_router}; + + /// Reads one successful JSON response from the in-process mock router. + async fn get_json(router: &Router, path: &str) -> serde_json::Value { + let response = router + .clone() + .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + serde_json::from_slice(&body).unwrap() + } + + /// HPE OEM ordering round-trips without corrupting standard BootOption IDs. + #[tokio::test] + async fn hpe_boot_order_is_persisted_separately_from_standard_boot_order() { + let router = machine_router( + &host_info(HardwareType::HpeProliantDl380aGen11), + Arc::new(NoopCallbacks), + "test-host-id".to_string(), + false, + MachineRouterOptions::default(), + ) + .0 + .layer(NormalizePathLayer::trim_trailing_slash()); + let boot_path = hpe_boot_resource("1").odata_id; + let initial = get_json(&router, &boot_path).await; + assert_eq!( + initial["PersistentBootConfigOrder"], + json!(["NIC.BootOption.Boot0000", "HD.BootOption.Boot0001",]) + ); + + let updated_order = json!(["HD.BootOption.Boot0001", "NIC.BootOption.Boot0000",]); + let response = router + .clone() + .oneshot( + Request::builder() + .method(Method::PATCH) + .uri(format!("{boot_path}settings/")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + json!({"PersistentBootConfigOrder": updated_order}).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let updated = get_json(&router, &boot_path).await; + assert_eq!(updated["PersistentBootConfigOrder"], updated_order); + let system = get_json(&router, &resource("1").odata_id).await; + assert_eq!(system["Boot"]["BootOrder"], json!(["Boot0000", "Boot0001"])); + } +} diff --git a/crates/bmc-proxy/Cargo.toml b/crates/bmc-proxy/Cargo.toml index d5d56a5dad..b0a1ba7ca8 100644 --- a/crates/bmc-proxy/Cargo.toml +++ b/crates/bmc-proxy/Cargo.toml @@ -38,6 +38,7 @@ carbide-tls = { path = "../tls" } logfmt = { path = "../logfmt" } metrics-endpoint = { path = "../metrics-endpoint" } carbide-utils = { path = "../utils" } +trace-propagation = { path = "../trace-propagation" } libc = { workspace = true } # DO NOT PUT DEPENDENCIES OTHER THAN LOCAL DEPS HERE, THEY SHOULD ALL HAVE 'path =' IN THEM. @@ -74,6 +75,8 @@ reqwest = { workspace = true, default-features = false, features = [ "rustls", "stream", ] } +reqwest-middleware = { workspace = true } +reqwest-tracing = { workspace = true } rustls-pemfile = { workspace = true } rustls-pki-types = { workspace = true } thiserror = { workspace = true } diff --git a/crates/bmc-proxy/src/bmc_proxy.rs b/crates/bmc-proxy/src/bmc_proxy.rs index e126b1760b..96ab252611 100644 --- a/crates/bmc-proxy/src/bmc_proxy.rs +++ b/crates/bmc-proxy/src/bmc_proxy.rs @@ -52,6 +52,8 @@ use tokio_rustls::rustls::{RootCertStore, ServerConfig}; use tokio_rustls::{TlsAcceptor, rustls}; use tokio_util::sync::CancellationToken; use tower_http::add_extension::AddExtensionLayer; +use trace_propagation::{is_propagated_header, set_span_parent_from_headers}; +use tracing::Instrument; use crate::config::{AuthConfig, TlsConfig}; use crate::metrics::{ @@ -92,7 +94,7 @@ struct BmcProxyState { } type CredentialCache = Arc>>; -type HttpClientCache = Arc>>; +type HttpClientCache = Arc>>; type LookupToIpCache = Arc>>; #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -423,7 +425,7 @@ impl BmcProxy { .expect("could not spawn task to handle HTTP connection"); } - tracing::info!("carbide-bmc-proxy shutting down"); + tracing::info!("nico-bmc-proxy shutting down"); } } @@ -572,6 +574,50 @@ async fn root_url() -> &'static str { async fn proxy_request( State(state): State, request: Request, +) -> Result, Response> { + let request_span = bmc_proxy_request_span(&request); + + let result = proxy_request_inner(state, request) + .instrument(request_span.clone()) + .await; + let status = match &result { + Ok(response) | Err(response) => response.status(), + }; + request_span.record("http.response.status_code", status.as_u16()); + request_span.record("otel.status_code", span_status(status)); + result +} + +fn bmc_proxy_request_span(request: &Request) -> tracing::Span { + let request_span = tracing::info_span!( + parent: None, + "bmc_proxy_request", + http.request.method = %request.method(), + url.path = %request.uri().path(), + http.response.status_code = tracing::field::Empty, + otel.status_code = tracing::field::Empty, + bmc.ip_address = tracing::field::Empty, + logfmt.suppress = true, + ); + set_span_parent_from_headers(&request_span, request.headers()); + request_span +} + +/// The OpenTelemetry status for a proxied request that answered with `status`. +/// +/// Only a 5xx marks the span failed: a rejected or malformed request is the caller's error, and +/// counting it against the proxy would bury the hops that actually broke. +fn span_status(status: StatusCode) -> &'static str { + if status.is_server_error() { + "error" + } else { + "ok" + } +} + +async fn proxy_request_inner( + state: BmcProxyState, + request: Request, ) -> Result, Response> { if !state.allows(&request) { return Ok(error_response((StatusCode::FORBIDDEN, "Forbidden").into())); @@ -611,6 +657,8 @@ async fn proxy_request( } }; + tracing::Span::current().record("bmc.ip_address", target_ip.to_string()); + let path_and_query = parts .uri .into_parts() @@ -816,6 +864,9 @@ fn build_response( fn copy_request_headers(source: &HeaderMap, dest: &mut HeaderMap) { for (name, value) in source { if is_hop_by_hop_header(name.as_str()) + // Trace context describes the caller's hop; the upstream client's tracing middleware + // re-injects the proxy's own hop on egress. + || is_propagated_header(name.as_str()) || *name == axum::http::header::HOST || *name == axum::http::header::AUTHORIZATION || name.as_str().eq_ignore_ascii_case("forwarded") @@ -922,7 +973,7 @@ impl From<(StatusCode, &'static str)> for ProxyError { } struct BmcClientInfo { - pub http_client: reqwest::Client, + pub http_client: reqwest_middleware::ClientWithMiddleware, pub header_map: HeaderMap, pub credentials: BmcCredentials, pub base_upstream_uri: Uri, @@ -937,8 +988,8 @@ enum BmcCredentials { impl BmcCredentials { fn apply_to_request( self, - request: reqwest::RequestBuilder, - ) -> Result { + request: reqwest_middleware::RequestBuilder, + ) -> Result { match self { Self::UsernamePassword { username, password } => { Ok(request.basic_auth(username, Some(password))) @@ -1081,8 +1132,8 @@ async fn get_bmc_credentials( Ok(credentials) } -fn build_http_client() -> Result { - reqwest::Client::builder() +fn build_http_client() -> Result { + let client = reqwest::Client::builder() .danger_accept_invalid_certs(true) .redirect(reqwest::redirect::Policy::limited(5)) .connect_timeout(std::time::Duration::from_secs(5)) // Limit connections to 5 seconds @@ -1092,13 +1143,16 @@ fn build_http_client() -> Result { .map_err(|err| { tracing::error!(error = %err, "build_http_client"); BmcProxyError::InternalProxying(format!("Http building failed: {err}")) - }) + })?; + Ok(reqwest_middleware::ClientBuilder::new(client) + .with(reqwest_tracing::TracingMiddleware::default()) + .build()) } async fn get_http_client( ip: IpAddr, client_cache: &HttpClientCache, -) -> Result { +) -> Result { let mut client_cache = client_cache.lock().await; if let Some(client) = client_cache.get(&ip) { tracing::debug!(bmc_ip_address = %ip, "Using cached BMC HTTP client"); @@ -1149,10 +1203,10 @@ mod tests { use super::{ BmcCredentials, BmcProxyState, ConnectionFailReason, CredentialCache, ForwardedTarget, TcpAcceptFailed, TlsCertificateReloadFailed, TlsConnectionFailed, - authorize_principal_allow_list, build_authority, build_response, copy_request_headers, - create_client, evict_cached_credentials, forwarded_header_value, get_http_client, - ip_for_forwarded_target, is_hop_by_hop_header, method_supports_body, - parse_forwarded_host_value, request_principal_ids, + authorize_principal_allow_list, bmc_proxy_request_span, build_authority, build_response, + copy_request_headers, create_client, evict_cached_credentials, forwarded_header_value, + get_http_client, ip_for_forwarded_target, is_hop_by_hop_header, method_supports_body, + parse_forwarded_host_value, request_principal_ids, span_status, }; use crate::metrics::MethodLabel; @@ -1216,6 +1270,8 @@ mod tests { ContentLength, Connection, Upgrade, + TraceParent, + TraceState, } #[derive(Clone, Copy)] @@ -1442,10 +1498,26 @@ mod tests { axum::http::header::UPGRADE, HeaderValue::from_static("websocket"), ), + HeaderCopyCase::TraceParent => ( + HeaderName::from_static("traceparent"), + HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"), + ), + HeaderCopyCase::TraceState => ( + HeaderName::from_static("tracestate"), + HeaderValue::from_static("vendor=value"), + ), } } fn copied_header_names(case: HeaderCopyCase) -> Vec { + // Trace-header filtering asks the global propagator which headers are its own, so the + // propagator `setup_logging` installs at startup has to be in place for the trace cases to + // mean anything. Installing it here rather than relying on another test having run keeps + // this independent of test ordering. + opentelemetry::global::set_text_map_propagator( + opentelemetry_sdk::propagation::TraceContextPropagator::new(), + ); + let (name, value) = header_for_copy_case(case); let mut source = HeaderMap::new(); source.insert(name, value); @@ -1735,6 +1807,107 @@ mod tests { "upgrade filtered" { HeaderCopyCase::Upgrade => vec![], } + + "traceparent filtered" { + HeaderCopyCase::TraceParent => vec![], + } + + "tracestate filtered" { + HeaderCopyCase::TraceState => vec![], + } + ); + } + + #[test] + fn proxy_request_span_continues_inbound_trace_on_upstream_inject() { + use opentelemetry::trace::{SpanId, TraceContextExt, TraceId, TracerProvider}; + use opentelemetry_sdk::propagation::TraceContextPropagator; + use opentelemetry_sdk::trace::{InMemorySpanExporter, Sampler, SdkTracerProvider}; + use trace_propagation::{extract_context, inject_current_context}; + use tracing_subscriber::layer::SubscriberExt; + + opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new()); + + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_sampler(Sampler::AlwaysOn) + .with_simple_exporter(exporter.clone()) + .build(); + let tracer = provider.tracer("nico-bmc-proxy-test"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + + let inbound_trace = 0x42u128; + let inbound_span = 0x55u64; + let mut inbound_headers = http::HeaderMap::new(); + inbound_headers.insert( + "traceparent", + format!("00-{:032x}-{:016x}-01", inbound_trace, inbound_span) + .parse() + .unwrap(), + ); + + let mut egress_headers = http::HeaderMap::new(); + tracing::subscriber::with_default(subscriber, || { + let request = Request::builder() + .uri("/redfish/v1/Systems") + .header("traceparent", inbound_headers["traceparent"].clone()) + .body(()) + .unwrap(); + let request_span = bmc_proxy_request_span(&request); + let _entered = request_span.enter(); + inject_current_context(&mut egress_headers); + }); + + let egress_context = extract_context(&egress_headers); + assert_eq!( + egress_context.span().span_context().trace_id(), + TraceId::from(inbound_trace), + ); + assert_ne!( + egress_context.span().span_context().span_id(), + SpanId::from(inbound_span), + ); + + let spans = exporter.get_finished_spans().expect("finished spans"); + let request = spans + .iter() + .find(|span| span.name == "bmc_proxy_request") + .expect("request span exported"); + assert_eq!( + request.span_context.trace_id(), + TraceId::from(inbound_trace) + ); + assert_eq!(request.parent_span_id, SpanId::from(inbound_span)); + } + + #[test] + fn proxy_request_span_reports_only_server_errors_as_failed() { + value_scenarios!( + run = span_status; + "success" { + StatusCode::OK => "ok", + } + + "redirect" { + StatusCode::TEMPORARY_REDIRECT => "ok", + } + + "rejected by the allow list" { + StatusCode::FORBIDDEN => "ok", + } + + "malformed request" { + StatusCode::BAD_REQUEST => "ok", + } + + "upstream unreachable" { + StatusCode::BAD_GATEWAY => "error", + } + + "proxy failure" { + StatusCode::INTERNAL_SERVER_ERROR => "error", + } ); } @@ -1980,7 +2153,8 @@ mod tests { #[test] fn bmc_username_password_credentials_use_basic_auth() { - let request = reqwest::Client::new().get("https://example.com/redfish/v1"); + let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(); + let request = client.get("https://example.com/redfish/v1"); let request = BmcCredentials::UsernamePassword { username: "admin".to_string(), password: "secret".to_string(), @@ -1999,7 +2173,8 @@ mod tests { #[test] fn bmc_session_token_credentials_use_redfish_token_header() { - let request = reqwest::Client::new().get("https://example.com/redfish/v1"); + let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(); + let request = client.get("https://example.com/redfish/v1"); let request = BmcCredentials::SessionToken { token: "token-123".to_string(), } diff --git a/crates/bmc-proxy/src/config.rs b/crates/bmc-proxy/src/config.rs index 7d18dfe809..fe7e58c0ac 100644 --- a/crates/bmc-proxy/src/config.rs +++ b/crates/bmc-proxy/src/config.rs @@ -55,6 +55,21 @@ pub struct Config { #[serde(default)] pub carbide_api: CarbideApiConfig, pub bmc_proxy: Option, + #[serde(default)] + pub tracing: TracingConfig, +} + +/// OpenTelemetry trace export settings for proxied BMC requests. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct TracingConfig { + /// Whether to record and export OTLP spans. Default: false. + #[serde(default)] + pub enabled: bool, + /// Collector endpoint for OTLP/gRPC traces. Overridden by the standard + /// `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` and `OTEL_EXPORTER_OTLP_ENDPOINT` + /// variables when either is set. + #[serde(default)] + pub otlp_endpoint: Option, } struct Defaults; @@ -170,6 +185,7 @@ mod tests { ProxyPortOnly, ProxyHostAndPort, ExplicitCarbideApi, + TracingSection, } #[derive(Debug, PartialEq)] @@ -183,6 +199,8 @@ mod tests { service_base_paths: Vec, carbide_api_url: String, bmc_proxy: Option, + tracing_enabled: bool, + tracing_otlp_endpoint: Option, } fn config_source(case: ConfigCase) -> String { @@ -223,6 +241,13 @@ mod tests { api_url = "https://api.example.com:1079" "# } + ConfigCase::TracingSection => { + r#" + [tracing] + enabled = true + otlp_endpoint = "http://collector.example.com:4317" + "# + } }; format!("{extra}\n{MINIMAL_TLS}") @@ -245,6 +270,8 @@ mod tests { service_base_paths: config.auth.trust.spiffe_service_base_paths, carbide_api_url: config.carbide_api.api_url.to_string(), bmc_proxy: config.bmc_proxy.map(|pair| pair.to_string()), + tracing_enabled: config.tracing.enabled, + tracing_otlp_endpoint: config.tracing.otlp_endpoint, } } @@ -267,6 +294,8 @@ mod tests { carbide_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079/" .to_string(), bmc_proxy: None, + tracing_enabled: false, + tracing_otlp_endpoint: None, }, } @@ -285,6 +314,8 @@ mod tests { carbide_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079/" .to_string(), bmc_proxy: None, + tracing_enabled: false, + tracing_otlp_endpoint: None, }, } @@ -306,6 +337,8 @@ mod tests { carbide_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079/" .to_string(), bmc_proxy: None, + tracing_enabled: false, + tracing_otlp_endpoint: None, }, } @@ -324,6 +357,8 @@ mod tests { carbide_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079/" .to_string(), bmc_proxy: Some("proxy.local".to_string()), + tracing_enabled: false, + tracing_otlp_endpoint: None, }, } @@ -342,6 +377,8 @@ mod tests { carbide_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079/" .to_string(), bmc_proxy: Some("8443".to_string()), + tracing_enabled: false, + tracing_otlp_endpoint: None, }, } @@ -360,6 +397,8 @@ mod tests { carbide_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079/" .to_string(), bmc_proxy: Some("proxy.local:8443".to_string()), + tracing_enabled: false, + tracing_otlp_endpoint: None, }, } @@ -377,6 +416,28 @@ mod tests { ], carbide_api_url: "https://api.example.com:1079/".to_string(), bmc_proxy: None, + tracing_enabled: false, + tracing_otlp_endpoint: None, + }, + } + + "tracing section" { + ConfigCase::TracingSection => ConfigSummary { + listen: "[::]:1079".to_string(), + metrics_endpoint: "[::]:1080".to_string(), + allowed_principals: vec![], + identity_pemfile_path: "/tls/cert.pem".to_string(), + root_cafile_path: "/tls/ca.pem".to_string(), + trust_domain: "nico.local".to_string(), + service_base_paths: vec![ + "/forge-system/sa/".to_string(), + "/default/sa/".to_string(), + ], + carbide_api_url: "https://carbide-api.forge-system.svc.cluster.local:1079/" + .to_string(), + bmc_proxy: None, + tracing_enabled: true, + tracing_otlp_endpoint: Some("http://collector.example.com:4317".to_string()), }, } ); diff --git a/crates/bmc-proxy/src/main.rs b/crates/bmc-proxy/src/main.rs index 93a8874389..28f5b2e279 100644 --- a/crates/bmc-proxy/src/main.rs +++ b/crates/bmc-proxy/src/main.rs @@ -70,9 +70,6 @@ async fn main() -> Result<(), Error> { return Ok(()); } - let debug = args.debug; - setup_logging(debug)?; - let config = tokio::fs::read_to_string(&args.config_path) .await .map_err(|e| { @@ -83,6 +80,8 @@ async fn main() -> Result<(), Error> { }) .and_then(|s| Config::parse(&s))?; + let tracing_guard = setup_logging(args.debug, &config.tracing)?; + let mut join_set = JoinSet::new(); let cancel_token = CancellationToken::new(); @@ -118,5 +117,7 @@ async fn main() -> Result<(), Error> { // Wait until tasks are complete, propagating any panics join_set.join_all().await; + tracing_guard.shutdown().await; + Ok(()) } diff --git a/crates/bmc-proxy/src/metrics.rs b/crates/bmc-proxy/src/metrics.rs index 1608cb5a58..80b68d02db 100644 --- a/crates/bmc-proxy/src/metrics.rs +++ b/crates/bmc-proxy/src/metrics.rs @@ -280,7 +280,7 @@ impl From for UpstreamStatus { impl UpstreamStatus { /// The class of a completed forward: the response's status class, or /// `Error` when the request never produced a response. - pub(crate) fn from_result(result: &Result) -> Self { + pub(crate) fn from_result(result: &Result) -> Self { match result { Ok(response) => response.status().into(), Err(_) => Self::Error, @@ -591,14 +591,16 @@ mod tests { .expect("response builds"), ); assert_eq!( - UpstreamStatus::from_result(&Ok(response)), + UpstreamStatus::from_result::(&Ok(response)), UpstreamStatus::Http5xx ); - let error = reqwest::Client::new() - .get("http://") - .build() - .expect_err("an empty host cannot build a request"); + let error = reqwest_middleware::Error::from( + reqwest::Client::new() + .get("http://") + .build() + .expect_err("an empty host cannot build a request"), + ); assert_eq!( UpstreamStatus::from_result(&Err(error)), UpstreamStatus::Error diff --git a/crates/bmc-proxy/src/setup.rs b/crates/bmc-proxy/src/setup.rs index add40ff7d6..56d6daa452 100644 --- a/crates/bmc-proxy/src/setup.rs +++ b/crates/bmc-proxy/src/setup.rs @@ -16,10 +16,17 @@ */ use metrics_endpoint::MetricsSetup; -use tracing_subscriber::Layer; +use opentelemetry::trace::TracerProvider; +use opentelemetry::{KeyValue, global}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::Resource; +use opentelemetry_sdk::propagation::TraceContextPropagator; use tracing_subscriber::filter::{EnvFilter, LevelFilter}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{Layer, filter}; + +use crate::config::TracingConfig; #[derive(thiserror::Error, Debug)] pub enum SetupError { @@ -33,24 +40,80 @@ pub enum SetupError { pub type SetupResult = Result; -pub fn setup_logging(debug: bool) -> SetupResult<()> { - // Default log level if RUST_LOG is not set +/// Owns the OTLP tracer provider for the lifetime of the process. The batch +/// span processor only exports on its own schedule, so the provider has to be +/// shut down explicitly at exit or the final batch is dropped. +pub struct TracingGuard(Option); + +impl TracingGuard { + /// Flushes and shuts down the OTLP exporter. `SdkTracerProvider::shutdown` + /// blocks the calling thread for up to five seconds waiting on the batch + /// processor's worker, so it must not run on a runtime worker. + pub async fn shutdown(self) { + let Some(provider) = self.0 else { + return; + }; + + match tokio::task::spawn_blocking(move || provider.shutdown()).await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::warn!(%error, "failed to flush OpenTelemetry spans on shutdown"); + } + Err(error) => { + tracing::warn!(%error, "OpenTelemetry shutdown task failed"); + } + } + } +} + +pub fn setup_logging(debug: bool, tracing_config: &TracingConfig) -> SetupResult { + // W3C propagation must be installed before any inbound extract or outbound inject (#2438). + // Without it, OpenTelemetry's default propagator is a no-op. + global::set_text_map_propagator(TraceContextPropagator::new()); + let default_log_level = if debug { LevelFilter::DEBUG } else { LevelFilter::INFO }; - // Ignore certain spans and events from 3rd party frameworks let log_filter = dep_log_filter( EnvFilter::builder() .with_default_directive(default_log_level.into()) .from_env()?, ); + let tracing_enabled = debug || tracing_config.enabled; + let trace_filter = filter::filter_fn(move |metadata: &tracing::Metadata<'_>| { + tracing_enabled && should_accept_span_or_event(metadata) + }) + .with_max_level_hint(default_log_level); + + let endpoint = otlp_endpoint( + |var| std::env::var(var).ok(), + tracing_config.otlp_endpoint.as_deref(), + ); + + // Span export is a diagnostic aid, not part of the proxy's contract, so a + // rejected endpoint degrades to no tracing rather than failing startup and + // cutting off BMC access. The subscriber does not exist yet, so the failure + // is carried past initialization and reported below. + let (tracer_provider, exporter_error) = match endpoint.as_deref().map(build_span_exporter) { + None => (None, None), + Some(Ok(exporter)) => (Some(build_tracer_provider(exporter)), None), + Some(Err(error)) => (None, Some(error)), + }; + + let maybe_otel_tracing_layer = tracer_provider.as_ref().map(|provider| { + tracing_opentelemetry::layer() + .with_tracer(provider.tracer("nico-bmc-proxy")) + .with_filter(trace_filter) + }); + let log_events = carbide_instrument::LogEventsMetric::new("nico-bmc-proxy"); tracing_subscriber::registry() .with(log_events.layer().with_filter(log_filter.clone())) + .with(maybe_otel_tracing_layer) .with( logfmt::layer() .with_event_fields([logfmt::EventField::with_default( @@ -61,8 +124,82 @@ pub fn setup_logging(debug: bool) -> SetupResult<()> { ) .try_init()?; - tracing::info!(configured_log_level = %LevelFilter::current(), "current log level"); - Ok(()) + tracing::info!( + configured_log_level = %LevelFilter::current(), + tracing_enabled, + "current log level" + ); + + if let Some(error) = exporter_error { + tracing::warn!( + %error, + endpoint = endpoint.as_deref().unwrap_or_default(), + "OpenTelemetry span export disabled; proxy continues without tracing" + ); + } + + Ok(TracingGuard(tracer_provider)) +} + +/// The standard OTLP endpoint variables, in the order `TonicExporterBuilder::resolve_endpoint` +/// itself consults them: signal-specific first, then the one covering every signal. +const OTLP_ENDPOINT_VARS: [&str; 2] = [ + opentelemetry_otlp::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, + opentelemetry_otlp::OTEL_EXPORTER_OTLP_ENDPOINT, +]; + +/// Resolves the collector endpoint, preferring the standard OTLP variables over the config TOML. +/// `env` is a parameter so the precedence is testable without mutating the process environment. +/// +/// The variables are read here, mirroring the exporter builder's own order, rather than left to the +/// builder: absent any configuration it defaults to `http://localhost:4317` and would ship spans at +/// a collector nobody asked for. Returning `None` is what holds span export off entirely, and that +/// decision needs the same view of the environment the builder has. +fn otlp_endpoint( + env: impl Fn(&str) -> Option, + config_endpoint: Option<&str>, +) -> Option { + // An empty endpoint counts as unset on either path, matching how the builder treats an empty + // programmatic value. + OTLP_ENDPOINT_VARS + .iter() + .find_map(|var| env(var).filter(|endpoint| !endpoint.is_empty())) + .or_else(|| { + config_endpoint + .filter(|endpoint| !endpoint.is_empty()) + .map(str::to_string) + }) +} + +fn build_span_exporter( + endpoint: &str, +) -> Result { + // `with_tonic` already selects OTLP/gRPC. The rest of the transport — timeout, compression, + // TLS, headers — is left to the standard `OTEL_EXPORTER_OTLP_*` variables the builder reads. + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .build() +} + +fn build_tracer_provider( + exporter: opentelemetry_otlp::SpanExporter, +) -> opentelemetry_sdk::trace::SdkTracerProvider { + opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .with_resource( + Resource::builder() + .with_attributes([KeyValue::new("service.name", "nico-bmc-proxy")]) + .build(), + ) + .build() +} + +/// Tokio runtime spans are not closed reliably; exporting them would leak memory. +fn should_accept_span_or_event(metadata: &tracing::Metadata<'_>) -> bool { + !metadata + .module_path() + .is_some_and(|path| path.starts_with("tokio")) } pub fn setup_metrics() -> SetupResult { @@ -168,6 +305,136 @@ mod tests { ); } + // The tonic channel is built lazily on the ambient runtime, so the + // accepted-endpoint case needs a runtime even though nothing connects. + #[tokio::test] + async fn span_exporter_build_validates_endpoint_eagerly() { + value_scenarios!( + run = |endpoint| build_span_exporter(endpoint).is_ok(); + "well-formed collector endpoint is accepted" { + "http://otel-collector.observability.svc.cluster.local:4317" => true, + } + + "malformed endpoint is rejected at build time rather than at first export" { + "http://otel collector:4317" => false, + } + ); + } + + /// The inputs [`otlp_endpoint`] weighs against each other, named so a scenario reads as the + /// deployment it stands for. + #[derive(Clone, Copy)] + struct EndpointInputs { + signal_var: Option<&'static str>, + generic_var: Option<&'static str>, + config: Option<&'static str>, + } + + fn resolved_endpoint(inputs: EndpointInputs) -> Option { + otlp_endpoint( + |var| { + if var == opentelemetry_otlp::OTEL_EXPORTER_OTLP_TRACES_ENDPOINT { + inputs.signal_var.map(str::to_string) + } else if var == opentelemetry_otlp::OTEL_EXPORTER_OTLP_ENDPOINT { + inputs.generic_var.map(str::to_string) + } else { + panic!("unexpected endpoint variable: {var}") + } + }, + inputs.config, + ) + } + + #[test] + fn otlp_endpoint_prefers_standard_variables_over_config() { + // Distinct per source so a failure names the one that wrongly won. Syntactically valid + // endpoints, since the exporter builder rejects malformed ones. + const SIGNAL_ENDPOINT: &str = "http://signal-collector:4317"; + const GENERIC_ENDPOINT: &str = "http://generic-collector:4317"; + const CONFIG_ENDPOINT: &str = "http://config-collector:4317"; + const NOTHING_SET: EndpointInputs = EndpointInputs { + signal_var: None, + generic_var: None, + config: None, + }; + + value_scenarios!( + run = resolved_endpoint; + "nothing configured leaves span export off" { + NOTHING_SET => None, + } + + "signal-specific variable is used" { + EndpointInputs { signal_var: Some(SIGNAL_ENDPOINT), ..NOTHING_SET } + => Some(SIGNAL_ENDPOINT.to_string()), + } + + "generic variable is honored too" { + EndpointInputs { generic_var: Some(GENERIC_ENDPOINT), ..NOTHING_SET } + => Some(GENERIC_ENDPOINT.to_string()), + } + + "signal-specific variable wins over the generic one" { + EndpointInputs { + signal_var: Some(SIGNAL_ENDPOINT), + generic_var: Some(GENERIC_ENDPOINT), + ..NOTHING_SET + } => Some(SIGNAL_ENDPOINT.to_string()), + } + + "signal-specific variable overrides the config file" { + EndpointInputs { + signal_var: Some(SIGNAL_ENDPOINT), + config: Some(CONFIG_ENDPOINT), + ..NOTHING_SET + } => Some(SIGNAL_ENDPOINT.to_string()), + } + + "generic variable overrides the config file" { + EndpointInputs { + generic_var: Some(GENERIC_ENDPOINT), + config: Some(CONFIG_ENDPOINT), + ..NOTHING_SET + } => Some(GENERIC_ENDPOINT.to_string()), + } + + "config file is used when no variable is set" { + EndpointInputs { config: Some(CONFIG_ENDPOINT), ..NOTHING_SET } + => Some(CONFIG_ENDPOINT.to_string()), + } + + "empty config endpoint counts as unset" { + EndpointInputs { config: Some(""), ..NOTHING_SET } + => None, + } + + "empty variable does not shadow the generic one" { + EndpointInputs { + signal_var: Some(""), + generic_var: Some(GENERIC_ENDPOINT), + ..NOTHING_SET + } => Some(GENERIC_ENDPOINT.to_string()), + } + + "empty variable falls through to the config file" { + EndpointInputs { + signal_var: Some(""), + config: Some(CONFIG_ENDPOINT), + ..NOTHING_SET + } => Some(CONFIG_ENDPOINT.to_string()), + } + ); + } + + #[test] + fn should_accept_span_or_event_accepts_application_spans() { + assert!(should_accept_span_or_event( + tracing::info_span!(target: "carbide_bmc_proxy", "bmc_proxy_request") + .metadata() + .unwrap() + )); + } + #[test] fn metrics_setup_initializes_health_controller() { // Mirrors setup_metrics() without its global-meter install: the diff --git a/crates/credential-rotation/src/lib.rs b/crates/credential-rotation/src/lib.rs index c864fcd552..dce3329f4a 100644 --- a/crates/credential-rotation/src/lib.rs +++ b/crates/credential-rotation/src/lib.rs @@ -97,6 +97,7 @@ use libredfish::model::service_root::RedfishVendor; use mac_address::MacAddress; use model::bmc_info::BmcInfo; use model::machine::Machine; +use model::power_shelf::PowerShelf; use model::switch::Switch; use sqlx::PgPool; @@ -227,13 +228,40 @@ pub enum RotateOutcome { NoWork, } +/// How the engine obtains the dispatch vendor `set_bmc_root_password` branches +/// on for a rotation target. +/// +/// The precise Redfish dispatch vendor is not persisted for any device family +/// (the stored hardware vendor is DMI-derived and too coarse). The distinction +/// is instead about *who* resolves it: a caller that already holds the exact +/// vendor hands it over (`Fixed`), while a caller that would otherwise have to +/// probe an authenticated BMC to learn it defers that probe to the engine +/// (`Probe`), so the probe runs inside the engine's quarantine-on-failure +/// envelope. +#[derive(Debug, Clone)] +pub enum DispatchVendor { + /// The caller already holds the precise vendor: the switch controller (a + /// compile-time constant, NVIDIA MGX) and the machine controller (which + /// probes it in its own controller layer before calling). Nothing persists + /// it -- `Fixed` only means "resolved by the caller, not the engine". + Fixed(RedfishVendor), + /// Resolve the vendor at rotation time by probing the BMC's Chassis + /// manufacturer ([`RedfishClientPool::probe_bmc_vendor`]) -- power-shelf + /// PMCs (Lite-On/Delta), which do *not* expose a recognized vendor in their + /// Redfish service root. The probe runs *inside* the engine's + /// quarantine-on-failure envelope and reuses the same credential candidates + /// as the rotation itself, so a failed probe records backoff (lockout-safe) + /// rather than looping the entry guard, and a stale per-device secret does + /// not deadlock recovery. + Probe, +} + /// Where and how to reach a single device's BMC for rotation. /// /// `device_mac` is the BMC MAC that keys both the `device_credential_rotation` -/// row and the per-device Vault secret. `vendor` is resolved by the caller, -/// which for switch / power-shelf BMCs means probing at rotation time via -/// [`RedfishClientPool::probe_bmc_vendor`] (the precise dispatch vendor is not -/// persisted anywhere). +/// row and the per-device Vault secret. `vendor` is either the caller-known +/// dispatch vendor or a directive to probe it at rotation time (see +/// [`DispatchVendor`]). #[derive(Debug, Clone)] pub struct BmcRotationTarget { /// BMC MAC keying the rotation row and the per-device secret. @@ -242,8 +270,8 @@ pub struct BmcRotationTarget { pub host: String, /// BMC port, when non-default. pub port: Option, - /// Precise dispatch vendor `set_bmc_root_password` branches on. - pub vendor: RedfishVendor, + /// How to obtain the dispatch vendor `set_bmc_root_password` branches on. + pub vendor: DispatchVendor, } /// A reachable, keyable BMC endpoint: the MAC that keys both the @@ -286,9 +314,25 @@ impl BmcEndpoint { Self::from_bmc_info(switch.bmc_info.as_ref()?) } - /// Pair this endpoint with a caller-resolved dispatch `vendor` to form the - /// [`BmcRotationTarget`] the engine rotates. + /// The BMC (PMC) endpoint of a power shelf, or `None` when the power shelf + /// has no BMC info or it is unkeyable / unreachable. + pub fn from_power_shelf(power_shelf: &PowerShelf) -> Option { + Self::from_bmc_info(power_shelf.bmc_info.as_ref()?) + } + + /// Pair this endpoint with a caller-known dispatch `vendor` to form the + /// [`BmcRotationTarget`] the engine rotates (machine + switch controllers). pub fn into_target(self, vendor: RedfishVendor) -> BmcRotationTarget { + self.into_target_with(DispatchVendor::Fixed(vendor)) + } + + /// Pair this endpoint with a directive to probe the dispatch vendor at + /// rotation time (power-shelf controller: Lite-On/Delta PMCs). + pub fn into_target_probing_vendor(self) -> BmcRotationTarget { + self.into_target_with(DispatchVendor::Probe) + } + + fn into_target_with(self, vendor: DispatchVendor) -> BmcRotationTarget { BmcRotationTarget { device_mac: self.device_mac, host: self.host, @@ -784,7 +828,15 @@ async fn converge_bmc_password( password: new_password.clone(), }; - let convergence = change_or_recover(redfish_pool, bmc, rotate_from, rotate_to) + // Resolve the dispatch vendor before touching the password. For a probe + // target this is an authenticated Chassis read, so it must go through the + // same redaction and (via the caller) quarantine-on-failure path as the + // rotation itself. + let vendor = resolve_dispatch_vendor(redfish_pool, bmc, &rotate_from, &rotate_to) + .await + .map_err(|e| redact(e, &[¤t_password, &new_password]))?; + + let convergence = change_or_recover(redfish_pool, bmc, vendor, rotate_from, rotate_to) .await .map_err(|e| redact(e, &[¤t_password, &new_password]))? .redacted(&[¤t_password, &new_password]); @@ -824,6 +876,7 @@ async fn converge_bmc_password( async fn change_or_recover( redfish_pool: &dyn RedfishClientPool, bmc: &BmcRotationTarget, + vendor: RedfishVendor, rotate_from: Credentials, rotate_to: Credentials, ) -> Result { @@ -836,7 +889,7 @@ async fn change_or_recover( .set_bmc_root_password( &bmc.host, bmc.port, - bmc.vendor, + vendor, rotate_from, new_password.clone(), ) @@ -863,6 +916,45 @@ async fn change_or_recover( } } +/// Resolve the dispatch vendor `set_bmc_root_password` branches on. +/// +/// A [`DispatchVendor::Fixed`] target returns immediately. A +/// [`DispatchVendor::Probe`] target (power-shelf PMCs) probes the BMC's Chassis +/// manufacturer, trying the per-device secret first and the rotate-TO value +/// second -- the same two candidates the change path uses. That ordering means +/// a crash-recovered device (whose per-device secret still lags the hardware) +/// still resolves its vendor via the rotate-TO value rather than deadlocking, +/// and it is bounded to at most two logins so it stays clear of BMC lockout. A +/// probe that never authenticates returns `Err`, which the caller records as a +/// quarantine with backoff. Returns an already-`to_string`-ed error (still to be +/// redacted by the caller); never returns a secret-bearing string itself. +async fn resolve_dispatch_vendor( + redfish_pool: &dyn RedfishClientPool, + bmc: &BmcRotationTarget, + rotate_from: &Credentials, + rotate_to: &Credentials, +) -> Result { + match &bmc.vendor { + DispatchVendor::Fixed(vendor) => Ok(*vendor), + DispatchVendor::Probe => { + let mut last_err = None; + for candidate in [rotate_from, rotate_to] { + match redfish_pool + .probe_bmc_vendor(&bmc.host, bmc.port, candidate.clone()) + .await + { + Ok(vendor) => return Ok(vendor), + Err(e) => last_err = Some(e.to_string()), + } + } + Err(format!( + "probe BMC dispatch vendor: {}", + last_err.unwrap_or_else(|| "no credential candidates".to_string()) + )) + } + } +} + /// Replace every non-empty secret in `message` with `REDACTED`. Defense in /// depth on top of the Redfish layer's own redaction, so no password reaches a /// log line or the `rotate_last_error_redacted` column. @@ -901,7 +993,8 @@ mod tests { use super::{ BMC, BmcCredentialRotationConverged, BmcCredentialRotationQuarantined, BmcCredentialRotationRecovered, BmcRotationGate, BmcRotationTarget, CredentialConvergence, - RotateOutcome, change_or_recover, needs_rotation, redact, rotate_bmc, + DispatchVendor, RotateOutcome, change_or_recover, needs_rotation, redact, + resolve_dispatch_vendor, rotate_bmc, }; const BMC_ROTATION_RESULTS_METRIC: &str = "carbide_bmc_credential_rotation_results_total"; @@ -924,7 +1017,16 @@ mod tests { device_mac: test_mac(), host: "127.0.0.1".to_string(), port: Some(443), - vendor: RedfishVendor::NvidiaGBx00, + vendor: DispatchVendor::Fixed(RedfishVendor::NvidiaGBx00), + } + } + + /// A probe-vendor variant of [`target`] for power-shelf-style PMCs, whose + /// dispatch vendor the engine resolves at rotation time. + fn probe_target() -> BmcRotationTarget { + BmcRotationTarget { + vendor: DispatchVendor::Probe, + ..target() } } @@ -1192,10 +1294,15 @@ mod tests { // The BMC is on "old"; the change to "new" authenticates and succeeds. let sim = bmc_on_password("old"); - let convergence = - change_or_recover(&sim, &target(), creds("root", "old"), creds("root", "new")) - .await - .expect("the change should succeed"); + let convergence = change_or_recover( + &sim, + &target(), + RedfishVendor::NvidiaGBx00, + creds("root", "old"), + creds("root", "new"), + ) + .await + .expect("the change should succeed"); assert_eq!( convergence, @@ -1217,10 +1324,15 @@ mod tests { // re-issuing the change. let sim = bmc_on_password("new"); - let convergence = - change_or_recover(&sim, &target(), creds("root", "old"), creds("root", "new")) - .await - .expect("an already-converged BMC must be recovered"); + let convergence = change_or_recover( + &sim, + &target(), + RedfishVendor::NvidiaGBx00, + creds("root", "old"), + creds("root", "new"), + ) + .await + .expect("an already-converged BMC must be recovered"); assert!( matches!(convergence, CredentialConvergence::Recovered { .. }), @@ -1234,9 +1346,15 @@ mod tests { // nor the probe (with "new") authenticates: the change error is surfaced. let sim = bmc_on_password("mystery"); - change_or_recover(&sim, &target(), creds("root", "old"), creds("root", "new")) - .await - .expect_err("neither credential authenticating must surface an error"); + change_or_recover( + &sim, + &target(), + RedfishVendor::NvidiaGBx00, + creds("root", "old"), + creds("root", "new"), + ) + .await + .expect_err("neither credential authenticating must surface an error"); } #[tokio::test] @@ -1248,9 +1366,15 @@ mod tests { sim.set_change_password_error("change boom"); sim.set_get_accounts_error(true); - let err = change_or_recover(&sim, &target(), creds("root", "old"), creds("root", "new")) - .await - .expect_err("a failed change plus a failed probe must surface an error"); + let err = change_or_recover( + &sim, + &target(), + RedfishVendor::NvidiaGBx00, + creds("root", "old"), + creds("root", "new"), + ) + .await + .expect_err("a failed change plus a failed probe must surface an error"); assert!( err.contains("probe also failed"), @@ -1258,6 +1382,47 @@ mod tests { ); } + #[tokio::test] + async fn resolve_dispatch_vendor_returns_the_fixed_vendor_without_probing() { + // A Fixed target (machine / switch) returns its vendor verbatim; the sim's + // service root would resolve to a different vendor, so a returned NvidiaGBx00 + // proves the fixed value short-circuits the probe. + let sim = RedfishSim::default(); + let vendor = resolve_dispatch_vendor( + &sim, + &target(), + &creds("root", "old"), + &creds("root", "new"), + ) + .await + .expect("a fixed vendor resolves without touching the BMC"); + assert_eq!(vendor, RedfishVendor::NvidiaGBx00); + } + + #[tokio::test] + async fn resolve_dispatch_vendor_probes_the_chassis_manufacturer_for_power_shelves() { + // A Probe target with an unrecognized service-root vendor falls back to the + // Chassis manufacturer, the standard power-shelf (Lite-On / Delta) + // determination. + for (manufacturer, expected) in [ + ("Lite-On Technology Corp.", RedfishVendor::LiteOnPowerShelf), + ("Delta Electronics", RedfishVendor::DeltaPowerShelf), + ] { + let sim = RedfishSim::default(); + sim.set_service_root_vendor(Some("Contoso".to_string())); + sim.set_chassis_manufacturer(Some(manufacturer.to_string())); + let vendor = resolve_dispatch_vendor( + &sim, + &probe_target(), + &creds("root", "old"), + &creds("root", "new"), + ) + .await + .expect("a Lite-On/Delta chassis must resolve a power-shelf vendor"); + assert_eq!(vendor, expected, "manufacturer {manufacturer:?}"); + } + } + #[test] fn needs_rotation_only_when_behind_and_not_quarantined() { let mut status = DeviceRotationStatus { diff --git a/crates/dpf/src/flavor.rs b/crates/dpf/src/flavor.rs index 1f11c17d66..08df1faf7f 100644 --- a/crates/dpf/src/flavor.rs +++ b/crates/dpf/src/flavor.rs @@ -1213,7 +1213,7 @@ fn get_bf4_astra_nvconfig() -> DpuFlavorNvconfig { let parameters = vec![ "PF_BAR2_ENABLE=0".to_string(), "PER_PF_NUM_SF=1".to_string(), - "PF_TOTAL_SF=20".to_string(), + "PF_TOTAL_SF=30".to_string(), "PF_SF_BAR_SIZE=14".to_string(), "NUM_PF_MSIX_VALID=0".to_string(), "PF_NUM_PF_MSIX_VALID=1".to_string(), @@ -1578,11 +1578,11 @@ mod tests { ) => true, } - "Astra nvconfig requests 20 total SFs and 46 VFs" { + "Astra nvconfig requests 30 total SFs and 46 VFs" { ( nvconfig_parameters .iter() - .any(|parameter| parameter == "PF_TOTAL_SF=20") + .any(|parameter| parameter == "PF_TOTAL_SF=30") && nvconfig_parameters .iter() .any(|parameter| parameter == "NUM_OF_VFS=46") diff --git a/crates/dpf/src/lib.rs b/crates/dpf/src/lib.rs index 73749d92d7..11fc76aab9 100644 --- a/crates/dpf/src/lib.rs +++ b/crates/dpf/src/lib.rs @@ -70,6 +70,7 @@ mod test; // Re-exports for convenience pub use error::DpfError; +pub use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; pub use repository::{DpfRepository, KubeRepository}; pub use sdk::{ DpfSdk, DpfSdkBuilder, DpuProvisioningSource, NoLabels, ResourceLabeler, build_deployment, diff --git a/crates/dpf/src/sdk.rs b/crates/dpf/src/sdk.rs index 79292ef715..34eb613e2e 100644 --- a/crates/dpf/src/sdk.rs +++ b/crates/dpf/src/sdk.rs @@ -743,10 +743,15 @@ pub fn build_service_configuration( let service_daemon_set = DpuServiceConfigurationServiceConfigurationServiceDaemonSet { annotations: svc.service_daemon_set_annotations.clone(), labels: None, - resources: None, + resources: svc.service_daemon_set_resources.clone(), update_strategy: Some( DpuServiceConfigurationServiceConfigurationServiceDaemonSetUpdateStrategy { - rolling_update: Some(DpuServiceConfigurationServiceConfigurationServiceDaemonSetUpdateStrategyRollingUpdate{ max_surge: None, max_unavailable: Some(IntOrString::String("100%".to_string())) }), + rolling_update: Some( + DpuServiceConfigurationServiceConfigurationServiceDaemonSetUpdateStrategyRollingUpdate { + max_surge: None, + max_unavailable: Some(IntOrString::String("100%".to_string())), + }, + ), r#type: Some("RollingUpdate".into()), }, ), diff --git a/crates/dpf/src/types.rs b/crates/dpf/src/types.rs index 121c7b75e1..12616cdd11 100644 --- a/crates/dpf/src/types.rs +++ b/crates/dpf/src/types.rs @@ -20,6 +20,7 @@ use std::collections::BTreeMap; use std::net::IpAddr; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use serde::{Deserialize, Serialize}; use crate::crds::dpus_generated::DpuStatusPhase; @@ -186,6 +187,8 @@ pub struct ServiceDefinition { pub service_chain_switches: Vec, /// Optional annotations for the service DaemonSet (e.g. Multus CNI networks). pub service_daemon_set_annotations: Option>, + /// Optional extended resources requested by the service DaemonSet. + pub service_daemon_set_resources: Option>, /// Optional service Network Attachment Definition specification pub service_nad: Option, } diff --git a/crates/machine-a-tron/src/api_client.rs b/crates/machine-a-tron/src/api_client.rs index 2814fcb921..5794039f15 100644 --- a/crates/machine-a-tron/src/api_client.rs +++ b/crates/machine-a-tron/src/api_client.rs @@ -487,6 +487,7 @@ impl ApiClient { network_virtualization_type: network_virtualization_type.map(|t| t as i32), vni: None, routing_profile_type: None, + routing_profile_overrides: None, metadata: Some(rpc::forge::Metadata { name: format!("vpc_{vpc_count}"), description: "".to_string(), diff --git a/crates/machine-a-tron/src/machine_state_machine.rs b/crates/machine-a-tron/src/machine_state_machine.rs index 15d7df5764..d7fc81e1a2 100644 --- a/crates/machine-a-tron/src/machine_state_machine.rs +++ b/crates/machine-a-tron/src/machine_state_machine.rs @@ -24,8 +24,9 @@ use std::time::Duration; use bmc_mock::injection::InjectionStore; use bmc_mock::ipmi_sim::IpmiEndpoint; use bmc_mock::{ - BmcCommand, BmcState, BootOptionKind, Callbacks, HostnameQuerying, MachineInfo, MockPowerState, - POWER_CYCLE_DELAY, SetSystemPowerError, SetSystemPowerResult, SystemPowerControl, + BmcCommand, BmcEvent, BmcState, BootOptionKind, Callbacks, HostnameQuerying, MachineInfo, + MockPowerState, POWER_CYCLE_DELAY, SetSystemPowerError, SetSystemPowerResult, + SystemPowerControl, }; use carbide_network::virtualization::build_dual_stack_list; use carbide_uuid::machine::MachineId; @@ -100,17 +101,38 @@ impl DhcpRetryState { } } -/// Abandon a queued machine DHCP request when the machine powers off or cycles -/// so that we can actively drain everything behind the Dhcp(_) action barrier. -/// If any future action needs to run regardless of DHCP status, need to include -/// that action here. -fn abandon_machine_dhcp_on_power_change( +/// Abandon queued work for the current boot when the machine powers off or +/// cycles. A retrying in-band action would otherwise block the power-change +/// cleanup and timer queued behind it. +/// +/// BMC initialization remains valid because the BMC stays powered independently +/// of the machine. Preserve an existing power-off cleanup in case another power +/// change arrives before that action runs. +fn abandon_machine_actions_on_power_change( actions: &mut VecDeque, dhcp_retry: &mut DhcpRetryState, ) { - let is_machine_dhcp = |action: &FsmAction| matches!(action, FsmAction::Dhcp(DhcpType::Machine)); - if actions.iter().any(is_machine_dhcp) { - actions.retain(|action| !is_machine_dhcp(action)); + let abandoned_machine_dhcp = actions + .iter() + .any(|action| matches!(action, FsmAction::Dhcp(DhcpType::Machine))); + + actions.retain(|action| match action { + FsmAction::SetupBmc | FsmAction::Dhcp(DhcpType::Bmc) | FsmAction::CleanupOnPowerOff => true, + FsmAction::SetTimer( + Timer::PowerCycle + | Timer::MachineOn + | Timer::ScoutAgentControlPoll + | Timer::DpuAgentControlPoll, + ) + | FsmAction::Dhcp(DhcpType::Machine) + | FsmAction::PxeBootRequest + | FsmAction::InitialDiscoveryRequest(_) + | FsmAction::AgentControlRequest(_) + | FsmAction::DpuAgentNetworkObservation + | FsmAction::BmcEvent(BmcEvent::PowerOn | BmcEvent::BootCompleted) => false, + }); + + if abandoned_machine_dhcp { dhcp_retry.reset(); } } @@ -605,8 +627,13 @@ impl MachineStateMachine { fn fsm_event(&mut self, event: Event) { if matches!(event, Event::PowerCycle | Event::PowerOff) { - abandon_machine_dhcp_on_power_change(&mut self.actions, &mut self.dhcp_retry); + abandon_machine_actions_on_power_change(&mut self.actions, &mut self.dhcp_retry); + + self.machine_on_deadline = None; + self.power_cycle_deadline = None; + self.agent_polling_deadline = None; } + let old_state = self.fsm; let (new_state, actions) = self.fsm.event(event); tracing::info!(previous_state = ?old_state, ?event, next_state = ?new_state, ?actions, "machine FSM step"); @@ -1522,7 +1549,7 @@ mod tests { } #[test] - fn power_change_abandons_a_queued_machine_dhcp() { + fn power_change_abandons_queued_machine_actions() { let queued = |actions: &[FsmAction]| VecDeque::from(actions.to_vec()); check_values( @@ -1537,11 +1564,21 @@ mod tests { }, }, Check { - scenario: "unrelated queued actions are preserved", + scenario: "in-band work from the previous boot is abandoned", input: ( queued(&[ FsmAction::SetupBmc, FsmAction::Dhcp(DhcpType::Machine), + FsmAction::SetTimer(Timer::PowerCycle), + FsmAction::SetTimer(Timer::MachineOn), + FsmAction::SetTimer(Timer::ScoutAgentControlPoll), + FsmAction::SetTimer(Timer::DpuAgentControlPoll), + FsmAction::PxeBootRequest, + FsmAction::InitialDiscoveryRequest(OsImage::Scout), + FsmAction::AgentControlRequest(OsImage::Scout), + FsmAction::DpuAgentNetworkObservation, + FsmAction::BmcEvent(BmcEvent::PowerOn), + FsmAction::BmcEvent(BmcEvent::BootCompleted), FsmAction::CleanupOnPowerOff, ]), 1, @@ -1565,10 +1602,16 @@ mod tests { }, }, Check { - scenario: "power change without a queued DHCP changes nothing", - input: (queued(&[FsmAction::PxeBootRequest]), 0), + scenario: "power change without queued in-band work changes nothing", + input: ( + queued(&[FsmAction::SetupBmc, FsmAction::CleanupOnPowerOff]), + 0, + ), expect: PowerChangeOutcome { - remaining_actions: vec![format!("{:?}", FsmAction::PxeBootRequest)], + remaining_actions: vec![ + format!("{:?}", FsmAction::SetupBmc), + format!("{:?}", FsmAction::CleanupOnPowerOff), + ], attempt: 0, backoff_pending: false, }, @@ -1580,7 +1623,8 @@ mod tests { for _ in 0..failures { retry.schedule_next(now, 0); } - abandon_machine_dhcp_on_power_change(&mut actions, &mut retry); + + abandon_machine_actions_on_power_change(&mut actions, &mut retry); outcome(&actions, &retry) }, ); diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 89c3cf6b0a..2966c9eea8 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -80,11 +80,13 @@ use model::machine::{ MachineLastRebootRequested, MachineLastRebootRequestedMode, MachineNextStateResolver, MachineState, MachineValidationContext, ManagedHostState, ManagedHostStateSnapshot, MeasuringState, NetworkConfigUpdateState, NextStateBFBSupport, PerformPowerOperation, - PowerDrainState, PowerState, ReprovisionState, RetryInfo, SecureEraseBossContext, - SecureEraseBossState, SetBootOrderInfo, SetBootOrderState, SetSecureBootState, - SpdmMeasuringState, StateMachineArea, UefiSetupInfo, UefiSetupState, UnlockHostState, - ValidationState, dpf_based_dpu_provisioning_possible, get_display_ids, + PowerDrainState, PowerState, ReadyBootConfigState, ReadyBootConfigTerminalFailure, + ReprovisionState, RetryInfo, SecureEraseBossContext, SecureEraseBossState, SetBootOrderInfo, + SetBootOrderState, SetSecureBootState, SpdmMeasuringState, StateMachineArea, UefiSetupInfo, + UefiSetupState, UnlockHostState, ValidationState, dpf_based_dpu_provisioning_possible, + get_display_ids, }; +use model::machine_boot_interface::MachineBootInterfaceTarget; use model::power_manager::PowerHandlingOutcome; use model::predicted_machine_interface::PredictedMachineInterface; use model::resource_pool::common::CommonPools; @@ -136,8 +138,9 @@ use helpers::{ }; use host_boot_config::{ HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, - HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, - initial_set_boot_order_info, run_host_boot_config_stage, should_skip_boot_order_remediation, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, decide_host_boot_config, + initial_set_boot_order_info, inspect_host_boot_config, run_host_boot_config_stage, + should_skip_boot_order_remediation, }; use state_controller::db_write_batch::DbWriteBatch; @@ -757,27 +760,71 @@ impl MachineStateHandler { if !matches!(mh_state, ManagedHostState::Failed { .. }) && let Some((machine_id, details)) = get_failed_state(mh_snapshot) { - tracing::error!( - host_id = %mh_snapshot.host_snapshot.id, - dpu_ids = %get_display_ids(&mh_snapshot.dpu_snapshots), - failed_machine_id = %machine_id, - ?details, - "ManagedHost is moved to Failed state", + let already_relocking_machine_failure = matches!( + &mh_state, + ManagedHostState::BootConfiguring { + boot_config_state: + ReadyBootConfigState::LockHost { + terminal_failure: + Some(ReadyBootConfigTerminalFailure::Machine { + machine_id: pending_machine_id, + details: pending_details, + }), + }, + .. + } if *pending_machine_id == machine_id && *pending_details == details ); - let next_state = match mh_state { - ManagedHostState::Assigned { .. } => ManagedHostState::Assigned { - instance_state: InstanceState::Failed { + + // Boot reconciliation may have opened lockdown. Preserve a newly + // observed host or DPU failure, but restore lockdown before moving + // the managed host into its terminal state. + if already_relocking_machine_failure { + tracing::info!( + host_machine_id = %mh_snapshot.host_snapshot.id, + failed_machine_id = %machine_id, + "Deferring global failure transition until boot reconciliation restores lockdown", + ); + } else { + tracing::error!( + host_id = %mh_snapshot.host_snapshot.id, + dpu_ids = %get_display_ids(&mh_snapshot.dpu_snapshots), + failed_machine_id = %machine_id, + ?details, + "ManagedHost is moved to Failed state", + ); + let next_state = match &mh_state { + ManagedHostState::Assigned { .. } => ManagedHostState::Assigned { + instance_state: InstanceState::Failed { + details, + machine_id, + }, + }, + ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface, + post_lock_verification_retry_count, + boot_config_state, + } if !matches!(boot_config_state, ReadyBootConfigState::Failed { .. }) => { + ready_boot_config_locking( + Versioned { + value: desired_boot_interface.clone(), + version: *desired_version, + }, + *post_lock_verification_retry_count, + Some(ReadyBootConfigTerminalFailure::Machine { + machine_id, + details, + }), + ) + } + _ => ManagedHostState::Failed { details, machine_id, + retry_count: 0, }, - }, - _ => ManagedHostState::Failed { - details, - machine_id, - retry_count: 0, - }, - }; - return Ok(StateHandlerOutcome::transition(next_state)); + }; + return Ok(StateHandlerOutcome::transition(next_state)); + } } match &mh_state { @@ -852,7 +899,9 @@ impl MachineStateHandler { return Ok(outcome); } - // Check if instance to be created. + // An already-committed instance wins before disruptive boot + // reconciliation. Allocation locks the machine row, while its + // eligibility check rejects an earlier pending desired version. if mh_snapshot.instance.is_some() { return Ok(StateHandlerOutcome::transition( ManagedHostState::PreAssignedMeasuring { @@ -861,6 +910,12 @@ impl MachineStateHandler { )); } + if let Some(next_state) = + pending_ready_boot_config_state(&mh_snapshot.host_snapshot) + { + return Ok(StateHandlerOutcome::transition(next_state)); + } + if let Some(outcome) = handle_bom_validation_requested( &self.host_handler.host_handler_params, mh_snapshot, @@ -1033,6 +1088,26 @@ impl MachineStateHandler { Ok(StateHandlerOutcome::do_nothing()) } + ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface, + post_lock_verification_retry_count, + boot_config_state, + } => { + handle_ready_boot_config( + ctx, + mh_snapshot, + &self.host_handler.host_handler_params.reachability_params, + Versioned { + value: desired_boot_interface.clone(), + version: *desired_version, + }, + *post_lock_verification_retry_count, + boot_config_state.clone(), + ) + .await + } + ManagedHostState::RotatingBmc { retry_count } => { // One tick converges every BMC that needs work: force-requested // devices (bypassing backoff) and, when site-wide rotation is @@ -2295,6 +2370,7 @@ pub async fn check_restart_in_logs( "The server is restarted by chassis control command.", // Lenovo "DPU Warm Reset", // Bluefield "BMC IP Address Deleted", // Bluefield + "The property ResetType was assigned the value 'ForceWarmReboot' due to modification by the service.", // GB200 ]); // Generic reset keywords @@ -2411,10 +2487,36 @@ impl StateHandler for MachineStateHandler { .is_empty() && mh_snapshot.dpu_snapshots.is_empty() { - tracing::error!("No DPU snapshot found for host {}", host_machine_id); - return Err(StateHandlerError::GenericError(eyre!( - "no DPU snapshot found" - ))); + if let Some(next_state) = + ready_boot_config_missing_dpu_recovery(&mh_snapshot.managed_state) + { + tracing::error!( + machine_id = %host_machine_id, + "DPU snapshots disappeared during boot reconciliation; restoring lockdown before parking the repair", + ); + return Ok(StateHandlerOutcome::transition(next_state)); + } + + let can_continue_without_dpu = match &mh_snapshot.managed_state { + ManagedHostState::BootConfiguring { + boot_config_state, .. + } => !ready_boot_config_may_have_opened_lockdown(boot_config_state), + _ => false, + }; + if !can_continue_without_dpu { + tracing::error!(machine_id = %host_machine_id, "No DPU snapshot found for host"); + return Err(StateHandlerError::GenericError(eyre!( + "no DPU snapshot found" + ))); + } + + // Prepare and Failed must still process desired-state changes, and + // LockHost only needs host Redfish. Keep those recovery paths + // dispatchable through a transient DPU snapshot gap. + tracing::warn!( + machine_id = %host_machine_id, + "Continuing boot reconciliation recovery without DPU snapshots", + ); } self.record_metrics(mh_snapshot, ctx); @@ -5394,41 +5496,917 @@ pub struct HostMachineStateHandler { host_handler_params: HostHandlerParams, } -impl HostMachineStateHandler { - pub fn new(host_handler_params: HostHandlerParams) -> Self { - Self { - host_handler_params, - } - } -} +impl HostMachineStateHandler { + pub fn new(host_handler_params: HostHandlerParams) -> Self { + Self { + host_handler_params, + } + } +} + +fn managed_host_network_config_version_synced_and_dpu_healthy( + dpu_snapshot: &Machine, + host_version: ConfigVersion, +) -> bool { + if !dpu_snapshot.managed_host_network_config_version_synced(host_version) { + return false; + } + + let Some(dpu_health) = dpu_snapshot.dpu_agent_health_report() else { + return false; + }; + + // Note that DPU alerts may be surpressed (classifications removed) in the aggregate health + // report so the individual DPU's report is used. + !dpu_health + .has_classification(&health_report::HealthAlertClassification::prevent_host_state_changes()) +} + +fn check_host_health_for_alerts(state: &ManagedHostStateSnapshot) -> Result<(), StateHandlerError> { + // In some states, DPU alerts may be surpressed (classifications removed) in the aggregate health report. + // Since this is not called from a state that supresses DPU alerts, this is ok here. + match state + .aggregate_health + .has_classification(&health_report::HealthAlertClassification::prevent_host_state_changes()) + { + true => Err(StateHandlerError::HealthProbeAlert), + false => Ok(()), + } +} + +/// Whether a captured desired target can be replaced before this substate runs. +/// +/// Pure checks and pre-write states can adopt newer intent. Vendor jobs, +/// polling, unlock choreography, and cleanup must finish against the captured +/// target so a concurrent write cannot orphan an in-flight Redfish operation. +fn ready_boot_config_can_adopt_latest(state: &ReadyBootConfigState) -> bool { + match state { + ReadyBootConfigState::Prepare + | ReadyBootConfigState::CheckHostConfig + | ReadyBootConfigState::ConfigureBios { .. } => true, + ReadyBootConfigState::SetBootOrder { + set_boot_order_info, + } => matches!( + set_boot_order_info.set_boot_order_state, + SetBootOrderState::SetBootOrder | SetBootOrderState::CheckBootOrder + ), + ReadyBootConfigState::UnlockHost { .. } + | ReadyBootConfigState::WaitingForBiosJob { .. } + | ReadyBootConfigState::PollingBiosSetup { .. } + | ReadyBootConfigState::LockHost { .. } + | ReadyBootConfigState::Failed { .. } => false, + } +} + +// One complete persisted retry tolerates a transient final observation. +// Repeated drift after lockdown strongly suggests that lockdown or another +// actor is undoing the configuration. +const MAX_POST_LOCK_VERIFICATION_RETRIES: u32 = 1; + +/// Returns the state required by newer intent, or `None` while the captured +/// desired version remains current. +fn ready_boot_config_superseded_state( + captured_version: ConfigVersion, + current_desired: Option>, +) -> Option { + if current_desired + .as_ref() + .is_some_and(|desired| desired.version == captured_version) + { + return None; + } + + Some(current_desired.map_or(ManagedHostState::Ready, |desired| { + ready_boot_configuring(desired, 0, ReadyBootConfigState::Prepare) + })) +} + +fn ready_boot_config_state_from_stage(stage: HostBootConfigStage) -> ReadyBootConfigState { + match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + ReadyBootConfigState::ConfigureBios { retry_count } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + ReadyBootConfigState::WaitingForBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + ReadyBootConfigState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => ReadyBootConfigState::SetBootOrder { + set_boot_order_info, + }, + } +} + +/// Returns whether a Ready boot substate may have left lockdown disabled. +/// +/// Keep this match exhaustive so a new substate cannot silently skip cleanup. +fn ready_boot_config_may_have_opened_lockdown(state: &ReadyBootConfigState) -> bool { + match state { + ReadyBootConfigState::Prepare + | ReadyBootConfigState::LockHost { .. } + | ReadyBootConfigState::Failed { .. } => false, + ReadyBootConfigState::UnlockHost { .. } + | ReadyBootConfigState::CheckHostConfig + | ReadyBootConfigState::ConfigureBios { .. } + | ReadyBootConfigState::WaitingForBiosJob { .. } + | ReadyBootConfigState::PollingBiosSetup { .. } + | ReadyBootConfigState::SetBootOrder { .. } => true, + } +} + +/// Routes an active repair through cleanup when expected DPU snapshots vanish. +/// +/// `Prepare` has not opened lockdown, `LockHost` is already cleanup, and +/// `Failed` is reached only after cleanup. Every other substate may have +/// disabled lockdown. +fn ready_boot_config_missing_dpu_recovery(state: &ManagedHostState) -> Option { + let ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface, + post_lock_verification_retry_count, + boot_config_state, + } = state + else { + return None; + }; + if !ready_boot_config_may_have_opened_lockdown(boot_config_state) { + return None; + } + + Some(ready_boot_config_locking( + Versioned { + value: desired_boot_interface.clone(), + version: *desired_version, + }, + *post_lock_verification_retry_count, + Some(ReadyBootConfigTerminalFailure::Convergence { + failure: "expected DPU snapshots disappeared while boot-interface reconciliation may have left lockdown disabled" + .to_string(), + }), + )) +} + +fn ready_boot_config_requires_timeout_cleanup( + state: &ReadyBootConfigState, + time_in_state: chrono::Duration, +) -> bool { + let cleanup_timeout = chrono::Duration::from_std(model::machine::slas::BOOT_CONFIGURING) + .expect("BootConfiguring SLA fits chrono::Duration"); + time_in_state >= cleanup_timeout && ready_boot_config_may_have_opened_lockdown(state) +} + +fn ready_boot_configuring( + desired: Versioned, + post_lock_verification_retry_count: u32, + boot_config_state: ReadyBootConfigState, +) -> ManagedHostState { + ManagedHostState::BootConfiguring { + desired_version: desired.version, + desired_boot_interface: desired.value, + post_lock_verification_retry_count, + boot_config_state, + } +} + +/// Builds the convergence state for a machine whose desired boot-interface +/// version has not yet been verified. +fn pending_ready_boot_config_state(machine: &Machine) -> Option { + let desired = machine.config.desired_boot_interface.as_ref()?; + machine + .pending_boot_interface_config_version() + .map(|_| ready_boot_configuring(desired.clone(), 0, ReadyBootConfigState::Prepare)) +} + +fn ready_boot_config_locking( + desired: Versioned, + post_lock_verification_retry_count: u32, + terminal_failure: Option, +) -> ManagedHostState { + ready_boot_configuring( + desired, + post_lock_verification_retry_count, + ReadyBootConfigState::LockHost { terminal_failure }, + ) +} + +fn ready_boot_config_after_post_lock_drift( + desired: Versioned, + post_lock_verification_retry_count: u32, + machine_id: MachineId, +) -> ManagedHostState { + if post_lock_verification_retry_count < MAX_POST_LOCK_VERIFICATION_RETRIES { + return ready_boot_configuring( + desired, + post_lock_verification_retry_count + 1, + ReadyBootConfigState::Prepare, + ); + } + + ready_boot_configuring( + desired, + post_lock_verification_retry_count, + ReadyBootConfigState::Failed { + failure: format!( + "Boot configuration on host {machine_id} repeatedly drifted after lockdown was restored; manual intervention required" + ), + }, + ) +} + +/// Maps the shared HostInit boot driver onto persisted Ready substates. +/// +/// Terminal driver failures pass through `LockHost` so convergence cannot park +/// while lockdown may be open. +async fn handle_ready_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + desired: Versioned, + post_lock_verification_retry_count: u32, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let boot_interface: BootInterfaceTarget = desired.value.clone().into(); + let outcome = match run_host_boot_config_stage( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + Some(&boot_interface), + stage, + ) + .await + { + Ok(outcome) => outcome, + Err(StateHandlerError::ManualInterventionRequired(failure)) => { + return Ok(StateHandlerOutcome::transition(ready_boot_config_locking( + desired, + post_lock_verification_retry_count, + Some(ReadyBootConfigTerminalFailure::Convergence { failure }), + ))); + } + Err(error) => return Err(error), + }; + + match outcome { + HostBootConfigOutcome::Continue(stage) => { + Ok(StateHandlerOutcome::transition(ready_boot_configuring( + desired, + post_lock_verification_retry_count, + ready_boot_config_state_from_stage(stage), + ))) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ready_boot_config_locking(desired, post_lock_verification_retry_count, None), + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + Ok(StateHandlerOutcome::transition(ready_boot_config_locking( + desired, + post_lock_verification_retry_count, + Some(ReadyBootConfigTerminalFailure::Convergence { failure }), + ))) + } + } +} + +/// Converges an unassigned Ready host to its persisted boot-interface target. +/// +/// The outer state captures one target and desired version. Safe boundaries +/// may adopt newer intent, while an in-flight vendor job or cleanup finishes +/// against the captured target before the controller switches versions. +async fn handle_ready_boot_config( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + desired: Versioned, + post_lock_verification_retry_count: u32, + boot_config_state: ReadyBootConfigState, +) -> Result, StateHandlerError> { + // Only states that can adopt replacement intent need an unlocked read. + // LockHost re-reads under the machine-row lock before it commits a + // transition, and in-flight vendor stages deliberately finish their + // captured generation. + let current_desired = if matches!(boot_config_state, ReadyBootConfigState::Failed { .. }) + || ready_boot_config_can_adopt_latest(&boot_config_state) + { + let mut conn = ctx.services.db_pool.acquire().await?; + db::machine_desired_boot_interface::get(conn.as_mut(), &mh_snapshot.host_snapshot.id) + .await? + } else { + None + }; + let captured_boot_interface: BootInterfaceTarget = desired.value.clone().into(); + + if ready_boot_config_requires_timeout_cleanup( + &boot_config_state, + mh_snapshot.host_snapshot.state.version.since_state_change(), + ) { + let failure = format!( + "boot-interface reconciliation stopped progressing in {boot_config_state:?} for longer than its {}-second cleanup deadline", + model::machine::slas::BOOT_CONFIGURING.as_secs(), + ); + tracing::error!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + ?boot_config_state, + reason = %failure, + "Restoring lockdown before parking timed-out boot reconciliation", + ); + return Ok(StateHandlerOutcome::transition(ready_boot_config_locking( + desired, + post_lock_verification_retry_count, + Some(ReadyBootConfigTerminalFailure::Convergence { failure }), + ))); + } + + // A terminal failure stays scoped to the captured desired generation. + // A maintenance request can take control, while changing intent creates a + // new version and restarts convergence. + if matches!(boot_config_state, ReadyBootConfigState::Failed { .. }) { + if let Some(outcome) = maintenance::maintenance_transition_if_requested(mh_snapshot) { + return Ok(outcome); + } + return Ok( + match ready_boot_config_superseded_state(desired.version, current_desired) { + Some(next_state) => StateHandlerOutcome::transition(next_state), + None => StateHandlerOutcome::do_nothing(), + }, + ); + } + + if ready_boot_config_can_adopt_latest(&boot_config_state) + && current_desired.as_ref().map(|current| current.version) != Some(desired.version) + { + return Ok(match current_desired { + Some(current_desired) => { + let next_state = if matches!(boot_config_state, ReadyBootConfigState::Prepare) { + ReadyBootConfigState::Prepare + } else { + // Re-observe the replacement before any Redfish write. + ReadyBootConfigState::CheckHostConfig + }; + StateHandlerOutcome::transition(ready_boot_configuring( + current_desired, + 0, + next_state, + )) + } + None if matches!(boot_config_state, ReadyBootConfigState::Prepare) => { + StateHandlerOutcome::transition(ManagedHostState::Ready) + } + None => { + // The target disappeared after lockdown may have opened. + StateHandlerOutcome::transition(ready_boot_config_locking( + desired, + post_lock_verification_retry_count, + None, + )) + } + }); + } + + if matches!(boot_config_state, ReadyBootConfigState::Prepare) + && !mh_snapshot + .host_snapshot + .associated_dpu_machine_ids() + .is_empty() + && mh_snapshot.dpu_snapshots.is_empty() + { + // Prepare has not opened lockdown, so it can safely process target + // replacement or removal above. Do not let the shared boot check + // mistake a transiently empty snapshot list for a zero-DPU host. + return Ok(StateHandlerOutcome::wait( + "Waiting for expected DPU snapshots before boot-interface reconciliation".to_string(), + )); + } + + match boot_config_state { + ReadyBootConfigState::Prepare => { + if mh_snapshot.host_snapshot.bmc_vendor().is_supermicro() { + // A locked Supermicro BMC can report stale boot order. Reboot + // after disabling lockdown before performing the exact read. + return Ok(StateHandlerOutcome::transition(ready_boot_configuring( + desired, + post_lock_verification_retry_count, + ReadyBootConfigState::UnlockHost { + unlock_host_state: UnlockHostState::DisableLockdown, + }, + ))); + } + + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let preflight_decision = match check_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + reachability_params, + HostBootConfigDpuFreshness::CurrentHostState, + Some(&captured_boot_interface), + ctx, + ) + .await? + { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(decision) => decision, + }; + let preflight_complete = matches!(preflight_decision, HostBootConfigDecision::Complete); + + let next_state = if preflight_complete { + // Avoid opening an ordinary host that is already correct. + ReadyBootConfigState::LockHost { + terminal_failure: None, + } + } else { + match redfish_client.lockdown_status().await { + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support checking lockdown status during Ready boot repair", + ); + ReadyBootConfigState::CheckHostConfig + } + Err(error) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %error, + "Failed to fetch lockdown status during Ready boot repair", + ); + return Ok(StateHandlerOutcome::wait(format!( + "Failed to fetch lockdown status: {error}" + ))); + } + Ok(lockdown_status) if !lockdown_status.is_fully_disabled() => { + ReadyBootConfigState::UnlockHost { + unlock_host_state: UnlockHostState::DisableLockdown, + } + } + Ok(_) => ReadyBootConfigState::CheckHostConfig, + } + }; + + Ok(StateHandlerOutcome::transition(ready_boot_configuring( + desired, + post_lock_verification_retry_count, + next_state, + ))) + } + ReadyBootConfigState::UnlockHost { unlock_host_state } => { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let next_state = match unlock_host_state { + UnlockHostState::DisableLockdown => { + match redfish_client.lockdown_bmc(EnabledDisabled::Disabled).await { + Ok(()) => {} + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support disabling lockdown during Ready boot repair", + ); + } + Err(error) => return Err(redfish_error("lockdown_bmc", error)), + } + + if mh_snapshot.host_snapshot.bmc_vendor().is_supermicro() { + ReadyBootConfigState::UnlockHost { + unlock_host_state: UnlockHostState::RebootHost, + } + } else { + ReadyBootConfigState::CheckHostConfig + } + } + UnlockHostState::RebootHost => { + host_power_control( + redfish_client.as_ref(), + &mh_snapshot.host_snapshot, + SystemPowerControl::ForceRestart, + ctx, + ) + .await + .map_err(|error| { + StateHandlerError::GenericError(eyre!( + "failed to ForceRestart host after disabling BMC lockdown: {error}" + )) + })?; + + ReadyBootConfigState::UnlockHost { + unlock_host_state: UnlockHostState::WaitForUefiBoot, + } + } + UnlockHostState::WaitForUefiBoot => { + let entered_at = mh_snapshot.host_snapshot.state.version.timestamp(); + if wait(&entered_at, reachability_params.uefi_boot_wait) { + return Ok(StateHandlerOutcome::wait(format!( + "Waiting for UEFI boot to complete on {} after post-unlock reboot", + mh_snapshot.host_snapshot.id + ))); + } + ReadyBootConfigState::CheckHostConfig + } + }; + + Ok(StateHandlerOutcome::transition(ready_boot_configuring( + desired, + post_lock_verification_retry_count, + next_state, + ))) + } + ReadyBootConfigState::CheckHostConfig => { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let next_state = match check_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + reachability_params, + HostBootConfigDpuFreshness::CurrentHostState, + Some(&captured_boot_interface), + ctx, + ) + .await? + { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + ReadyBootConfigState::ConfigureBios { retry_count: 0 } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + ReadyBootConfigState::SetBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + ReadyBootConfigState::LockHost { + terminal_failure: None, + } + } + }; + + Ok(StateHandlerOutcome::transition(ready_boot_configuring( + desired, + post_lock_verification_retry_count, + next_state, + ))) + } + ReadyBootConfigState::ConfigureBios { retry_count } => { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_ready_boot_config_stage( + ctx, + mh_snapshot, + reachability_params, + redfish_client.as_ref(), + desired, + post_lock_verification_retry_count, + HostBootConfigStage::ConfigureBios { retry_count }, + ) + .await + } + ReadyBootConfigState::WaitingForBiosJob { bios_config_info } => { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_ready_boot_config_stage( + ctx, + mh_snapshot, + reachability_params, + redfish_client.as_ref(), + desired, + post_lock_verification_retry_count, + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + ) + .await + } + ReadyBootConfigState::PollingBiosSetup { retry_count } => { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_ready_boot_config_stage( + ctx, + mh_snapshot, + reachability_params, + redfish_client.as_ref(), + desired, + post_lock_verification_retry_count, + HostBootConfigStage::PollingBiosSetup { retry_count }, + ) + .await + } + ReadyBootConfigState::SetBootOrder { + set_boot_order_info, + } => { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_ready_boot_config_stage( + ctx, + mh_snapshot, + reachability_params, + redfish_client.as_ref(), + desired, + post_lock_verification_retry_count, + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }, + ) + .await + } + ReadyBootConfigState::LockHost { terminal_failure } => { + let lockdown_disabled = mh_snapshot.host_snapshot.host_profile.disable_lockdown; + + // A profile that deliberately leaves lockdown disabled has no + // cleanup barrier. Terminal failures can therefore be published + // without requiring Redfish access. Successful convergence still + // performs the final exact-target observation below. + if lockdown_disabled && let Some(terminal_failure) = &terminal_failure { + match terminal_failure { + ReadyBootConfigTerminalFailure::Machine { + machine_id, + details, + } => { + return Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: details.clone(), + machine_id: *machine_id, + retry_count: 0, + })); + } + ReadyBootConfigTerminalFailure::Convergence { failure } => { + let mut txn = ctx.services.db_pool.begin().await?; + let current_desired = db::machine_desired_boot_interface::lock( + txn.as_mut(), + &mh_snapshot.host_snapshot.id, + ) + .await?; + let next_state = + ready_boot_config_superseded_state(desired.version, current_desired) + .unwrap_or_else(|| { + ready_boot_configuring( + desired, + post_lock_verification_retry_count, + ReadyBootConfigState::Failed { + failure: failure.clone(), + }, + ) + }); + return Ok(StateHandlerOutcome::transition(next_state).with_txn(txn)); + } + } + } + + let redfish_client = match ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await + { + Ok(redfish_client) => redfish_client, + Err(error) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %error, + "Waiting for Redfish access before completing Ready boot repair cleanup", + ); + return Ok(StateHandlerOutcome::wait( + "Waiting for host Redfish access before completing Ready boot repair cleanup" + .to_string(), + )); + } + }; + + if lockdown_disabled { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Skipping lockdown re-enable after Ready boot repair per expected-machine config", + ); + } else { + let (lockdown_command_required, verify_after_command, require_supported_command) = + match redfish_client.lockdown_status().await { + Ok(lockdown_status) if lockdown_status.is_fully_enabled() => { + (false, false, false) + } + Ok(lockdown_status) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + ?lockdown_status, + "Restoring lockdown after Ready boot repair", + ); + (true, true, true) + } + Err(RedfishError::NotSupported(_)) => { + // The command may still be supported even when the + // vendor has no corresponding status read. + (true, false, false) + } + Err(error) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %error, + "Could not read lockdown status before Ready boot repair cleanup; attempting restoration", + ); + (true, true, true) + } + }; + + let lockdown_command_supported = if lockdown_command_required { + match redfish_client.lockdown_bmc(EnabledDisabled::Enabled).await { + Ok(()) => true, + Err(RedfishError::NotSupported(_)) if require_supported_command => { + return Ok(StateHandlerOutcome::wait( + "BMC lockdown status is not verified and the BMC does not support restoring it" + .to_string(), + )); + } + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support re-enabling lockdown after Ready boot repair", + ); + false + } + Err(error) => return Err(redfish_error("lockdown_bmc", error)), + } + } else { + false + }; + + if lockdown_command_supported && verify_after_command { + match redfish_client.lockdown_status().await { + Ok(lockdown_status) if lockdown_status.is_fully_enabled() => {} + Ok(lockdown_status) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + ?lockdown_status, + "Waiting for lockdown policy restoration after Ready boot repair", + ); + return Ok(StateHandlerOutcome::wait(format!( + "Waiting for lockdown to be fully enabled after Ready boot repair; current status: {lockdown_status:?}" + ))); + } + Err(RedfishError::NotSupported(_)) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC stopped supporting lockdown verification after reporting a readable status", + ); + return Ok(StateHandlerOutcome::wait( + "BMC lockdown restoration could not be verified".to_string(), + )); + } + Err(error) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %error, + "Failed to verify lockdown after Ready boot repair", + ); + return Ok(StateHandlerOutcome::wait(format!( + "Failed to verify lockdown after Ready boot repair: {error}" + ))); + } + } + } + } -fn managed_host_network_config_version_synced_and_dpu_healthy( - dpu_snapshot: &Machine, - host_version: ConfigVersion, -) -> bool { - if !dpu_snapshot.managed_host_network_config_version_synced(host_version) { - return false; - } + if let Some(terminal_failure) = terminal_failure { + match terminal_failure { + ReadyBootConfigTerminalFailure::Machine { + machine_id, + details, + } => { + return Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details, + machine_id, + retry_count: 0, + })); + } + ReadyBootConfigTerminalFailure::Convergence { failure } => { + let mut txn = ctx.services.db_pool.begin().await?; + let current_desired = db::machine_desired_boot_interface::lock( + txn.as_mut(), + &mh_snapshot.host_snapshot.id, + ) + .await?; + let next_state = + ready_boot_config_superseded_state(desired.version, current_desired) + .unwrap_or_else(|| { + ready_boot_configuring( + desired, + post_lock_verification_retry_count, + ReadyBootConfigState::Failed { failure }, + ) + }); + return Ok(StateHandlerOutcome::transition(next_state).with_txn(txn)); + } + } + } - let Some(dpu_health) = dpu_snapshot.dpu_agent_health_report() else { - return false; - }; + let boot_config_verified = + if mh_snapshot.host_snapshot.bmc_vendor().is_supermicro() && !lockdown_disabled { + // For a still-current desired generation, entry into LockHost + // durably records an exact read while Supermicro lockdown was + // disabled. Locked boot-order reads can be stale, so that + // checkpoint is stronger evidence than another Redfish read. + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + "Using the pre-lock Supermicro boot verification", + ); + true + } else { + // This state is restartable, so re-observe after lockdown is + // restored rather than trusting an earlier pre-crash read. + let inspection = inspect_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + &captured_boot_interface, + ) + .await?; + matches!( + decide_host_boot_config(inspection), + HostBootConfigDecision::Complete + ) + }; + if !boot_config_verified { + let mut txn = ctx.services.db_pool.begin().await?; + let current_desired = db::machine_desired_boot_interface::lock( + txn.as_mut(), + &mh_snapshot.host_snapshot.id, + ) + .await?; + let next_state = + ready_boot_config_superseded_state(desired.version, current_desired) + .unwrap_or_else(|| { + if post_lock_verification_retry_count + < MAX_POST_LOCK_VERIFICATION_RETRIES + { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + post_lock_verification_retry_count = + post_lock_verification_retry_count + 1, + "Boot configuration drifted after lockdown restoration; retrying reconciliation", + ); + } else { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + post_lock_verification_retry_count, + "Boot configuration repeatedly drifted after lockdown restoration; parking reconciliation", + ); + } + ready_boot_config_after_post_lock_drift( + desired, + post_lock_verification_retry_count, + mh_snapshot.host_snapshot.id, + ) + }); + return Ok(StateHandlerOutcome::transition(next_state).with_txn(txn)); + } - // Note that DPU alerts may be surpressed (classifications removed) in the aggregate health - // report so the individual DPU's report is used. - !dpu_health - .has_classification(&health_report::HealthAlertClassification::prevent_host_state_changes()) -} + let mut txn = ctx.services.db_pool.begin().await?; + let verified = db::machine_desired_boot_interface::mark_verified( + txn.as_mut(), + &mh_snapshot.host_snapshot.id, + desired.version, + Utc::now(), + ) + .await?; + let next_state = if verified { + ManagedHostState::Ready + } else { + match db::machine_desired_boot_interface::get( + txn.as_mut(), + &mh_snapshot.host_snapshot.id, + ) + .await? + { + Some(current_desired) => { + ready_boot_configuring(current_desired, 0, ReadyBootConfigState::Prepare) + } + None => ManagedHostState::Ready, + } + }; -fn check_host_health_for_alerts(state: &ManagedHostStateSnapshot) -> Result<(), StateHandlerError> { - // In some states, DPU alerts may be surpressed (classifications removed) in the aggregate health report. - // Since this is not called from a state that supresses DPU alerts, this is ok here. - match state - .aggregate_health - .has_classification(&health_report::HealthAlertClassification::prevent_host_state_changes()) - { - true => Err(StateHandlerError::HealthProbeAlert), - false => Ok(()), + Ok(StateHandlerOutcome::transition(next_state).with_txn(txn)) + } + // Handled above so superseding intent can restart a parked failure. + ReadyBootConfigState::Failed { .. } => Ok(StateHandlerOutcome::do_nothing()), } } @@ -5498,6 +6476,107 @@ async fn handle_host_init_boot_config_stage( } } +/// Finishes HostInit lockdown and records a fresh target-specific verification +/// in the same state-transition transaction when the final Redfish read agrees. +/// +/// A failed or mismatched read does not block ingestion. The desired version +/// remains pending, keeps the host out of allocation, and is handled by the +/// Ready reconciliation path. If operator intent changes before the exact +/// version is committed, the replacement likewise remains pending. +async fn complete_host_init_lockdown( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + next_state: ManagedHostState, +) -> Result, StateHandlerError> { + let outcome = StateHandlerOutcome::transition(next_state); + let Some(desired) = mh_snapshot + .host_snapshot + .config + .desired_boot_interface + .as_ref() + else { + return Ok(outcome); + }; + + if mh_snapshot.host_snapshot.bmc_vendor().is_supermicro() + && !mh_snapshot.host_snapshot.host_profile.disable_lockdown + { + // Supermicro boot-order reads can become stale after lockdown. HostInit + // has no persisted pre-lock verification boundary, so leave this + // generation pending for the Ready convergence flow. + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + "Deferring Supermicro boot verification until Ready convergence", + ); + return Ok(outcome); + } + + let redfish_client = match ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await + { + Ok(redfish_client) => redfish_client, + Err(error) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + error = %error, + "Could not create a Redfish client for HostInit boot verification; leaving it pending for Ready reconciliation", + ); + return Ok(outcome); + } + }; + let boot_interface: BootInterfaceTarget = desired.value.clone().into(); + let inspection = match inspect_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + &boot_interface, + ) + .await + { + Ok(inspection) => inspection, + Err(error) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + error = %error, + "Could not verify boot configuration after HostInit lockdown; leaving it pending for Ready reconciliation", + ); + return Ok(outcome); + } + }; + if !matches!( + decide_host_boot_config(inspection), + HostBootConfigDecision::Complete + ) { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + "Boot configuration differs after HostInit lockdown; leaving it pending for Ready reconciliation", + ); + return Ok(outcome); + } + + let mut txn = ctx.services.db_pool.begin().await?; + let verified = db::machine_desired_boot_interface::mark_verified( + txn.as_mut(), + &mh_snapshot.host_snapshot.id, + desired.version, + Utc::now(), + ) + .await?; + if !verified { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + desired_version = %desired.version, + "Desired boot interface changed during HostInit verification; leaving it pending", + ); + } + Ok(outcome.with_txn(txn)) +} + /// TODO: we need to handle the case where the job is deleted for some reason /// Resolve the current site-wide UEFI target version (host_uefi or dpu_uefi) /// from `sitewide_credential_rotation.target_version` so ingestion drives a @@ -6083,7 +7162,9 @@ impl StateHandler for HostMachineStateHandler { machine_id = %host_machine_id, "Lockdown disabled per expected-machine config, skipping lockdown enable" ); - return Ok(StateHandlerOutcome::transition( + return complete_host_init_lockdown( + ctx, + mh_snapshot, ManagedHostState::BomValidating { bom_validating_state: BomValidating::MatchingSku( BomValidatingContext { @@ -6094,7 +7175,8 @@ impl StateHandler for HostMachineStateHandler { }, ), }, - )); + ) + .await; } tracing::info!( @@ -6138,23 +7220,39 @@ impl StateHandler for HostMachineStateHandler { } LockdownState::TimeWaitForDPUDown => { if !mh_snapshot.has_managed_dpus() { - // No DPU to wait for going down/up -- skip - // straight to BomValidating. Covers - // `Nic`/`Ignore` hosts and anything else - // with no DPU snapshots; otherwise we'd - // wait `dpu_wait_time` for a DPU that's - // never going to come up. - let next_state = ManagedHostState::BomValidating { - bom_validating_state: BomValidating::MatchingSku( - BomValidatingContext { - machine_validation_context: Some( - MachineValidationContext::Discovery, + // There is no DPU power cycle to observe for + // `Nic`/`Ignore` hosts. Disable still needs the + // shared status poll to reach platform + // configuration. Preserve the established + // Enable path, which does not require a + // separately readable lockdown status. + if lockdown_info.mode == LockdownMode::Enable { + return complete_host_init_lockdown( + ctx, + mh_snapshot, + ManagedHostState::BomValidating { + bom_validating_state: BomValidating::MatchingSku( + BomValidatingContext { + machine_validation_context: Some( + MachineValidationContext::Discovery, + ), + ..BomValidatingContext::default() + }, ), - reboot_retry_count: None, }, - ), - }; - return Ok(StateHandlerOutcome::transition(next_state)); + ) + .await; + } + return Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForLockdown { + lockdown_info: LockdownInfo { + state: LockdownState::PollingLockdownStatus, + mode: lockdown_info.mode.clone(), + }, + }, + }, + )); } // Lets wait for some time before checking if DPU is up or not. // Waiting is needed because DPU takes some time to go down. If we check DPU @@ -6268,7 +7366,16 @@ impl StateHandler for HostMachineStateHandler { mode = ?lockdown_info.mode, "Lockdown status verified successfully" ); - Ok(StateHandlerOutcome::transition(next_state)) + if lockdown_info.mode == LockdownMode::Enable { + complete_host_init_lockdown( + ctx, + mh_snapshot, + next_state, + ) + .await + } else { + Ok(StateHandlerOutcome::transition(next_state)) + } } else { Ok(StateHandlerOutcome::wait(format!( "Polling lockdown status, waiting for {:?} to be applied. Current status: {:?}", @@ -6281,7 +7388,12 @@ impl StateHandler for HostMachineStateHandler { machine_id = %host_machine_id, "BMC vendor does not support checking lockdown status." ); - Ok(StateHandlerOutcome::transition(next_state)) + if lockdown_info.mode == LockdownMode::Enable { + complete_host_init_lockdown(ctx, mh_snapshot, next_state) + .await + } else { + Ok(StateHandlerOutcome::transition(next_state)) + } } Err(e) => { tracing::warn!( @@ -6305,7 +7417,10 @@ impl StateHandler for HostMachineStateHandler { // or Measuring state, depending on if machine attestation // is enabled or not. if rebooted(&mh_snapshot.host_snapshot) || *skip_reboot { - Ok(StateHandlerOutcome::transition(ManagedHostState::Ready)) + let next_state = + pending_ready_boot_config_state(&mh_snapshot.host_snapshot) + .unwrap_or(ManagedHostState::Ready); + Ok(StateHandlerOutcome::transition(next_state)) } else { let status = trigger_reboot_if_needed( &mh_snapshot.host_snapshot, @@ -12069,6 +13184,211 @@ mod tests { use super::*; + #[test] + fn terminal_ready_boot_config_failure_is_deferred_until_lockdown_restoration() { + let desired_version = ConfigVersion::initial(); + let desired_boot_interface = + MachineBootInterfaceTarget::MacOnly("02:00:00:00:00:01".parse().unwrap()); + let failure = "BIOS job retries exhausted".to_string(); + + assert_eq!( + ready_boot_config_locking( + Versioned::new(desired_boot_interface.clone(), desired_version), + 0, + Some(ReadyBootConfigTerminalFailure::Convergence { + failure: failure.clone(), + }), + ), + ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface, + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: Some(ReadyBootConfigTerminalFailure::Convergence { failure }), + }, + } + ); + } + + #[test] + fn missing_dpus_during_ready_boot_config_fails_closed() { + let desired_version = ConfigVersion::initial(); + let desired_boot_interface = + MachineBootInterfaceTarget::MacOnly("02:00:00:00:00:01".parse().unwrap()); + let active = ready_boot_configuring( + Versioned::new(desired_boot_interface.clone(), desired_version), + 0, + ReadyBootConfigState::CheckHostConfig, + ); + + assert!(matches!( + ready_boot_config_missing_dpu_recovery(&active), + Some(ManagedHostState::BootConfiguring { + desired_version: version, + desired_boot_interface: target, + boot_config_state: ReadyBootConfigState::LockHost { + terminal_failure: + Some(ReadyBootConfigTerminalFailure::Convergence { failure }), + }, + .. + }) if version == desired_version + && target == desired_boot_interface + && failure.contains("DPU snapshots disappeared") + )); + + for safe_state in [ + ReadyBootConfigState::Prepare, + ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + ReadyBootConfigState::Failed { + failure: "already parked".to_string(), + }, + ] { + let state = ready_boot_configuring( + Versioned::new(desired_boot_interface.clone(), desired_version), + 0, + safe_state, + ); + assert_eq!(ready_boot_config_missing_dpu_recovery(&state), None); + } + } + + #[test] + fn stalled_ready_boot_config_cleans_up_only_after_work_can_start() { + let timed_out = chrono::Duration::from_std(model::machine::slas::BOOT_CONFIGURING).unwrap(); + let still_within_sla = timed_out - chrono::Duration::seconds(1); + + assert!(!ready_boot_config_requires_timeout_cleanup( + &ReadyBootConfigState::CheckHostConfig, + still_within_sla, + )); + assert!(ready_boot_config_requires_timeout_cleanup( + &ReadyBootConfigState::CheckHostConfig, + timed_out, + )); + + for safe_state in [ + ReadyBootConfigState::Prepare, + ReadyBootConfigState::LockHost { + terminal_failure: None, + }, + ReadyBootConfigState::Failed { + failure: "already parked".to_string(), + }, + ] { + assert!(!ready_boot_config_requires_timeout_cleanup( + &safe_state, + timed_out, + )); + } + } + + #[test] + fn post_lock_drift_retries_once_then_requires_intervention() { + let machine_id = + MachineId::from_str("fm100htes3rn1npvbtm5qd57dkilaag7ljugl1llmm7rfuq1ov50i0rpl30") + .unwrap(); + let desired_version = ConfigVersion::initial(); + let desired_boot_interface = + MachineBootInterfaceTarget::MacOnly("02:00:00:00:00:01".parse().unwrap()); + let desired = Versioned::new(desired_boot_interface.clone(), desired_version); + + assert_eq!( + ready_boot_config_after_post_lock_drift(desired.clone(), 0, machine_id), + ManagedHostState::BootConfiguring { + desired_version, + desired_boot_interface: desired_boot_interface.clone(), + post_lock_verification_retry_count: 1, + boot_config_state: ReadyBootConfigState::Prepare, + } + ); + + assert!(matches!( + ready_boot_config_after_post_lock_drift(desired, 1, machine_id), + ManagedHostState::BootConfiguring { + desired_version: version, + desired_boot_interface: target, + post_lock_verification_retry_count: 1, + boot_config_state: ReadyBootConfigState::Failed { failure }, + } if version == desired_version + && target == desired_boot_interface + && failure.contains("repeatedly drifted") + )); + } + + #[test] + fn ready_boot_config_adopts_targets_only_at_safe_boundaries() { + let set_boot_order = |set_boot_order_state| ReadyBootConfigState::SetBootOrder { + set_boot_order_info: SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state, + retry_count: 0, + }, + }; + + for state in [ + ReadyBootConfigState::Prepare, + ReadyBootConfigState::CheckHostConfig, + ReadyBootConfigState::ConfigureBios { retry_count: 0 }, + set_boot_order(SetBootOrderState::SetBootOrder), + set_boot_order(SetBootOrderState::CheckBootOrder), + ] { + assert!(ready_boot_config_can_adopt_latest(&state), "{state:?}"); + } + + for state in [ + ReadyBootConfigState::UnlockHost { + unlock_host_state: UnlockHostState::RebootHost, + }, + ReadyBootConfigState::PollingBiosSetup { retry_count: 0 }, + set_boot_order(SetBootOrderState::WaitForSetBootOrderJobCompletion), + ReadyBootConfigState::LockHost { + terminal_failure: Some(ReadyBootConfigTerminalFailure::Convergence { + failure: "exhausted".to_string(), + }), + }, + ReadyBootConfigState::Failed { + failure: "exhausted".to_string(), + }, + ] { + assert!(!ready_boot_config_can_adopt_latest(&state), "{state:?}"); + } + } + + #[test] + fn failed_ready_boot_config_restarts_only_for_superseding_intent() { + let failed_version = ConfigVersion::initial(); + let next_version = failed_version.increment(); + let failed_target = + MachineBootInterfaceTarget::MacOnly("02:00:00:00:00:01".parse().unwrap()); + let next_target = MachineBootInterfaceTarget::MacOnly("02:00:00:00:00:02".parse().unwrap()); + + assert_eq!( + ready_boot_config_superseded_state( + failed_version, + Some(Versioned::new(failed_target, failed_version)), + ), + None, + ); + assert_eq!( + ready_boot_config_superseded_state( + failed_version, + Some(Versioned::new(next_target.clone(), next_version)), + ), + Some(ManagedHostState::BootConfiguring { + desired_version: next_version, + desired_boot_interface: next_target, + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::Prepare, + }), + ); + assert_eq!( + ready_boot_config_superseded_state(failed_version, None), + Some(ManagedHostState::Ready), + ); + } + #[test] fn ipmi_socket_address_uses_reported_or_default_port() { let ip_address = IpAddr::V4("192.0.2.10".parse().unwrap()); diff --git a/crates/machine-controller/src/handler/rotation.rs b/crates/machine-controller/src/handler/rotation.rs index 5bb4f5bf9b..29febebaae 100644 --- a/crates/machine-controller/src/handler/rotation.rs +++ b/crates/machine-controller/src/handler/rotation.rs @@ -35,10 +35,7 @@ //! under live tenancy (`Assigned/RotatingBmc`). use carbide_credential_rotation::{BmcEndpoint, BmcRotationTick, RotateOutcome, rotate_bmc}; -use carbide_secrets::credentials::{BmcCredentialType, CredentialKey, Credentials}; use carbide_uuid::machine::MachineId; -use libredfish::model::service_root::RedfishVendor; -use mac_address::MacAddress; use model::machine::{Machine, ManagedHostStateSnapshot}; use sqlx::PgTransaction; use state_controller::state_handler::StateHandlerError; @@ -197,8 +194,11 @@ async fn rotate_endpoint( endpoint: BmcEndpoint, force: bool, ) -> BmcRotationTick { - let vendor = resolve_dispatch_vendor(services, &endpoint).await; - let target = endpoint.into_target(vendor); + // The precise `RedfishVendor` is not persisted for a machine BMC (the stored + // hardware vendor is DMI-derived and too coarse), so defer resolution to the + // engine's probe, which runs inside the same quarantine-on-failure envelope + // as the rotation and reuses its credential candidates. + let target = endpoint.into_target_probing_vendor(); match rotate_bmc( &services.db_pool, services.credential_manager.as_ref(), @@ -232,67 +232,11 @@ async fn rotate_endpoint( } } -/// Resolve the precise dispatch vendor `set_bmc_root_password` branches on by -/// probing at rotation time. -/// -/// The stored hardware `BMCVendor` is deliberately not used: it is derived from -/// DMI `sys_vendor` and is too coarse (every NVIDIA `RedfishVendor` -- DPU, -/// GBx00, GH200, ... -- collapses to `BMCVendor::Nvidia`), and the precise -/// `RedfishVendor` is not persisted anywhere. Probing is exactly what the switch -/// and power-shelf controllers do and what the engine's `BmcRotationTarget` -/// contract expects. A probe failure falls back to `RedfishVendor::Unknown`, -/// which the engine surfaces as a device-level error and quarantines with -/// backoff -- so an unreachable BMC backs off rather than hot-looping the -/// controller through Ready -> RotatingBmc -> Ready every sweep. -async fn resolve_dispatch_vendor( - services: &MachineStateHandlerServices, - endpoint: &BmcEndpoint, -) -> RedfishVendor { - let Some(credentials) = per_device_bmc_credentials(services, endpoint.device_mac).await else { - return RedfishVendor::Unknown; - }; - match services - .redfish_client_pool - .probe_bmc_vendor(&endpoint.host, endpoint.port, credentials) - .await - { - Ok(vendor) => vendor, - Err(e) => { - tracing::warn!( - mac = %endpoint.device_mac, - error = %e, - "BMC vendor probe failed; rotation engine will quarantine the device" - ); - RedfishVendor::Unknown - } - } -} - -/// Read the current per-device BMC root secret, used only to satisfy the vendor -/// probe's Chassis-fallback authentication. The engine re-reads it under its own -/// crash-safe path; a missing secret here just yields an `Unknown` vendor. -async fn per_device_bmc_credentials( - services: &MachineStateHandlerServices, - mac: MacAddress, -) -> Option { - let key = CredentialKey::BmcCredentials { - credential_type: BmcCredentialType::BmcRoot { - bmc_mac_address: mac, - }, - }; - match services.credential_manager.get_credentials(&key).await { - Ok(credentials) => credentials, - Err(e) => { - tracing::warn!(%mac, error = %e, "failed reading per-device BMC secret for vendor probe"); - None - } - } -} - #[cfg(test)] mod tests { use std::net::IpAddr; + use mac_address::MacAddress; use model::test_support::machine_snapshot::managed_host_state_snapshot; use super::*; diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index 0a84cf7062..e4695decb9 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -28,7 +28,8 @@ use model::machine::slas::MachineSlaConfig; use model::machine::{ self, AttestationMode, DpuDiscoveringState, DpuInitState, HostHealthConfig, MachineMaintenanceOperation, MachineValidatingState, ManagedHostState, - ManagedHostStateSnapshot, MeasuringState, SpdmMeasuringState, ValidationState, + ManagedHostStateSnapshot, MeasuringState, ReadyBootConfigState, SpdmMeasuringState, + ValidationState, }; use sqlx::PgConnection; use state_controller::io::StateControllerIO; @@ -276,6 +277,21 @@ impl StateControllerIO for MachineStateControllerIO { MachineValidatingState::LockAfterBootRepair { .. } => "lockafterbootrepair", } } + + fn ready_boot_config_state_name(state: &ReadyBootConfigState) -> &'static str { + match state { + ReadyBootConfigState::Prepare => "prepare", + ReadyBootConfigState::UnlockHost { .. } => "unlockhost", + ReadyBootConfigState::CheckHostConfig => "checkhostconfig", + ReadyBootConfigState::ConfigureBios { .. } => "configurebios", + ReadyBootConfigState::WaitingForBiosJob { .. } => "waitingforbiosjob", + ReadyBootConfigState::PollingBiosSetup { .. } => "pollingbiossetup", + ReadyBootConfigState::SetBootOrder { .. } => "setbootorder", + ReadyBootConfigState::LockHost { .. } => "lockhost", + ReadyBootConfigState::Failed { .. } => "failed", + } + } + match state { ManagedHostState::DpuDiscoveringState { dpu_states } => { // Min state indicates the least processed DPU. The state machine is blocked @@ -299,6 +315,12 @@ impl StateControllerIO for MachineStateControllerIO { ("hostnotready", machine_state_name(machine_state)) } ManagedHostState::Ready => ("ready", ""), + ManagedHostState::BootConfiguring { + boot_config_state, .. + } => ( + "bootconfiguring", + ready_boot_config_state_name(boot_config_state), + ), ManagedHostState::Maintenance { operation } => { let op = match operation { MachineMaintenanceOperation::PowerOn => "power_on", @@ -376,6 +398,10 @@ impl StateControllerIO for MachineStateControllerIO { { Some(details.cause.metric_label()) } + ManagedHostState::BootConfiguring { + boot_config_state: ReadyBootConfigState::Failed { .. }, + .. + } => Some("boot_config_convergence_failed"), _ => None, } } @@ -394,3 +420,29 @@ impl StateControllerIO for MachineStateControllerIO { ) } } + +#[cfg(test)] +mod tests { + use model::machine_boot_interface::MachineBootInterfaceTarget; + + use super::*; + + #[test] + fn terminal_ready_boot_config_requires_manual_intervention() { + let state = ManagedHostState::BootConfiguring { + desired_version: ConfigVersion::initial(), + desired_boot_interface: MachineBootInterfaceTarget::MacOnly( + "02:00:00:00:00:01".parse().unwrap(), + ), + post_lock_verification_retry_count: 0, + boot_config_state: ReadyBootConfigState::Failed { + failure: "exhausted".to_string(), + }, + }; + + assert_eq!( + ::manual_intervention_reason(&state), + Some("boot_config_convergence_failed"), + ); + } +} diff --git a/crates/machine-controller/src/metrics.rs b/crates/machine-controller/src/metrics.rs index 232854ffd3..4ff6b9a6b2 100644 --- a/crates/machine-controller/src/metrics.rs +++ b/crates/machine-controller/src/metrics.rs @@ -56,6 +56,7 @@ pub struct MachineMetrics { /// - the Machine has not yet been target of an instance creation request /// - no health alerts which classification `PreventAllocations` to be set /// - the machine not to be in Maintenance Mode + /// - the desired boot-interface generation to have a matching observation pub is_usable_as_instance: bool, /// is the host's bios password set pub is_host_bios_password_set: bool, diff --git a/crates/machine-controller/tests/integration/bmc_rotation.rs b/crates/machine-controller/tests/integration/bmc_rotation.rs index f9854ae92a..709d25ba1e 100644 --- a/crates/machine-controller/tests/integration/bmc_rotation.rs +++ b/crates/machine-controller/tests/integration/bmc_rotation.rs @@ -93,7 +93,7 @@ async fn ready_host_converges_bmc_to_site_target( .build() .await .0; - mh.advance_state(ManagedHostState::Ready).await; + mh.advance_to_converged_ready().await; let host_mac = mh .host @@ -236,7 +236,7 @@ async fn feature_flag_off_suppresses_passive_rotation( .build() .await .0; - mh.advance_state(ManagedHostState::Ready).await; + mh.advance_to_converged_ready().await; let host_mac = mh .host @@ -297,7 +297,7 @@ async fn force_request_converges_quarantined_bmc_when_disabled( .build() .await .0; - mh.advance_state(ManagedHostState::Ready).await; + mh.advance_to_converged_ready().await; let host = mh.host.machine().await; let machine_id = host.id; diff --git a/crates/machine-controller/tests/integration/maintenance.rs b/crates/machine-controller/tests/integration/maintenance.rs index f23e6f37aa..04ad21d2bc 100644 --- a/crates/machine-controller/tests/integration/maintenance.rs +++ b/crates/machine-controller/tests/integration/maintenance.rs @@ -193,7 +193,7 @@ async fn create_ready_host( .build() .await .0; - host.advance_state(ManagedHostState::Ready).await; + host.advance_to_converged_ready().await; host } diff --git a/crates/machine-controller/tests/integration/power_management.rs b/crates/machine-controller/tests/integration/power_management.rs index 130e4f01de..963b88e517 100644 --- a/crates/machine-controller/tests/integration/power_management.rs +++ b/crates/machine-controller/tests/integration/power_management.rs @@ -67,7 +67,7 @@ impl TestContext { .build() .await .0; - mh.advance_state(ManagedHostState::Ready).await; + mh.advance_to_converged_ready().await; Self { env, mh } } } diff --git a/crates/power-shelf-controller/Cargo.toml b/crates/power-shelf-controller/Cargo.toml index 99af068df0..9f319c2c9a 100644 --- a/crates/power-shelf-controller/Cargo.toml +++ b/crates/power-shelf-controller/Cargo.toml @@ -25,7 +25,9 @@ repository.workspace = true [dependencies] carbide-api-db = { path = "../api-db", default-features = false } carbide-api-model = { path = "../api-model", default-features = false } +carbide-credential-rotation = { path = "../credential-rotation", default-features = false } carbide-health-metrics = { path = "../health-metrics", default-features = false } +carbide-redfish = { path = "../redfish", default-features = false } carbide-rack = { path = "../rack", default-features = false } carbide-secrets = { path = "../secrets", default-features = false } carbide-utils = { path = "../utils", default-features = false } diff --git a/crates/power-shelf-controller/src/context.rs b/crates/power-shelf-controller/src/context.rs index 779a4707f9..6caa6cd9d9 100644 --- a/crates/power-shelf-controller/src/context.rs +++ b/crates/power-shelf-controller/src/context.rs @@ -17,7 +17,9 @@ use std::sync::Arc; +use carbide_credential_rotation::BmcRotationGate; use carbide_health_metrics::PerObjectMetricsRegistry; +use carbide_redfish::libredfish::RedfishClientPool; use carbide_secrets::credentials::CredentialManager; use component_manager::component_manager::ComponentManager; use sqlx::PgPool; @@ -39,6 +41,17 @@ pub struct PowerShelfStateHandlerServices { /// `false` so power shelves stay out of rack firmware wait unless explicitly /// enabled. pub rack_firmware_reprovisioning_enabled: bool, + /// Libredfish pool used to converge the power shelf BMC (PMC) credential + /// (REQ-2). The same shared instance the machine- and switch-controllers use. + pub redfish_client_pool: Arc, + /// Short-TTL cache of the site-wide BMC rotation aggregate, shared across + /// this replica's per-object ticks so the steady state costs one aggregate + /// query per TTL window rather than a per-device query every sweep. + pub bmc_rotation_gate: BmcRotationGate, + /// Site-wide kill-switch for passive BMC credential rotation. When `false`, + /// a Ready power shelf never enters `RotatingBmc` on its own; the operator + /// force-converge escape hatch still works regardless. + pub bmc_rotation_enabled: bool, } impl StateHandlerContextObjects for PowerShelfStateHandlerContextObjects { diff --git a/crates/power-shelf-controller/src/handler.rs b/crates/power-shelf-controller/src/handler.rs index 49ed324649..23fe31dbed 100644 --- a/crates/power-shelf-controller/src/handler.rs +++ b/crates/power-shelf-controller/src/handler.rs @@ -35,6 +35,7 @@ use crate::initializing::handle_initializing; use crate::maintenance::handle_maintenance; use crate::ready::handle_ready; use crate::reprovisioning::handle_reprovisioning; +use crate::rotating_bmc::handle_rotating_bmc; /// The actual PowerShelf State handler (structure mirrors SwitchStateHandler). #[derive(Debug, Default, Clone)] @@ -80,6 +81,9 @@ impl PowerShelfStateHandler { handle_configuring(power_shelf_id, state, ctx).await } PowerShelfControllerState::Ready => handle_ready(power_shelf_id, state, ctx).await, + PowerShelfControllerState::RotatingBmc { retry_count } => { + handle_rotating_bmc(power_shelf_id, state, *retry_count, ctx).await + } PowerShelfControllerState::Maintenance { .. } => { handle_maintenance(power_shelf_id, state, ctx).await } diff --git a/crates/power-shelf-controller/src/io.rs b/crates/power-shelf-controller/src/io.rs index f1ee2bbc2a..4803d1dd50 100644 --- a/crates/power-shelf-controller/src/io.rs +++ b/crates/power-shelf-controller/src/io.rs @@ -153,6 +153,7 @@ impl StateControllerIO for PowerShelfStateControllerIO { PowerShelfControllerState::FetchingData => ("fetching_data", ""), PowerShelfControllerState::Configuring => ("configuring", ""), PowerShelfControllerState::Ready => ("ready", ""), + PowerShelfControllerState::RotatingBmc { .. } => ("rotatingbmc", ""), PowerShelfControllerState::Maintenance { operation } => { let op = match operation { model::power_shelf::PowerShelfMaintenanceOperation::PowerOn => "power_on", diff --git a/crates/power-shelf-controller/src/lib.rs b/crates/power-shelf-controller/src/lib.rs index d8502c6ab8..6648450e8b 100644 --- a/crates/power-shelf-controller/src/lib.rs +++ b/crates/power-shelf-controller/src/lib.rs @@ -29,3 +29,4 @@ pub mod maintenance; pub mod metrics; pub mod ready; pub mod reprovisioning; +pub mod rotating_bmc; diff --git a/crates/power-shelf-controller/src/ready.rs b/crates/power-shelf-controller/src/ready.rs index 2d242384c0..94c94b024f 100644 --- a/crates/power-shelf-controller/src/ready.rs +++ b/crates/power-shelf-controller/src/ready.rs @@ -29,6 +29,7 @@ use state_controller::state_handler::{ use crate::context::PowerShelfStateHandlerContextObjects; use crate::maintenance::build_power_shelf_endpoint; use crate::reprovisioning::first_reprovisioning_state; +use crate::rotating_bmc::should_enter_bmc_rotation; /// Handles the Ready state for a power shelf. /// @@ -129,6 +130,16 @@ pub async fn handle_ready( )); } + // Lowest precedence: only converge the PMC credential once the power shelf is + // otherwise idle in Ready, so rotation never contends with maintenance or + // reprovisioning. The site-flag gate and the operator force-converge override + // live in `should_enter_bmc_rotation`. + if should_enter_bmc_rotation(ctx.services, state).await? { + return Ok(StateHandlerOutcome::transition( + PowerShelfControllerState::RotatingBmc { retry_count: 0 }, + )); + } + let txn = poll_power_state(power_shelf_id, state, ctx).await; Ok(StateHandlerOutcome::do_nothing().with_txn_opt(txn)) diff --git a/crates/power-shelf-controller/src/rotating_bmc.rs b/crates/power-shelf-controller/src/rotating_bmc.rs new file mode 100644 index 0000000000..60e4dc0922 --- /dev/null +++ b/crates/power-shelf-controller/src/rotating_bmc.rs @@ -0,0 +1,184 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Power-shelf-controller BMC (PMC) credential rotation (REQ-2). +//! +//! The shared [`carbide_credential_rotation`] engine owns the password dance, +//! backoff, and crash-safety; this module is the thin power-shelf-controller +//! adapter, mirroring the switch-controller's `rotating_bmc.rs`. A power shelf +//! has exactly one BMC (its PMC), so there is no per-device fan-out: one +//! endpoint, one [`rotate_bmc`] call per tick. +//! +//! - *Should we enter rotation?* [`should_enter_bmc_rotation`] honors an operator +//! force-converge request first (even when the site-wide flag is off), then the +//! passive gate (`bmc_rotation_enabled` && the cached [`BmcRotationGate`] says +//! the PMC lags the staged target). +//! - *Do one rotation tick.* [`handle_rotating_bmc`] converges the PMC via +//! [`rotate_bmc`], folds the outcome through the shared state-neutral +//! [`advance`] retry seam, and returns to `Ready` (clearing a satisfied force +//! request) or re-enters `RotatingBmc` with an incremented retry budget. +//! +//! Unlike the switch path (all NVIDIA MGX, so a fixed dispatch vendor), a power +//! shelf's Lite-On/Delta PMC vendor is not persisted and is not exposed in the +//! Redfish service root, so the engine probes it at rotation time +//! ([`BmcEndpoint::into_target_probing_vendor`]) -- the repo's standard +//! power-shelf vendor determination. +//! +//! A BMC password change never touches the power shelf's power delivery, so this +//! is safe in `Ready`. + +use carbide_credential_rotation::{ + BmcEndpoint, BmcRotationTick, RotateOutcome, RotationStep, advance, rotate_bmc, +}; +use carbide_uuid::power_shelf::PowerShelfId; +use model::power_shelf::{PowerShelf, PowerShelfControllerState}; +use state_controller::state_handler::{ + StateHandlerContext, StateHandlerError, StateHandlerOutcome, +}; + +use crate::context::{PowerShelfStateHandlerContextObjects, PowerShelfStateHandlerServices}; + +/// Whether a Ready power shelf should enter `PowerShelfControllerState::RotatingBmc` +/// now. +/// +/// An operator force-converge request always wins -- the ops escape hatch is +/// honored even when the site-wide flag is off. Otherwise the passive gate fires +/// only when BMC rotation is enabled site-wide *and* the PMC lags the staged +/// target; the flag is checked first so a disabled site never runs the per-device +/// gate query. +pub async fn should_enter_bmc_rotation( + services: &PowerShelfStateHandlerServices, + power_shelf: &PowerShelf, +) -> Result { + if power_shelf.bmc_credential_rotation_requested { + return Ok(true); + } + if !services.bmc_rotation_enabled { + return Ok(false); + } + let Some(endpoint) = BmcEndpoint::from_power_shelf(power_shelf) else { + return Ok(false); + }; + services + .bmc_rotation_gate + .bmc_rotation_needed(&services.db_pool, endpoint.device_mac) + .await + .map_err(|e| StateHandlerError::GenericError(eyre::eyre!("bmc rotation gate query: {e}"))) +} + +/// Run one rotation tick over the power shelf's PMC and decide the next state. +/// +/// `force` (an operator escape-hatch request on this power shelf row) bypasses +/// the device's backoff quarantine inside [`rotate_bmc`]. A settled tick returns +/// to `Ready` and clears a satisfied force request; a transient bookkeeping +/// failure re-enters `RotatingBmc` bounded by the shared retry budget; an +/// exhausted budget (`GaveUp`) returns to `Ready` but leaves a pending force +/// request set so the entry guard re-attempts on a later sweep. +pub async fn handle_rotating_bmc( + power_shelf_id: &PowerShelfId, + power_shelf: &PowerShelf, + retry_count: u32, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, +) -> Result, StateHandlerError> { + let force = power_shelf.bmc_credential_rotation_requested; + + let tick = match BmcEndpoint::from_power_shelf(power_shelf) { + Some(endpoint) => rotate_power_shelf_bmc(ctx.services, endpoint, force).await, + // A forced request announces a missing PMC so its one-shot flag still + // clears; a passive sweep silently settles an unaddressable power shelf + // (the entry guard never selects it). + None => { + if force { + tracing::warn!( + %power_shelf_id, + "force-converge request on a power shelf with no addressable BMC; clearing the request without action" + ); + } + BmcRotationTick::Settled + } + }; + + match advance(tick, retry_count, power_shelf_id) { + step @ (RotationStep::Settled | RotationStep::GaveUp) => { + // Only a settled tick clears a one-shot force request: the forced + // attempt genuinely fired. GaveUp exhausted the transient-retry + // budget without the forced attempt cleanly running, so leave the + // flag set and let the entry guard re-attempt on a later sweep rather + // than silently drop the operator's request. + let txn = if force && matches!(step, RotationStep::Settled) { + let mut t = ctx.services.db_pool.begin().await?; + db::power_shelf::clear_bmc_credential_rotation_requested(&mut t, *power_shelf_id) + .await?; + Some(t) + } else { + None + }; + Ok(StateHandlerOutcome::transition(PowerShelfControllerState::Ready).with_txn_opt(txn)) + } + RotationStep::Retry { retry_count } => Ok(StateHandlerOutcome::transition( + PowerShelfControllerState::RotatingBmc { retry_count }, + )), + } +} + +/// Rotate the power shelf's PMC toward the staged target. `force` bypasses the +/// device's backoff quarantine (operator escape hatch). Returns +/// [`BmcRotationTick::Retry`] only on a transient bookkeeping error; device +/// faults are quarantined inside [`rotate_bmc`] and reported as `Settled`. +async fn rotate_power_shelf_bmc( + services: &PowerShelfStateHandlerServices, + endpoint: BmcEndpoint, + force: bool, +) -> BmcRotationTick { + // A power shelf's Lite-On/Delta PMC vendor is neither persisted nor exposed + // in the Redfish service root, so the engine resolves it at rotation time by + // probing the Chassis manufacturer (the repo's standard power-shelf vendor + // determination). The probe runs inside the engine's quarantine-on-failure + // envelope, so a failed probe records backoff rather than looping the guard. + let target = endpoint.into_target_probing_vendor(); + match rotate_bmc( + &services.db_pool, + services.credential_manager.as_ref(), + services.redfish_client_pool.as_ref(), + &target, + force, + ) + .await + { + Ok(RotateOutcome::Converged) => { + tracing::info!(mac = %target.device_mac, force, "power shelf BMC (PMC) converged to site-wide rotation target"); + BmcRotationTick::Settled + } + Ok(RotateOutcome::Quarantined { until }) => { + tracing::warn!( + mac = %target.device_mac, + %until, + "power shelf BMC (PMC) rotation attempt failed; quarantined until backoff elapses" + ); + BmcRotationTick::Settled + } + Ok(RotateOutcome::NoWork) => BmcRotationTick::Settled, + Err(e) => { + tracing::warn!( + mac = %target.device_mac, + error = %e, + "transient power shelf BMC (PMC) rotation bookkeeping failure; will retry the tick" + ); + BmcRotationTick::Retry + } + } +} diff --git a/crates/redfish/src/libredfish/test_support.rs b/crates/redfish/src/libredfish/test_support.rs index dc744e76da..e6a212921e 100644 --- a/crates/redfish/src/libredfish/test_support.rs +++ b/crates/redfish/src/libredfish/test_support.rs @@ -58,6 +58,10 @@ struct RedfishSimState { machine_setup_bios_job_id: Option, is_bios_setup: Option, default_lockdown: Option, + /// Override whether `lockdown_bmc` changes the observed state. `None` + /// preserves the normal successful behavior; `Some(false)` models a BMC + /// accepting the write without applying the requested policy. + lockdown_bmc_applies: Option, job_state_sequence: VecDeque, /// Offset (in seconds) applied to the BMC `DateTime` returned by /// `get_manager`, relative to the controller's `Utc::now()`. Defaults to 0 @@ -309,6 +313,11 @@ impl RedfishSim { } } + /// Control whether `lockdown_bmc` updates the observed lockdown state. + pub fn set_lockdown_bmc_applies(&self, applies: bool) { + self.state.lock().unwrap().lockdown_bmc_applies = Some(applies); + } + /// Set the offset (in seconds) applied to the BMC `DateTime` returned by /// `get_manager`, relative to the controller clock. Use a value larger than /// the time-sync threshold to simulate an out-of-sync BMC clock. @@ -1667,8 +1676,10 @@ impl Redfish for RedfishSimClient { ) -> libredfish::RedfishFuture<'a, Result<(), RedfishError>> { Box::pin(async move { let mut state = self.state.lock().unwrap(); - let host_state = state.hosts.get_mut(&self._host).unwrap(); - host_state.lockdown = target; + if state.lockdown_bmc_applies.unwrap_or(true) { + let host_state = state.hosts.get_mut(&self._host).unwrap(); + host_state.lockdown = target; + } Ok(()) }) } diff --git a/crates/rpc/build.rs b/crates/rpc/build.rs index e1aecde03a..41fd43cf05 100644 --- a/crates/rpc/build.rs +++ b/crates/rpc/build.rs @@ -463,6 +463,19 @@ fn main() -> Result<(), Box> { .type_attribute("forge.DpaInterfaceList", "#[derive(serde::Serialize)]") .type_attribute("forge.Vpc", "#[derive(serde::Serialize)]") .type_attribute("forge.VpcConfig", "#[derive(serde::Serialize)]") + .type_attribute( + "forge.VpcRoutingProfileOverrides", + "#[derive(serde::Serialize)]", + ) + .type_attribute( + "forge.VpcEffectiveRoutingProfile", + "#[derive(serde::Serialize)]", + ) + .type_attribute("common.RouteTargets", "#[derive(serde::Serialize)]") + .type_attribute( + "forge.PrefixFilterPolicyEntries", + "#[derive(serde::Serialize)]", + ) .type_attribute("forge.VpcStatus", "#[derive(serde::Serialize)]") .type_attribute("forge.VpcList", "#[derive(serde::Serialize)]") .type_attribute( diff --git a/crates/rpc/proto/common.proto b/crates/rpc/proto/common.proto index 841ff6f72a..1112f4b14d 100644 --- a/crates/rpc/proto/common.proto +++ b/crates/rpc/proto/common.proto @@ -46,6 +46,12 @@ message RouteTarget { uint32 vni = 2; } +// A list of route targets. Unlike a repeated field, this message can be optional, +// allowing containing messages to distinguish an omitted list from an empty list. +message RouteTargets { + repeated RouteTarget values = 1; +} + // Placeholder ID types: These are not code-generated, but overridden by our own type-safe UUID wrappers. They are // identical to common.UUID over the wire, for backwards compatibility. message DomainId { diff --git a/crates/rpc/proto/forge.proto b/crates/rpc/proto/forge.proto index 0c41c74f7f..1b2ae8f642 100644 --- a/crates/rpc/proto/forge.proto +++ b/crates/rpc/proto/forge.proto @@ -1645,6 +1645,40 @@ message TenantSearchQuery { optional string tenantOrganizationId = 1; // protolint:disable:this FIELD_NAMES_LOWER_SNAKE_CASE } +// A presence-aware list of prefix-filter entries. +message PrefixFilterPolicyEntries { + repeated PrefixFilterPolicyEntry values = 1; +} + +// Routing-profile properties set directly on a VPC. Present properties +// override its named routing profile; omitted properties inherit from it. +// `internal` and `access_tier` are intentionally not exposed because allocation +// and access controls must always come from the named base profile. +message VpcRoutingProfileOverrides { + optional common.RouteTargets route_target_imports = 1; + optional common.RouteTargets route_targets_on_exports = 2; + optional bool leak_default_route_from_underlay = 3; + optional bool leak_tenant_host_routes_to_underlay = 4; + optional bool tenant_leak_communities_accepted = 5; + optional PrefixFilterPolicyEntries accepted_leaks_from_underlay = 6; + optional PrefixFilterPolicyEntries allowed_anycast_prefixes = 7; +} + +// The fully resolved routing profile currently applied to an FNN VPC. +// Unlike VpcRoutingProfileOverrides, this includes the protected properties +// inherited from the named base profile and has no presence-aware fields. +message VpcEffectiveRoutingProfile { + repeated common.RouteTarget route_target_imports = 1; + repeated common.RouteTarget route_targets_on_exports = 2; + bool leak_default_route_from_underlay = 3; + bool leak_tenant_host_routes_to_underlay = 4; + bool tenant_leak_communities_accepted = 5; + repeated PrefixFilterPolicyEntry accepted_leaks_from_underlay = 6; + repeated PrefixFilterPolicyEntry allowed_anycast_prefixes = 7; + bool internal = 8; + uint32 access_tier = 9; +} + message VpcConfig { string tenant_organization_id = 1; optional string tenant_keyset_id = 2; @@ -1654,6 +1688,7 @@ message VpcConfig { // Desired VNI for this VPC. Only populated when explicitly requested during creation. optional uint32 vni = 6; optional string routing_profile_type = 7; + optional VpcRoutingProfileOverrides routing_profile_overrides = 8; } message VpcStatus { @@ -1661,6 +1696,10 @@ message VpcStatus { // explicitly requested, we'd expect this to match config.vni. // If not explicitly requested, we'd expect config.vni to be unset. optional uint32 vni = 1; + // Computed from the API server's current named profile and the persisted + // VPC overrides. This value is not persisted and may change when the API + // server configuration changes. + optional VpcEffectiveRoutingProfile effective_routing_profile = 2; } message Vpc { @@ -1749,6 +1788,10 @@ message VpcCreationRequest { // For example, an internal tenant can request internal or external VPCs, // but an external tenant can only request external VPCs. optional string routing_profile_type = 17; + + // Properties to overlay on the selected named routing profile. Properties + // left unset inherit from the base profile. + optional VpcRoutingProfileOverrides routing_profile_overrides = 18; } enum VpcVirtualizationType { @@ -5898,15 +5941,15 @@ message BmcCredentialRotationRequest { } Mode mode = 1; - // ID of the device that owns the target BMC. Machine and switch IDs are - // supported. Power shelf IDs are valid DeviceIds and are accepted for forward - // compatibility, but power shelf BMC rotation is not yet supported. + // ID of the device that owns the target BMC: a machine (host or DPU BMC), a + // switch (switch BMC), or a power shelf (its PMC). Each such device has + // exactly one BMC, so any one id uniquely names the device. common.DeviceId device_id = 3; - // MAC of the BMC to target, resolved server-side to its owning machine or - // switch. May be given alone, or alongside a `device_id` as a cross-check - // (they must identify the same device). At least one of `device_id` / - // `bmc_mac` must be set (enforced server-side). + // MAC of the BMC to target, resolved server-side to its owning machine, + // switch, or power shelf. May be given alone, or alongside a `device_id` as a + // cross-check (they must identify the same device). At least one of + // `device_id` / `bmc_mac` must be set (enforced server-side). optional string bmc_mac = 4; } diff --git a/crates/rpc/src/model/power_shelf.rs b/crates/rpc/src/model/power_shelf.rs index 229f8ddbe7..a463a2a805 100644 --- a/crates/rpc/src/model/power_shelf.rs +++ b/crates/rpc/src/model/power_shelf.rs @@ -183,6 +183,7 @@ mod tests { }, controller_state_outcome: None, bmc_mac_address: None, + bmc_credential_rotation_requested: false, bmc_info: None, rack_id: None, power_shelf_maintenance_requested: None, diff --git a/crates/rpc/src/model/vpc.rs b/crates/rpc/src/model/vpc.rs index 4008a0f7b2..47b7075f1c 100644 --- a/crates/rpc/src/model/vpc.rs +++ b/crates/rpc/src/model/vpc.rs @@ -21,7 +21,8 @@ use carbide_uuid::network_security_group::NetworkSecurityGroupIdParseError; use config_version::ConfigVersion; use model::metadata::{LabelFilter, Metadata}; use model::vpc::{ - NewVpc, UpdateVpc, UpdateVpcVirtualization, Vpc, VpcPeering, VpcSearchFilter, VpcStatus, + NewVpc, PrefixFilterPolicyEntry, RouteTargetConfig, UpdateVpc, UpdateVpcVirtualization, Vpc, + VpcPeering, VpcRoutingProfileOverrides, VpcSearchFilter, VpcStatus, }; use crate as rpc; @@ -48,6 +49,7 @@ impl From for rpc::forge::Vpc { .config .network_security_group_id .map(|nsg_id| nsg_id.to_string()); + let routing_profile_overrides = src.config.routing_profile_overrides.map(Into::into); let metadata = Some(rpc::Metadata { name: src.metadata.name, description: src.metadata.description, @@ -78,6 +80,7 @@ impl From for rpc::forge::Vpc { default_nvlink_logical_partition_id: src.config.default_nvlink_logical_partition_id, vni: desired_vni, routing_profile_type: src.config.routing_profile_type.clone(), + routing_profile_overrides, }), status: Some(rpc::forge::VpcStatus::from(src.status)), @@ -100,6 +103,120 @@ impl From for rpc::forge::VpcStatus { rpc::forge::VpcStatus { // This is the pattern we have elsewhere because a VNI should never be negative. vni: src.vni.map(|x| x as u32), + // The API handler resolves this from the current runtime config. + effective_routing_profile: None, + } + } +} + +impl TryFrom for VpcRoutingProfileOverrides { + type Error = RpcDataConversionError; + + fn try_from(profile: rpc::forge::VpcRoutingProfileOverrides) -> Result { + Ok(Self { + route_target_imports: profile.route_target_imports.map(|targets| { + targets + .values + .into_iter() + .map(|target| RouteTargetConfig { + asn: target.asn, + vni: target.vni, + }) + .collect() + }), + route_targets_on_exports: profile.route_targets_on_exports.map(|targets| { + targets + .values + .into_iter() + .map(|target| RouteTargetConfig { + asn: target.asn, + vni: target.vni, + }) + .collect() + }), + leak_default_route_from_underlay: profile.leak_default_route_from_underlay, + leak_tenant_host_routes_to_underlay: profile.leak_tenant_host_routes_to_underlay, + tenant_leak_communities_accepted: profile.tenant_leak_communities_accepted, + accepted_leaks_from_underlay: profile + .accepted_leaks_from_underlay + .map(|entries| { + entries + .values + .into_iter() + .map(|entry| { + Ok(PrefixFilterPolicyEntry { + prefix: entry.prefix.parse()?, + }) + }) + .collect::, RpcDataConversionError>>() + }) + .transpose()?, + allowed_anycast_prefixes: profile + .allowed_anycast_prefixes + .map(|entries| { + entries + .values + .into_iter() + .map(|entry| { + Ok(PrefixFilterPolicyEntry { + prefix: entry.prefix.parse()?, + }) + }) + .collect::, RpcDataConversionError>>() + }) + .transpose()?, + }) + } +} + +impl From for rpc::forge::VpcRoutingProfileOverrides { + fn from(profile: VpcRoutingProfileOverrides) -> Self { + Self { + route_target_imports: profile.route_target_imports.map(|targets| { + rpc::common::RouteTargets { + values: targets + .into_iter() + .map(|target| rpc::common::RouteTarget { + asn: target.asn, + vni: target.vni, + }) + .collect(), + } + }), + route_targets_on_exports: profile.route_targets_on_exports.map(|targets| { + rpc::common::RouteTargets { + values: targets + .into_iter() + .map(|target| rpc::common::RouteTarget { + asn: target.asn, + vni: target.vni, + }) + .collect(), + } + }), + leak_default_route_from_underlay: profile.leak_default_route_from_underlay, + leak_tenant_host_routes_to_underlay: profile.leak_tenant_host_routes_to_underlay, + tenant_leak_communities_accepted: profile.tenant_leak_communities_accepted, + accepted_leaks_from_underlay: profile.accepted_leaks_from_underlay.map(|entries| { + rpc::forge::PrefixFilterPolicyEntries { + values: entries + .into_iter() + .map(|entry| rpc::forge::PrefixFilterPolicyEntry { + prefix: entry.prefix.to_string(), + }) + .collect(), + } + }), + allowed_anycast_prefixes: profile.allowed_anycast_prefixes.map(|entries| { + rpc::forge::PrefixFilterPolicyEntries { + values: entries + .into_iter() + .map(|entry| rpc::forge::PrefixFilterPolicyEntry { + prefix: entry.prefix.to_string(), + }) + .collect(), + } + }), } } } @@ -145,6 +262,10 @@ impl TryFrom for NewVpc { RpcDataConversionError::InvalidNetworkSecurityGroupId(e.value()) })?, routing_profile_type: None, + routing_profile_overrides: value + .routing_profile_overrides + .map(TryInto::try_into) + .transpose()?, network_virtualization_type: virt_type, metadata, }) @@ -267,6 +388,7 @@ mod tests { default_nvlink_logical_partition_id: None, vni: Some(42), routing_profile_type: Some("EXTERNAL".to_string()), + routing_profile_overrides: None, }, status: VpcStatus { vni: Some(100) }, metadata: Metadata::new_with_default_name(), diff --git a/crates/test-harness/src/managed_host.rs b/crates/test-harness/src/managed_host.rs index baa15fb32f..eddc3149d9 100644 --- a/crates/test-harness/src/managed_host.rs +++ b/crates/test-harness/src/managed_host.rs @@ -22,6 +22,7 @@ use carbide_api_core::test_support::Api; use carbide_api_core::test_support::fixture_config::FixtureDefault as _; use carbide_site_explorer::test_support::TestSiteExplorer; use carbide_uuid::machine::MachineId; +use chrono::Utc; use mac_address::MacAddress; use model::expected_machine::{ExpectedMachine, ExpectedMachineData}; use model::hardware_info::HardwareInfo; @@ -90,6 +91,45 @@ impl TestManagedHost { .expect("database transaction should commit"); } + /// Advances this fixture to a stable Ready state after recording its + /// current desired boot interface as verified. + /// + /// This models the invariant established by the machine controller before + /// it returns a host to Ready. Use [`Self::advance_state`] instead when a + /// test intentionally needs a Ready host with pending boot configuration. + pub async fn advance_to_converged_ready(&self) { + let mut txn = self + .api + .database_connection + .begin() + .await + .expect("database transaction should start"); + let machine = self.host.db_machine(&mut txn).await; + let desired_version = machine + .config + .desired_boot_interface + .as_ref() + .expect("test host should have a desired boot interface") + .version; + assert!( + db::machine_desired_boot_interface::mark_verified( + txn.as_mut(), + &self.host.id, + desired_version, + Utc::now(), + ) + .await + .expect("boot-interface verification should be recorded"), + "test host's desired boot interface should still be current" + ); + machine + .advance_state(&mut txn, ManagedHostState::Ready) + .await; + txn.commit() + .await + .expect("database transaction should commit"); + } + pub async fn report_dpu_network_status(&self) { for dpu in &self.dpus { dpu.record_network_status().await; diff --git a/crates/trace-propagation/src/lib.rs b/crates/trace-propagation/src/lib.rs index 09bdf041fd..eee2f7aade 100644 --- a/crates/trace-propagation/src/lib.rs +++ b/crates/trace-propagation/src/lib.rs @@ -33,6 +33,9 @@ //! - normally, use the off-the-shelf `reqwest-tracing` middleware (a separate crate). //! - if you only hold the already-built request (so it can't go through a middleware client), //! call [`inject_current_context`] on its header map. +//! - **Proxying** — a service that copies inbound headers onto an outbound request has to drop the +//! propagator's own headers first, or the upstream parents under the caller instead of the proxy; +//! [`is_propagated_header`] identifies them without assuming which propagator is installed. //! //! [`extract_context`] (read) and [`inject_context`] (write) are the underlying primitives, //! exposed for direct use when the helpers above don't fit. @@ -72,6 +75,26 @@ pub fn set_span_parent_from_headers(span: &tracing::Span, headers: &http::Header } } +// --- Proxying --- + +/// Whether `name` is a header owned by the globally configured text-map propagator. +/// +/// A proxy that copies inbound headers onto its upstream request has to skip these: egress injects +/// the local hop's context, and a copied-through header would either be overwritten or survive and +/// parent the upstream under the original caller instead of the proxy. Asking the propagator for its +/// own field set keeps that filter correct for whichever propagator is installed, rather than +/// assuming W3C's `traceparent`/`tracestate`. +/// +/// Always false when no propagator is installed, since the default no-op propagator claims no +/// headers. +pub fn is_propagated_header(name: &str) -> bool { + global::get_text_map_propagator(|propagator| { + propagator + .fields() + .any(|field| field.eq_ignore_ascii_case(name)) + }) +} + // --- Egress --- /// Inject `cx` into outbound request headers using the globally configured text-map propagator. @@ -216,6 +239,25 @@ mod tests { }); } + // --- Proxying --- + + #[test] + fn propagated_headers_are_the_installed_propagator_fields() { + install_w3c_propagator(); + + // The W3C propagator's own two headers. Header names arrive in whatever case the caller + // sent, while the propagator advertises them lowercase, so matching ignores case. + assert!(is_propagated_header("traceparent")); + assert!(is_propagated_header("tracestate")); + assert!(is_propagated_header("TraceParent")); + + // Everything else is the proxied request's own payload and has to keep flowing. + assert!(!is_propagated_header("content-type")); + assert!(!is_propagated_header("x-request-id")); + // A near-miss, to pin that this is an exact field match and not a prefix test. + assert!(!is_propagated_header("trace")); + } + // --- Egress --- #[test] diff --git a/crates/trace-propagation/tests/no_global_propagator.rs b/crates/trace-propagation/tests/no_global_propagator.rs index 005555be0a..43ed68245a 100644 --- a/crates/trace-propagation/tests/no_global_propagator.rs +++ b/crates/trace-propagation/tests/no_global_propagator.rs @@ -20,7 +20,7 @@ use opentelemetry::global; use tower::{Service, ServiceExt}; use trace_propagation::{ TraceInjectService, extract_context, inject_context, inject_current_context, - set_span_parent_from_headers, + is_propagated_header, set_span_parent_from_headers, }; const SAMPLE_TRACEPARENT: &str = "00-1111111111111111aaaaaaaaaaaaaaaa-2222222222222222-01"; @@ -63,6 +63,18 @@ fn ingress_and_egress_are_noops_without_a_global_propagator() { assert_no_global_propagator(); } +#[test] +fn no_header_is_propagator_owned_without_a_global_propagator() { + assert_no_global_propagator(); + + // A process that propagates nothing owns no hop, so a proxy built on this has nothing to strip + // and forwards trace headers as the opaque payload they are to it. + assert!(!is_propagated_header("traceparent")); + assert!(!is_propagated_header("tracestate")); + + assert_no_global_propagator(); +} + #[tokio::test] async fn trace_inject_service_forwards_unchanged_without_a_global_propagator() { assert_no_global_propagator(); diff --git a/crates/trace-propagation/tests/non_w3c_propagator.rs b/crates/trace-propagation/tests/non_w3c_propagator.rs new file mode 100644 index 0000000000..75c2a89053 --- /dev/null +++ b/crates/trace-propagation/tests/non_w3c_propagator.rs @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! Verifies [`trace_propagation::is_propagated_header`] reports whatever the *installed* propagator +//! advertises rather than a hardcoded W3C field list, so a proxy built on it keeps filtering +//! correctly if a deployment ever configures a different format (B3, Jaeger, a composite, ...). +//! +//! This is its own test binary (separate process) because it installs a non-W3C global propagator, +//! which would otherwise leak into every other test in the crate. + +use opentelemetry::propagation::text_map_propagator::FieldIter; +use opentelemetry::propagation::{Extractor, Injector, TextMapPropagator}; +use opentelemetry::{Context, global}; +use trace_propagation::is_propagated_header; + +/// Carries its context in one non-W3C header, standing in for any alternative format. Only +/// [`TextMapPropagator::fields`] matters here, so inject/extract are left inert. +#[derive(Debug)] +struct SingleHeaderPropagator { + fields: Vec, +} + +impl SingleHeaderPropagator { + const HEADER: &'static str = "x-trace-id"; + + fn new() -> Self { + Self { + fields: vec![Self::HEADER.to_string()], + } + } +} + +impl TextMapPropagator for SingleHeaderPropagator { + fn inject_context(&self, _cx: &Context, _injector: &mut dyn Injector) {} + + fn extract_with_context(&self, cx: &Context, _extractor: &dyn Extractor) -> Context { + cx.clone() + } + + fn fields(&self) -> FieldIter<'_> { + FieldIter::new(&self.fields) + } +} + +#[test] +fn propagated_headers_follow_a_non_w3c_propagator() { + global::set_text_map_propagator(SingleHeaderPropagator::new()); + + assert!(is_propagated_header(SingleHeaderPropagator::HEADER)); + + // W3C's headers belong to no installed propagator here, so a proxy treats them as ordinary + // request payload. Hardcoding `traceparent`/`tracestate` would get both of these wrong. + assert!(!is_propagated_header("traceparent")); + assert!(!is_propagated_header("tracestate")); +} diff --git a/crates/uuid/src/device/mod.rs b/crates/uuid/src/device/mod.rs index c019f5ee02..5e0ac5a861 100644 --- a/crates/uuid/src/device/mod.rs +++ b/crates/uuid/src/device/mod.rs @@ -233,6 +233,36 @@ impl DeviceId { #[allow(deprecated)] Self::Machine(MachineId::default()) } + + /// The kind of device this id names, independent of the id value itself. + /// Useful for kind-labeled messages, since [`DeviceId`]'s own [`Display`] + /// prints only the bare id without its kind. + pub fn kind(&self) -> DeviceKind { + match self { + DeviceId::Machine(_) => DeviceKind::Machine, + DeviceId::Switch(_) => DeviceKind::Switch, + DeviceId::PowerShelf(_) => DeviceKind::PowerShelf, + } + } +} + +/// The kind of a [`DeviceId`], without its id value. Its [`Display`] renders a +/// human-readable label (e.g. for error messages). +#[derive(Copy, Clone, Eq, Hash, PartialEq, Ord, PartialOrd, Debug)] +pub enum DeviceKind { + Machine, + Switch, + PowerShelf, +} + +impl Display for DeviceKind { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + DeviceKind::Machine => write!(f, "machine"), + DeviceKind::Switch => write!(f, "switch"), + DeviceKind::PowerShelf => write!(f, "power-shelf"), + } + } } impl From for DeviceId { diff --git a/docs/configuration/component-manager-rms.md b/docs/configuration/component-manager-rms.md index c7ce9a647c..651a4d999e 100644 --- a/docs/configuration/component-manager-rms.md +++ b/docs/configuration/component-manager-rms.md @@ -1,4 +1,4 @@ -# Component Manager RMS Backends (Day 1) +# Component Manager RMS Backends (Day 1) v2.0 New Operator guide for configuring **Rack Manager Service (RMS)** backends in the `[component_manager]` section of `nico-api` site config, and the **rack profile** diff --git a/docs/configuration/dns.md b/docs/configuration/dns.md index aa0a0522c2..1319af3e0c 100644 --- a/docs/configuration/dns.md +++ b/docs/configuration/dns.md @@ -1,4 +1,4 @@ -# DNS +# DNS v2.0 New NICo answers DNS for everything it manages. Records are never authored by hand: they derive from the machine, BMC, and instance inventory in the `nico-api` database, appear when an interface or instance gains an address, and disappear when it loses one. This page covers the names NICo serves, how the site zone and per-segment subdomains are configured, and how reverse (PTR) resolution works. For the deployment side - the `nico-dns` service, the recursive resolver in front of it, and the fixed infrastructure service names - refer to [IP and Network Configuration](../provisioning/ip-and-network-configuration.md#3-dns-configuration). diff --git a/docs/configuration/machine_identity.md b/docs/configuration/machine_identity.md index 5855fae0d6..6f0507268e 100644 --- a/docs/configuration/machine_identity.md +++ b/docs/configuration/machine_identity.md @@ -1,4 +1,4 @@ -# Machine Identity (Day 1) +# Machine Identity (Day 1) v2.0 New Operator guide for per-organization **machine identity** configuration: JWT-SVID issuance for tenant workloads, optional RFC 8693 token delegation, discovery endpoints, verification, and signing-key rotation. @@ -118,12 +118,12 @@ When token delegation is configured, NICo issues a short-lived **intermediate** **Endpoint:** `PUT /v2/org/{org}/nico/site/{siteID}/tenant-identity/token-delegation` -> **Recommendation:** Token delegation causes `nico-api` to call the org-configured `tokenEndpoint` over HTTP(S). For external token exchange URLs, configure site-level egress controls in `[machine_identity]` during [Day 0](../getting-started/installation-options/day0-machine-identity.md): -> -> - `token_endpoint_http_proxy` — route outbound token-exchange HTTP through a controlled egress proxy -> - `token_endpoint_domain_allowlist` — restrict which hostnames tenants may register on `tokenEndpoint` -> -> Together these mitigate SSRF-style risk if a tenant admin supplies an endpoint the API should not reach. They are optional at install time but **strongly recommended** for production sites that delegate to external hosts. +**Recommendation:** Token delegation causes `nico-api` to call the org-configured `tokenEndpoint` over HTTP(S). For external token exchange URLs, configure site-level egress controls in `[machine_identity]` during [Day 0](../getting-started/installation-options/day0-machine-identity.md): + +- `token_endpoint_http_proxy` — route outbound token-exchange HTTP through a controlled egress proxy +- `token_endpoint_domain_allowlist` — restrict which hostnames tenants may register on `tokenEndpoint` + +Together these mitigate SSRF-style risk if a tenant admin supplies an endpoint the API should not reach. They are optional at install time but **strongly recommended** for production sites that delegate to external hosts. Example (adjust fields to match your STS): @@ -142,9 +142,11 @@ curl -sS -X PUT \ }' ``` -> **PUT is full replace:** omitting `clientSecretBasic` on an update clears stored credentials. Re-supply secrets on every update that should keep basic auth. +**PUT is full replace:** omitting `clientSecretBasic` on an update clears stored credentials. Re-supply secrets on every update that should keep basic auth. -> **Note:** `tokenEndpoint` may use `http://` with an IP address (for example a node-local sidecar). NICo allows this for in-instance STS; use allowlists and network policy in production where appropriate. + +`tokenEndpoint` may use `http://` with an IP address (for example a node-local sidecar). NICo allows this for in-instance STS; use allowlists and network policy in production where appropriate. + ```bash # Remove delegation (return to direct signing) diff --git a/docs/configuration/secrets-storage.md b/docs/configuration/secrets-storage.md index 34e489ffb1..1419af2ffd 100644 --- a/docs/configuration/secrets-storage.md +++ b/docs/configuration/secrets-storage.md @@ -1,4 +1,4 @@ -# Secrets Storage +# Secrets Storage v2.0 New NICo keeps the credentials it manages (BMC logins, switch and UFM accounts, factory defaults, and so on) in a credentials store. Vault is the historical and default backend. NICo can also store credentials in Postgres, encrypted per credential with envelope encryption, and can read both backends side by side; that combination is what makes a gradual, reversible migration off Vault possible. diff --git a/docs/hcl.md b/docs/hcl.md index f698a52b92..6749f069f5 100644 --- a/docs/hcl.md +++ b/docs/hcl.md @@ -8,59 +8,59 @@ inquiries or technical specifications, please contact the original hardware vend **Last Updated: 2/20/2026** -| Host Machine | BMC/Management Firmware Version | BIOS/UEFI Version | Misc. Firmware Version (FPGA, CPLD, LXPM, etc.) | -|-------------------------------------|-------------------------------------|---------------------|-------------------------------------------------| -| GB200 NVL - Wiwynn | 25.06-2_NV_WW_02 | 1.3.2GA | 1.3.2GA | -| NVSwitch Tray - Wiwynn | 1.3.2GA | 1.3.2GA | 1.3.2GA | -| GB200 Compute Tray (1RU) | 1.3.2GA | 1.3.2GA | 1.3.2GA | -| NVSwitch Tray DGX | 1.3.2GA | 1.3.2GA | 1.3.2GA | -| DGX H100 | 25.06.27 (DGXH100_H200_25.06.4 pkg) | 1.06.07 (DGXH100_H200_25.06.4 pkg) | | -| Lenovo ThinkSystem SR670 V2 | 6.10 | 3.30 | 3.31.01 | -| Lenovo ThinkSystem SR675 V3 | 14.10 | 8.30 | 4.20.03 | -| Lenovo ThinkSystem SR675 V3 OVX* | 14.10 | 8.30 | 4.20.03 | -| Lenovo ThinkSystem SR650 | 10.40 | 4.30 | 2.13 | -| Lenovo ThinkSystem SR650 V3 | 6.92 | 3.70 | 4.21.01 | -| Lenovo ThinkSystem SR650 V2 | 5.70 | 3.60 | 3.31.01 | -| Lenovo ThinkSystem SR650 V2 OVX* | 5.70 | 3.60 | 3.31.01 | -| Lenovo ThinkSystem SR655 V3 | 5.80 | 5.70 | 4.20.03 | -| Lenovo ThinkSystem SR655 V3 OVX* | 5.80 | 5.70 | 4.20.03 | -| Lenovo ThinkSystem SR665 V3 OVX* | 5.80 | 5.70 | 4.20.03 | -| Lenovo SR650 V4 | 1.90 | 1.30 | 5.03.00 | -| Lenovo HS350X V3 | 1.20 | 2.17.0 | | -| Dell PowerEdge XE9680 | iDRAC 7.20.60.50 | 2.7.4 | 1.6.0 | -| Dell PowerEdge R750 | iDRAC 7.20.60.50 | 1.18.1 | 1.1.1 | -| SYS-221H-TNR | 1.03.18 | 2.7 | SAA Ver = 1.3.0-p7 | -| Dell PowerEdge R760 | iDRAC 7.20.60.50 | 2.7.5 | 1.2.6 | -| ARS-121L-DNR | 01.08.02 / 01.03.16 (LCC) | 2.2a / 2.0 (LCC) | SAA Ver = 1.2.0-p6 / SUM = 2.14.0-p6 (LCC) | -| SYS-221H-TN24R | X1.05.10 | 2.7 | SAA Ver = 1.3.0-p5 | -| ARS-221GL-NR | 1.03.16 | 2.0 | | -| HPE ProLiant DL385 Gen10 Plus v2 | 3.15 | 3.80_09-05-2025 | | -| DL380 Gen12 | 1.20.00 | 1.62_02-06-2026 | | -| SSG-121E-NES24R | 01.04.19 | 2.7 | SAA Ver = 1.3.0-p1 | -| SYS-121H-TNR | X1.05.10 | 2.7 | SAA Ver = 1.3.0-p5 | -| SYS-821GE-TNHR | 1.03.18 | 2.7 | SAA Ver = 1.3.0-p7 | -| Dell R760xd2 | iDRAC 7.20.80.50 | 2.9.4 | 1.1.2 | -| Dell R670 | iDRAC 1.20.80.51 | 1.7.5 | | -| Dell R770 | iDRAC 1.20.80.51 | 1.7.5 | | -| SYS-421GE-TNRT | 1.03.19 | 2.6 | SAA Ver = 1.2.0-p8 | -| Dell PowerEdge R640 | iDRAC 7.00.00.182 | 2.24.0 | 1.0.6 | +| Host Machine | BMC/Management Firmware Version | BIOS/UEFI Version | Misc. Firmware Version (FPGA, CPLD, LXPM, etc.) | +| ------------ | ------------------------------- | ----------------- | ----------------------------------------------- | +| GB200 NVL - Wiwynn | 25.06-2_NV_WW_02 | 1.3.2GA | 1.3.2GA | +| NVSwitch Tray - Wiwynn | 1.3.2GA | 1.3.2GA | 1.3.2GA | +| GB200 Compute Tray (1RU) | 1.3.2GA | 1.3.2GA | 1.3.2GA | +| NVSwitch Tray DGX | 1.3.2GA | 1.3.2GA | 1.3.2GA | +| DGX H100 | 25.06.27 (DGXH100_H200_25.06.4 pkg) | 1.06.07 (DGXH100_H200_25.06.4 pkg) | | +| Lenovo ThinkSystem SR670 V2 | 6.10 | 3.30 | 3.31.01 | +| Lenovo ThinkSystem SR675 V3 | 14.10 | 8.30 | 4.20.03 | +| Lenovo ThinkSystem SR675 V3 OVX* | 14.10 | 8.30 | 4.20.03 | +| Lenovo ThinkSystem SR650 | 10.40 | 4.30 | 2.13 | +| Lenovo ThinkSystem SR650 V3 | 6.92 | 3.70 | 4.21.01 | +| Lenovo ThinkSystem SR650 V2 | 5.70 | 3.60 | 3.31.01 | +| Lenovo ThinkSystem SR650 V2 OVX* | 5.70 | 3.60 | 3.31.01 | +| Lenovo ThinkSystem SR655 V3 | 5.80 | 5.70 | 4.20.03 | +| Lenovo ThinkSystem SR655 V3 OVX* | 5.80 | 5.70 | 4.20.03 | +| Lenovo ThinkSystem SR665 V3 OVX* | 5.80 | 5.70 | 4.20.03 | +| Lenovo SR650 V4 | 1.90 | 1.30 | 5.03.00 | +| Lenovo HS350X V3 | 1.20 | 2.17.0 | | +| Dell PowerEdge XE9680 | iDRAC 7.20.60.50 | 2.7.4 | 1.6.0 | +| Dell PowerEdge R750 | iDRAC 7.20.60.50 | 1.18.1 | 1.1.1 | +| SYS-221H-TNR | 1.03.18 | 2.7 | SAA Ver = 1.3.0-p7 | +| Dell PowerEdge R760 | iDRAC 7.20.60.50 | 2.7.5 | 1.2.6 | +| ARS-121L-DNR | 01.08.02 / 01.03.16 (LCC) | 2.2a / 2.0 (LCC) | SAA Ver = 1.2.0-p6 / SUM = 2.14.0-p6 (LCC) | +| SYS-221H-TN24R | X1.05.10 | 2.7 | SAA Ver = 1.3.0-p5 | +| ARS-221GL-NR | 1.03.16 | 2.0 | | +| HPE ProLiant DL385 Gen10 Plus v2 | 3.15 | 3.80_09-05-2025 | | +| DL380 Gen12 | 1.20.00 | 1.62_02-06-2026 | | +| SSG-121E-NES24R | 01.04.19 | 2.7 | SAA Ver = 1.3.0-p1 | +| SYS-121H-TNR | X1.05.10 | 2.7 | SAA Ver = 1.3.0-p5 | +| SYS-821GE-TNHR | 1.03.18 | 2.7 | SAA Ver = 1.3.0-p7 | +| Dell R760xd2 | iDRAC 7.20.80.50 | 2.9.4 | 1.1.2 | +| Dell R670 | iDRAC 1.20.80.51 | 1.7.5 | | +| Dell R770 | iDRAC 1.20.80.51 | 1.7.5 | | +| SYS-421GE-TNRT | 1.03.19 | 2.6 | SAA Ver = 1.2.0-p8 | +| Dell PowerEdge R640 | iDRAC 7.00.00.182 | 2.24.0 | 1.0.6 | \* OVX may not show up as an option; check the Server Serial Number to confirm. -### Hosts -- Under Development +### Hosts -- Under Development Upcoming This list outlines platforms that are under development and have not undergone full unit testing. -| Host Machine | BMC/Management Firmware Version | BIOS/UEFI Version | Provisioning Manager Version | -|-------------------------------------|-------------------------------------|---------------------|--------------------------------------------| -| Lenovo GB300 Compute Tray | 3.0.0 | 1.0.0GA | 1.0.0GA | +| Host Machine | BMC/Management Firmware Version | BIOS/UEFI Version | Provisioning Manager Version | +| ------------ | ------------------------------- | ----------------- | ---------------------------- | +| Lenovo GB300 Compute Tray | 3.0.0 | 1.0.0GA | 1.0.0GA | ## DPUs -| DPU | Firmware / Software Version | -|--------------|---------------------------------------------------| -| BlueField 2 | DOCA 3.2.0 | -| BlueField 3 | DOCA 3.2.0 | +| DPU | Firmware / Software Version | +| --- | --------------------------- | +| BlueField 2 | DOCA 3.2.0 | +| BlueField 3 | DOCA 3.2.0 | ## GPUs @@ -71,4 +71,3 @@ Unlike host platforms, GPU compatibility is not validated on a per-platform basi | GPU Family | Support | | ---------- | ------- | | Current NVIDIA Data Center GPUs | Supported | - diff --git a/docs/manuals/vpc/flat_vpcs_zero_dpu.md b/docs/manuals/vpc/flat_vpcs_zero_dpu.md index 9f1a71c417..9597459977 100644 --- a/docs/manuals/vpc/flat_vpcs_zero_dpu.md +++ b/docs/manuals/vpc/flat_vpcs_zero_dpu.md @@ -1,4 +1,4 @@ -# Flat VPCs and Zero-DPU Hosts +# Flat VPCs and Zero-DPU Hosts v2.0 New `Flat` is a VPC virtualization type for tenant instances that run on hosts **without a NICo-managed DPU** — either hosts with no DPU hardware at all, or @@ -371,7 +371,7 @@ Rules enforced at allocation: Read instance status through the REST API or `nicocli`: -``` +```text GET /v2/org/{org}/nico/instance/{id} nicocli instance get ``` diff --git a/docs/observability/tracing.md b/docs/observability/tracing.md index a71cb4f3ba..aa8b16d5aa 100644 --- a/docs/observability/tracing.md +++ b/docs/observability/tracing.md @@ -8,7 +8,8 @@ How NICo component tracing works, what it covers, how to turn it on and off and - **nico-api** (the `carbide-api` binary) is NICo's primary tracing source and the subject of this document. **nico-dns** also emits traces, but with a separate simpler always-on setup. - No other NICo component emits traces. + **nico-bmc-proxy** emits traces for each proxied BMC request when configured (see + [nico-bmc-proxy tracing](#16-nico-bmc-proxy-tracing)). - **nico-api traces are off by default**; two things must both be true before any spans are emitted: - An OTLP endpoint is configured at startup, either in the nico-api config TOML: @@ -35,7 +36,7 @@ How NICo component tracing works, what it covers, how to turn it on and off and - nico-api **propagates W3C trace context** at its network boundaries: it reads `traceparent`/ `tracestate` from inbound REST and gRPC requests and continues that trace, injecting the same headers into its outbound requests. Propagation links traces across services, but does not by itself - enable recording (see [1.6](#16-w3c-trace-context-propagation)). + enable recording (see [W3C trace-context propagation](#17-w3c-trace-context-propagation)). --- @@ -43,13 +44,16 @@ How NICo component tracing works, what it covers, how to turn it on and off and ### 1.1 Which components emit traces -Two binaries build an OTLP span exporter: +The following binaries build an OTLP span exporter: - **nico-api** (`crates/api-core/src/logging/setup.rs`) - the rich, control-plane tracing this document is mostly about, off by default behind endpoint plus enabled-flag configuration - **nico-dns** (`crates/dns/src/main.rs`) - a separate, much simpler **always-on** setup. +- **nico-bmc-proxy** (`crates/bmc-proxy/src/setup.rs`) - one span per proxied BMC request, off by + default behind endpoint plus `[tracing] enabled` (see + [nico-bmc-proxy tracing](#16-nico-bmc-proxy-tracing)). -The other binaries (nico-pxe, nico-dhcp, nico-bmc-proxy, nico-hardware-health, nico-ssh-console-rs, and +The other binaries (nico-pxe, nico-dhcp, nico-hardware-health, nico-ssh-console-rs, and nico-dsx-exchange-consumer) carry the OpenTelemetry crates in the workspace but do not build a span exporter, so they do not emit traces. @@ -89,7 +93,8 @@ nico-api uses a custom `CarbideSpanSampler`: - **In-process child spans inherit the root's decision**, so once a trace is sampled the whole call tree beneath it is captured - **except tokio spans, which are always dropped** (they leak and would exhaust memory). - For a span parented to a **remote** (ingress-extracted) trace, the decision stays local: an inbound `sampled` - flag does not override `tracing_enabled` (see [1.6](#16-w3c-trace-context-propagation)). + flag does not override `tracing_enabled` (see + [W3C trace-context propagation](#17-w3c-trace-context-propagation)). - The exporter resource is `service.name = carbide-api`; the tracer is named `carbide`. ### 1.4 How traces leave nico-api @@ -116,7 +121,51 @@ nico-api's: - **Resource / output:** `service.name = carbide-dns`; logs are JSON on stdout (not logfmt). - **Same transport constraints:** OTLP/gRPC, plaintext (`with_tonic`, no `tls` feature) -### 1.6 W3C trace-context propagation +### 1.6 nico-bmc-proxy tracing + +nico-bmc-proxy traces each proxied Redfish request through the BMC credential proxy +(`crates/bmc-proxy/src/bmc_proxy.rs`). It follows the same W3C propagation model as nico-api +(issue [#2438](https://github.com/NVIDIA/infra-controller/issues/2438)) so a call from nico-api or +DPS stays one trace across the proxy hop (issue +[#2355](https://github.com/NVIDIA/infra-controller/issues/2355)). + +- **Off by default.** Spans are exported only when an OTLP endpoint is configured **and** + `[tracing] enabled = true` (or the process is started with `--debug`). There is no runtime + toggle on this binary. +- **Endpoint.** Set the standard `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, or + `OTEL_EXPORTER_OTLP_ENDPOINT` to cover every signal at once; the trace-specific variable wins when + both are set. `[tracing] otlp_endpoint` in the proxy TOML is the fallback for when neither variable + is set. The remaining standard OTLP transport settings (`OTEL_EXPORTER_OTLP_TIMEOUT`, + `OTEL_EXPORTER_OTLP_COMPRESSION`, `OTEL_EXPORTER_OTLP_HEADERS`, ...) are read by the exporter + itself and apply as well. A malformed endpoint is rejected when the + exporter is built; the proxy logs a warning and serves BMC traffic without tracing rather than + refusing to start. +- **Ingress.** Each proxied request opens a `bmc_proxy_request` span and adopts any inbound + `traceparent`/`tracestate` via `trace_propagation::set_span_parent_from_headers`. +- **Egress to BMC.** Upstream Redfish calls use a `reqwest-tracing` client so the active proxy + span's W3C context is injected on the BMC leg. The inbound headers are dropped before the upstream + request is assembled — `trace_propagation::is_propagated_header` asks the configured propagator + which headers are its own — so the BMC parents under the proxy's span rather than the caller's. +- **Egress to nico-api (gRPC).** Credential lookup uses the shared `ForgeApiClient`, which already + wraps the transport with `TraceInjectService`. +- **Resource / tracer:** `service.name = nico-bmc-proxy`, tracer name `nico-bmc-proxy`. +- **Span fields:** HTTP method and request path, the status the proxy answered its caller with (not + the BMC's — a request the proxy rejects never reaches one), and BMC target IP (span attribute, not + a Prometheus label). Only a 5xx sets the span status to error; a 4xx is the caller's error. + +Example config: + +```toml +[tracing] +enabled = true +otlp_endpoint = "http://otel-collector.observability.svc.cluster.local:4317" +``` + +Point this at the same collector nico-api uses: spans only join into one trace if every +hop's exporter reaches the same backend. The components stay distinguishable by their +`service.name`. + +### 1.7 W3C trace-context propagation nico-api accepts and produces **W3C Trace Context** headers (`traceparent` and `tracestate`) at its network boundaries, so a request already traced by another service stays one trace as it passes @@ -145,7 +194,7 @@ through nico-api. The standard `TraceContextPropagator` is installed once at sta this hop. **This is a known limitation.** - **Scope.** Trace context only (`traceparent` or `tracestate`). -### 1.7 Adding a new network client +### 1.8 Adding a new network client Propagation is automatic on ingress but opt-in on egress. Keep the following in mind when adding code: diff --git a/docs/operations/firmware-updates.md b/docs/operations/firmware-updates.md index a681047e89..105a5d052b 100644 --- a/docs/operations/firmware-updates.md +++ b/docs/operations/firmware-updates.md @@ -1,4 +1,4 @@ -# Firmware Updates +# Firmware Updates New This guide first gives a high-level view of firmware updates in NICo and helps you choose the workflow that applies to your hardware. The linked workflow diff --git a/docs/operations/monitoring-health.md b/docs/operations/monitoring-health.md index 658583d531..23039323e5 100644 --- a/docs/operations/monitoring-health.md +++ b/docs/operations/monitoring-health.md @@ -1,4 +1,4 @@ -# Monitoring and Health +# Monitoring and Health New This page covers monitoring and health workflows for NICo sites after deployment: hardware health, DPU health, aggregate host health, health diff --git a/docs/operations/nvos-password-rotation.md b/docs/operations/nvos-password-rotation.md index a94473f1d0..a96f8a0b3e 100644 --- a/docs/operations/nvos-password-rotation.md +++ b/docs/operations/nvos-password-rotation.md @@ -1,4 +1,4 @@ -# NVOS Password Rotation +# NVOS Password Rotation v2.0 New Use this workflow to rotate the NVOS admin password across managed NVLink switches, monitor site-wide convergence, and recover switches that do not reach the published target. diff --git a/docs/operations/tenant-lifecycle-cleanup.md b/docs/operations/tenant-lifecycle-cleanup.md index b4284ec40a..84f8cdca3d 100644 --- a/docs/operations/tenant-lifecycle-cleanup.md +++ b/docs/operations/tenant-lifecycle-cleanup.md @@ -1,4 +1,4 @@ -# Tenant Lifecycle Cleanup +# Tenant Lifecycle Cleanup New Use this workflow to release an instance, track NICo cleanup progress, and verify that the host is ready for reuse. diff --git a/docs/provisioning/boot-interfaces-and-dpu-modes.md b/docs/provisioning/boot-interfaces-and-dpu-modes.md index aef0e9ebdd..33d517c5d8 100644 --- a/docs/provisioning/boot-interfaces-and-dpu-modes.md +++ b/docs/provisioning/boot-interfaces-and-dpu-modes.md @@ -1,4 +1,4 @@ -# Boot Interfaces and DPU Policies +# Boot Interfaces and DPU Policies v2.0 New This guide explains how NICo decides **which interface a host boots from**, how a host's **DPUs are managed**, and how operators configure both through the Expected Machines table. It is the deep companion to [Ingesting Hosts](ingesting-hosts.md): that page covers the end-to-end ingest flow and the basic `expected_machines.json`; this page covers the per-host and per-NIC knobs (`dpu_policy`, `host_nics`), **what the defaults do when you set nothing**, and how a boot device is chosen and applied behind the scenes. diff --git a/docs/provisioning/machine-validation.md b/docs/provisioning/machine-validation.md index 3ec36e7807..faae2aaaa2 100644 --- a/docs/provisioning/machine-validation.md +++ b/docs/provisioning/machine-validation.md @@ -1,4 +1,4 @@ -# Machine Validation +# Machine Validation v2.0 Machine Validation is NVIDIA Infra Controller's in-band validation framework for checking a machine before it is made available to tenants. NICo uses Scout to run diff --git a/fern/changelog/2026-01-23.mdx b/fern/changelog/2026-01-23.mdx index e2d58681f4..97ccd3ba78 100644 --- a/fern/changelog/2026-01-23.mdx +++ b/fern/changelog/2026-01-23.mdx @@ -2,7 +2,7 @@ tags: ["EOL", "deployment", "gb200", "nvlink"] --- -## NVIDIA Infra Controller EA End of Life +## NVIDIA Infra Controller EA EOL NVIDIA Infra Controller EA introduced a self-hosted, disconnected deployment model with source and build instructions that do not depend on NGC. diff --git a/fern/changelog/2026-02-12.mdx b/fern/changelog/2026-02-12.mdx index 6e027a50dd..41989fd06c 100644 --- a/fern/changelog/2026-02-12.mdx +++ b/fern/changelog/2026-02-12.mdx @@ -1,8 +1,8 @@ --- -tags: ["EOL", "rest-api", "provisioning", "nvlink"] +tags: ["EOL", "nvlink", "provisioning", "rest-api"] --- -## NVIDIA Infra Controller v0.2 End of Life +## NVIDIA Infra Controller v0.2 EOL NVIDIA Infra Controller v0.2 made the project available as open-source software and expanded its APIs, orchestration, and hardware lifecycle support. diff --git a/fern/changelog/2026-05-15.mdx b/fern/changelog/2026-05-15.mdx index bee429f76f..6d475e516f 100644 --- a/fern/changelog/2026-05-15.mdx +++ b/fern/changelog/2026-05-15.mdx @@ -1,8 +1,8 @@ --- -tags: ["current", "deployment", "rack-level-administration", "gb200"] +tags: ["maintenance", "deployment", "gb200", "rack-level-administration"] --- -## NVIDIA Infra Controller v0.8 Current +## NVIDIA Infra Controller v0.8 Maintenance NVIDIA Infra Controller v0.8 introduced a unified documentation site, simplified deployment, and expanded rack-level administration for GB200 systems. diff --git a/fern/changelog/2026-07-31.mdx b/fern/changelog/2026-07-31.mdx index 74e6365797..8a55e50c0e 100644 --- a/fern/changelog/2026-07-31.mdx +++ b/fern/changelog/2026-07-31.mdx @@ -1,8 +1,12 @@ --- -tags: ["next", "deployment", "ntp", "security", "database"] +tags: ["current", "database", "deployment", "dpu", "machine-identity", "multi-arch", "networking", "ntp", "rest-api", "security", "telemetry", "validation", "zero-dpu"] --- -## NVIDIA Infra Controller v2.0 Next +## NVIDIA Infra Controller v2.0 Current + +NICo v2.0 expands bare-metal lifecycle management with built-in NTP, broader hardware telemetry, multi-architecture support, and stronger security and operational controls. + +Before upgrading an existing deployment, review the REST API database migration requirements below. The NICo REST API database has moved from a standalone PostgreSQL instance to the shared Zalando-managed `nico-pg-cluster`. A new `nico_rest` database and `nico-rest.nico` user are provisioned automatically (gated on `rest.enabled: true`). Database credentials are now synced by External Secrets Operator as the `nico-rest-pg-creds` secret in the `nico-rest` namespace, replacing the former `db-creds` secret. diff --git a/fern/docs.yml b/fern/docs.yml index ceaaa2bc4a..6bf942cd1d 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -44,5 +44,11 @@ redirects: destination: "/infra-controller/documentation/getting-started/quick-start-guide" - source: "/infra-controller/documentation/introduction" destination: "/infra-controller/documentation/overview/what-is-nico" + - source: "/infra-controller/documentation/operations-day-2/firmware-updates" + destination: "/infra-controller/documentation/operations-day-2/firmware-updates/overview" + - source: "/infra-controller/documentation/operations-day-2/observability/core-metrics" + destination: "/infra-controller/documentation/operations-day-2/observability/core-metrics-list" + - source: "/infra-controller/documentation/provisioning-day-0/boot-interfaces-and-dpu-modes" + destination: "/infra-controller/documentation/provisioning-day-0/boot-interfaces-and-dpu-policies" - source: "/infra-controller/documentation/reference/release-notes" destination: "/infra-controller/documentation/release-notes" diff --git a/fern/fern.config.json b/fern/fern.config.json index 22bc767300..162a343345 100644 --- a/fern/fern.config.json +++ b/fern/fern.config.json @@ -1,4 +1,4 @@ { "organization": "nvidia", - "version": "5.80.4" + "version": "5.89.1" } diff --git a/rest-api/api/pkg/api/handler/instancetype.go b/rest-api/api/pkg/api/handler/instancetype.go index 08d51abed4..339a014e56 100644 --- a/rest-api/api/pkg/api/handler/instancetype.go +++ b/rest-api/api/pkg/api/handler/instancetype.go @@ -96,7 +96,7 @@ func (cith CreateInstanceTypeHandler) Handle(c echo.Context) error { // Validate role, only Provider Admins are allowed to create Instance Types ok = auth.ValidateUserRoles(dbUser, org, nil, auth.ProviderAdminRole) if !ok { - logger.Warn().Msg("user does not have Tenant Admin role, access denied") + logger.Warn().Msg("user does not have Provider Admin role, access denied") return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "User does not have Provider Admin role with org", nil) } diff --git a/rest-api/api/pkg/api/handler/operatingsystem.go b/rest-api/api/pkg/api/handler/operatingsystem.go index 75128ae184..ce2c4215b0 100644 --- a/rest-api/api/pkg/api/handler/operatingsystem.go +++ b/rest-api/api/pkg/api/handler/operatingsystem.go @@ -26,7 +26,6 @@ import ( "github.com/NVIDIA/infra-controller/rest-api/api/pkg/api/model" "github.com/NVIDIA/infra-controller/rest-api/api/pkg/api/pagination" sc "github.com/NVIDIA/infra-controller/rest-api/api/pkg/client/site" - auth "github.com/NVIDIA/infra-controller/rest-api/auth/pkg/authorization" cutil "github.com/NVIDIA/infra-controller/rest-api/common/pkg/util" cdb "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db" cdbm "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/model" @@ -236,6 +235,26 @@ func validateIpxeTemplateAvailableAtSites(ctx context.Context, dbSession *cdb.Se return nil } +// getTenantSiteIDs returns the IDs of all sites the tenant has access to, +// regardless of site status. Used to scope provider-owned Operating System +// visibility for tenant admins. +func getTenantSiteIDs(ctx context.Context, dbSession *cdb.Session, tenantID uuid.UUID) ([]uuid.UUID, error) { + tsDAO := cdbm.NewTenantSiteDAO(dbSession) + tss, _, err := tsDAO.GetAll(ctx, nil, + cdbm.TenantSiteFilterInput{TenantIDs: []uuid.UUID{tenantID}}, + cdbp.PageInput{Limit: cutil.GetPtr(cdbp.TotalLimit)}, + nil, + ) + if err != nil { + return nil, err + } + ids := make([]uuid.UUID, len(tss)) + for i, ts := range tss { + ids[i] = ts.SiteID + } + return ids, nil +} + // ~~~~~ Create Handler ~~~~~ // // CreateOperatingSystemHandler is the API Handler for creating new OperatingSystem @@ -278,47 +297,37 @@ func (csh CreateOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve current user", nil) } - // Validate org - ok, err := auth.ValidateOrgMembership(dbUser, org) - if !ok { - if err != nil { - logger.Error().Err(err).Msg("error validating org membership for User in request") - } else { - logger.Warn().Msg("could not validate org membership for user, access denied") - } - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, fmt.Sprintf("Failed to validate membership for org: %s", org), nil) - } - - // Validate role, only Tenant Admins are allowed to create OperatingSystem - ok = auth.ValidateUserRoles(dbUser, org, nil, auth.TenantAdminRole) - if !ok { - logger.Warn().Msg("user does not have Tenant Admin role, access denied") - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "User does not have Tenant Admin role with org", nil) + ip, tenant, apiError := common.IsProviderOrTenant(ctx, logger, csh.dbSession, org, dbUser, false, nil) + if apiError != nil { + return cutil.NewAPIErrorResponse(c, apiError.Code, apiError.Message, apiError.Data) } // Validate request // Bind request data to API model apiRequest := model.APIOperatingSystemCreateRequest{} - err = c.Bind(&apiRequest) + err := c.Bind(&apiRequest) if err != nil { logger.Warn().Err(err).Msg("error binding request data into API model") return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to parse request data, potentially invalid structure", nil) } - // Validate the tenant for which this OperatingSystem is being created - tenant, err := common.GetTenantForOrg(ctx, nil, csh.dbSession, org) - if err != nil { - if err == common.ErrOrgTenantNotFound { - logger.Warn().Err(err).Msg("Org does not have a Tenant associated") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Org does not have a Tenant associated", nil) - } - logger.Error().Err(err).Msg("unable to retrieve tenant for org") - return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve tenant for org", nil) + + // Infer OS type from the provided source fields (ipxeScript -> iPXE, + // ipxeTemplateId -> Templated iPXE, otherwise Image). + osType := apiRequest.GetOperatingSystemType() + + // Provider Admin is limited to iPXE Template-based OSes. When both roles + // allow the action, Provider Admin takes priority (provider-owned OS). + allowedByProvider := ip != nil && osType == cdbm.OperatingSystemTypeTemplatedIPXE + allowedByTenant := tenant != nil + if !allowedByProvider && !allowedByTenant { + logger.Warn().Msg("caller is not permitted to create this Operating System") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Provider Admin can only create iPXE Template-based Operating Systems", nil) } - // Default TenantID to org's Tenant when nil; validate when set - if apiRequest.TenantID == nil { - apiRequest.TenantID = cutil.GetPtr(tenant.ID.String()) - } else if *apiRequest.TenantID != tenant.ID.String() { + // If the caller provided an explicit tenantId in the body (deprecated), validate + // it matches the org's tenant. + // TODO: tenantId as parameter is deprecated and will need to be removed by 2026-10-01. + if tenant != nil && apiRequest.TenantID != nil && *apiRequest.TenantID != tenant.ID.String() { logger.Warn().Str("tenantId", *apiRequest.TenantID).Msg("TenantID in request does not match org's Tenant") return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "TenantID specified in request does not match org's Tenant", nil) } @@ -337,98 +346,121 @@ func (csh CreateOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Error validating user data in Operating System creation request", verr) } - // check for name uniqueness for the tenant, ie, tenant cannot have another os with same name + // Check for name uniqueness within the owner's scope (provider or tenant). // TODO consider doing this with an advisory lock for correctness osDAO := cdbm.NewOperatingSystemDAO(csh.dbSession) - oss, tot, err := osDAO.GetAll( - ctx, - nil, - cdbm.OperatingSystemFilterInput{ - TenantIDs: []uuid.UUID{tenant.ID}, - Names: []string{apiRequest.Name}, - }, - cdbp.PageInput{}, - nil, - ) + uniquenessFilter := cdbm.OperatingSystemFilterInput{Names: []string{apiRequest.Name}} + if allowedByProvider { + uniquenessFilter.InfrastructureProviderID = &ip.ID + } else { + uniquenessFilter.TenantIDs = []uuid.UUID{tenant.ID} + } + oss, tot, err := osDAO.GetAll(ctx, nil, uniquenessFilter, cdbp.PageInput{}, nil) if err != nil { - logger.Error().Err(err).Msg("db error checking for name uniqueness of tenant os") + logger.Error().Err(err).Msg("db error checking for name uniqueness of os") return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to create OperatingSystem due to DB error", nil) } if tot > 0 { - logger.Warn().Str("tenantId", tenant.ID.String()).Str("name", apiRequest.Name).Msg("Operating System with same name already exists for tenant") - return cutil.NewAPIErrorResponse(c, http.StatusConflict, "Another Operating System with specified name already exists for Tenant", validation.Errors{ + logger.Warn().Str("name", apiRequest.Name).Msg("Operating System with same name already exists") + return cutil.NewAPIErrorResponse(c, http.StatusConflict, "Another Operating System with specified name already exists", validation.Errors{ "id": errors.New(oss[0].ID.String()), }) } - // Infer OS type from the provided source fields (ipxeScript -> iPXE, - // ipxeTemplateId -> Templated iPXE, otherwise Image). - osType := apiRequest.GetOperatingSystemType() - // Set the phoneHomeEnabled if provided in request phoneHomeEnabled := false if apiRequest.PhoneHomeEnabled != nil { phoneHomeEnabled = *apiRequest.PhoneHomeEnabled } - // Verify or validate site + // Resolve and validate target sites. Site ownership differs by caller: + // provider-owned OSes (Templated iPXE only) target the provider's own sites; + // tenant-owned OSes target sites the tenant has access to. tsDAO := cdbm.NewTenantSiteDAO(csh.dbSession) rdbst := []cdbm.Site{} sttsmap := map[uuid.UUID]*cdbm.TenantSite{} dbossd := []cdbm.StatusDetail{} - // Get all TenantSite records for the Tenant - tss, _, err := tsDAO.GetAll( - ctx, - nil, - cdbm.TenantSiteFilterInput{ - TenantIDs: []uuid.UUID{tenant.ID}, - }, - cdbp.PageInput{ - Limit: cutil.GetPtr(cdbp.TotalLimit), - }, - nil, - ) - if err != nil { - logger.Error().Err(err).Msg("db error retrieving TenantSite records for Tenant") - return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Site associations for Tenant, DB error", nil) - } - for _, ts := range tss { - cts := ts - sttsmap[ts.SiteID] = &cts + requestedSiteIDs := make([]uuid.UUID, 0, len(apiRequest.SiteIDs)) + for _, siteID := range apiRequest.SiteIDs { + parsedSiteID, perr := uuid.Parse(siteID) + if perr != nil { + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Failed to create Operating System, invalid Site ID: %s", siteID), nil) + } + requestedSiteIDs = append(requestedSiteIDs, parsedSiteID) } - // Validate the site for which this image based Operating System is being created - for _, stID := range apiRequest.SiteIDs { - site, serr := common.GetSiteFromIDString(ctx, nil, stID, csh.dbSession) + sitesByID := make(map[uuid.UUID]*cdbm.Site, len(requestedSiteIDs)) + if len(requestedSiteIDs) > 0 { + siteDAO := cdbm.NewSiteDAO(csh.dbSession) + sites, _, serr := siteDAO.GetAll( + ctx, + nil, + cdbm.SiteFilterInput{SiteIDs: requestedSiteIDs}, + cdbp.PageInput{Limit: cutil.GetPtr(cdbp.TotalLimit)}, + nil, + ) if serr != nil { - if serr == common.ErrInvalidID { - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Failed to create Operating System, invalid Site ID: %s", stID), nil) - } - if serr == cdb.ErrDoesNotExist { - return cutil.NewAPIErrorResponse(c, http.StatusNotFound, fmt.Sprintf("Failed to create Operating System, could not find Site with ID: %s ", stID), nil) + logger.Error().Err(serr).Msg("error retrieving Sites from DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Sites specified in request, DB error", nil) + } + for i := range sites { + site := &sites[i] + sitesByID[site.ID] = site + } + for _, siteID := range requestedSiteIDs { + if _, ok := sitesByID[siteID]; !ok { + return cutil.NewAPIErrorResponse(c, http.StatusNotFound, fmt.Sprintf("Failed to create Operating System, could not find Site with ID: %s", siteID.String()), nil) } - logger.Warn().Err(serr).Str("Site ID", stID).Msg("error retrieving Site from DB by ID") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Failed to create Operating System, could not retrieve Site with ID: %s, DB error", stID), nil) } + } - if site.Status != cdbm.SiteStatusRegistered { - logger.Warn().Msg(fmt.Sprintf("Unable to associate Operating System to Site: %s. Site is not in Registered state", site.ID.String())) - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Failed to create Operating System, Site: %s specified in request is not in Registered state", site.ID.String()), nil) + if !allowedByProvider && len(requestedSiteIDs) > 0 { + // Tenant-owned: retrieve the requested TenantSite associations in one query. + tss, _, terr := tsDAO.GetAll( + ctx, + nil, + cdbm.TenantSiteFilterInput{ + TenantIDs: []uuid.UUID{tenant.ID}, + SiteIDs: requestedSiteIDs, + }, + cdbp.PageInput{ + Limit: cutil.GetPtr(cdbp.TotalLimit), + }, + nil, + ) + if terr != nil { + logger.Error().Err(terr).Msg("db error retrieving TenantSite records for Tenant") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Site associations for Tenant, DB error", nil) } - - // Validate the TenantSite exists for current tenant and this site - _, ok := sttsmap[site.ID] - if !ok { - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Unable to associate Operating System with Site: %s, Tenant does not have access to Site", stID), nil) + for _, ts := range tss { + cts := ts + sttsmap[ts.SiteID] = &cts } + } - // Validate the Site has the ImageBasedOperatingSystem capability enabled for Image based Operating Systems - if osType == cdbm.OperatingSystemTypeImage && (site.Config == nil || !site.Config.ImageBasedOperatingSystem) { - logger.Warn().Str("siteId", stID).Msg("Image based Operating System is not supported for Site, ImageBasedOperatingSystem capability is not enabled") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Creation of Image based Operating Systems is not supported. Site must have ImageBasedOperatingSystem capability enabled.", nil) + for _, siteID := range requestedSiteIDs { + site := sitesByID[siteID] + if site.Status != cdbm.SiteStatusRegistered { + logger.Warn().Str("siteId", siteID.String()).Msg("Unable to associate Operating System to Site; Site is not in Registered state") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Failed to create Operating System, Site: %s specified in request is not in Registered state", siteID.String()), nil) } + if allowedByProvider { + if site.InfrastructureProviderID != ip.ID { + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Unable to associate Operating System with Site: %s, Site does not belong to provider", siteID.String()), nil) + } + } else { + // Validate the TenantSite exists for current tenant and this site + if _, ok := sttsmap[siteID]; !ok { + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("Unable to associate Operating System with Site: %s, Tenant does not have access to Site", siteID.String()), nil) + } + // Validate the Site has the ImageBasedOperatingSystem capability enabled for Image based Operating Systems + if osType == cdbm.OperatingSystemTypeImage && (site.Config == nil || !site.Config.ImageBasedOperatingSystem) { + logger.Warn().Str("siteId", siteID.String()).Msg("Image based Operating System is not supported for Site, ImageBasedOperatingSystem capability is not enabled") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Creation of Image based Operating Systems is not supported. Site must have ImageBasedOperatingSystem capability enabled.", nil) + } + } rdbst = append(rdbst, *site) } @@ -456,6 +488,18 @@ func (csh CreateOperatingSystemHandler) Handle(c echo.Context) error { osStatusMessage = "received Operating System creation request, syncing" } + // Assign ownership: provider-owned OSes carry InfrastructureProviderID + // (tenant_id=nil); tenant-owned OSes carry TenantID + // (infrastructure_provider_id=nil). When pushed to nico-core, tenant-owned + // OSes carry tenant_organization_id while provider-owned OSes omit it. + var ownerTenantID *uuid.UUID + var ownerProviderID *uuid.UUID + if allowedByProvider { + ownerProviderID = &ip.ID + } else { + ownerTenantID = &tenant.ID + } + // Values needed after the transaction closure var os *cdbm.OperatingSystem var dbossa []cdbm.OperatingSystemSiteAssociation @@ -467,28 +511,29 @@ func (csh CreateOperatingSystemHandler) Handle(c echo.Context) error { err = cdb.WithTx(ctx, csh.dbSession, func(tx *cdb.Tx) error { // Create the db record for Operating System osInput := cdbm.OperatingSystemCreateInput{ - Name: apiRequest.Name, - Description: apiRequest.Description, - Org: org, - TenantID: &tenant.ID, - OsType: osType, - ImageURL: apiRequest.ImageURL, - ImageSHA: apiRequest.ImageSHA, - ImageAuthType: apiRequest.ImageAuthType, - ImageAuthToken: apiRequest.ImageAuthToken, - ImageDisk: apiRequest.ImageDisk, - RootFsId: apiRequest.RootFsID, - RootFsLabel: apiRequest.RootFsLabel, - IpxeScript: apiRequest.IpxeScript, - IpxeTemplateId: apiRequest.IpxeTemplateId, - IpxeTemplateParameters: apiRequest.IpxeTemplateParameters.ToDBModel(), - IpxeTemplateArtifacts: apiRequest.IpxeTemplateArtifacts.ToDBModel(), - UserData: apiRequest.UserData, - AllowOverride: apiRequest.AllowOverride, - EnableBlockStorage: apiRequest.EnableBlockStorage, - PhoneHomeEnabled: phoneHomeEnabled, - Status: osStatus, - CreatedBy: dbUser.ID, + Name: apiRequest.Name, + Description: apiRequest.Description, + Org: org, + TenantID: ownerTenantID, + InfrastructureProviderID: ownerProviderID, + OsType: osType, + ImageURL: apiRequest.ImageURL, + ImageSHA: apiRequest.ImageSHA, + ImageAuthType: apiRequest.ImageAuthType, + ImageAuthToken: apiRequest.ImageAuthToken, + ImageDisk: apiRequest.ImageDisk, + RootFsId: apiRequest.RootFsID, + RootFsLabel: apiRequest.RootFsLabel, + IpxeScript: apiRequest.IpxeScript, + IpxeTemplateId: apiRequest.IpxeTemplateId, + IpxeTemplateParameters: apiRequest.IpxeTemplateParameters.ToDBModel(), + IpxeTemplateArtifacts: apiRequest.IpxeTemplateArtifacts.ToDBModel(), + UserData: apiRequest.UserData, + AllowOverride: apiRequest.AllowOverride, + EnableBlockStorage: apiRequest.EnableBlockStorage, + PhoneHomeEnabled: phoneHomeEnabled, + Status: osStatus, + CreatedBy: dbUser.ID, } createdOs, derr := osDAO.Create(ctx, tx, osInput) if derr != nil { @@ -527,7 +572,7 @@ func (csh CreateOperatingSystemHandler) Handle(c echo.Context) error { ) if derr != nil { logger.Error().Err(derr).Msg("unable to create the Operating System association record in DB") - return cutil.NewAPIError(http.StatusInternalServerError, "Failed to associate Operating System with one or more Sites, DB error", nil) + return cutil.NewAPIError(http.StatusInternalServerError, "Failed to associate Operating System with Site, DB error", nil) } // Create Status details @@ -581,7 +626,7 @@ func (csh CreateOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIError(http.StatusInternalServerError, "Failed to retrieve client for Site", nil) } - createOsRequest := apiRequest.ToProto(os, tenant.Org) + createOsRequest := apiRequest.ToImageProto(os, tenant.Org) workflowOptions := temporalClient.StartWorkflowOptions{ ID: "image-os-create-" + ossa.SiteID.String() + "-" + os.ID.String() + "-" + *ossa.Version, @@ -649,7 +694,7 @@ func (csh CreateOperatingSystemHandler) Handle(c echo.Context) error { // generic Core gRPC proxy (Image OSes are synced in-transaction above). Per-site // failures are recorded on the association status and do not fail the request. if cdbm.IsIPXEType(os.Type) && len(dbossa) > 0 { - req := model.BuildCreateOperatingSystemRequest(os) + req := apiRequest.ToProto(os) siteErrors := syncOperatingSystemToSitesViaProxy(ctx, logger, csh.dbSession, csh.scp, dbossa, createOperatingSystemMethod, req) if aerr := updateOperatingSystemAggregateStatus(ctx, logger, csh.dbSession, os.ID, siteErrors > 0, aggregateSyncMessage(siteErrors > 0)); aerr != nil { logger.Error().Err(aerr).Msg("failed to update aggregate Operating System status after create sync") @@ -747,27 +792,14 @@ func (gash GetAllOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve current user", nil) } - // Validate org - ok, err := auth.ValidateOrgMembership(dbUser, org) - if !ok { - if err != nil { - logger.Error().Err(err).Msg("error validating org membership for User in request") - } else { - logger.Warn().Msg("could not validate org membership for user, access denied") - } - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, fmt.Sprintf("Failed to validate membership for org: %s", org), nil) - } - - // Validate role, only Tenant Admins are allowed to retrieve OperatingSystems - ok = auth.ValidateUserRoles(dbUser, org, nil, auth.TenantAdminRole) - if !ok { - logger.Warn().Msg("user does not have Tenant Admin role, access denied") - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "User does not have Tenant Admin role with org", nil) + ip, tenant, apiError := common.IsProviderOrTenant(ctx, logger, gash.dbSession, org, dbUser, false, nil) + if apiError != nil { + return cutil.NewAPIErrorResponse(c, apiError.Code, apiError.Message, apiError.Data) } // Validate pagination request pageRequest := pagination.PageRequest{} - err = c.Bind(&pageRequest) + err := c.Bind(&pageRequest) if err != nil { logger.Warn().Err(err).Msg("error binding pagination request data into API model") return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to parse request pagination data", nil) @@ -780,20 +812,29 @@ func (gash GetAllOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to validate pagination request data", err) } - // Validate the tenant associated with the org - tenant, err := common.GetTenantForOrg(ctx, nil, gash.dbSession, org) - if err != nil { - if err == common.ErrOrgTenantNotFound { - logger.Warn().Err(err).Msg("Org does not have a Tenant associated") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Org does not have a Tenant associated", nil) - } - logger.Error().Err(err).Msg("unable to retrieve tenant for org") - return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve tenant for org", nil) - } + // Visibility rules: + // Provider admin: sees only provider-created entries (no tenant entries). + // Tenant admin: sees own entries + provider entries at tenant-accessible sites. + // Dual-role: visibility is the union of both (own tenant + own provider). + filter := cdbm.OperatingSystemFilterInput{} + var tenantVisibleProviderSiteIDs []uuid.UUID - filter := cdbm.OperatingSystemFilterInput{ - TenantIDs: []uuid.UUID{tenant.ID}, - Orgs: []string{org}, + switch { + case ip != nil && tenant == nil: + // Provider admin only: sees only provider-created entries. + filter.InfrastructureProviderID = &ip.ID + case tenant != nil && ip == nil: + // Tenant admin only: own entries + provider entries at tenant-accessible sites. + tenantSiteIDs, tsErr := getTenantSiteIDs(ctx, gash.dbSession, tenant.ID) + if tsErr != nil { + logger.Error().Err(tsErr).Msg("error retrieving tenant site IDs for visibility filter") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to determine site access for tenant", nil) + } + tenantVisibleProviderSiteIDs = tenantSiteIDs + case tenant != nil && ip != nil: + // Dual-role: own tenant + own provider entries, no site restriction. + filter.TenantIDs = []uuid.UUID{tenant.ID} + filter.InfrastructureProviderID = &ip.ID } // Get and validate includeRelation params @@ -806,26 +847,73 @@ func (gash GetAllOperatingSystemHandler) Handle(c echo.Context) error { // now check siteID in query tsDAO := cdbm.NewTenantSiteDAO(gash.dbSession) + var tenantSites []cdbm.TenantSite + tenantSitesLoaded := false qSiteID := qParams["siteId"] if len(qSiteID) > 0 { + requestedSiteIDs := make([]uuid.UUID, 0, len(qSiteID)) for _, siteID := range qSiteID { - site, err := common.GetSiteFromIDString(ctx, nil, siteID, gash.dbSession) - if err != nil { - logger.Warn().Err(err).Msg("error getting Site from query string") + parsedSiteID, perr := uuid.Parse(siteID) + if perr != nil { + logger.Warn().Err(perr).Str("siteId", siteID).Msg("error parsing Site ID from query string") return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to retrieve Site specified in query", nil) } + requestedSiteIDs = append(requestedSiteIDs, parsedSiteID) + } - // Determine if tenant has access to requested site - _, err = tsDAO.GetByTenantIDAndSiteID(ctx, nil, tenant.ID, site.ID, nil) - if err != nil { - if err == cdb.ErrDoesNotExist { - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Tenant is not associated with Site specified in query", nil) - } - logger.Warn().Err(err).Msg("error retrieving Tenant Site association from DB") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to determine if Tenant has access to Site specified in query, DB error", nil) + siteDAO := cdbm.NewSiteDAO(gash.dbSession) + sites, _, serr := siteDAO.GetAll( + ctx, + nil, + cdbm.SiteFilterInput{SiteIDs: requestedSiteIDs}, + cdbp.PageInput{Limit: cutil.GetPtr(cdbp.TotalLimit)}, + nil, + ) + if serr != nil { + logger.Error().Err(serr).Msg("error retrieving Sites specified in query") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Sites specified in query, DB error", nil) + } + sitesByID := make(map[uuid.UUID]*cdbm.Site, len(sites)) + for i := range sites { + site := &sites[i] + sitesByID[site.ID] = site + } + + tenantSiteIDs := make(map[uuid.UUID]struct{}) + if tenant != nil { + var terr error + tenantSites, _, terr = tsDAO.GetAll( + ctx, + nil, + cdbm.TenantSiteFilterInput{ + TenantIDs: []uuid.UUID{tenant.ID}, + SiteIDs: requestedSiteIDs, + }, + cdbp.PageInput{Limit: cutil.GetPtr(cdbp.TotalLimit)}, + nil, + ) + if terr != nil { + logger.Error().Err(terr).Msg("error retrieving Tenant Site associations specified in query") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to determine if Tenant has access to Sites specified in query, DB error", nil) + } + tenantSitesLoaded = true + for _, ts := range tenantSites { + tenantSiteIDs[ts.SiteID] = struct{}{} + } + } + + for _, siteID := range requestedSiteIDs { + site, ok := sitesByID[siteID] + if !ok { + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to retrieve Site specified in query", nil) } - filter.SiteIDs = append(filter.SiteIDs, site.ID) + _, tenantHasAccess := tenantSiteIDs[siteID] + providerHasAccess := ip != nil && site.InfrastructureProviderID == ip.ID + if !tenantHasAccess && !providerHasAccess { + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Caller is not associated with Site specified in query", nil) + } + filter.SiteIDs = append(filter.SiteIDs, siteID) } } @@ -869,6 +957,53 @@ func (gash GetAllOperatingSystemHandler) Handle(c echo.Context) error { osDAO := cdbm.NewOperatingSystemDAO(gash.dbSession) ossaDAO := cdbm.NewOperatingSystemSiteAssociationDAO(gash.dbSession) + if tenant != nil && ip == nil { + mergedOSIDs := make(map[uuid.UUID]struct{}) + selectionPage := cdbp.PageInput{Limit: cutil.GetPtr(cdbp.TotalLimit)} + + tenantFilter := filter + tenantFilter.TenantIDs = []uuid.UUID{tenant.ID} + tenantOperatingSystems, _, terr := osDAO.GetAll(ctx, nil, tenantFilter, selectionPage, nil) + if terr != nil { + logger.Error().Err(terr).Msg("error retrieving tenant-owned Operating Systems from DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve tenant-owned Operating Systems", nil) + } + for _, os := range tenantOperatingSystems { + mergedOSIDs[os.ID] = struct{}{} + } + + providerSiteIDs := tenantVisibleProviderSiteIDs + if filter.SiteIDs != nil { + providerSiteIDs = filter.SiteIDs + } + if len(providerSiteIDs) > 0 { + providerAssociations, _, perr := ossaDAO.GetAll( + ctx, + nil, + cdbm.OperatingSystemSiteAssociationFilterInput{SiteIDs: providerSiteIDs}, + selectionPage, + []string{cdbm.OperatingSystemRelationName}, + ) + if perr != nil { + logger.Error().Err(perr).Msg("error retrieving Operating System Site associations visible to Tenant from DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve provider-owned Operating Systems", nil) + } + for _, association := range providerAssociations { + if association.OperatingSystem != nil && association.OperatingSystem.InfrastructureProviderID != nil { + mergedOSIDs[association.OperatingSystemID] = struct{}{} + } + } + } + + mergedIDs := make([]uuid.UUID, 0, len(mergedOSIDs)) + for id := range mergedOSIDs { + mergedIDs = append(mergedIDs, id) + } + filter.TenantIDs = nil + filter.InfrastructureProviderID = nil + filter.OperatingSystemIds = mergedIDs + } + // Create response oss, total, err := osDAO.GetAll( ctx, @@ -934,30 +1069,35 @@ func (gash GetAllOperatingSystemHandler) Handle(c echo.Context) error { dbossaMap[dbossa.OperatingSystemID] = append(dbossaMap[dbossa.OperatingSystemID], curVal) } - // Get all TenantSite records for the Tenant + // Get all TenantSite records for the Tenant (only relevant when the caller + // is acting as a Tenant; provider-only admins have no tenant-site context). sttsmap := map[uuid.UUID]*cdbm.TenantSite{} - tsDAO = cdbm.NewTenantSiteDAO(gash.dbSession) - tss, _, err := tsDAO.GetAll( - ctx, - nil, - cdbm.TenantSiteFilterInput{ - TenantIDs: []uuid.UUID{tenant.ID}, - SiteIDs: siteIDs, - }, - cdbp.PageInput{ - Limit: cutil.GetPtr(cdbp.TotalLimit), - }, - nil, - ) - if err != nil { - logger.Error().Err(err).Msg("db error retrieving TenantSite records for Tenant") - return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Site associations for Tenant, DB error", nil) - } + if tenant != nil { + if !tenantSitesLoaded { + var tserr error + tenantSites, _, tserr = tsDAO.GetAll( + ctx, + nil, + cdbm.TenantSiteFilterInput{ + TenantIDs: []uuid.UUID{tenant.ID}, + SiteIDs: siteIDs, + }, + cdbp.PageInput{ + Limit: cutil.GetPtr(cdbp.TotalLimit), + }, + nil, + ) + if tserr != nil { + logger.Error().Err(tserr).Msg("db error retrieving TenantSite records for Tenant") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Site associations for Tenant, DB error", nil) + } + } - for _, ts := range tss { - curVal := ts - sttsmap[ts.SiteID] = &curVal + for _, ts := range tenantSites { + curVal := ts + sttsmap[ts.SiteID] = &curVal + } } // Create response @@ -1026,22 +1166,9 @@ func (gsh GetOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve current user", nil) } - // Validate org - ok, err := auth.ValidateOrgMembership(dbUser, org) - if !ok { - if err != nil { - logger.Error().Err(err).Msg("error validating org membership for User in request") - } else { - logger.Warn().Msg("could not validate org membership for user, access denied") - } - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, fmt.Sprintf("Failed to validate membership for org: %s", org), nil) - } - - // Validate role, only Tenant Admins are allowed to retrieve OperatingSystem - ok = auth.ValidateUserRoles(dbUser, org, nil, auth.TenantAdminRole) - if !ok { - logger.Warn().Msg("user does not have Tenant Admin role, access denied") - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "User does not have Tenant Admin role with org", nil) + ip, tenant, apiError := common.IsProviderOrTenant(ctx, logger, gsh.dbSession, org, dbUser, false, nil) + if apiError != nil { + return cutil.NewAPIErrorResponse(c, apiError.Code, apiError.Message, apiError.Data) } // Get and validate includeRelation params @@ -1065,17 +1192,6 @@ func (gsh GetOperatingSystemHandler) Handle(c echo.Context) error { osDAO := cdbm.NewOperatingSystemDAO(gsh.dbSession) - // Validate the tenant for which this OperatingSystem is being retrieved - tenant, err := common.GetTenantForOrg(ctx, nil, gsh.dbSession, org) - if err != nil { - if err == common.ErrOrgTenantNotFound { - logger.Warn().Err(err).Msg("Org does not have a Tenant associated") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Org does not have a Tenant associated", nil) - } - logger.Error().Err(err).Msg("unable to retrieve tenant for org") - return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve tenant for org", nil) - } - // Check that operating system exists os, err := osDAO.GetByID(ctx, nil, sID, qIncludeRelations) if err != nil { @@ -1086,10 +1202,60 @@ func (gsh GetOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Could not retrieve OperatingSystem to update", nil) } - // verify tenant matches - if os.TenantID == nil || tenant.ID != *os.TenantID { - logger.Warn().Msg("tenant in org does not match tenant in operating system") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Tenant for OperatingSystem in request does not match tenant in org", nil) + // Visibility check with role-based rules: + // Provider admin: can only see provider-owned entries. + // Tenant admin: can see own entries + provider entries at accessible sites. + // Dual-role: can see both tenant and provider entries. + ownedByTenant := tenant != nil && os.TenantID != nil && *os.TenantID == tenant.ID + ownedByProvider := ip != nil && os.InfrastructureProviderID != nil && *os.InfrastructureProviderID == ip.ID + providerVisibilityNeedsSiteCheck := tenant != nil && ip == nil && os.InfrastructureProviderID != nil + + if !ownedByTenant && !ownedByProvider && !providerVisibilityNeedsSiteCheck { + logger.Warn().Msg("operating system does not belong to the tenant or provider in org") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Operating System does not belong to the tenant or infrastructure provider in org", nil) + } + + // If caller has dual role (Tenant+Provider) we already know we can go forward. + // Otherwise we need additional checks: + if !(tenant != nil && ip != nil) { + if ip != nil && !ownedByProvider { + logger.Warn().Msg("provider admin cannot view tenant-owned operating system") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Operating System does not belong to the infrastructure provider in org", nil) + } + if providerVisibilityNeedsSiteCheck { + // Tenant admin seeing a provider-owned entry: verify site-scoped visibility. + ossaDAO := cdbm.NewOperatingSystemSiteAssociationDAO(gsh.dbSession) + ossas, _, ossaErr := ossaDAO.GetAll(ctx, nil, + cdbm.OperatingSystemSiteAssociationFilterInput{OperatingSystemIDs: []uuid.UUID{os.ID}}, + cdbp.PageInput{Limit: cutil.GetPtr(cdbp.TotalLimit)}, + nil, + ) + if ossaErr != nil { + logger.Error().Err(ossaErr).Msg("error retrieving OS site associations for visibility check") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to verify site access for Operating System", nil) + } + + tenantSiteIDs, tsErr := getTenantSiteIDs(ctx, gsh.dbSession, tenant.ID) + if tsErr != nil { + logger.Error().Err(tsErr).Msg("error retrieving tenant site IDs for visibility check") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to determine site access for tenant", nil) + } + tsSet := make(map[uuid.UUID]struct{}, len(tenantSiteIDs)) + for _, sid := range tenantSiteIDs { + tsSet[sid] = struct{}{} + } + visible := false + for _, ossa := range ossas { + if _, ok := tsSet[ossa.SiteID]; ok { + visible = true + break + } + } + if !visible { + logger.Warn().Msg("provider-owned OS has no site associations at sites accessible to the tenant") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Operating System is not associated with any site accessible to the caller", nil) + } + } } // get status details for the response @@ -1194,22 +1360,9 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve current user", nil) } - // Validate org - ok, err := auth.ValidateOrgMembership(dbUser, org) - if !ok { - if err != nil { - logger.Error().Err(err).Msg("error validating org membership for User in request") - } else { - logger.Warn().Msg("could not validate org membership for user, access denied") - } - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, fmt.Sprintf("Failed to validate membership for org: %s", org), nil) - } - - // Validate role, only Tenant Admins are allowed to update OperatingSystem - ok = auth.ValidateUserRoles(dbUser, org, nil, auth.TenantAdminRole) - if !ok { - logger.Warn().Msg("user does not have Tenant Admin role, access denied") - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "User does not have Tenant Admin role with org", nil) + ip, tenant, apiError := common.IsProviderOrTenant(ctx, logger, ush.dbSession, org, dbUser, false, nil) + if apiError != nil { + return cutil.NewAPIErrorResponse(c, apiError.Code, apiError.Message, apiError.Data) } // Get os ID from URL param @@ -1258,41 +1411,45 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Error validating user data in Operating System creation request", verr) } - // Validate the tenant for which this OperatingSystem is being updated - tenant, err := common.GetTenantForOrg(ctx, nil, ush.dbSession, org) - if err != nil { - if err == common.ErrOrgTenantNotFound { - logger.Warn().Err(err).Msg("Org does not have a Tenant associated") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Org does not have a Tenant associated", nil) + // Enforce ownership: both roles are evaluated independently so a dual-role + // caller is permitted if either role authorizes the operation. + ownedByTenant := tenant != nil && os.TenantID != nil && *os.TenantID == tenant.ID && os.InfrastructureProviderID == nil + ownedByProvider := false + if ip != nil && os.InfrastructureProviderID != nil { + if *os.InfrastructureProviderID != ip.ID { + logger.Warn().Msg("provider admin cannot update operating system owned by a different provider") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Provider Admin can only update Operating Systems owned by their own provider", nil) } - logger.Error().Err(err).Msg("unable to retrieve tenant for org") - return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve tenant for org", nil) + ownedByProvider = true } - - // verify tenant matches - if os.TenantID == nil || tenant.ID != *os.TenantID { - logger.Warn().Msg("tenant in os does not belong to tenant in org") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Tenant for OperatingSystem in request does not match tenant in org", nil) + if !ownedByProvider && !ownedByTenant { + if ip != nil && tenant == nil { + logger.Warn().Msg("provider admin cannot update tenant-owned operating system") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Provider Admin can only update provider-owned Operating Systems", nil) + } + if tenant != nil && ip == nil { + logger.Warn().Msg("tenant admin cannot update provider-owned operating system") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Tenant Admin can only update their own Operating Systems", nil) + } + logger.Warn().Msg("user does not have permission to update this operating system") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Operating System does not belong to your tenant or infrastructure provider", nil) } - // check for name uniqueness for the tenant, ie, tenant cannot have another os with same name + // Check for name uniqueness within the owner's scope (provider or tenant). if apiRequest.Name != nil && *apiRequest.Name != os.Name { - oss, tot, serr := osDAO.GetAll( - ctx, - nil, - cdbm.OperatingSystemFilterInput{ - TenantIDs: []uuid.UUID{tenant.ID}, - Names: []string{*apiRequest.Name}, - }, - cdbp.PageInput{}, - nil, - ) + uniquenessFilter := cdbm.OperatingSystemFilterInput{Names: []string{*apiRequest.Name}} + if os.InfrastructureProviderID != nil { + uniquenessFilter.InfrastructureProviderID = os.InfrastructureProviderID + } else { + uniquenessFilter.TenantIDs = []uuid.UUID{tenant.ID} + } + oss, tot, serr := osDAO.GetAll(ctx, nil, uniquenessFilter, cdbp.PageInput{}, nil) if serr != nil { - logger.Error().Err(serr).Msg("db error checking for name uniqueness of tenant os") + logger.Error().Err(serr).Msg("db error checking for name uniqueness of os") return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to update OperatingSystem due to DB error", nil) } if tot > 0 { - return cutil.NewAPIErrorResponse(c, http.StatusConflict, "Another Operating System with specified name already exists for Tenant", validation.Errors{ + return cutil.NewAPIErrorResponse(c, http.StatusConflict, "Another Operating System with specified name already exists", validation.Errors{ "id": errors.New(oss[0].ID.String()), }) } @@ -1358,9 +1515,9 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { // For a Templated iPXE Operating System, verify the effective iPXE template (the // request's template when changing it, otherwise the current one) is available at - // every Site the OS is synced to before updating and re-pushing it. This mirrors - // the create-time check and also catches a request switching to a template that - // is not present at the OS's Sites. + // its associated Site before updating and re-pushing it. This mirrors the + // create-time check and also catches a request switching to a template that is + // not present at the OS's Site. if os.Type == cdbm.OperatingSystemTypeTemplatedIPXE { templatedOssas, _, oerr := ossaDAO.GetAll(ctx, nil, cdbm.OperatingSystemSiteAssociationFilterInput{OperatingSystemIDs: []uuid.UUID{os.ID}}, @@ -1369,11 +1526,15 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { logger.Error().Err(oerr).Msg("error retrieving Operating System Site associations for iPXE template validation") return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Operating System Site associations from DB", nil) } + if len(templatedOssas) != 1 { + logger.Warn().Int("siteAssociationCount", len(templatedOssas)).Msg("unable to update Templated iPXE Operating System unless it is associated with exactly one Site") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Templated iPXE Operating System must be associated with exactly one Site to be updated", nil) + } effectiveTemplateID := os.IpxeTemplateId if apiRequest.IpxeTemplateId != nil { effectiveTemplateID = apiRequest.IpxeTemplateId } - if effectiveTemplateID != nil && len(templatedOssas) > 0 { + if effectiveTemplateID != nil { targetSiteIDs := make([]uuid.UUID, 0, len(templatedOssas)) for _, o := range templatedOssas { targetSiteIDs = append(targetSiteIDs, o.SiteID) @@ -1505,7 +1666,7 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIError(http.StatusInternalServerError, "Failed to retrieve client for Site", nil) } - updateOsRequest := apiRequest.ToProto(uos, tenant.Org) + updateOsRequest := apiRequest.ToImageProto(uos, tenant.Org) workflowOptions := temporalClient.StartWorkflowOptions{ ID: "image-os-update-" + updatedOssa.SiteID.String() + "-" + uos.ID.String() + "-" + *updatedOssa.Version, @@ -1577,12 +1738,12 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { dbossas = refreshedOssas } - // Templated iPXE updates re-push the definition to every associated Site via - // the Core proxy after commit. Mark each association (and its status detail) + // Templated iPXE updates re-push the definition to its associated Site via + // the Core proxy after commit. Mark the association (and its status detail) // Syncing inside this transaction so the in-flight state is durable before any // proxy update runs: validateTemplatedIpxeOsForSite gates Instance selection on // a Synced association, so this prevents an Instance from being created against a - // definition that is mid-update. The post-commit proxy sync transitions each + // definition that is mid-update. The post-commit proxy sync transitions the // association to Synced or Error. if uos.Type == cdbm.OperatingSystemTypeTemplatedIPXE { tmplOssas, _, derr := ossaDAO.GetAll( @@ -1596,6 +1757,10 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { logger.Error().Err(derr).Msg("error retrieving Operating System Site associations for templated iPXE update") return cutil.NewAPIError(http.StatusInternalServerError, "Failed to retrieve Operating System Site associations, DB error", nil) } + if len(tmplOssas) != 1 { + logger.Warn().Int("siteAssociationCount", len(tmplOssas)).Msg("unable to update Templated iPXE Operating System unless it is associated with exactly one Site") + return cutil.NewAPIError(http.StatusBadRequest, "Templated iPXE Operating System must be associated with exactly one Site to be updated", nil) + } for _, tossa := range tmplOssas { if _, derr := ossaDAO.Update(ctx, tx, cdbm.OperatingSystemSiteAssociationUpdateInput{ OperatingSystemSiteAssociationID: tossa.ID, @@ -1644,7 +1809,7 @@ func (ush UpdateOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Operating System Site associations from DB", nil) } if len(ipxeOssas) > 0 { - req := model.BuildUpdateOperatingSystemRequest(uos) + req := apiRequest.ToProto(uos) siteErrors := syncOperatingSystemToSitesViaProxy(ctx, logger, ush.dbSession, ush.scp, ipxeOssas, updateOperatingSystemMethod, req) if aerr := updateOperatingSystemAggregateStatus(ctx, logger, ush.dbSession, uos.ID, siteErrors > 0, aggregateSyncMessage(siteErrors > 0)); aerr != nil { logger.Error().Err(aerr).Msg("failed to update aggregate Operating System status after update sync") @@ -1701,22 +1866,9 @@ func (dsh DeleteOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve current user", nil) } - // Validate org - ok, err := auth.ValidateOrgMembership(dbUser, org) - if !ok { - if err != nil { - logger.Error().Err(err).Msg("error validating org membership for User in request") - } else { - logger.Warn().Msg("could not validate org membership for user, access denied") - } - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, fmt.Sprintf("Failed to validate membership for org: %s", org), nil) - } - - // Validate role, only Tenant Admins are allowed to delete OperatingSystem - ok = auth.ValidateUserRoles(dbUser, org, nil, auth.TenantAdminRole) - if !ok { - logger.Warn().Msg("user does not have Tenant Admin role, access denied") - return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "User does not have Tenant Admin role with org", nil) + ip, tenant, apiError := common.IsProviderOrTenant(ctx, logger, dsh.dbSession, org, dbUser, false, nil) + if apiError != nil { + return cutil.NewAPIErrorResponse(c, apiError.Code, apiError.Message, apiError.Data) } // Get operating system ID from URL param @@ -1730,17 +1882,6 @@ func (dsh DeleteOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Invalid Operating System ID in URL", nil) } - // Validate the tenant for which this OperatingSystem is being updated - tenant, err := common.GetTenantForOrg(ctx, nil, dsh.dbSession, org) - if err != nil { - if err == common.ErrOrgTenantNotFound { - logger.Warn().Err(err).Msg("Org does not have a Tenant associated") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Org does not have a Tenant associated", nil) - } - logger.Error().Err(err).Msg("unable to retrieve tenant for org") - return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve tenant for org", nil) - } - // Check that operating system exists osDAO := cdbm.NewOperatingSystemDAO(dsh.dbSession) os, err := osDAO.GetByID(ctx, nil, osID, nil) @@ -1752,10 +1893,20 @@ func (dsh DeleteOperatingSystemHandler) Handle(c echo.Context) error { return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Could not retrieve Operating System to delete", nil) } - // verify tenant matches - if os.TenantID == nil || tenant.ID != *os.TenantID { - logger.Warn().Msg("tenant in os does not belong to tenant in org") - return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Tenant for Operating System in request does not match tenant in org", nil) + // Enforce ownership: both roles are evaluated independently so a dual-role + // caller is permitted if either role authorizes the operation. + ownedByTenant := tenant != nil && os.TenantID != nil && *os.TenantID == tenant.ID && os.InfrastructureProviderID == nil + ownedByProvider := false + if ip != nil && os.InfrastructureProviderID != nil { + if *os.InfrastructureProviderID != ip.ID { + logger.Warn().Msg("provider admin cannot delete operating system owned by a different provider") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Provider Admin can only delete Operating Systems owned by their own provider", nil) + } + ownedByProvider = true + } + if !ownedByProvider && !ownedByTenant { + logger.Warn().Msg("user does not have permission to delete this operating system") + return cutil.NewAPIErrorResponse(c, http.StatusForbidden, "Operating System does not belong to current org's Tenant or Infrastructure Provider", nil) } // Verify if tenant associated with Site in case of Image based OS @@ -1795,7 +1946,11 @@ func (dsh DeleteOperatingSystemHandler) Handle(c echo.Context) error { // verify no instances are using the os isDAO := cdbm.NewInstanceDAO(dsh.dbSession) - instances, _, err := isDAO.GetAll(ctx, nil, cdbm.InstanceFilterInput{TenantIDs: []uuid.UUID{tenant.ID}, OperatingSystemIDs: []uuid.UUID{os.ID}}, paginator.PageInput{}, nil) + instanceFilter := cdbm.InstanceFilterInput{OperatingSystemIDs: []uuid.UUID{os.ID}} + if tenant != nil { + instanceFilter.TenantIDs = []uuid.UUID{tenant.ID} + } + instances, _, err := isDAO.GetAll(ctx, nil, instanceFilter, paginator.PageInput{}, nil) if err != nil { logger.Error().Err(err).Msg("error retrieving Instances for Operating System from DB") return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve Instances for deleting operatingsystem", nil) @@ -1871,7 +2026,7 @@ func (dsh DeleteOperatingSystemHandler) Handle(c echo.Context) error { } // Prepare the delete/release request workflow object - deleteOsRequest := os.ToDeletionRequestProto(tenant.Org) + deleteOsRequest := os.ToImageDeletionRequestProto(tenant.Org) workflowOptions := temporalClient.StartWorkflowOptions{ ID: "image-os-delete-" + ossa.SiteID.String() + "-" + os.ID.String() + "-" + *ossa.Version, @@ -2012,7 +2167,7 @@ func (dsh DeleteOperatingSystemHandler) Handle(c echo.Context) error { // OS once every site is cleaned up. A not-found object on a site is treated as // already deleted. if os.Type == cdbm.OperatingSystemTypeTemplatedIPXE && len(ossasToDelete) > 0 { - req := model.BuildDeleteOperatingSystemRequest(os) + req := os.ToDeletionRequestProto() remaining := 0 for _, ossa := range ossasToDelete { slogger := logger.With().Str("Site ID", ossa.SiteID.String()).Logger() diff --git a/rest-api/api/pkg/api/handler/operatingsystem_templated_proxy_test.go b/rest-api/api/pkg/api/handler/operatingsystem_templated_proxy_test.go index ca86d851ba..30f76c11cb 100644 --- a/rest-api/api/pkg/api/handler/operatingsystem_templated_proxy_test.go +++ b/rest-api/api/pkg/api/handler/operatingsystem_templated_proxy_test.go @@ -48,7 +48,10 @@ type templatedProxyFixture struct { ipOrg string tnOrg string + ipu *cdbm.User tnu *cdbm.User + ip *cdbm.InfrastructureProvider + tn *cdbm.Tenant site *cdbm.Site tmpl *cdbm.IpxeTemplate } @@ -66,7 +69,6 @@ func buildTemplatedProxyFixture(t *testing.T) *templatedProxyFixture { tnOrg := "tmpl-proxy-tn-org" ipu := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{ipOrg}, []string{authz.ProviderAdminRole}) - _ = ipu tnu := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{tnOrg}, []string{authz.TenantAdminRole}) ip := testMachineBuildInfrastructureProvider(t, dbSession, ipOrg, "tmpl-proxy-provider") @@ -103,7 +105,10 @@ func buildTemplatedProxyFixture(t *testing.T) *templatedProxyFixture { tracer: tracer, ipOrg: ipOrg, tnOrg: tnOrg, + ipu: ipu, tnu: tnu, + ip: ip, + tn: tenant, site: site, tmpl: tmpl, } @@ -147,6 +152,10 @@ func (f *templatedProxyFixture) bindProxyClient(psc *proxySiteClient) { } func (f *templatedProxyFixture) newEchoContext(method, body string, params map[string]string) (echo.Context, *httptest.ResponseRecorder) { + return f.newEchoContextForUser(method, body, params, f.tnu) +} + +func (f *templatedProxyFixture) newEchoContextForUser(method, body string, params map[string]string, user *cdbm.User) (echo.Context, *httptest.ResponseRecorder) { e := echo.New() req := httptest.NewRequest(method, "/", strings.NewReader(body)) req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) @@ -160,7 +169,7 @@ func (f *templatedProxyFixture) newEchoContext(method, body string, params map[s } ec.SetParamNames(names...) ec.SetParamValues(values...) - ec.Set("user", f.tnu) + ec.Set("user", user) reqCtx := context.WithValue(f.ctx, otelecho.TracerKey, f.tracer) ec.SetRequest(ec.Request().WithContext(reqCtx)) return ec, rec @@ -328,6 +337,98 @@ func TestOperatingSystemHandler_TemplatedIPXE_Proxy(t *testing.T) { }) } +func TestOperatingSystemHandler_TemplatedIPXE_ProviderCreateOmitsTenantOrganizationID(t *testing.T) { + f := buildTemplatedProxyFixture(t) + psc := newProxySiteClient(t, createOperatingSystemMethod, nil, nil) + f.bindProxyClient(psc) + + createReq := model.APIOperatingSystemCreateRequest{ + Name: "tmpl-proxy-provider-os", + Description: cutil.GetPtr("provider-owned templated OS"), + IpxeTemplateId: cutil.GetPtr(f.tmpl.ID.String()), + SiteIDs: []string{f.site.ID.String()}, + } + body, err := json.Marshal(createReq) + require.NoError(t, err) + + ec, rec := f.newEchoContextForUser( + http.MethodPost, + string(body), + map[string]string{"orgName": f.ipOrg}, + f.ipu, + ) + h := CreateOperatingSystemHandler{dbSession: f.dbSession, tc: f.tc, scp: f.scp, cfg: f.cfg} + require.NoError(t, h.Handle(ec)) + require.Equal(t, http.StatusCreated, rec.Code, "body: %s", rec.Body.String()) + + psc.client.AssertExpectations(t) + psc.workflow.AssertExpectations(t) + var coreReq corev1.CreateOperatingSystemRequest + require.NoError(t, protojson.Unmarshal(psc.captured.RequestJSON, &coreReq)) + assert.Equal(t, "tmpl-proxy-provider-os", coreReq.Name) + assert.Nil(t, coreReq.TenantOrganizationId, "provider-owned OS must omit tenant_organization_id") + + var rsp model.APIOperatingSystem + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &rsp)) + osID, err := uuid.Parse(rsp.ID) + require.NoError(t, err) + osDAO := cdbm.NewOperatingSystemDAO(f.dbSession) + persisted, err := osDAO.GetByID(f.ctx, nil, osID, nil) + require.NoError(t, err) + require.NotNil(t, persisted.InfrastructureProviderID) + assert.Equal(t, f.site.InfrastructureProviderID, *persisted.InfrastructureProviderID) + assert.Nil(t, persisted.TenantID) +} + +func TestOperatingSystemHandler_TemplatedIPXE_UpdateRejectsMultipleSites(t *testing.T) { + f := buildTemplatedProxyFixture(t) + + secondSite := testMachineBuildSite(t, f.dbSession, f.ip, "tmpl-proxy-second-site", cdbm.SiteStatusRegistered) + testBuildTenantSiteAssociation(t, f.dbSession, f.tnOrg, f.tn.ID, secondSite.ID, f.tnu.ID) + itsaDAO := cdbm.NewIpxeTemplateSiteAssociationDAO(f.dbSession) + _, err := itsaDAO.Create(f.ctx, nil, cdbm.IpxeTemplateSiteAssociationCreateInput{ + IpxeTemplateID: f.tmpl.ID, + SiteID: secondSite.ID, + }) + require.NoError(t, err) + + originalDescription := "multi-site templated OS" + osDAO := cdbm.NewOperatingSystemDAO(f.dbSession) + os, err := osDAO.Create(f.ctx, nil, cdbm.OperatingSystemCreateInput{ + Name: "tmpl-proxy-multi-site-os", + Description: &originalDescription, + Org: f.tnOrg, + TenantID: &f.tn.ID, + OsType: cdbm.OperatingSystemTypeTemplatedIPXE, + IpxeTemplateId: cutil.GetPtr(f.tmpl.ID.String()), + Status: cdbm.OperatingSystemStatusReady, + CreatedBy: f.tnu.ID, + }) + require.NoError(t, err) + common.TestBuildOperatingSystemSiteAssociation(t, f.dbSession, os.ID, f.site.ID, cutil.GetPtr("site-1"), cdbm.OperatingSystemSiteAssociationStatusSynced, f.tnu) + common.TestBuildOperatingSystemSiteAssociation(t, f.dbSession, os.ID, secondSite.ID, cutil.GetPtr("site-2"), cdbm.OperatingSystemSiteAssociationStatusSynced, f.tnu) + + updateReq := model.APIOperatingSystemUpdateRequest{ + Description: cutil.GetPtr("must not be persisted"), + } + body, err := json.Marshal(updateReq) + require.NoError(t, err) + ec, rec := f.newEchoContext( + http.MethodPatch, + string(body), + map[string]string{"orgName": f.tnOrg, "id": os.ID.String()}, + ) + + h := UpdateOperatingSystemHandler{dbSession: f.dbSession, tc: f.tc, scp: f.scp, cfg: f.cfg} + require.NoError(t, h.Handle(ec)) + require.Equal(t, http.StatusBadRequest, rec.Code, "body: %s", rec.Body.String()) + + persisted, err := osDAO.GetByID(f.ctx, nil, os.ID, nil) + require.NoError(t, err) + require.NotNil(t, persisted.Description) + assert.Equal(t, originalDescription, *persisted.Description) +} + func TestOperatingSystemHandler_TemplatedIPXE_ProxyCreateExecuteError(t *testing.T) { f := buildTemplatedProxyFixture(t) diff --git a/rest-api/api/pkg/api/handler/operatingsystem_test.go b/rest-api/api/pkg/api/handler/operatingsystem_test.go index 87d8182edf..ef4d7d5d9d 100644 --- a/rest-api/api/pkg/api/handler/operatingsystem_test.go +++ b/rest-api/api/pkg/api/handler/operatingsystem_test.go @@ -1607,7 +1607,7 @@ func TestOperatingSystemHandler_Update(t *testing.T) { user: user, osID: os2.ID.String(), expectedErr: true, - expectedStatus: http.StatusBadRequest, + expectedStatus: http.StatusForbidden, }, { name: "error when req body doesnt bind", @@ -2158,7 +2158,7 @@ func TestOperatingSystemHandler_Delete(t *testing.T) { user: tnu, osID: os3.ID.String(), expectedErr: true, - expectedStatus: http.StatusBadRequest, + expectedStatus: http.StatusForbidden, }, { name: "error when instance present for os", @@ -2278,3 +2278,503 @@ func TestOperatingSystemHandler_Delete(t *testing.T) { }) } } + +// buildRawIpxeProviderOS creates a provider-owned raw iPXE OS (no site +// associations) via the DAO. Raw iPXE avoids any post-commit site sync, so the +// write handlers exercise ownership enforcement without Temporal/proxy mocks. +func buildRawIpxeProviderOS(t *testing.T, ctx context.Context, osDAO cdbm.OperatingSystemDAO, org string, providerID uuid.UUID, name string, createdBy uuid.UUID) *cdbm.OperatingSystem { + os, err := osDAO.Create(ctx, nil, cdbm.OperatingSystemCreateInput{ + Name: name, + Description: cutil.GetPtr("test"), + Org: org, + InfrastructureProviderID: &providerID, + OsType: cdbm.OperatingSystemTypeIPXE, + IpxeScript: cutil.GetPtr("ipxe"), + Status: cdbm.OperatingSystemStatusReady, + CreatedBy: createdBy, + }) + require.NoError(t, err) + require.NotNil(t, os) + return os +} + +func buildRawIpxeTenantOS(t *testing.T, ctx context.Context, osDAO cdbm.OperatingSystemDAO, org string, tenantID uuid.UUID, name string, createdBy uuid.UUID) *cdbm.OperatingSystem { + os, err := osDAO.Create(ctx, nil, cdbm.OperatingSystemCreateInput{ + Name: name, + Description: cutil.GetPtr("test"), + Org: org, + TenantID: &tenantID, + OsType: cdbm.OperatingSystemTypeIPXE, + IpxeScript: cutil.GetPtr("ipxe"), + Status: cdbm.OperatingSystemStatusReady, + CreatedBy: createdBy, + }) + require.NoError(t, err) + require.NotNil(t, os) + return os +} + +// TestOperatingSystemHandler_Create_Ownership asserts that a Provider Admin may +// only create iPXE Template-based Operating Systems. +func TestOperatingSystemHandler_Create_Ownership(t *testing.T) { + ctx := context.Background() + dbSession := testMachineInitDB(t) + defer dbSession.Close() + common.TestSetupSchema(t, dbSession) + + cfg := common.GetTestConfig() + tcfg, _ := cfg.GetTemporalConfig() + scp := sc.NewClientPool(tcfg) + tempClient := &tmocks.Client{} + + provOrg := "own-provider-org" + provUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{provOrg}, []string{authz.ProviderAdminRole}) + testMachineBuildInfrastructureProvider(t, dbSession, provOrg, "own-ip") + + tracer, _, ctx := common.TestCommonTraceProviderSetup(t, ctx) + + tests := []struct { + name string + reqBody model.APIOperatingSystemCreateRequest + expectedStatus int + }{ + { + name: "provider admin cannot create image OS", + reqBody: model.APIOperatingSystemCreateRequest{Name: "prov-image", Description: cutil.GetPtr("test"), ImageURL: cutil.GetPtr("https://example.com/img.iso")}, + expectedStatus: http.StatusForbidden, + }, + { + name: "provider admin cannot create raw ipxe OS", + reqBody: model.APIOperatingSystemCreateRequest{Name: "prov-ipxe", Description: cutil.GetPtr("test"), IpxeScript: cutil.GetPtr("ipxe")}, + expectedStatus: http.StatusForbidden, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + body, err := json.Marshal(tc.reqBody) + require.NoError(t, err) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(body))) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + + ec := e.NewContext(req, rec) + ec.SetParamNames("orgName") + ec.SetParamValues(provOrg) + ec.Set("user", provUser) + ec.SetRequest(ec.Request().WithContext(context.WithValue(ctx, otelecho.TracerKey, tracer))) + + ch := CreateOperatingSystemHandler{dbSession: dbSession, tc: tempClient, cfg: cfg, scp: scp} + require.NoError(t, ch.Handle(ec)) + require.Equal(t, tc.expectedStatus, rec.Code) + }) + } +} + +// TestOperatingSystemHandler_Update_Ownership asserts ownership enforcement for +// updates across provider and tenant roles. +func TestOperatingSystemHandler_Update_Ownership(t *testing.T) { + ctx := context.Background() + dbSession := testMachineInitDB(t) + defer dbSession.Close() + common.TestSetupSchema(t, dbSession) + + cfg := common.GetTestConfig() + tcfg, _ := cfg.GetTemporalConfig() + scp := sc.NewClientPool(tcfg) + tempClient := &tmocks.Client{} + osDAO := cdbm.NewOperatingSystemDAO(dbSession) + + provOrg := "own-provider-org" + provUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{provOrg}, []string{authz.ProviderAdminRole}) + ip := testMachineBuildInfrastructureProvider(t, dbSession, provOrg, "own-ip") + provOS := buildRawIpxeProviderOS(t, ctx, osDAO, provOrg, ip.ID, "prov-os-update", provUser.ID) + + sharedOrg := "own-shared-org" + tnUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{sharedOrg}, []string{authz.TenantAdminRole}) + sharedProvUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{sharedOrg}, []string{authz.ProviderAdminRole}) + ip2 := testMachineBuildInfrastructureProvider(t, dbSession, sharedOrg, "own-ip-2") + tn := testMachineBuildTenant(t, dbSession, sharedOrg, "own-tenant") + provOSShared := buildRawIpxeProviderOS(t, ctx, osDAO, sharedOrg, ip2.ID, "prov-os-shared-update", sharedProvUser.ID) + tnOS := buildRawIpxeTenantOS(t, ctx, osDAO, sharedOrg, tn.ID, "tenant-os-update", tnUser.ID) + + tracer, _, ctx := common.TestCommonTraceProviderSetup(t, ctx) + + tests := []struct { + name string + reqOrgName string + user *cdbm.User + os *cdbm.OperatingSystem + expectedStatus int + }{ + { + name: "provider admin updates own provider OS", + reqOrgName: provOrg, + user: provUser, + os: provOS, + expectedStatus: http.StatusOK, + }, + { + name: "tenant admin cannot update provider-owned OS", + reqOrgName: sharedOrg, + user: tnUser, + os: provOSShared, + expectedStatus: http.StatusForbidden, + }, + { + name: "provider admin cannot update tenant-owned OS", + reqOrgName: sharedOrg, + user: sharedProvUser, + os: tnOS, + expectedStatus: http.StatusForbidden, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + body, err := json.Marshal(model.APIOperatingSystemUpdateRequest{Description: cutil.GetPtr("updated description")}) + require.NoError(t, err) + + e := echo.New() + req := httptest.NewRequest(http.MethodPut, "/", strings.NewReader(string(body))) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + + ec := e.NewContext(req, rec) + ec.SetParamNames("orgName", "id") + ec.SetParamValues(tc.reqOrgName, tc.os.ID.String()) + ec.Set("user", tc.user) + ec.SetRequest(ec.Request().WithContext(context.WithValue(ctx, otelecho.TracerKey, tracer))) + + uh := UpdateOperatingSystemHandler{dbSession: dbSession, tc: tempClient, cfg: cfg, scp: scp} + require.NoError(t, uh.Handle(ec)) + require.Equal(t, tc.expectedStatus, rec.Code) + }) + } +} + +// TestOperatingSystemHandler_Delete_Ownership asserts ownership enforcement for +// deletes across provider and tenant roles. +func TestOperatingSystemHandler_Delete_Ownership(t *testing.T) { + ctx := context.Background() + dbSession := testMachineInitDB(t) + defer dbSession.Close() + common.TestSetupSchema(t, dbSession) + + cfg := common.GetTestConfig() + tcfg, _ := cfg.GetTemporalConfig() + scp := sc.NewClientPool(tcfg) + tempClient := &tmocks.Client{} + osDAO := cdbm.NewOperatingSystemDAO(dbSession) + + provOrg := "own-provider-org" + provUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{provOrg}, []string{authz.ProviderAdminRole}) + ip := testMachineBuildInfrastructureProvider(t, dbSession, provOrg, "own-ip") + provOS := buildRawIpxeProviderOS(t, ctx, osDAO, provOrg, ip.ID, "prov-os-delete", provUser.ID) + + sharedOrg := "own-shared-org" + tnUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{sharedOrg}, []string{authz.TenantAdminRole}) + sharedProvUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{sharedOrg}, []string{authz.ProviderAdminRole}) + ip2 := testMachineBuildInfrastructureProvider(t, dbSession, sharedOrg, "own-ip-2") + tn := testMachineBuildTenant(t, dbSession, sharedOrg, "own-tenant") + provOSShared := buildRawIpxeProviderOS(t, ctx, osDAO, sharedOrg, ip2.ID, "prov-os-shared-delete", sharedProvUser.ID) + tnOS := buildRawIpxeTenantOS(t, ctx, osDAO, sharedOrg, tn.ID, "tenant-os-delete", tnUser.ID) + + tracer, _, ctx := common.TestCommonTraceProviderSetup(t, ctx) + + tests := []struct { + name string + reqOrgName string + user *cdbm.User + os *cdbm.OperatingSystem + expectedStatus int + }{ + { + name: "tenant admin cannot delete provider-owned OS", + reqOrgName: sharedOrg, + user: tnUser, + os: provOSShared, + expectedStatus: http.StatusForbidden, + }, + { + name: "provider admin cannot delete tenant-owned OS", + reqOrgName: sharedOrg, + user: sharedProvUser, + os: tnOS, + expectedStatus: http.StatusForbidden, + }, + { + name: "provider admin deletes own provider OS", + reqOrgName: provOrg, + user: provUser, + os: provOS, + expectedStatus: http.StatusAccepted, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := echo.New() + req := httptest.NewRequest(http.MethodDelete, "/", nil) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + + ec := e.NewContext(req, rec) + ec.SetParamNames("orgName", "id") + ec.SetParamValues(tc.reqOrgName, tc.os.ID.String()) + ec.Set("user", tc.user) + ec.SetRequest(ec.Request().WithContext(context.WithValue(ctx, otelecho.TracerKey, tracer))) + + dh := DeleteOperatingSystemHandler{dbSession: dbSession, tc: tempClient, cfg: cfg, scp: scp} + require.NoError(t, dh.Handle(ec)) + require.Equal(t, tc.expectedStatus, rec.Code) + assert.NotEqual(t, "", tc.name) + }) + } +} + +// buildProviderOS creates a provider-owned OS (no tenant) for the given provider. +func buildProviderOS(t *testing.T, ctx context.Context, osDAO cdbm.OperatingSystemDAO, org string, providerID uuid.UUID, name string, createdBy uuid.UUID) *cdbm.OperatingSystem { + os, err := osDAO.Create(ctx, nil, cdbm.OperatingSystemCreateInput{ + Name: name, + Description: cutil.GetPtr("test"), + Org: org, + InfrastructureProviderID: &providerID, + OsType: cdbm.OperatingSystemTypeIPXE, + IpxeScript: cutil.GetPtr("ipxe"), + Status: cdbm.OperatingSystemStatusReady, + CreatedBy: createdBy, + }) + require.NoError(t, err) + require.NotNil(t, os) + return os +} + +// buildTenantOS creates a tenant-owned OS for the given tenant. +func buildTenantOS(t *testing.T, ctx context.Context, osDAO cdbm.OperatingSystemDAO, org string, tenantID uuid.UUID, name string, createdBy uuid.UUID) *cdbm.OperatingSystem { + os, err := osDAO.Create(ctx, nil, cdbm.OperatingSystemCreateInput{ + Name: name, + Description: cutil.GetPtr("test"), + Org: org, + TenantID: &tenantID, + OsType: cdbm.OperatingSystemTypeIPXE, + IpxeScript: cutil.GetPtr("ipxe"), + Status: cdbm.OperatingSystemStatusReady, + CreatedBy: createdBy, + AllowOverride: false, + }) + require.NoError(t, err) + require.NotNil(t, os) + return os +} + +// TestOperatingSystemHandler_GetAll_Visibility exercises provider-admin listing +// and tenant cross-visibility of provider-owned OSes at accessible sites. +func TestOperatingSystemHandler_GetAll_Visibility(t *testing.T) { + ctx := context.Background() + dbSession := testMachineInitDB(t) + defer dbSession.Close() + common.TestSetupSchema(t, dbSession) + + cfg := common.GetTestConfig() + tempClient := &tmocks.Client{} + osDAO := cdbm.NewOperatingSystemDAO(dbSession) + + // Provider-only org. + provOrg := "vis-provider-org" + provUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{provOrg}, []string{authz.ProviderAdminRole}) + ip := testMachineBuildInfrastructureProvider(t, dbSession, provOrg, "vis-ip") + siteA := testMachineBuildSite(t, dbSession, ip, "vis-site-a", cdbm.SiteStatusRegistered) + siteB := testMachineBuildSite(t, dbSession, ip, "vis-site-b", cdbm.SiteStatusRegistered) + + provOSA := buildProviderOS(t, ctx, osDAO, provOrg, ip.ID, "prov-os-a", provUser.ID) + common.TestBuildOperatingSystemSiteAssociation(t, dbSession, provOSA.ID, siteA.ID, cutil.GetPtr("test"), cdbm.OperatingSystemSiteAssociationStatusSynced, provUser) + provOSB := buildProviderOS(t, ctx, osDAO, provOrg, ip.ID, "prov-os-b", provUser.ID) + common.TestBuildOperatingSystemSiteAssociation(t, dbSession, provOSB.ID, siteB.ID, cutil.GetPtr("test"), cdbm.OperatingSystemSiteAssociationStatusSynced, provUser) + + // The tenant can see provider-owned entries from a different org only when + // they are associated with a Site the tenant can access. + tenantOrg := "vis-tenant-org" + servingProviderOrg := "vis-serving-provider-org" + tnUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{tenantOrg}, []string{authz.TenantAdminRole}) + ip2 := testMachineBuildInfrastructureProvider(t, dbSession, servingProviderOrg, "vis-ip-2") + siteC := testMachineBuildSite(t, dbSession, ip2, "vis-site-c", cdbm.SiteStatusRegistered) + siteD := testMachineBuildSite(t, dbSession, ip2, "vis-site-d", cdbm.SiteStatusRegistered) + tn := testMachineBuildTenant(t, dbSession, tenantOrg, "vis-tenant") + tsC := testBuildTenantSiteAssociation(t, dbSession, tenantOrg, tn.ID, siteC.ID, tnUser.ID) + assert.NotNil(t, tsC) + + buildTenantOS(t, ctx, osDAO, tenantOrg, tn.ID, "tenant-os-1", tnUser.ID) + buildTenantOS(t, ctx, osDAO, tenantOrg, tn.ID, "tenant-os-2", tnUser.ID) + provC := buildProviderOS(t, ctx, osDAO, servingProviderOrg, ip2.ID, "prov-os-c", tnUser.ID) + common.TestBuildOperatingSystemSiteAssociation(t, dbSession, provC.ID, siteC.ID, cutil.GetPtr("test"), cdbm.OperatingSystemSiteAssociationStatusSynced, tnUser) + provD := buildProviderOS(t, ctx, osDAO, servingProviderOrg, ip2.ID, "prov-os-d", tnUser.ID) + common.TestBuildOperatingSystemSiteAssociation(t, dbSession, provD.ID, siteD.ID, cutil.GetPtr("test"), cdbm.OperatingSystemSiteAssociationStatusSynced, tnUser) + buildProviderOS(t, ctx, osDAO, servingProviderOrg, ip2.ID, "prov-os-without-site", tnUser.ID) + + tracer, _, ctx := common.TestCommonTraceProviderSetup(t, ctx) + + tests := []struct { + name string + reqOrgName string + user *cdbm.User + target string + expectedNames []string + }{ + { + name: "provider admin sees only provider-owned OSes", + reqOrgName: provOrg, + user: provUser, + expectedNames: []string{"prov-os-a", "prov-os-b"}, + }, + { + name: "tenant admin sees own OSes plus provider OSes at accessible sites", + reqOrgName: tenantOrg, + user: tnUser, + expectedNames: []string{"tenant-os-1", "tenant-os-2", "prov-os-c"}, + }, + { + name: "tenant visibility union is paginated after merging", + reqOrgName: tenantOrg, + user: tnUser, + target: "/?pageSize=2&orderBy=NAME_ASC", + expectedNames: []string{"prov-os-c", "tenant-os-1"}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := echo.New() + target := tc.target + if target == "" { + target = "/" + } + req := httptest.NewRequest(http.MethodGet, target, nil) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + + ec := e.NewContext(req, rec) + ec.SetParamNames("orgName") + ec.SetParamValues(tc.reqOrgName) + ec.Set("user", tc.user) + + reqCtx := context.WithValue(ctx, otelecho.TracerKey, tracer) + ec.SetRequest(ec.Request().WithContext(reqCtx)) + + mh := GetAllOperatingSystemHandler{dbSession: dbSession, tc: tempClient, cfg: cfg} + err := mh.Handle(ec) + assert.Nil(t, err) + require.Equal(t, http.StatusOK, rec.Code) + + rsp := []model.APIOperatingSystem{} + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &rsp)) + gotNames := make([]string, len(rsp)) + for i, os := range rsp { + gotNames[i] = os.Name + } + assert.ElementsMatch(t, tc.expectedNames, gotNames) + }) + } +} + +// TestOperatingSystemHandler_GetByID_Visibility exercises role-based access to a +// single OS: provider admins may only read provider-owned entries, tenant admins +// may read own entries plus provider entries at accessible sites. +func TestOperatingSystemHandler_GetByID_Visibility(t *testing.T) { + ctx := context.Background() + dbSession := testMachineInitDB(t) + defer dbSession.Close() + common.TestSetupSchema(t, dbSession) + + cfg := common.GetTestConfig() + tempClient := &tmocks.Client{} + osDAO := cdbm.NewOperatingSystemDAO(dbSession) + + provOrg := "vis-provider-org" + provUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{provOrg}, []string{authz.ProviderAdminRole}) + ip := testMachineBuildInfrastructureProvider(t, dbSession, provOrg, "vis-ip") + siteA := testMachineBuildSite(t, dbSession, ip, "vis-site-a", cdbm.SiteStatusRegistered) + provOSA := buildProviderOS(t, ctx, osDAO, provOrg, ip.ID, "prov-os-a", provUser.ID) + common.TestBuildOperatingSystemSiteAssociation(t, dbSession, provOSA.ID, siteA.ID, cutil.GetPtr("test"), cdbm.OperatingSystemSiteAssociationStatusSynced, provUser) + + tenantOrg := "vis-tenant-org" + servingProviderOrg := "vis-serving-provider-org" + tnUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{tenantOrg}, []string{authz.TenantAdminRole}) + servingProviderUser := testMachineBuildUser(t, dbSession, uuid.NewString(), []string{servingProviderOrg}, []string{authz.ProviderAdminRole}) + ip2 := testMachineBuildInfrastructureProvider(t, dbSession, servingProviderOrg, "vis-ip-2") + siteC := testMachineBuildSite(t, dbSession, ip2, "vis-site-c", cdbm.SiteStatusRegistered) + siteD := testMachineBuildSite(t, dbSession, ip2, "vis-site-d", cdbm.SiteStatusRegistered) + tn := testMachineBuildTenant(t, dbSession, tenantOrg, "vis-tenant") + testBuildTenantSiteAssociation(t, dbSession, tenantOrg, tn.ID, siteC.ID, tnUser.ID) + + tnOS := buildTenantOS(t, ctx, osDAO, tenantOrg, tn.ID, "tenant-os-1", tnUser.ID) + provC := buildProviderOS(t, ctx, osDAO, servingProviderOrg, ip2.ID, "prov-os-c", tnUser.ID) + common.TestBuildOperatingSystemSiteAssociation(t, dbSession, provC.ID, siteC.ID, cutil.GetPtr("test"), cdbm.OperatingSystemSiteAssociationStatusSynced, tnUser) + provD := buildProviderOS(t, ctx, osDAO, servingProviderOrg, ip2.ID, "prov-os-d", tnUser.ID) + common.TestBuildOperatingSystemSiteAssociation(t, dbSession, provD.ID, siteD.ID, cutil.GetPtr("test"), cdbm.OperatingSystemSiteAssociationStatusSynced, tnUser) + + tracer, _, ctx := common.TestCommonTraceProviderSetup(t, ctx) + + tests := []struct { + name string + reqOrgName string + user *cdbm.User + os *cdbm.OperatingSystem + expectedStatus int + }{ + { + name: "provider admin can read provider-owned OS", + reqOrgName: provOrg, + user: provUser, + os: provOSA, + expectedStatus: http.StatusOK, + }, + { + name: "tenant admin can read provider OS at accessible site", + reqOrgName: tenantOrg, + user: tnUser, + os: provC, + expectedStatus: http.StatusOK, + }, + { + name: "tenant admin cannot read provider OS at inaccessible site", + reqOrgName: tenantOrg, + user: tnUser, + os: provD, + expectedStatus: http.StatusForbidden, + }, + { + name: "tenant admin can read own OS", + reqOrgName: tenantOrg, + user: tnUser, + os: tnOS, + expectedStatus: http.StatusOK, + }, + { + name: "provider admin cannot read tenant-owned OS", + reqOrgName: servingProviderOrg, + user: servingProviderUser, + os: tnOS, + expectedStatus: http.StatusForbidden, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + + ec := e.NewContext(req, rec) + ec.SetParamNames("orgName", "id") + ec.SetParamValues(tc.reqOrgName, tc.os.ID.String()) + ec.Set("user", tc.user) + + reqCtx := context.WithValue(ctx, otelecho.TracerKey, tracer) + ec.SetRequest(ec.Request().WithContext(reqCtx)) + + gh := GetOperatingSystemHandler{dbSession: dbSession, tc: tempClient, cfg: cfg} + err := gh.Handle(ec) + assert.Nil(t, err) + require.Equal(t, tc.expectedStatus, rec.Code) + }) + } +} diff --git a/rest-api/api/pkg/api/handler/sku.go b/rest-api/api/pkg/api/handler/sku.go index 4666c84550..c5b441ab91 100644 --- a/rest-api/api/pkg/api/handler/sku.go +++ b/rest-api/api/pkg/api/handler/sku.go @@ -8,15 +8,18 @@ import ( "errors" "fmt" "net/http" + "reflect" "github.com/NVIDIA/infra-controller/rest-api/api/internal/config" "github.com/NVIDIA/infra-controller/rest-api/api/pkg/api/handler/util/common" "github.com/NVIDIA/infra-controller/rest-api/api/pkg/api/model" "github.com/NVIDIA/infra-controller/rest-api/api/pkg/api/pagination" + sc "github.com/NVIDIA/infra-controller/rest-api/api/pkg/client/site" cutil "github.com/NVIDIA/infra-controller/rest-api/common/pkg/util" cdb "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db" cdbm "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/model" "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/paginator" + corev1 "github.com/NVIDIA/infra-controller/rest-api/proto/core/gen/v1" "github.com/google/uuid" "github.com/labstack/echo/v4" "go.opentelemetry.io/otel/attribute" @@ -277,3 +280,404 @@ func (gsh GetSkuHandler) Handle(c echo.Context) error { logger.Info().Msg("finishing API handler") return c.JSON(http.StatusOK, apiSku) } + +// CreateSkuHandler creates one SKU on a Site's Core service. +type CreateSkuHandler struct { + dbSession *cdb.Session + scp *sc.ClientPool + tracerSpan *cutil.TracerSpan +} + +// NewCreateSkuHandler initializes and returns a new CreateSkuHandler. +func NewCreateSkuHandler(dbSession *cdb.Session, scp *sc.ClientPool) CreateSkuHandler { + return CreateSkuHandler{ + dbSession: dbSession, + scp: scp, + tracerSpan: cutil.NewTracerSpan(), + } +} + +// Handle godoc +// @Summary Create SKU +// @Description Create a SKU on the selected Site's Core service. +// @Tags SKU +// @Accept json +// @Produce json +// @Security ApiKeyAuth +// @Param org path string true "Name of NGC organization" +// @Param request body model.APISkuCreateRequest true "SKU create request" +// @Success 201 {object} model.APISku +// @Router /v2/org/{org}/nico/sku [post] +func (csh CreateSkuHandler) Handle(c echo.Context) error { + org, dbUser, ctx, logger, handlerSpan := common.SetupHandler("SKU", "Create", c, csh.tracerSpan) + if handlerSpan != nil { + defer handlerSpan.End() + } + + // Validate request data + // Bind request data to API model + apiReq := model.APISkuCreateRequest{} + err := c.Bind(&apiReq) + if err != nil { + logger.Warn().Err(err).Msg("error binding request data into API model") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to parse request data, potentially invalid structure", nil) + } + + // Validate request attributes + verr := apiReq.Validate() + if verr != nil { + logger.Warn().Err(verr).Msg("error validating SKU create request data") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Error validating SKU create request data", verr) + } + + stc, siteID, apiErr := common.AuthorizeProviderSiteForCore(common.AuthorizeProviderSiteForCoreInput{ + Ctx: ctx, + Logger: logger, + DBSession: csh.dbSession, + SCP: csh.scp, + Org: org, + User: dbUser, + SiteID: apiReq.SiteID, + }) + if apiErr != nil { + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, apiErr.Data) + } + + logger.Info().Str("skuID", apiReq.ID).Str("siteID", siteID).Msg("creating SKU via Core proxy") + var ids corev1.SkuIdList + apiErr = common.ExecuteCoreGRPC(ctx, stc, corev1.Forge_CreateSku_FullMethodName, apiReq.ToProto(), &ids, siteID) + if apiErr != nil { + logAPIError(logger, apiErr, "failed to create SKU via Core proxy") + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, nil) + } + if len(ids.Ids) != 1 { + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Core returned an unexpected SKU create response", nil) + } + + var response corev1.SkuList + apiErr = common.ExecuteCoreGRPC( + ctx, + stc, + corev1.Forge_FindSkusByIds_FullMethodName, + &corev1.SkusByIdsRequest{Ids: []string{ids.Ids[0]}}, + &response, + siteID, + ) + if apiErr == nil && len(response.Skus) != 1 { + apiErr = cutil.NewAPIError(http.StatusNotFound, "Could not find SKU with the specified ID", nil) + } + + var skuToPersist *corev1.Sku + if apiErr != nil { + logger.Warn().Err(apiErr).Str("skuID", ids.Ids[0]).Str("siteID", siteID). + Msg("SKU created but post-create retrieval failed; returning request-derived response") + createdSKU := apiReq.ToProto().Skus[0] + createdSKU.Id = ids.Ids[0] + skuToPersist = createdSKU + } else { + skuToPersist = response.Skus[0] + } + + if skuToPersist == nil || skuToPersist.Id == "" { + err = errors.New("cannot persist an empty Core SKU") + } else { + projected := &cdbm.SKU{} + projected.FromProto(skuToPersist, uuid.MustParse(siteID)) + skuDAO := cdbm.NewSkuDAO(csh.dbSession) + err = cdb.WithTx(ctx, csh.dbSession, func(tx *cdb.Tx) error { + err := tx.TryAcquireAdvisoryLock(ctx, cdb.GetAdvisoryLockIDFromString(projected.ID), nil) + if err != nil { + return err + } + + _, err = skuDAO.Create(ctx, tx, cdbm.SkuCreateInput{ + SkuID: projected.ID, + SiteID: projected.SiteID, + Description: projected.Description, + SchemaVersion: projected.SchemaVersion, + Components: projected.Components, + DeviceType: projected.DeviceType, + AssociatedMachineIds: projected.AssociatedMachineIds, + }) + return err + }) + } + if err != nil { + errorChecker := cdb.PostgresErrorChecker{} + if errorChecker.IsUniqueConstraintError(err) { + message := fmt.Sprintf( + "SKU %q already exists; inspect it with GET /v2/org/%s/nico/sku/%s and retry changes as an update", + skuToPersist.Id, + org, + skuToPersist.Id, + ) + return cutil.NewAPIErrorResponse(c, http.StatusConflict, message, nil) + } + logger.Error().Err(err).Str("skuID", ids.Ids[0]).Str("siteID", siteID). + Msg("SKU created in Core but failed to update REST DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "SKU was created in Core but failed to update REST DB", nil) + } + if apiErr != nil { + return c.JSON(http.StatusCreated, model.NewAPISkuFromCreateRequest(apiReq, ids.Ids[0], siteID)) + } + return c.JSON(http.StatusCreated, model.NewAPISkuFromProto(skuToPersist, siteID)) +} + +// UpdateSkuHandler partially updates one SKU on a Site's Core service. +type UpdateSkuHandler struct { + dbSession *cdb.Session + scp *sc.ClientPool + tracerSpan *cutil.TracerSpan +} + +// NewUpdateSkuHandler returns a new UpdateSkuHandler. +func NewUpdateSkuHandler(dbSession *cdb.Session, scp *sc.ClientPool) UpdateSkuHandler { + return UpdateSkuHandler{ + dbSession: dbSession, + scp: scp, + tracerSpan: cutil.NewTracerSpan(), + } +} + +// Handle godoc +// @Summary Update SKU +// @Description Update selected mutable fields on a SKU. +// @Tags SKU +// @Accept json +// @Produce json +// @Security ApiKeyAuth +// @Param org path string true "Name of NGC organization" +// @Param id path string true "SKU ID" +// @Param request body model.APISkuUpdateRequest true "SKU update request" +// @Success 200 {object} model.APISku +// @Router /v2/org/{org}/nico/sku/{id} [patch] +func (ush UpdateSkuHandler) Handle(c echo.Context) error { + org, dbUser, ctx, logger, handlerSpan := common.SetupHandler("SKU", "Update", c, ush.tracerSpan) + if handlerSpan != nil { + defer handlerSpan.End() + } + + skuID := c.Param("id") + if skuID == "" { + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "SKU ID must be specified", nil) + } + ush.tracerSpan.SetAttribute(handlerSpan, attribute.String("sku_id", skuID), logger) + + apiReq := model.APISkuUpdateRequest{} + err := c.Bind(&apiReq) + if err != nil { + logger.Warn().Err(err).Msg("error binding request data into API model") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Failed to parse request data, potentially invalid structure", nil) + } + + err = apiReq.Validate() + if err != nil { + logger.Warn().Err(err).Msg("error validating SKU update request data") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "Error validating SKU update request data", err) + } + + skuDAO := cdbm.NewSkuDAO(ush.dbSession) + savedSKU, err := skuDAO.Get(ctx, nil, skuID) + if errors.Is(err, cdb.ErrDoesNotExist) { + return cutil.NewAPIErrorResponse(c, http.StatusNotFound, "Could not find SKU with the specified ID", nil) + } + if err != nil { + logger.Error().Err(err).Str("skuID", skuID).Msg("error retrieving SKU from DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve SKU, DB error", nil) + } + + stc, siteID, apiErr := common.AuthorizeProviderSiteForCore(common.AuthorizeProviderSiteForCoreInput{ + Ctx: ctx, + Logger: logger, + DBSession: ush.dbSession, + SCP: ush.scp, + Org: org, + User: dbUser, + SiteID: savedSKU.SiteID.String(), + }) + if apiErr != nil { + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, apiErr.Data) + } + + var response corev1.SkuList + apiErr = common.ExecuteCoreGRPC( + ctx, + stc, + corev1.Forge_FindSkusByIds_FullMethodName, + &corev1.SkusByIdsRequest{Ids: []string{skuID}}, + &response, + siteID, + ) + if apiErr == nil && len(response.Skus) != 1 { + apiErr = cutil.NewAPIError(http.StatusNotFound, "Could not find SKU with the specified ID", nil) + } + if apiErr != nil { + logAPIError(logger, apiErr, "failed to retrieve SKU before update") + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, nil) + } + + apiReq.SkuID = skuID + + updatedSKU := &corev1.Sku{} + if apiReq.Components == nil { + updatedSKU = apiReq.ApplyMetadataToProto(response.Skus[0]) + logger.Info().Str("skuID", skuID).Str("siteID", siteID).Msg("updating SKU metadata via Core proxy") + apiErr = common.ExecuteCoreGRPC(ctx, stc, corev1.Forge_UpdateSkuMetadata_FullMethodName, apiReq.ToMetadataProto(), nil, siteID) + if apiErr != nil { + logAPIError(logger, apiErr, "failed to update SKU metadata via Core proxy") + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, nil) + } + } else { + updatedReq := apiReq.ToReplacementProto(response.Skus[0]) + logger.Info().Str("skuID", skuID).Str("siteID", siteID).Msg("updating SKU via Core proxy") + apiErr = common.ExecuteCoreGRPC(ctx, stc, corev1.Forge_ReplaceSku_FullMethodName, updatedReq, updatedSKU, siteID) + if apiErr != nil { + logAPIError(logger, apiErr, "failed to update SKU via Core proxy") + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, nil) + } + } + + if updatedSKU == nil || updatedSKU.Id == "" { + err = errors.New("cannot persist an empty Core SKU") + } else { + projected := &cdbm.SKU{} + projected.FromProto(updatedSKU, savedSKU.SiteID) + + err = cdb.WithTx(ctx, ush.dbSession, func(tx *cdb.Tx) error { + err := tx.TryAcquireAdvisoryLock(ctx, cdb.GetAdvisoryLockIDFromString(projected.ID), nil) + if err != nil { + return err + } + + existing, err := skuDAO.Get(ctx, tx, projected.ID) + if err != nil { + return err + } + if existing.SiteID != projected.SiteID { + return fmt.Errorf("SKU %q already exists for Site %q", projected.ID, existing.SiteID) + } + + components := projected.Components + if existing.Components != nil && components == nil { + // DAO update uses nil to mean "not supplied", so use an empty value + // to clear stale components when Core reports none. + components = &cdbm.SkuComponents{SkuComponents: &corev1.SkuComponents{}} + } + associatedMachineIDs := projected.AssociatedMachineIds + if associatedMachineIDs == nil { + // DAO update uses nil to mean "not supplied"; Core nil means there + // are no associations in the authoritative projection. + associatedMachineIDs = []string{} + } + if existing.Description == projected.Description && + existing.SchemaVersion == projected.SchemaVersion && + existing.Components.Equal(components) && + reflect.DeepEqual(existing.DeviceType, projected.DeviceType) && + reflect.DeepEqual(existing.AssociatedMachineIds, associatedMachineIDs) { + return nil + } + + _, err = skuDAO.Update(ctx, tx, cdbm.SkuUpdateInput{ + SkuID: projected.ID, + Description: &projected.Description, + SchemaVersion: &projected.SchemaVersion, + Components: components, + DeviceType: projected.DeviceType, + AssociatedMachineIds: associatedMachineIDs, + }) + return err + }) + } + if err != nil { + logger.Error().Err(err).Str("skuID", skuID).Str("siteID", siteID). + Msg("SKU updated in Core but failed to update REST DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "SKU was updated in Core but failed to update REST DB", nil) + } + + return c.JSON(http.StatusOK, model.NewAPISkuFromProto(updatedSKU, siteID)) +} + +// DeleteSkuHandler deletes one unused SKU from a Site's Core service. +type DeleteSkuHandler struct { + dbSession *cdb.Session + scp *sc.ClientPool + tracerSpan *cutil.TracerSpan +} + +// NewDeleteSkuHandler returns a new DeleteSkuHandler. +func NewDeleteSkuHandler(dbSession *cdb.Session, scp *sc.ClientPool) DeleteSkuHandler { + return DeleteSkuHandler{ + dbSession: dbSession, + scp: scp, + tracerSpan: cutil.NewTracerSpan(), + } +} + +// Handle godoc +// @Summary Delete SKU +// @Description Delete an unused SKU. +// @Tags SKU +// @Produce json +// @Security ApiKeyAuth +// @Param org path string true "Name of NGC organization" +// @Param id path string true "SKU ID" +// @Success 204 +// @Router /v2/org/{org}/nico/sku/{id} [delete] +func (dsh DeleteSkuHandler) Handle(c echo.Context) error { + org, dbUser, ctx, logger, handlerSpan := common.SetupHandler("SKU", "Delete", c, dsh.tracerSpan) + if handlerSpan != nil { + defer handlerSpan.End() + } + + skuID := c.Param("id") + if skuID == "" { + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "SKU ID must be specified", nil) + } + + dsh.tracerSpan.SetAttribute(handlerSpan, attribute.String("sku_id", skuID), logger) + + skuDAO := cdbm.NewSkuDAO(dsh.dbSession) + savedSKU, err := skuDAO.Get(ctx, nil, skuID) + if errors.Is(err, cdb.ErrDoesNotExist) { + return cutil.NewAPIErrorResponse(c, http.StatusNotFound, "Could not find SKU with the specified ID", nil) + } + if err != nil { + logger.Error().Err(err).Str("skuID", skuID).Msg("error retrieving SKU from DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to retrieve SKU, DB error", nil) + } + if len(savedSKU.AssociatedMachineIds) > 0 { + logger.Warn().Str("skuID", skuID).Msg("SKU is associated with machines and cannot be deleted") + return cutil.NewAPIErrorResponse(c, http.StatusBadRequest, "SKU is associated with machines and cannot be deleted", nil) + } + + stc, siteID, apiErr := common.AuthorizeProviderSiteForCore(common.AuthorizeProviderSiteForCoreInput{ + Ctx: ctx, + Logger: logger, + DBSession: dsh.dbSession, + SCP: dsh.scp, + Org: org, + User: dbUser, + SiteID: savedSKU.SiteID.String(), + }) + if apiErr != nil { + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, apiErr.Data) + } + + logger.Info().Str("skuID", skuID).Str("siteID", siteID).Msg("deleting SKU via Core proxy") + apiErr = common.ExecuteCoreGRPC(ctx, stc, corev1.Forge_DeleteSku_FullMethodName, &corev1.SkuIdList{Ids: []string{skuID}}, nil, siteID) + if apiErr != nil { + if apiErr.Code != http.StatusNotFound { + logAPIError(logger, apiErr, "failed to delete SKU via Core proxy") + return cutil.NewAPIErrorResponse(c, apiErr.Code, apiErr.Message, nil) + } + logger.Warn().Str("skuID", skuID).Str("siteID", siteID).Msg("SKU not found in Core, removing stale REST DB record") + } + err = skuDAO.Delete(ctx, nil, skuID) + if err != nil { + logger.Error().Err(err).Str("skuID", skuID).Msg("failed to delete SKU from REST DB") + return cutil.NewAPIErrorResponse(c, http.StatusInternalServerError, "Failed to delete SKU from REST DB", nil) + } + + logger.Info().Str("skuID", skuID).Str("siteID", siteID).Msg("finishing API handler") + return c.NoContent(http.StatusNoContent) +} diff --git a/rest-api/api/pkg/api/handler/sku_test.go b/rest-api/api/pkg/api/handler/sku_test.go index b5eb205e66..8157e9a712 100644 --- a/rest-api/api/pkg/api/handler/sku_test.go +++ b/rest-api/api/pkg/api/handler/sku_test.go @@ -6,21 +6,36 @@ package handler import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" + "strings" "testing" "github.com/NVIDIA/infra-controller/rest-api/api/internal/config" + "github.com/NVIDIA/infra-controller/rest-api/api/pkg/api/handler/util/common" "github.com/NVIDIA/infra-controller/rest-api/api/pkg/api/model" + sc "github.com/NVIDIA/infra-controller/rest-api/api/pkg/client/site" authz "github.com/NVIDIA/infra-controller/rest-api/auth/pkg/authorization" + "github.com/NVIDIA/infra-controller/rest-api/common/pkg/coreproxy" cutil "github.com/NVIDIA/infra-controller/rest-api/common/pkg/util" cdb "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db" cdbm "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/model" cdbu "github.com/NVIDIA/infra-controller/rest-api/db/pkg/util" + corev1 "github.com/NVIDIA/infra-controller/rest-api/proto/core/gen/v1" + swe "github.com/NVIDIA/infra-controller/rest-api/site-workflow/pkg/error" "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/uptrace/bun/extra/bundebug" + tmocks "go.temporal.io/sdk/mocks" + tp "go.temporal.io/sdk/temporal" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" ) // testSkuInitDB initializes a test database session (pattern from tenant_test.go) @@ -81,6 +96,7 @@ func testSkuSetupTestData(t *testing.T, dbSession *cdb.Session, org string) (*cd sku1 := &cdbm.SKU{ ID: "test-sku-1", SiteID: site.ID, + Description: "First test SKU", DeviceType: &deviceType1, AssociatedMachineIds: []string{"machine-1", "machine-2"}, } @@ -91,6 +107,7 @@ func testSkuSetupTestData(t *testing.T, dbSession *cdb.Session, org string) (*cd sku2 := &cdbm.SKU{ ID: "test-sku-2", SiteID: site.ID, + Description: "Second test SKU", DeviceType: &deviceType2, AssociatedMachineIds: []string{"machine-3"}, } @@ -272,6 +289,7 @@ func TestGetAllSkuHandler_Handle(t *testing.T) { for _, sku := range response { assert.Equal(t, site.ID.String(), sku.SiteID, "All results should be from the specified site") assert.NotEqual(t, unmanagedSku.ID, sku.ID, "Unmanaged SKU should not be in response") + assert.NotEmpty(t, sku.Description) } }, }, @@ -544,6 +562,7 @@ func TestGetSkuHandler_Handle(t *testing.T) { assert.Nil(t, err) assert.Equal(t, sku1.ID, response.ID, "SKU ID should match") assert.Equal(t, site.ID.String(), response.SiteID, "Site ID should match") + assert.Equal(t, sku1.Description, response.Description, "SKU description should match") }, }, { @@ -664,3 +683,719 @@ func TestGetSkuHandler_Handle(t *testing.T) { }) } } + +func TestCreateSkuHandler(t *testing.T) { + t.Run("proxies create and returns created SKU", func(t *testing.T) { + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + skipPersistedSKU: true, + }) + req := validSkuCreateRequest(fixture.siteID) + + rec := fixture.request(t, http.MethodPost, "", req, fixture.createHandler.Handle) + require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_CreateSku_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[1].FullMethod) + + var coreReq corev1.SkuList + require.NoError(t, protojson.Unmarshal(fixture.requests[0].RequestJSON, &coreReq)) + require.Len(t, coreReq.Skus, 1) + assert.Equal(t, req.ID, coreReq.Skus[0].Id) + assert.Equal(t, model.CoreSkuSchemaVersion, coreReq.Skus[0].SchemaVersion) + require.Len(t, coreReq.Skus[0].Components.Storage, 1) + assert.Empty(t, coreReq.Skus[0].Components.Storage[0].Vendor) + assert.Zero(t, coreReq.Skus[0].Components.Storage[0].CapacityMb) + assert.Equal(t, uint32(3_600_000), coreReq.Skus[0].Components.Storage[0].GetMinSizeMb()) + assert.Equal(t, uint32(3_900_000), coreReq.Skus[0].Components.Storage[0].GetMaxSizeMb()) + assert.Equal(t, []string{`^/devices/pci.*nvme[0-1]$`}, coreReq.Skus[0].Components.Storage[0].PciPatterns) + + var response model.APISku + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &response)) + assert.Equal(t, req.ID, response.ID) + assert.Equal(t, fixture.siteID, response.SiteID) + assert.Empty(t, response.AssociatedMachineIds) + + saved, err := cdbm.NewSkuDAO(fixture.createHandler.dbSession).Get(context.Background(), nil, req.ID) + require.NoError(t, err) + assert.Equal(t, uuid.MustParse(fixture.siteID), saved.SiteID) + assert.Equal(t, response.Description, saved.Description) + assert.Equal(t, response.SchemaVersion, saved.SchemaVersion) + assert.Equal(t, response.DeviceType, saved.DeviceType) + require.NotNil(t, saved.Components) + require.NotNil(t, saved.Components.Chassis) + // The post-create Core response is authoritative, even when it differs + // from the create request used by this test fixture. + assert.Equal(t, "existing chassis", saved.Components.Chassis.Model) + }) + + t.Run("returns created when post-create fetch fails", func(t *testing.T) { + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + findError: errors.New("post-create fetch failed"), + skipPersistedSKU: true, + }) + req := validSkuCreateRequest(fixture.siteID) + + rec := fixture.request(t, http.MethodPost, "", req, fixture.createHandler.Handle) + require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_CreateSku_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[1].FullMethod) + + var response model.APISku + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &response)) + assert.Equal(t, req.ID, response.ID) + assert.Equal(t, fixture.siteID, response.SiteID) + assert.Equal(t, *req.Description, response.Description) + assert.Equal(t, model.CoreSkuSchemaVersion, response.SchemaVersion) + assert.Equal(t, req.DeviceType, response.DeviceType) + assert.Equal(t, model.NewAPISkuComponents(req.Components.ToProto()), response.Components) + assert.Empty(t, response.AssociatedMachineIds) + assert.Nil(t, response.Created) + + saved, err := cdbm.NewSkuDAO(fixture.createHandler.dbSession).Get(context.Background(), nil, req.ID) + require.NoError(t, err) + assert.Equal(t, uuid.MustParse(fixture.siteID), saved.SiteID) + assert.Equal(t, *req.Description, saved.Description) + assert.Equal(t, model.CoreSkuSchemaVersion, saved.SchemaVersion) + assert.Equal(t, req.DeviceType, saved.DeviceType) + require.NotNil(t, saved.Components) + require.Len(t, saved.Components.Storage, 1) + assert.Equal(t, uint32(3_600_000), saved.Components.Storage[0].GetMinSizeMb()) + assert.Equal(t, uint32(3_900_000), saved.Components.Storage[0].GetMaxSizeMb()) + }) + + t.Run("uses Core as the authority for duplicate IDs", func(t *testing.T) { + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + createError: status.Error(codes.AlreadyExists, "SKU already exists: sku-1"), + }) + + rec := fixture.request(t, http.MethodPost, "", validSkuCreateRequest(fixture.siteID), fixture.createHandler.Handle) + + require.Equal(t, http.StatusConflict, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 1) + assert.Equal(t, corev1.Forge_CreateSku_FullMethodName, fixture.requests[0].FullMethod) + }) + + t.Run("returns conflict when the REST projection already exists", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + + rec := fixture.request(t, http.MethodPost, "", validSkuCreateRequest(fixture.siteID), fixture.createHandler.Handle) + + require.Equal(t, http.StatusConflict, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_CreateSku_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[1].FullMethod) + assert.Contains(t, rec.Body.String(), "sku-1") + assert.Contains(t, rec.Body.String(), "inspect") + assert.Contains(t, rec.Body.String(), "update") + }) + + t.Run("returns error when Core create fails", func(t *testing.T) { + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + createError: errors.New("Core unavailable"), + skipPersistedSKU: true, + }) + + rec := fixture.request(t, http.MethodPost, "", validSkuCreateRequest(fixture.siteID), fixture.createHandler.Handle) + + require.Equal(t, http.StatusInternalServerError, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 1) + assert.Equal(t, corev1.Forge_CreateSku_FullMethodName, fixture.requests[0].FullMethod) + _, err := cdbm.NewSkuDAO(fixture.createHandler.dbSession).Get(context.Background(), nil, "sku-1") + assert.ErrorIs(t, err, cdb.ErrDoesNotExist) + }) + + t.Run("rejects invalid Site ID", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + req := validSkuCreateRequest("invalid-site-id") + + rec := fixture.request(t, http.MethodPost, "", req, fixture.createHandler.Handle) + + require.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "siteId") + assert.Empty(t, fixture.requests) + }) + + t.Run("rejects legacy storage mutation fields", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + + for _, field := range []string{"vendor", "capacityMb"} { + t.Run(field, func(t *testing.T) { + rec := fixture.request(t, http.MethodPost, "", map[string]any{ + "siteId": fixture.siteID, + "id": "sku-legacy-storage", + "components": map[string]any{ + "storage": []map[string]any{{ + "model": "legacy", + "count": 1, + field: 0, + }}, + }, + }, fixture.createHandler.Handle) + + assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) + }) + } + assert.Empty(t, fixture.requests) + }) + + t.Run("rejects inverted storage size range", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + req := validSkuCreateRequest(fixture.siteID) + req.Components.Storage[0].MinSizeMiB = cutil.GetPtr(uint32(4_000_000)) + req.Components.Storage[0].MaxSizeMiB = cutil.GetPtr(uint32(3_800_000)) + + rec := fixture.request(t, http.MethodPost, "", req, fixture.createHandler.Handle) + + assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "minSizeMiB") + assert.Empty(t, fixture.requests) + }) + + t.Run("rejects tenant admin", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.TenantAdminRole}) + + rec := fixture.request(t, http.MethodPost, "", validSkuCreateRequest(fixture.siteID), fixture.createHandler.Handle) + assert.Equal(t, http.StatusForbidden, rec.Code) + assert.Empty(t, fixture.requests) + }) +} + +func TestUpdateSkuHandler(t *testing.T) { + t.Run("uses metadata RPC for metadata patch", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + deviceType := "cpu-server" + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + DeviceType: &deviceType, + }, fixture.updateHandler.Handle) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_UpdateSkuMetadata_FullMethodName, fixture.requests[1].FullMethod) + + var coreReq corev1.SkuUpdateMetadataRequest + require.NoError(t, protojson.Unmarshal(fixture.requests[1].RequestJSON, &coreReq)) + assert.Equal(t, "sku-1", coreReq.SkuId) + assert.Equal(t, deviceType, coreReq.GetDeviceType()) + + var response model.APISku + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &response)) + require.NotNil(t, response.DeviceType) + assert.Equal(t, deviceType, *response.DeviceType) + assert.Equal(t, uint32(4), response.SchemaVersion) + + saved, err := cdbm.NewSkuDAO(fixture.updateHandler.dbSession).Get(context.Background(), nil, "sku-1") + require.NoError(t, err) + require.NotNil(t, saved.DeviceType) + assert.Equal(t, deviceType, *saved.DeviceType) + assert.Equal(t, uint32(4), saved.SchemaVersion) + require.NotNil(t, saved.Components) + require.NotNil(t, saved.Components.Chassis) + assert.Equal(t, "existing chassis", saved.Components.Chassis.Model) + }) + + t.Run("preserves projection when Core metadataupdate fails", func(t *testing.T) { + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + updateMetadataError: errors.New("Core unavailable"), + }) + deviceType := "cpu-server" + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + DeviceType: &deviceType, + }, fixture.updateHandler.Handle) + require.Equal(t, http.StatusInternalServerError, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_UpdateSkuMetadata_FullMethodName, fixture.requests[1].FullMethod) + + var coreReq corev1.SkuUpdateMetadataRequest + require.NoError(t, protojson.Unmarshal(fixture.requests[1].RequestJSON, &coreReq)) + assert.Equal(t, deviceType, coreReq.GetDeviceType()) + + saved, err := cdbm.NewSkuDAO(fixture.updateHandler.dbSession).Get(context.Background(), nil, "sku-1") + require.NoError(t, err) + assert.Nil(t, saved.DeviceType) + assert.Nil(t, saved.Components) + assert.Empty(t, saved.AssociatedMachineIds) + }) + + t.Run("does not recreate projection deleted after Core update", func(t *testing.T) { + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + afterUpdateMetadata: func(dbSession *cdb.Session) { + skuDAO := cdbm.NewSkuDAO(dbSession) + err := skuDAO.Delete(context.Background(), nil, "sku-1") + require.NoError(t, err) + }, + }) + description := "updated description" + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + Description: &description, + }, fixture.updateHandler.Handle) + + require.Equal(t, http.StatusInternalServerError, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "failed to update REST DB") + _, err := cdbm.NewSkuDAO(fixture.updateHandler.dbSession).Get(context.Background(), nil, "sku-1") + assert.ErrorIs(t, err, cdb.ErrDoesNotExist) + }) + + t.Run("replaces version five components", func(t *testing.T) { + existing := existingSkuProto() + existing.SchemaVersion = model.CoreSkuSchemaVersion + deviceType := "cpu-server" + replacementResponse := existingSkuProto() + replacementResponse.SchemaVersion = model.CoreSkuSchemaVersion + replacementResponse.DeviceType = &deviceType + replacementResponse.Components = validSkuCreateRequest("").ToProto().Skus[0].Components + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + findResponse: &corev1.SkuList{Skus: []*corev1.Sku{existing}}, + replaceResponse: replacementResponse, + }) + components := validSkuCreateRequest(fixture.siteID).Components + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + DeviceType: &deviceType, + Components: components, + }, fixture.updateHandler.Handle) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_ReplaceSku_FullMethodName, fixture.requests[1].FullMethod) + + var coreReq corev1.Sku + require.NoError(t, protojson.Unmarshal(fixture.requests[1].RequestJSON, &coreReq)) + assert.Equal(t, "sku-1", coreReq.Id) + assert.Equal(t, deviceType, coreReq.GetDeviceType()) + assert.Equal(t, model.CoreSkuSchemaVersion, coreReq.SchemaVersion) + require.NotNil(t, coreReq.Components) + require.Len(t, coreReq.Components.Storage, 1) + + saved, err := cdbm.NewSkuDAO(fixture.updateHandler.dbSession).Get(context.Background(), nil, "sku-1") + require.NoError(t, err) + require.NotNil(t, saved.DeviceType) + assert.Equal(t, deviceType, *saved.DeviceType) + require.NotNil(t, saved.Components) + require.Len(t, saved.Components.Storage, 1) + assert.Equal(t, uint32(3_600_000), saved.Components.Storage[0].GetMinSizeMb()) + assert.Equal(t, uint32(3_900_000), saved.Components.Storage[0].GetMaxSizeMb()) + }) + + t.Run("preserves projection when Core replace fails", func(t *testing.T) { + existing := existingSkuProto() + existing.SchemaVersion = model.CoreSkuSchemaVersion + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + findResponse: &corev1.SkuList{Skus: []*corev1.Sku{existing}}, + replaceError: errors.New("Core unavailable"), + }) + deviceType := "cpu-server" + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + DeviceType: &deviceType, + Components: validSkuCreateRequest(fixture.siteID).Components, + }, fixture.updateHandler.Handle) + + require.Equal(t, http.StatusInternalServerError, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_ReplaceSku_FullMethodName, fixture.requests[1].FullMethod) + + var coreReq corev1.Sku + require.NoError(t, protojson.Unmarshal(fixture.requests[1].RequestJSON, &coreReq)) + assert.Equal(t, deviceType, coreReq.GetDeviceType()) + + saved, err := cdbm.NewSkuDAO(fixture.updateHandler.dbSession).Get(context.Background(), nil, "sku-1") + require.NoError(t, err) + assert.Nil(t, saved.DeviceType) + assert.Nil(t, saved.Components) + }) + + t.Run("replaces components for SKU with associated machines", func(t *testing.T) { + existing := existingSkuProto() + existing.SchemaVersion = model.CoreSkuSchemaVersion + existing.AssociatedMachineIds = []*corev1.MachineId{{Id: "machine-1"}} + replacementResponse := proto.Clone(existing).(*corev1.Sku) + replacementResponse.Components = validSkuCreateRequest("").ToProto().Skus[0].Components + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + findResponse: &corev1.SkuList{Skus: []*corev1.Sku{existing}}, + replaceResponse: replacementResponse, + }) + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + Components: validSkuCreateRequest(fixture.siteID).Components, + }, fixture.updateHandler.Handle) + + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_ReplaceSku_FullMethodName, fixture.requests[1].FullMethod) + + var coreReq corev1.Sku + require.NoError(t, protojson.Unmarshal(fixture.requests[1].RequestJSON, &coreReq)) + require.Len(t, coreReq.AssociatedMachineIds, 1) + assert.Equal(t, "machine-1", coreReq.AssociatedMachineIds[0].GetId()) + + var response model.APISku + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &response)) + assert.Equal(t, []string{"machine-1"}, response.AssociatedMachineIds) + + saved, err := cdbm.NewSkuDAO(fixture.updateHandler.dbSession).Get(context.Background(), nil, "sku-1") + require.NoError(t, err) + assert.Equal(t, []string{"machine-1"}, saved.AssociatedMachineIds) + }) + + t.Run("sends version five when replacing legacy SKU components", func(t *testing.T) { + existing := existingSkuProto() + require.Equal(t, uint32(4), existing.SchemaVersion) + replacementResponse := proto.Clone(existing).(*corev1.Sku) + replacementResponse.SchemaVersion = model.CoreSkuSchemaVersion + replacementResponse.Components = validSkuCreateRequest("").ToProto().Skus[0].Components + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + findResponse: &corev1.SkuList{Skus: []*corev1.Sku{existing}}, + replaceResponse: replacementResponse, + }) + components := validSkuCreateRequest(fixture.siteID).Components + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + Components: components, + }, fixture.updateHandler.Handle) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 2) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[0].FullMethod) + assert.Equal(t, corev1.Forge_ReplaceSku_FullMethodName, fixture.requests[1].FullMethod) + + var coreReq corev1.Sku + require.NoError(t, protojson.Unmarshal(fixture.requests[1].RequestJSON, &coreReq)) + assert.Equal(t, model.CoreSkuSchemaVersion, coreReq.SchemaVersion) + assert.Equal(t, "sku-1", coreReq.Id) + require.NotNil(t, coreReq.Components) + require.Len(t, coreReq.Components.Storage, 1) + assert.True(t, proto.Equal(components.ToProto(), coreReq.Components)) + }) + + t.Run("rejects inverted storage size range", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + components := validSkuCreateRequest(fixture.siteID).Components + components.Storage[0].MinSizeMiB = cutil.GetPtr(uint32(4_000_000)) + components.Storage[0].MaxSizeMiB = cutil.GetPtr(uint32(3_800_000)) + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + Components: components, + }, fixture.updateHandler.Handle) + + assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "minSizeMiB") + assert.Empty(t, fixture.requests) + }) + + t.Run("returns Core not found without completing update", func(t *testing.T) { + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + findResponse: &corev1.SkuList{}, + }) + description := "updated description" + + rec := fixture.request(t, http.MethodPatch, "sku-1", model.APISkuUpdateRequest{ + Description: &description, + }, fixture.updateHandler.Handle) + require.Equal(t, http.StatusNotFound, rec.Code, rec.Body.String()) + assert.JSONEq(t, `{"source":"","message":"Could not find SKU with the specified ID","data":null}`, rec.Body.String()) + require.Len(t, fixture.requests, 1) + assert.Equal(t, corev1.Forge_FindSkusByIds_FullMethodName, fixture.requests[0].FullMethod) + }) + + t.Run("returns not found for SKU not present in REST database", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + description := "updated description" + + rec := fixture.request(t, http.MethodPatch, "missing-sku", model.APISkuUpdateRequest{ + Description: &description, + }, fixture.updateHandler.Handle) + require.Equal(t, http.StatusNotFound, rec.Code, rec.Body.String()) + assert.JSONEq(t, `{"source":"","message":"Could not find SKU with the specified ID","data":null}`, rec.Body.String()) + assert.Empty(t, fixture.requests) + }) + +} + +func TestDeleteSkuHandler(t *testing.T) { + t.Run("proxies delete", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + + rec := fixture.request(t, http.MethodDelete, "sku-1", nil, fixture.deleteHandler.Handle) + require.Equal(t, http.StatusNoContent, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 1) + assert.Equal(t, corev1.Forge_DeleteSku_FullMethodName, fixture.requests[0].FullMethod) + + var coreReq corev1.SkuIdList + require.NoError(t, protojson.Unmarshal(fixture.requests[0].RequestJSON, &coreReq)) + assert.Equal(t, []string{"sku-1"}, coreReq.Ids) + _, err := cdbm.NewSkuDAO(fixture.deleteHandler.dbSession).Get(context.Background(), nil, "sku-1") + assert.ErrorIs(t, err, cdb.ErrDoesNotExist) + }) + + t.Run("rejects SKU with associated machines", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + skuDAO := cdbm.NewSkuDAO(fixture.deleteHandler.dbSession) + _, err := skuDAO.Update(context.Background(), nil, cdbm.SkuUpdateInput{ + SkuID: "sku-1", + AssociatedMachineIds: []string{"machine-1"}, + }) + require.NoError(t, err) + + rec := fixture.request(t, http.MethodDelete, "sku-1", nil, fixture.deleteHandler.Handle) + require.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) + assert.JSONEq(t, `{"source":"","message":"SKU is associated with machines and cannot be deleted","data":null}`, rec.Body.String()) + assert.Empty(t, fixture.requests) + + saved, err := skuDAO.Get(context.Background(), nil, "sku-1") + require.NoError(t, err) + assert.Equal(t, []string{"machine-1"}, saved.AssociatedMachineIds) + }) + + t.Run("removes stale record when Core returns not found", func(t *testing.T) { + deleteErr := tp.NewApplicationErrorWithCause( + "SKU not found", + swe.ErrTypeNICoObjectNotFound, + status.Error(codes.NotFound, "SKU not found"), + ) + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + deleteError: deleteErr, + }) + + rec := fixture.request(t, http.MethodDelete, "sku-1", nil, fixture.deleteHandler.Handle) + require.Equal(t, http.StatusNoContent, rec.Code, rec.Body.String()) + require.Len(t, fixture.requests, 1) + assert.Equal(t, corev1.Forge_DeleteSku_FullMethodName, fixture.requests[0].FullMethod) + _, err := cdbm.NewSkuDAO(fixture.deleteHandler.dbSession).Get(context.Background(), nil, "sku-1") + assert.ErrorIs(t, err, cdb.ErrDoesNotExist) + }) + + t.Run("preserves record when Core delete fails", func(t *testing.T) { + deleteErr := tp.NewApplicationErrorWithCause( + "Core unavailable", + swe.ErrTypeNICoUnavailable, + status.Error(codes.Unavailable, "Core unavailable"), + ) + fixture := newSkuManagementFixtureWithOptions(t, []string{authz.ProviderAdminRole}, skuManagementFixtureOptions{ + deleteError: deleteErr, + }) + + rec := fixture.request(t, http.MethodDelete, "sku-1", nil, fixture.deleteHandler.Handle) + require.Equal(t, http.StatusServiceUnavailable, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "Core unavailable") + require.Len(t, fixture.requests, 1) + assert.Equal(t, corev1.Forge_DeleteSku_FullMethodName, fixture.requests[0].FullMethod) + _, err := cdbm.NewSkuDAO(fixture.deleteHandler.dbSession).Get(context.Background(), nil, "sku-1") + assert.NoError(t, err) + }) + + t.Run("returns not found for unsaved SKU", func(t *testing.T) { + fixture := newSkuManagementFixture(t, []string{authz.ProviderAdminRole}) + + rec := fixture.request(t, http.MethodDelete, "missing-sku", nil, fixture.deleteHandler.Handle) + require.Equal(t, http.StatusNotFound, rec.Code, rec.Body.String()) + assert.JSONEq(t, `{"source":"","message":"Could not find SKU with the specified ID","data":null}`, rec.Body.String()) + assert.Empty(t, fixture.requests) + }) +} + +type skuManagementFixture struct { + org string + siteID string + user *cdbm.User + createHandler CreateSkuHandler + updateHandler UpdateSkuHandler + deleteHandler DeleteSkuHandler + requests []coreproxy.Request +} + +type skuManagementFixtureOptions struct { + findResponse *corev1.SkuList + findError error + createError error + replaceResponse *corev1.Sku + replaceError error + updateMetadataError error + deleteError error + skipPersistedSKU bool + afterUpdateMetadata func(*cdb.Session) +} + +func newSkuManagementFixture(t *testing.T, roles []string) *skuManagementFixture { + return newSkuManagementFixtureWithOptions(t, roles, skuManagementFixtureOptions{}) +} + +func newSkuManagementFixtureWithOptions(t *testing.T, roles []string, options skuManagementFixtureOptions) *skuManagementFixture { + t.Helper() + dbSession := common.TestInitDB(t) + t.Cleanup(dbSession.Close) + common.TestSetupSchema(t, dbSession) + require.NoError(t, dbSession.DB.ResetModel(context.Background(), (*cdbm.SKU)(nil))) + + org := "test-org" + user := common.TestBuildUser(t, dbSession, uuid.NewString(), org, roles) + ip := common.TestBuildInfrastructureProvider(t, dbSession, "Test Provider", org, user) + site := common.TestBuildSite(t, dbSession, ip, "Test Site", user) + sDAO := cdbm.NewSiteDAO(dbSession) + _, err := sDAO.Update(context.Background(), nil, cdbm.SiteUpdateInput{ + SiteID: site.ID, + Status: cutil.GetPtr(cdbm.SiteStatusRegistered), + }) + require.NoError(t, err) + if !options.skipPersistedSKU { + skuDAO := cdbm.NewSkuDAO(dbSession) + _, err = skuDAO.Create(context.Background(), nil, cdbm.SkuCreateInput{ + SkuID: "sku-1", + SiteID: site.ID, + SchemaVersion: 4, + }) + require.NoError(t, err) + } + + fixture := &skuManagementFixture{org: org, siteID: site.ID.String(), user: user} + client := &tmocks.Client{} + existing := existingSkuProto() + if options.createError != nil { + fixture.addWorkflowError(client, corev1.Forge_CreateSku_FullMethodName, options.createError) + } else { + fixture.addWorkflow(t, client, corev1.Forge_CreateSku_FullMethodName, &corev1.SkuIdList{Ids: []string{"sku-1"}}) + } + if options.findError != nil { + fixture.addWorkflowError(client, corev1.Forge_FindSkusByIds_FullMethodName, options.findError) + } else { + findResponse := options.findResponse + if findResponse == nil { + findResponse = &corev1.SkuList{Skus: []*corev1.Sku{existing}} + } + fixture.addWorkflow(t, client, corev1.Forge_FindSkusByIds_FullMethodName, findResponse) + } + replaceResponse := options.replaceResponse + if replaceResponse == nil { + replaceResponse = existing + } + if options.replaceError != nil { + fixture.addWorkflowError(client, corev1.Forge_ReplaceSku_FullMethodName, options.replaceError) + } else { + fixture.addWorkflow(t, client, corev1.Forge_ReplaceSku_FullMethodName, replaceResponse) + } + if options.updateMetadataError != nil { + fixture.addWorkflowError(client, corev1.Forge_UpdateSkuMetadata_FullMethodName, options.updateMetadataError) + } else { + fixture.addWorkflow(t, client, corev1.Forge_UpdateSkuMetadata_FullMethodName, nil, func() { + if options.afterUpdateMetadata != nil { + options.afterUpdateMetadata(dbSession) + } + }) + } + if options.deleteError != nil { + fixture.addWorkflowError(client, corev1.Forge_DeleteSku_FullMethodName, options.deleteError) + } else { + fixture.addWorkflow(t, client, corev1.Forge_DeleteSku_FullMethodName, nil) + } + + scp := sc.NewClientPool(nil) + scp.IDClientMap[site.ID.String()] = client + fixture.createHandler = NewCreateSkuHandler(dbSession, scp) + fixture.updateHandler = NewUpdateSkuHandler(dbSession, scp) + fixture.deleteHandler = NewDeleteSkuHandler(dbSession, scp) + return fixture +} + +func (f *skuManagementFixture) addWorkflowError(client *tmocks.Client, method string, getErr error) { + run := &tmocks.WorkflowRun{} + run.On("Get", mock.Anything, mock.Anything).Return(getErr) + client.On( + "ExecuteWorkflow", + mock.Anything, + mock.Anything, + coreproxy.WorkflowName, + mock.MatchedBy(func(req coreproxy.Request) bool { return req.FullMethod == method }), + ).Run(func(args mock.Arguments) { + f.requests = append(f.requests, args.Get(3).(coreproxy.Request)) + }).Return(run, nil).Maybe() +} + +func (f *skuManagementFixture) addWorkflow(t *testing.T, client *tmocks.Client, method string, response proto.Message, afterGet ...func()) { + t.Helper() + run := &tmocks.WorkflowRun{} + var responseJSON []byte + if response != nil { + var err error + responseJSON, err = protojson.Marshal(response) + require.NoError(t, err) + } + run.On("Get", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + out, ok := args.Get(1).(*coreproxy.Response) + require.True(t, ok) + out.ResponseJSON = responseJSON + for _, callback := range afterGet { + callback() + } + }).Return(nil) + client.On( + "ExecuteWorkflow", + mock.Anything, + mock.Anything, + coreproxy.WorkflowName, + mock.MatchedBy(func(req coreproxy.Request) bool { return req.FullMethod == method }), + ).Run(func(args mock.Arguments) { + f.requests = append(f.requests, args.Get(3).(coreproxy.Request)) + }).Return(run, nil).Maybe() +} + +func (f *skuManagementFixture) request(t *testing.T, method, skuID string, body any, handler func(echo.Context) error) *httptest.ResponseRecorder { + t.Helper() + requestJSON, err := json.Marshal(body) + require.NoError(t, err) + req := httptest.NewRequest(method, "/", strings.NewReader(string(requestJSON))) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + ec := echo.New().NewContext(req, rec) + ec.SetParamNames("orgName", "id") + ec.SetParamValues(f.org, skuID) + ec.Set("user", f.user) + require.NoError(t, handler(ec)) + return rec +} + +func validSkuCreateRequest(siteID string) model.APISkuCreateRequest { + deviceType := "gpu-server" + return model.APISkuCreateRequest{ + SiteID: siteID, + ID: "sku-1", + Description: cutil.GetPtr("test SKU"), + DeviceType: &deviceType, + Components: &model.APISkuComponents{ + Chassis: &model.APISkuChassis{ + Vendor: "NVIDIA", + Model: "DGX H100", + Architecture: "x86_64", + }, + Storage: []model.APISkuStorage{{ + Model: "informational-model", + Count: 2, + MinSizeMiB: cutil.GetPtr(uint32(3_600_000)), + MaxSizeMiB: cutil.GetPtr(uint32(3_900_000)), + PciPatterns: []string{`^/devices/pci.*nvme[0-1]$`}, + }}, + }, + } +} + +func existingSkuProto() *corev1.Sku { + description := "old description" + deviceType := "gpu-server" + return &corev1.Sku{ + Id: "sku-1", + Description: &description, + SchemaVersion: 4, + DeviceType: &deviceType, + Components: &corev1.SkuComponents{ + Chassis: &corev1.SkuComponentChassis{ + Vendor: "NVIDIA", + Model: "existing chassis", + Architecture: "x86_64", + }, + }, + } +} diff --git a/rest-api/api/pkg/api/model/expectedmachine_test.go b/rest-api/api/pkg/api/model/expectedmachine_test.go index c004875c79..e5f0153e6a 100644 --- a/rest-api/api/pkg/api/model/expectedmachine_test.go +++ b/rest-api/api/pkg/api/model/expectedmachine_test.go @@ -1145,9 +1145,9 @@ func TestNewAPIExpectedMachineWithSkuComponents(t *testing.T) { // Validate Storage components assert.Len(t, apiEM.Sku.Components.Storage, 1) - assert.Equal(t, "Samsung", apiEM.Sku.Components.Storage[0].Vendor) + assert.Equal(t, cutil.GetPtr("Samsung"), apiEM.Sku.Components.Storage[0].Vendor) assert.Equal(t, "PM9A3", apiEM.Sku.Components.Storage[0].Model) - assert.Equal(t, uint32(3840000), apiEM.Sku.Components.Storage[0].CapacityMb) + assert.Equal(t, cutil.GetPtr(uint32(3840000)), apiEM.Sku.Components.Storage[0].CapacityMb) assert.Equal(t, uint32(4), apiEM.Sku.Components.Storage[0].Count) // Validate Chassis component @@ -1291,8 +1291,8 @@ func TestNewAPIExpectedMachineWithSkuComponents(t *testing.T) { // Validate multiple Storage components assert.Len(t, apiEM.Sku.Components.Storage, 2) - assert.Equal(t, "Samsung", apiEM.Sku.Components.Storage[0].Vendor) - assert.Equal(t, "Intel", apiEM.Sku.Components.Storage[1].Vendor) + assert.Equal(t, cutil.GetPtr("Samsung"), apiEM.Sku.Components.Storage[0].Vendor) + assert.Equal(t, cutil.GetPtr("Intel"), apiEM.Sku.Components.Storage[1].Vendor) }, }, { diff --git a/rest-api/api/pkg/api/model/operatingsystem.go b/rest-api/api/pkg/api/model/operatingsystem.go index a6b52f91b8..6b4fe28f1c 100644 --- a/rest-api/api/pkg/api/model/operatingsystem.go +++ b/rest-api/api/pkg/api/model/operatingsystem.go @@ -480,7 +480,7 @@ func (oscr *APIOperatingSystemCreateRequest) ValidateAndSetUserData(phonehomeUrl return nil } -// ToProto builds the workflow request that asks a Site to create the +// ToImageProto builds the workflow request that asks a Site to create the // OS image for this API request. `os` is the just-persisted DB record; // its `ToImageAttributesProto(tenantOrg)` is the source of every wire // field because the handler has already merged the request fields into @@ -495,7 +495,7 @@ func (oscr *APIOperatingSystemCreateRequest) ValidateAndSetUserData(phonehomeUrl // `ToImageAttributesProto` dereferences `ImageURL` and `ImageSHA`. // For iPXE-typed records there is no Site-side image workflow, so // this method should not be called. -func (oscr *APIOperatingSystemCreateRequest) ToProto(os *cdbm.OperatingSystem, tenantOrg string) *corev1.OsImageAttributes { +func (oscr *APIOperatingSystemCreateRequest) ToImageProto(os *cdbm.OperatingSystem, tenantOrg string) *corev1.OsImageAttributes { return os.ToImageAttributesProto(tenantOrg) } @@ -859,7 +859,7 @@ func (osur *APIOperatingSystemUpdateRequest) ValidateAndSetUserData(phonehomeUrl return nil } -// ToProto builds the workflow request that asks a Site to update the +// ToImageProto builds the workflow request that asks a Site to update the // OS image for this API request. `uos` is the post-update DB record; // its `ToImageAttributesProto(tenantOrg)` is the source of every wire // field, so unchanged fields stay populated and updated fields reflect @@ -871,14 +871,13 @@ func (osur *APIOperatingSystemUpdateRequest) ValidateAndSetUserData(phonehomeUrl // update workflows on the Site side, so this method delegates to the // entity-level method rather than building a distinct wire shape. The // request-level method exists so call sites stay uniform with the -// rest of the layered convention (handlers always invoke -// `apiRequest.ToProto(entity, ...)`). +// rest of the layered convention. // // As with the create variant, the method trusts that the request has // been Validated (Validate + ValidateAndSetUserData) and that the // handler has confirmed the OS is image-typed before this is called; // `ToImageAttributesProto` dereferences `ImageURL` and `ImageSHA`. -func (osur *APIOperatingSystemUpdateRequest) ToProto(uos *cdbm.OperatingSystem, tenantOrg string) *corev1.OsImageAttributes { +func (osur *APIOperatingSystemUpdateRequest) ToImageProto(uos *cdbm.OperatingSystem, tenantOrg string) *corev1.OsImageAttributes { return uos.ToImageAttributesProto(tenantOrg) } @@ -1032,7 +1031,7 @@ func NewAPIOperatingSystemSummary(dbos *cdbm.OperatingSystem) *APIOperatingSyste } // validateTemplatedIpxeOS fully validates a Templated iPXE create request: image -// fields must be absent, at least one target site must be specified (the site list +// fields must be absent, exactly one target site must be specified (the site list // is fixed at creation and is immutable thereafter), and the template // parameters/artifacts must be well-formed. func (oscr *APIOperatingSystemCreateRequest) validateTemplatedIpxeOS() error { @@ -1047,8 +1046,8 @@ func (oscr *APIOperatingSystemCreateRequest) validateTemplatedIpxeOS() error { return err } - if len(oscr.SiteIDs) == 0 { - return validation.Errors{"siteIds": errors.New("at least one siteId must be specified for Templated iPXE Operating Systems")} + if len(oscr.SiteIDs) != 1 { + return validation.Errors{"siteIds": errors.New("exactly one siteId must be specified for Templated iPXE Operating Systems")} } for _, siteID := range oscr.SiteIDs { if _, err := uuid.Parse(siteID); err != nil { @@ -1076,85 +1075,47 @@ var validCacheStrategies = func() map[string]struct{} { return m }() -// BuildCreateOperatingSystemRequest builds the forge.Forge CreateOperatingSystem -// request proto from a persisted Operating System record. It is used by the OS -// handler to push iPXE / Templated iPXE definitions to on-site NICo Core through -// the generic Core gRPC proxy. +// ToProto builds the forge.Forge CreateOperatingSystem request from the +// persisted entity. The handler has already validated and merged this API +// request into os, whose ToProto method supplies the canonical wire fields. // // Note: artifact authTokens are nested inside the repeated artifacts message and // are therefore carried as-is (the proxy cannot redact nested fields). -func BuildCreateOperatingSystemRequest(os *cdbm.OperatingSystem) *corev1.CreateOperatingSystemRequest { +func (oscr *APIOperatingSystemCreateRequest) ToProto(os *cdbm.OperatingSystem) *corev1.CreateOperatingSystemRequest { + protoOS := os.ToProto() return &corev1.CreateOperatingSystemRequest{ - Id: &corev1.OperatingSystemId{Value: os.ID.String()}, - Name: os.Name, - Description: os.Description, - TenantOrganizationId: tenantOrganizationIDProto(os.Org), - IsActive: os.IsActive, - AllowOverride: os.AllowOverride, - PhoneHomeEnabled: os.PhoneHomeEnabled, - UserData: os.UserData, - IpxeScript: os.IpxeScript, - IpxeTemplateId: ipxeTemplateIDProto(os.IpxeTemplateId), - IpxeTemplateParameters: ipxeParametersProto(os.IpxeTemplateParameters), - IpxeTemplateArtifacts: ipxeArtifactsProto(os.IpxeTemplateArtifacts), + Id: protoOS.Id, + Name: protoOS.Name, + Description: protoOS.Description, + TenantOrganizationId: protoOS.TenantOrganizationId, + IsActive: protoOS.IsActive, + AllowOverride: protoOS.AllowOverride, + PhoneHomeEnabled: protoOS.PhoneHomeEnabled, + UserData: protoOS.UserData, + IpxeScript: protoOS.IpxeScript, + IpxeTemplateId: protoOS.IpxeTemplateId, + IpxeTemplateParameters: protoOS.IpxeTemplateParameters, + IpxeTemplateArtifacts: protoOS.IpxeTemplateArtifacts, } } -// BuildUpdateOperatingSystemRequest builds the forge.Forge UpdateOperatingSystem -// request proto from a persisted Operating System record. -func BuildUpdateOperatingSystemRequest(os *cdbm.OperatingSystem) *corev1.UpdateOperatingSystemRequest { +// ToProto builds the forge.Forge UpdateOperatingSystem request from the +// post-update entity. The entity conversion keeps unchanged and updated fields +// populated consistently. +func (osur *APIOperatingSystemUpdateRequest) ToProto(os *cdbm.OperatingSystem) *corev1.UpdateOperatingSystemRequest { + protoOS := os.ToProto() return &corev1.UpdateOperatingSystemRequest{ - Id: &corev1.OperatingSystemId{Value: os.ID.String()}, - Name: &os.Name, - Description: os.Description, - IsActive: &os.IsActive, - AllowOverride: &os.AllowOverride, - PhoneHomeEnabled: &os.PhoneHomeEnabled, - UserData: os.UserData, - IpxeScript: os.IpxeScript, - IpxeTemplateId: ipxeTemplateIDProto(os.IpxeTemplateId), - IpxeTemplateParameters: &corev1.IpxeTemplateParameters{Items: ipxeParametersProto(os.IpxeTemplateParameters)}, - IpxeTemplateArtifacts: &corev1.IpxeTemplateArtifacts{Items: ipxeArtifactsProto(os.IpxeTemplateArtifacts)}, - IpxeTemplateDefinitionHash: os.IpxeTemplateDefinitionHash, + Id: protoOS.Id, + Name: &protoOS.Name, + Description: protoOS.Description, + IsActive: &protoOS.IsActive, + AllowOverride: &protoOS.AllowOverride, + PhoneHomeEnabled: &protoOS.PhoneHomeEnabled, + UserData: protoOS.UserData, + IpxeScript: protoOS.IpxeScript, + IpxeTemplateId: protoOS.IpxeTemplateId, + IpxeTemplateParameters: &corev1.IpxeTemplateParameters{Items: protoOS.IpxeTemplateParameters}, + IpxeTemplateArtifacts: &corev1.IpxeTemplateArtifacts{Items: protoOS.IpxeTemplateArtifacts}, + IpxeTemplateDefinitionHash: protoOS.IpxeTemplateDefinitionHash, } } - -// BuildDeleteOperatingSystemRequest builds the forge.Forge DeleteOperatingSystem -// request proto for a persisted Operating System record. -func BuildDeleteOperatingSystemRequest(os *cdbm.OperatingSystem) *corev1.DeleteOperatingSystemRequest { - return &corev1.DeleteOperatingSystemRequest{ - Id: &corev1.OperatingSystemId{Value: os.ID.String()}, - } -} - -// tenantOrganizationIDProto maps a persisted org string onto the optional Core -// field. Empty means provider-owned and must be omitted (Core rejects ""). -func tenantOrganizationIDProto(org string) *string { - if org == "" { - return nil - } - return &org -} - -func ipxeTemplateIDProto(id *string) *corev1.IpxeTemplateId { - if id == nil { - return nil - } - return &corev1.IpxeTemplateId{Value: *id} -} - -func ipxeParametersProto(params []cdbm.OperatingSystemIpxeParameter) []*corev1.IpxeTemplateParameter { - out := make([]*corev1.IpxeTemplateParameter, 0, len(params)) - for i := range params { - out = append(out, params[i].ToProto()) - } - return out -} - -func ipxeArtifactsProto(artifacts []cdbm.OperatingSystemIpxeArtifact) []*corev1.IpxeTemplateArtifact { - out := make([]*corev1.IpxeTemplateArtifact, 0, len(artifacts)) - for i := range artifacts { - out = append(out, artifacts[i].ToProto()) - } - return out -} diff --git a/rest-api/api/pkg/api/model/operatingsystem_templated_test.go b/rest-api/api/pkg/api/model/operatingsystem_templated_test.go index ed4f7278ea..9c7019fb35 100644 --- a/rest-api/api/pkg/api/model/operatingsystem_templated_test.go +++ b/rest-api/api/pkg/api/model/operatingsystem_templated_test.go @@ -31,15 +31,20 @@ func TestOperatingSystemCreateRequest_Validate_Templated(t *testing.T) { expectErr bool }{ { - desc: "templated requires at least one siteId", + desc: "templated requires exactly one siteId", obj: APIOperatingSystemCreateRequest{Name: "abc", IpxeTemplateId: tmplID}, expectErr: true, }, { - desc: "templated with siteIds is ok", + desc: "templated with one siteId is ok", obj: APIOperatingSystemCreateRequest{Name: "abc", IpxeTemplateId: tmplID, SiteIDs: siteIDs}, expectErr: false, }, + { + desc: "templated with multiple siteIds is rejected", + obj: APIOperatingSystemCreateRequest{Name: "abc", IpxeTemplateId: tmplID, SiteIDs: []string{uuid.NewString(), uuid.NewString()}}, + expectErr: true, + }, { desc: "templated with non-UUID ipxeTemplateId is rejected", obj: APIOperatingSystemCreateRequest{Name: "abc", IpxeTemplateId: cutil.GetPtr("not-a-uuid"), SiteIDs: siteIDs}, @@ -125,14 +130,18 @@ func TestOperatingSystemUpdateRequest_Validate_Template(t *testing.T) { }) } -func TestBuildOperatingSystemRequests(t *testing.T) { +func TestOperatingSystemRequest_ToProto(t *testing.T) { id := uuid.New() + tenantID := uuid.New() authToken := "secret-token" + createRequest := &APIOperatingSystemCreateRequest{} + updateRequest := &APIOperatingSystemUpdateRequest{} os := &cdbm.OperatingSystem{ ID: id, Name: "templated-os", Description: cutil.GetPtr("desc"), Org: "org-1", + TenantID: &tenantID, Type: cdbm.OperatingSystemTypeTemplatedIPXE, IsActive: true, AllowOverride: true, @@ -148,11 +157,12 @@ func TestBuildOperatingSystemRequests(t *testing.T) { IpxeTemplateDefinitionHash: cutil.GetPtr("hash-1"), } - t.Run("create request maps all fields", func(t *testing.T) { - req := BuildCreateOperatingSystemRequest(os) + t.Run("tenant-owned create request maps all fields and tenant org", func(t *testing.T) { + req := createRequest.ToProto(os) require.NotNil(t, req) assert.Equal(t, id.String(), req.GetId().GetValue()) assert.Equal(t, "templated-os", req.Name) + require.NotNil(t, req.TenantOrganizationId) assert.Equal(t, "org-1", req.GetTenantOrganizationId()) assert.True(t, req.IsActive) assert.True(t, req.AllowOverride) @@ -168,8 +178,19 @@ func TestBuildOperatingSystemRequests(t *testing.T) { assert.Nil(t, req.IpxeTemplateArtifacts[0].CachedUrl) }) + t.Run("provider-owned create request omits tenant org", func(t *testing.T) { + providerOS := *os + providerID := uuid.New() + providerOS.TenantID = nil + providerOS.InfrastructureProviderID = &providerID + + req := createRequest.ToProto(&providerOS) + require.NotNil(t, req) + assert.Nil(t, req.TenantOrganizationId) + }) + t.Run("update request maps all fields", func(t *testing.T) { - req := BuildUpdateOperatingSystemRequest(os) + req := updateRequest.ToProto(os) require.NotNil(t, req) assert.Equal(t, id.String(), req.GetId().GetValue()) require.NotNil(t, req.Name) @@ -184,7 +205,7 @@ func TestBuildOperatingSystemRequests(t *testing.T) { }) t.Run("delete request maps id", func(t *testing.T) { - req := BuildDeleteOperatingSystemRequest(os) + req := os.ToDeletionRequestProto() require.NotNil(t, req) assert.Equal(t, id.String(), req.GetId().GetValue()) }) diff --git a/rest-api/api/pkg/api/model/operatingsystem_test.go b/rest-api/api/pkg/api/model/operatingsystem_test.go index 42a965c0d3..fca567041e 100644 --- a/rest-api/api/pkg/api/model/operatingsystem_test.go +++ b/rest-api/api/pkg/api/model/operatingsystem_test.go @@ -921,7 +921,7 @@ func TestAPIOperatingSystemNew(t *testing.T) { } } -func TestAPIOperatingSystemCreateRequest_ToProto(t *testing.T) { +func TestAPIOperatingSystemCreateRequest_ToImageProto(t *testing.T) { id := uuid.New() url := "https://image" sha := "deadbeef" @@ -936,7 +936,7 @@ func TestAPIOperatingSystemCreateRequest_ToProto(t *testing.T) { } t.Run("delegates to ToImageAttributesProto with tenantOrg", func(t *testing.T) { req := APIOperatingSystemCreateRequest{} - got := req.ToProto(os, "org-1") + got := req.ToImageProto(os, "org-1") require.NotNil(t, got) require.NotNil(t, got.Id) assert.Equal(t, id.String(), got.Id.Value) @@ -959,14 +959,14 @@ func TestAPIOperatingSystemCreateRequest_ToProto(t *testing.T) { RootFsID: &rootFsID, } req := APIOperatingSystemCreateRequest{} - got := req.ToProto(osWithCtrl, "org-1") + got := req.ToImageProto(osWithCtrl, "org-1") require.NotNil(t, got) require.NotNil(t, got.Id) assert.Equal(t, ctrlID.String(), got.Id.Value) }) } -func TestAPIOperatingSystemUpdateRequest_ToProto(t *testing.T) { +func TestAPIOperatingSystemUpdateRequest_ToImageProto(t *testing.T) { id := uuid.New() url := "https://image-new" sha := "cafebabe" @@ -981,7 +981,7 @@ func TestAPIOperatingSystemUpdateRequest_ToProto(t *testing.T) { } t.Run("delegates to ToImageAttributesProto with tenantOrg", func(t *testing.T) { req := APIOperatingSystemUpdateRequest{} - got := req.ToProto(uos, "org-2") + got := req.ToImageProto(uos, "org-2") require.NotNil(t, got) require.NotNil(t, got.Id) assert.Equal(t, id.String(), got.Id.Value) diff --git a/rest-api/api/pkg/api/model/sku.go b/rest-api/api/pkg/api/model/sku.go index 7491c07dec..c4445fb746 100644 --- a/rest-api/api/pkg/api/model/sku.go +++ b/rest-api/api/pkg/api/model/sku.go @@ -4,10 +4,14 @@ package model import ( + "fmt" "time" cdbm "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/model" corev1 "github.com/NVIDIA/infra-controller/rest-api/proto/core/gen/v1" + validation "github.com/go-ozzo/ozzo-validation/v4" + validationis "github.com/go-ozzo/ozzo-validation/v4/is" + "google.golang.org/protobuf/proto" ) // APISku is the data structure to capture API representation of a SKU @@ -16,6 +20,12 @@ type APISku struct { ID string `json:"id"` // SiteID is the ID of the Site this SKU belongs to SiteID string `json:"siteId"` + // Description is the human-readable SKU description + Description string `json:"description"` + // SchemaVersion is the Core SKU schema version when known. + // When creating a new SKU or updating the components of an existing SKU, + // the schema version must be the current schema version. + SchemaVersion uint32 `json:"schemaVersion"` // DeviceType is the optional device type identifier DeviceType *string `json:"deviceType"` // AssociatedMachineIds is the list of machine IDs associated with this SKU @@ -23,9 +33,147 @@ type APISku struct { // Components contains the hardware components of this SKU Components *APISkuComponents `json:"components"` // Created is the date and time the entity was created - Created time.Time `json:"created"` + Created *time.Time `json:"created"` // Updated is the date and time the entity was last updated - Updated time.Time `json:"updated"` + Updated *time.Time `json:"updated"` +} + +// APISkuCreateRequest is the POST /sku request body. +type APISkuCreateRequest struct { + // SiteID is the Site whose Core service will own the SKU. + SiteID string `json:"siteId"` + // ID is the unique SKU identifier. + ID string `json:"id"` + // Description is the human-readable SKU description. + Description *string `json:"description"` + // DeviceType is the optional device type identifier. + DeviceType *string `json:"deviceType,omitempty"` + // Components is the expected hardware configuration. + Components *APISkuComponents `json:"components"` +} + +// APISkuUpdateRequest is the PATCH /sku/:id request body. +type APISkuUpdateRequest struct { + // SkuID is populated from the request path before proto conversion. + SkuID string `json:"-"` + // Description replaces the description when provided. + Description *string `json:"description,omitempty"` + // DeviceType replaces the device type when provided. + DeviceType *string `json:"deviceType,omitempty"` + // Components replaces the hardware configuration when provided. + Components *APISkuComponents `json:"components,omitempty"` +} + +// CoreSkuSchemaVersion is the Core wire format emitted by REST SKU mutations. +const CoreSkuSchemaVersion uint32 = 5 + +// Validate checks the create request before conversion to Core protobufs. +func (ascr APISkuCreateRequest) Validate() error { + return validation.ValidateStruct(&ascr, + validation.Field(&ascr.SiteID, + validation.Required.Error(validationErrorValueRequired), + validationis.UUID.Error(validationErrorInvalidUUID)), + validation.Field(&ascr.ID, validation.Required.Error(validationErrorValueRequired)), + validation.Field(&ascr.Components, validation.Required.Error(validationErrorValueRequired)), + ) +} + +// ToProto converts a validated create request into Core's single-item SkuList. +func (ascr APISkuCreateRequest) ToProto() *corev1.SkuList { + return &corev1.SkuList{Skus: []*corev1.Sku{{ + Id: ascr.ID, + Description: ascr.Description, + SchemaVersion: CoreSkuSchemaVersion, + DeviceType: ascr.DeviceType, + Components: ascr.Components.ToProto(), + }}} +} + +// Validate checks the update request and requires at least one mutable field. +func (asur APISkuUpdateRequest) Validate() error { + if asur.Description == nil && asur.DeviceType == nil && asur.Components == nil { + return validation.Errors{"request": validation.NewError("validation_required", "at least one mutable field is required")} + } + return validation.ValidateStruct(&asur, + validation.Field(&asur.Components), + ) +} + +// ToMetadataProto converts a validated metadata-only PATCH request. +func (asur APISkuUpdateRequest) ToMetadataProto() *corev1.SkuUpdateMetadataRequest { + return &corev1.SkuUpdateMetadataRequest{ + SkuId: asur.SkuID, + Description: asur.Description, + DeviceType: asur.DeviceType, + } +} + +// ApplyMetadataToProto merges metadata fields into a copy of the current Core SKU. +func (asur APISkuUpdateRequest) ApplyMetadataToProto(current *corev1.Sku) *corev1.Sku { + updated := proto.Clone(current).(*corev1.Sku) + updated.Id = asur.SkuID + if asur.Description != nil { + updated.Description = asur.Description + } + if asur.DeviceType != nil { + updated.DeviceType = asur.DeviceType + } + return updated +} + +// ToReplacementProto converts a component PATCH into Core's current SKU format. +func (asur APISkuUpdateRequest) ToReplacementProto(current *corev1.Sku) *corev1.Sku { + updated := asur.ApplyMetadataToProto(current) + updated.SchemaVersion = CoreSkuSchemaVersion + if asur.Components != nil { + updated.Components = asur.Components.ToProto() + } + return updated +} + +// NewAPISkuFromProto converts a Core SKU into the REST API representation. +func NewAPISkuFromProto(sku *corev1.Sku, siteID string) *APISku { + if sku == nil { + return nil + } + response := &APISku{ + ID: sku.Id, + SiteID: siteID, + Description: sku.GetDescription(), + SchemaVersion: sku.SchemaVersion, + DeviceType: sku.DeviceType, + AssociatedMachineIds: []string{}, + Components: NewAPISkuComponents(sku.Components), + } + for _, machineID := range sku.AssociatedMachineIds { + id := machineID.GetId() + if id != "" { + response.AssociatedMachineIds = append(response.AssociatedMachineIds, id) + } + } + if sku.Created != nil { + created := sku.Created.AsTime() + response.Created = &created + } + return response +} + +// NewAPISkuFromCreateRequest builds the best-known response +// after Core accepted a create request but the post-create read failed. +func NewAPISkuFromCreateRequest(req APISkuCreateRequest, skuID, siteID string) *APISku { + description := "" + if req.Description != nil { + description = *req.Description + } + return &APISku{ + ID: skuID, + SiteID: siteID, + Description: description, + SchemaVersion: CoreSkuSchemaVersion, + DeviceType: req.DeviceType, + AssociatedMachineIds: []string{}, + Components: NewAPISkuComponents(req.Components.ToProto()), + } } // NewAPISku accepts a DB layer SKU object and returns an API layer object @@ -37,10 +185,12 @@ func NewAPISku(dbSku *cdbm.SKU) *APISku { apiSku := &APISku{ ID: dbSku.ID, SiteID: dbSku.SiteID.String(), + Description: dbSku.Description, + SchemaVersion: dbSku.SchemaVersion, DeviceType: dbSku.DeviceType, AssociatedMachineIds: dbSku.AssociatedMachineIds, - Created: dbSku.Created, - Updated: dbSku.Updated, + Created: &dbSku.Created, + Updated: &dbSku.Updated, } // Map SKU Components if available @@ -51,7 +201,6 @@ func NewAPISku(dbSku *cdbm.SKU) *APISku { return apiSku } -// APISkuComponents is the data structure to capture API representation of SKU Components type APISkuComponents struct { // Cpus describes CPU components Cpus []APISkuCpu `json:"cpus"` @@ -71,6 +220,11 @@ type APISkuComponents struct { Tpm *APISkuTpm `json:"tpm"` } +// Validate checks every storage component. +func (c APISkuComponents) Validate() error { + return validation.Validate(c.Storage, validation.Each()) +} + // APISkuCpu represents a CPU component in the SKU type APISkuCpu struct { // Vendor describes the vendor of the CPU @@ -107,14 +261,45 @@ type APISkuMemory struct { // APISkuStorage represents a storage component in the SKU type APISkuStorage struct { - // Vendor describes the vendor of the storage device - Vendor string `json:"vendor"` - // Model describes the model of the storage device + // Vendor participates in storage matching for schema version 4 SKUs. It is + // read-only in REST mutation requests because component mutations use the + // current schema version. + Vendor *string `json:"vendor"` + // Model is informational starting with the 2.1 release. Model string `json:"model"` - // CapacityMb describes the capacity in megabytes - CapacityMb uint32 `json:"capacityMb"` + // CapacityMb participates in storage matching for schema version 4 SKUs. It + // is read-only in REST mutation requests because component mutations use the + // current schema version. + CapacityMb *uint32 `json:"capacityMb"` // Count describes the number of storage devices present Count uint32 `json:"count"` + // MinSizeMiB is the inclusive minimum capacity in mebibytes for each storage device. + // It is only used for matching for SKUs of schema version 5 and onwards. + MinSizeMiB *uint32 `json:"minSizeMiB"` + // MaxSizeMiB is the inclusive maximum capacity in mebibytes for each storage device. + // It is only used for matching for SKUs of schema version 5 and onwards. + MaxSizeMiB *uint32 `json:"maxSizeMiB"` + // PciPatterns contains regular expressions matched against storage PCI paths. + // It is only used for matching for SKUs of schema version 5 and onwards. + PciPatterns []string `json:"pciPatterns"` +} + +// Validate rejects read-only fields and invalid size bounds. +func (s APISkuStorage) Validate() error { + errs := validation.Errors{} + if s.Vendor != nil { + errs["vendor"] = fmt.Errorf("is a read-only SKU storage field") + } + if s.CapacityMb != nil { + errs["capacityMb"] = fmt.Errorf("is a read-only SKU storage field") + } + if s.MinSizeMiB != nil && s.MaxSizeMiB != nil && *s.MinSizeMiB > *s.MaxSizeMiB { + errs["minSizeMiB"] = fmt.Errorf("must be less than or equal to maxSizeMiB") + } + if len(errs) == 0 { + return nil + } + return errs } // APISkuChassis represents the chassis component in the SKU @@ -123,6 +308,8 @@ type APISkuChassis struct { Vendor string `json:"vendor"` // Model describes the model of the chassis Model string `json:"model"` + // Architecture describes the chassis architecture. + Architecture string `json:"architecture"` } // APISkuEthernetDevice represents an ethernet device component in the SKU @@ -133,6 +320,8 @@ type APISkuEthernetDevice struct { Model string `json:"model"` // Count describes the number of ethernet devices present Count uint32 `json:"count"` + // IsConnected reports whether the Ethernet device is connected. + IsConnected bool `json:"isConnected"` } // APISkuInfinibandDevice represents an infiniband device component in the SKU @@ -143,6 +332,8 @@ type APISkuInfinibandDevice struct { Model string `json:"model"` // Count describes the number of infiniband devices present Count uint32 `json:"count"` + // InactiveDevices contains zero-based indexes of inactive devices. + InactiveDevices []uint32 `json:"inactiveDevices"` } // APISkuTpm represents a TPM component in the SKU @@ -153,106 +344,149 @@ type APISkuTpm struct { Version string `json:"version"` } -// NewAPISkuComponents converts proto SkuComponents to API SkuComponents +// NewAPISkuComponents converts proto SkuComponents to API SkuComponents. func NewAPISkuComponents(protoComponents *corev1.SkuComponents) *APISkuComponents { if protoComponents == nil { return nil } - apiComponents := &APISkuComponents{} - - // Map CPU components - if len(protoComponents.Cpus) > 0 { - apiComponents.Cpus = []APISkuCpu{} - for _, cpu := range protoComponents.Cpus { - apiComponents.Cpus = append(apiComponents.Cpus, APISkuCpu{ - Vendor: cpu.Vendor, - Model: cpu.Model, - ThreadCount: cpu.ThreadCount, - Count: cpu.Count, - }) - } + components := &APISkuComponents{} + for _, cpu := range protoComponents.Cpus { + components.Cpus = append(components.Cpus, APISkuCpu{ + Vendor: cpu.Vendor, + Model: cpu.Model, + ThreadCount: cpu.ThreadCount, + Count: cpu.Count, + }) } - - // Map GPU components - if len(protoComponents.Gpus) > 0 { - apiComponents.Gpus = []APISkuGpu{} - for _, gpu := range protoComponents.Gpus { - apiComponents.Gpus = append(apiComponents.Gpus, APISkuGpu{ - Vendor: gpu.Vendor, - Model: gpu.Model, - TotalMemory: gpu.TotalMemory, - Count: gpu.Count, - }) - } + for _, gpu := range protoComponents.Gpus { + components.Gpus = append(components.Gpus, APISkuGpu{ + Vendor: gpu.Vendor, + Model: gpu.Model, + TotalMemory: gpu.TotalMemory, + Count: gpu.Count, + }) } - - // Map Memory components - if len(protoComponents.Memory) > 0 { - apiComponents.Memory = []APISkuMemory{} - for _, mem := range protoComponents.Memory { - apiComponents.Memory = append(apiComponents.Memory, APISkuMemory{ - CapacityMb: mem.CapacityMb, - MemoryType: mem.MemoryType, - Count: mem.Count, - }) - } + for _, memory := range protoComponents.Memory { + components.Memory = append(components.Memory, APISkuMemory{ + CapacityMb: memory.CapacityMb, + MemoryType: memory.MemoryType, + Count: memory.Count, + }) } - - // Map Storage components - if len(protoComponents.Storage) > 0 { - apiComponents.Storage = []APISkuStorage{} - for _, storage := range protoComponents.Storage { - apiComponents.Storage = append(apiComponents.Storage, APISkuStorage{ - Vendor: storage.Vendor, - Model: storage.Model, - CapacityMb: storage.CapacityMb, - Count: storage.Count, - }) - } + for _, storage := range protoComponents.Storage { + vendor := storage.Vendor + capacityMb := storage.CapacityMb + components.Storage = append(components.Storage, APISkuStorage{ + Vendor: &vendor, + Model: storage.Model, + CapacityMb: &capacityMb, + Count: storage.Count, + MinSizeMiB: storage.MinSizeMb, + MaxSizeMiB: storage.MaxSizeMb, + PciPatterns: storage.PciPatterns, + }) } - - // Map Chassis component (single object) if protoComponents.Chassis != nil { - apiComponents.Chassis = &APISkuChassis{ - Vendor: protoComponents.Chassis.Vendor, - Model: protoComponents.Chassis.Model, + components.Chassis = &APISkuChassis{ + Vendor: protoComponents.Chassis.Vendor, + Model: protoComponents.Chassis.Model, + Architecture: protoComponents.Chassis.Architecture, } } - - // Map EthernetDevices components - if len(protoComponents.EthernetDevices) > 0 { - apiComponents.EthernetDevices = []APISkuEthernetDevice{} - for _, ethDev := range protoComponents.EthernetDevices { - apiComponents.EthernetDevices = append(apiComponents.EthernetDevices, APISkuEthernetDevice{ - Vendor: ethDev.Vendor, - Model: ethDev.Model, - Count: ethDev.Count, - }) - } + for _, ethernet := range protoComponents.EthernetDevices { + components.EthernetDevices = append(components.EthernetDevices, APISkuEthernetDevice{ + Vendor: ethernet.Vendor, + Model: ethernet.Model, + Count: ethernet.Count, + IsConnected: ethernet.IsConnected, + }) } - - // Map InfinibandDevices components - if len(protoComponents.InfinibandDevices) > 0 { - apiComponents.InfinibandDevices = []APISkuInfinibandDevice{} - for _, ibDev := range protoComponents.InfinibandDevices { - apiComponents.InfinibandDevices = append(apiComponents.InfinibandDevices, APISkuInfinibandDevice{ - Vendor: ibDev.Vendor, - Model: ibDev.Model, - Count: ibDev.Count, - }) - } + for _, infiniband := range protoComponents.InfinibandDevices { + components.InfinibandDevices = append(components.InfinibandDevices, APISkuInfinibandDevice{ + Vendor: infiniband.Vendor, + Model: infiniband.Model, + Count: infiniband.Count, + InactiveDevices: infiniband.InactiveDevices, + }) } - - // Map Tpm components if protoComponents.Tpm != nil { - apiComponents.Tpm = &APISkuTpm{ + components.Tpm = &APISkuTpm{ Vendor: protoComponents.Tpm.Vendor, Version: protoComponents.Tpm.Version, } } + return components +} - return apiComponents +// ToProto converts API SKU components into the Core protobuf shape. +func (c *APISkuComponents) ToProto() *corev1.SkuComponents { + if c == nil { + return nil + } + components := &corev1.SkuComponents{} + if c.Chassis != nil { + components.Chassis = &corev1.SkuComponentChassis{ + Vendor: c.Chassis.Vendor, + Model: c.Chassis.Model, + Architecture: c.Chassis.Architecture, + } + } + for _, cpu := range c.Cpus { + components.Cpus = append(components.Cpus, &corev1.SkuComponentCpu{ + Vendor: cpu.Vendor, + Model: cpu.Model, + ThreadCount: cpu.ThreadCount, + Count: cpu.Count, + }) + } + for _, gpu := range c.Gpus { + components.Gpus = append(components.Gpus, &corev1.SkuComponentGpu{ + Vendor: gpu.Vendor, + Model: gpu.Model, + TotalMemory: gpu.TotalMemory, + Count: gpu.Count, + }) + } + for _, memory := range c.Memory { + components.Memory = append(components.Memory, &corev1.SkuComponentMemory{ + CapacityMb: memory.CapacityMb, + MemoryType: memory.MemoryType, + Count: memory.Count, + }) + } + for _, storage := range c.Storage { + components.Storage = append(components.Storage, &corev1.SkuComponentStorage{ + Model: storage.Model, + Count: storage.Count, + MinSizeMb: storage.MinSizeMiB, + MaxSizeMb: storage.MaxSizeMiB, + PciPatterns: storage.PciPatterns, + }) + } + for _, ethernet := range c.EthernetDevices { + components.EthernetDevices = append(components.EthernetDevices, &corev1.SkuComponentEthernetDevices{ + Vendor: ethernet.Vendor, + Model: ethernet.Model, + Count: ethernet.Count, + IsConnected: ethernet.IsConnected, + }) + } + for _, infiniband := range c.InfinibandDevices { + components.InfinibandDevices = append(components.InfinibandDevices, &corev1.SkuComponentInfinibandDevices{ + Vendor: infiniband.Vendor, + Model: infiniband.Model, + Count: infiniband.Count, + InactiveDevices: infiniband.InactiveDevices, + }) + } + if c.Tpm != nil { + components.Tpm = &corev1.SkuComponentTpm{ + Vendor: c.Tpm.Vendor, + Version: c.Tpm.Version, + } + } + return components } // APISkuSummary is the data structure to capture summary of a SKU diff --git a/rest-api/api/pkg/api/model/sku_test.go b/rest-api/api/pkg/api/model/sku_test.go index 24bfaa7346..31f6106796 100644 --- a/rest-api/api/pkg/api/model/sku_test.go +++ b/rest-api/api/pkg/api/model/sku_test.go @@ -4,15 +4,18 @@ package model import ( + "encoding/json" "fmt" "reflect" "testing" "time" + cutil "github.com/NVIDIA/infra-controller/rest-api/common/pkg/util" cdbm "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/model" corev1 "github.com/NVIDIA/infra-controller/rest-api/proto/core/gen/v1" "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNewAPISku(t *testing.T) { @@ -21,6 +24,7 @@ func TestNewAPISku(t *testing.T) { } siteID := uuid.New() + description := "Test SKU description" deviceType := "test-device-type" associatedMachineIds := []string{"machine-1", "machine-2"} createdTime := time.Now() @@ -30,6 +34,8 @@ func TestNewAPISku(t *testing.T) { dbSku := &cdbm.SKU{ ID: "test-sku-id", SiteID: siteID, + Description: description, + SchemaVersion: 5, DeviceType: &deviceType, AssociatedMachineIds: associatedMachineIds, Created: createdTime, @@ -52,10 +58,12 @@ func TestNewAPISku(t *testing.T) { want: &APISku{ ID: dbSku.ID, SiteID: siteID.String(), + Description: description, + SchemaVersion: 5, DeviceType: &deviceType, AssociatedMachineIds: associatedMachineIds, - Created: createdTime, - Updated: updatedTime, + Created: &createdTime, + Updated: &updatedTime, }, }, { @@ -83,6 +91,8 @@ func TestNewAPISku(t *testing.T) { // Compare basic fields assert.Equal(t, tt.want.ID, got.ID) assert.Equal(t, tt.want.SiteID, got.SiteID) + assert.Equal(t, tt.want.Description, got.Description) + assert.Equal(t, tt.want.SchemaVersion, got.SchemaVersion) assert.Equal(t, tt.want.DeviceType, got.DeviceType) assert.Equal(t, tt.want.AssociatedMachineIds, got.AssociatedMachineIds) assert.Equal(t, tt.want.Created, got.Created) @@ -114,6 +124,9 @@ func TestNewAPISkuWithFullComponents(t *testing.T) { deviceType := "gpu-server" createdTime := time.Now() updatedTime := time.Now() + minStorageSizeMb := uint32(7_600_000) + maxStorageSizeMb := uint32(7_800_000) + pciPattern := `^/devices/pci.*nvme[0-3]$` t.Run("complete GPU server with all component types", func(t *testing.T) { dbSku := &cdbm.SKU{ @@ -148,10 +161,13 @@ func TestNewAPISkuWithFullComponents(t *testing.T) { }, Storage: []*corev1.SkuComponentStorage{ { - Vendor: "Samsung", - Model: "PM9A3", - CapacityMb: 7680000, - Count: 4, + Vendor: "Samsung", + Model: "PM9A3", + CapacityMb: 7680000, + Count: 4, + MinSizeMb: &minStorageSizeMb, + MaxSizeMb: &maxStorageSizeMb, + PciPatterns: []string{pciPattern}, }, }, Chassis: &corev1.SkuComponentChassis{ @@ -199,10 +215,13 @@ func TestNewAPISkuWithFullComponents(t *testing.T) { // Validate Storage assert.Len(t, result.Components.Storage, 1) - assert.Equal(t, "Samsung", result.Components.Storage[0].Vendor) + assert.Equal(t, cutil.GetPtr("Samsung"), result.Components.Storage[0].Vendor) assert.Equal(t, "PM9A3", result.Components.Storage[0].Model) - assert.Equal(t, uint32(7680000), result.Components.Storage[0].CapacityMb) + assert.Equal(t, cutil.GetPtr(uint32(7680000)), result.Components.Storage[0].CapacityMb) assert.Equal(t, uint32(4), result.Components.Storage[0].Count) + assert.Equal(t, &minStorageSizeMb, result.Components.Storage[0].MinSizeMiB) + assert.Equal(t, &maxStorageSizeMb, result.Components.Storage[0].MaxSizeMiB) + assert.Equal(t, []string{pciPattern}, result.Components.Storage[0].PciPatterns) // Validate Chassis assert.NotNil(t, result.Components.Chassis) @@ -328,15 +347,15 @@ func TestNewAPISkuWithFullComponents(t *testing.T) { assert.Len(t, result.Components.Storage, 2) // Validate first storage type - assert.Equal(t, "Samsung", result.Components.Storage[0].Vendor) + assert.Equal(t, cutil.GetPtr("Samsung"), result.Components.Storage[0].Vendor) assert.Equal(t, "PM1733", result.Components.Storage[0].Model) - assert.Equal(t, uint32(15360000), result.Components.Storage[0].CapacityMb) + assert.Equal(t, cutil.GetPtr(uint32(15360000)), result.Components.Storage[0].CapacityMb) assert.Equal(t, uint32(24), result.Components.Storage[0].Count) // Validate second storage type - assert.Equal(t, "Intel", result.Components.Storage[1].Vendor) + assert.Equal(t, cutil.GetPtr("Intel"), result.Components.Storage[1].Vendor) assert.Equal(t, "P5520", result.Components.Storage[1].Model) - assert.Equal(t, uint32(7680000), result.Components.Storage[1].CapacityMb) + assert.Equal(t, cutil.GetPtr(uint32(7680000)), result.Components.Storage[1].CapacityMb) assert.Equal(t, uint32(4), result.Components.Storage[1].Count) // Validate memory configuration @@ -483,7 +502,7 @@ func TestNewAPISkuWithFullComponents(t *testing.T) { // Validate compact storage assert.Len(t, result.Components.Storage, 1) - assert.Equal(t, uint32(960000), result.Components.Storage[0].CapacityMb) + assert.Equal(t, cutil.GetPtr(uint32(960000)), result.Components.Storage[0].CapacityMb) }) } @@ -823,7 +842,7 @@ func TestAPISkuComponentsWithSpecialValues(t *testing.T) { assert.Equal(t, uint32(32768), result.Components.Memory[0].CapacityMb) assert.Len(t, result.Components.Storage, 3) - assert.Equal(t, "Samsung", result.Components.Storage[0].Vendor) + assert.Equal(t, cutil.GetPtr("Samsung"), result.Components.Storage[0].Vendor) assert.NotNil(t, result.Components.Tpm) assert.Equal(t, "Infineon", result.Components.Tpm.Vendor) @@ -876,3 +895,309 @@ func TestAPISkuSummaryEdgeCases(t *testing.T) { assert.Equal(t, specialType, *result.DeviceType) }) } + +func TestAPISkuCreateRequest(t *testing.T) { + t.Run("converts to proto", func(t *testing.T) { + deviceType := "gpu-server" + req := APISkuCreateRequest{ + SiteID: uuid.NewString(), + ID: "dgx-h100", + Description: cutil.GetPtr("DGX H100"), + DeviceType: &deviceType, + Components: testAPISkuComponents(), + } + + require.NoError(t, req.Validate()) + proto := req.ToProto() + require.Len(t, proto.Skus, 1) + sku := proto.Skus[0] + assert.Equal(t, "dgx-h100", sku.Id) + assert.Equal(t, "DGX H100", sku.GetDescription()) + assert.Equal(t, CoreSkuSchemaVersion, sku.SchemaVersion) + assert.Equal(t, "gpu-server", sku.GetDeviceType()) + require.NotNil(t, sku.Components) + require.NotNil(t, sku.Components.Chassis) + assert.Equal(t, "x86_64", sku.Components.Chassis.Architecture) + require.Len(t, sku.Components.InfinibandDevices, 1) + assert.Equal(t, []uint32{1}, sku.Components.InfinibandDevices[0].InactiveDevices) + require.Len(t, sku.Components.Storage, 1) + assert.Empty(t, sku.Components.Storage[0].Vendor) + assert.Zero(t, sku.Components.Storage[0].CapacityMb) + assert.Equal(t, "informational-model", sku.Components.Storage[0].Model) + assert.Equal(t, cutil.GetPtr(uint32(3_600_000)), sku.Components.Storage[0].MinSizeMb) + assert.Equal(t, cutil.GetPtr(uint32(3_900_000)), sku.Components.Storage[0].MaxSizeMb) + assert.Equal(t, []string{`^/devices/pci.*nvme0$`}, sku.Components.Storage[0].PciPatterns) + }) + + t.Run("preserves omitted description in proto", func(t *testing.T) { + req := APISkuCreateRequest{ + SiteID: uuid.NewString(), + ID: "dgx-h100", + Components: testAPISkuComponents(), + } + + require.NoError(t, req.Validate()) + proto := req.ToProto() + require.Len(t, proto.Skus, 1) + assert.Nil(t, proto.Skus[0].Description) + }) + + t.Run("validates required fields", func(t *testing.T) { + req := APISkuCreateRequest{SiteID: "not-a-uuid", ID: ""} + assert.Error(t, req.Validate()) + }) + + t.Run("rejects inverted storage size range", func(t *testing.T) { + components := testAPISkuComponents() + components.Storage[0].MinSizeMiB = cutil.GetPtr(uint32(4_000_000)) + components.Storage[0].MaxSizeMiB = cutil.GetPtr(uint32(3_800_000)) + req := APISkuCreateRequest{ + SiteID: uuid.NewString(), + ID: "dgx-h100", + Components: components, + } + + err := req.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "minSizeMiB: must be less than or equal to maxSizeMiB") + }) + + for _, test := range []struct { + name string + min *uint32 + max *uint32 + }{ + {name: "accepts equal storage size bounds", min: cutil.GetPtr(uint32(3_800_000)), max: cutil.GetPtr(uint32(3_800_000))}, + {name: "accepts only minimum size", min: cutil.GetPtr(uint32(3_800_000))}, + {name: "accepts only maximum size", max: cutil.GetPtr(uint32(4_000_000))}, + } { + t.Run(test.name, func(t *testing.T) { + components := testAPISkuComponents() + components.Storage[0].MinSizeMiB = test.min + components.Storage[0].MaxSizeMiB = test.max + req := APISkuCreateRequest{ + SiteID: uuid.NewString(), + ID: "dgx-h100", + Components: components, + } + + assert.NoError(t, req.Validate()) + }) + } +} + +func TestNewAPISkuFromCreateRequest_OmittedDescription(t *testing.T) { + response := NewAPISkuFromCreateRequest(APISkuCreateRequest{}, "dgx-h100", uuid.NewString()) + + assert.Empty(t, response.Description) +} + +func TestAPISku_MarshalJSON(t *testing.T) { + response := APISku{ + Components: &APISkuComponents{ + Storage: []APISkuStorage{{}}, + }, + } + + encoded, err := json.Marshal(response) + require.NoError(t, err) + assert.JSONEq(t, `{ + "id":"", + "siteId":"", + "description":"", + "schemaVersion":0, + "deviceType":null, + "associatedMachineIds":null, + "components":{ + "cpus":null, + "gpus":null, + "memory":null, + "storage":[{ + "vendor":null, + "model":"", + "capacityMb":null, + "count":0, + "minSizeMiB":null, + "maxSizeMiB":null, + "pciPatterns":null + }], + "chassis":null, + "ethernetDevices":null, + "infinibandDevices":null, + "tpm":null + }, + "created":null, + "updated":null + }`, string(encoded)) +} + +func TestAPISkuStorage_Validate(t *testing.T) { + t.Run("accepts schema version 5 fields", func(t *testing.T) { + var storage APISkuStorage + err := json.Unmarshal([]byte(`{ + "model":"informational-model", + "count":2, + "minSizeMiB":3600000, + "maxSizeMiB":3900000, + "pciPatterns":["^/devices/pci.*nvme[0-1]$"] + }`), &storage) + require.NoError(t, err) + require.NoError(t, storage.Validate()) + assert.Equal(t, "informational-model", storage.Model) + assert.Equal(t, uint32(2), storage.Count) + assert.Equal(t, cutil.GetPtr(uint32(3_600_000)), storage.MinSizeMiB) + assert.Equal(t, cutil.GetPtr(uint32(3_900_000)), storage.MaxSizeMiB) + assert.Equal(t, []string{`^/devices/pci.*nvme[0-1]$`}, storage.PciPatterns) + }) + + for name, body := range map[string]string{ + "rejects read-only vendor": `{"model":"legacy","count":1,"vendor":""}`, + "rejects read-only capacityMb": `{"model":"legacy","count":1,"capacityMb":0}`, + } { + t.Run(name, func(t *testing.T) { + var storage APISkuStorage + err := json.Unmarshal([]byte(body), &storage) + require.NoError(t, err) + + err = storage.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "read-only") + }) + } +} + +func TestAPISkuRequests_UnmarshalJSON(t *testing.T) { + t.Run("create", func(t *testing.T) { + var request APISkuCreateRequest + err := json.Unmarshal([]byte(`{ + "siteId":"60189e9c-7d12-438c-b9ca-6998d9c364b1", + "id":"sku-1", + "description":"description", + "deviceType":"gpu", + "components":{} + }`), &request) + + require.NoError(t, err) + assert.Equal(t, "60189e9c-7d12-438c-b9ca-6998d9c364b1", request.SiteID) + assert.Equal(t, "sku-1", request.ID) + assert.Equal(t, "description", *request.Description) + assert.Equal(t, "gpu", *request.DeviceType) + require.NotNil(t, request.Components) + }) + + t.Run("update", func(t *testing.T) { + var request APISkuUpdateRequest + err := json.Unmarshal([]byte(`{ + "description":"description", + "deviceType":"gpu", + "components":{} + }`), &request) + + require.NoError(t, err) + assert.Equal(t, "description", *request.Description) + assert.Equal(t, "gpu", *request.DeviceType) + require.NotNil(t, request.Components) + }) + +} + +func TestAPISkuUpdateRequest(t *testing.T) { + t.Run("converts metadata update to proto", func(t *testing.T) { + description := "updated description" + req := APISkuUpdateRequest{ + SkuID: "dgx-h100", + Description: &description, + } + existing := &corev1.Sku{ + Id: "dgx-h100", + Description: cutil.GetPtr("old description"), + SchemaVersion: 4, + DeviceType: cutil.GetPtr("gpu-server"), + Components: testAPISkuComponents().ToProto(), + AssociatedMachineIds: []*corev1.MachineId{{Id: "machine-1"}}, + } + + require.NoError(t, req.Validate()) + metadata := req.ToMetadataProto() + assert.Equal(t, "dgx-h100", metadata.SkuId) + assert.Equal(t, "updated description", metadata.GetDescription()) + + updated := req.ApplyMetadataToProto(existing) + assert.Equal(t, "updated description", updated.GetDescription()) + assert.Equal(t, uint32(4), updated.SchemaVersion) + assert.Equal(t, "gpu-server", updated.GetDeviceType()) + assert.Equal(t, existing.Components, updated.Components) + assert.Equal(t, existing.AssociatedMachineIds, updated.AssociatedMachineIds) + assert.NotSame(t, existing, updated) + }) + + t.Run("replaces components using current schema version", func(t *testing.T) { + description := "updated description" + req := APISkuUpdateRequest{ + SkuID: "dgx-h100", + Description: &description, + Components: testAPISkuComponents(), + } + existing := &corev1.Sku{ + Id: "dgx-h100", + Description: cutil.GetPtr("old description"), + SchemaVersion: CoreSkuSchemaVersion, + DeviceType: cutil.GetPtr("gpu-server"), + Components: testAPISkuComponents().ToProto(), + AssociatedMachineIds: []*corev1.MachineId{{Id: "machine-1"}}, + } + + require.NoError(t, req.Validate()) + updated := req.ToReplacementProto(existing) + assert.Equal(t, "updated description", updated.GetDescription()) + assert.Equal(t, CoreSkuSchemaVersion, updated.SchemaVersion) + assert.Equal(t, req.Components.ToProto(), updated.Components) + assert.Equal(t, existing.AssociatedMachineIds, updated.AssociatedMachineIds) + assert.NotSame(t, existing, updated) + }) + + t.Run("requires a mutable field", func(t *testing.T) { + req := APISkuUpdateRequest{} + assert.Error(t, req.Validate()) + }) + + t.Run("rejects inverted storage size range", func(t *testing.T) { + components := testAPISkuComponents() + components.Storage[0].MinSizeMiB = cutil.GetPtr(uint32(4_000_000)) + components.Storage[0].MaxSizeMiB = cutil.GetPtr(uint32(3_800_000)) + req := APISkuUpdateRequest{Components: components} + + err := req.Validate() + require.Error(t, err) + assert.Contains(t, err.Error(), "minSizeMiB: must be less than or equal to maxSizeMiB") + }) +} + +func testAPISkuComponents() *APISkuComponents { + return &APISkuComponents{ + Chassis: &APISkuChassis{ + Vendor: "NVIDIA", + Model: "DGX H100", + Architecture: "x86_64", + }, + Cpus: []APISkuCpu{{ + Vendor: "Intel", + Model: "Xeon", + ThreadCount: 112, + Count: 2, + }}, + Storage: []APISkuStorage{{ + Model: "informational-model", + Count: 2, + MinSizeMiB: cutil.GetPtr(uint32(3_600_000)), + MaxSizeMiB: cutil.GetPtr(uint32(3_900_000)), + PciPatterns: []string{`^/devices/pci.*nvme0$`}, + }}, + InfinibandDevices: []APISkuInfinibandDevice{{ + Vendor: "NVIDIA", + Model: "ConnectX-7", + Count: 2, + InactiveDevices: []uint32{1}, + }}, + } +} diff --git a/rest-api/api/pkg/api/routes.go b/rest-api/api/pkg/api/routes.go index c9c5017a54..870c88e5e2 100644 --- a/rest-api/api/pkg/api/routes.go +++ b/rest-api/api/pkg/api/routes.go @@ -979,6 +979,11 @@ func NewAPIRoutes(dbSession *cdb.Session, tc tClient.Client, tnc tClient.Namespa Handler: apiHandler.NewDeleteDpuExtensionServiceVersionHandler(dbSession, tc, scp, cfg), }, // SKU endpoints + { + Path: apiPathPrefix + "/sku", + Method: http.MethodPost, + Handler: apiHandler.NewCreateSkuHandler(dbSession, scp), + }, { Path: apiPathPrefix + "/sku", Method: http.MethodGet, @@ -989,6 +994,16 @@ func NewAPIRoutes(dbSession *cdb.Session, tc tClient.Client, tnc tClient.Namespa Method: http.MethodGet, Handler: apiHandler.NewGetSkuHandler(dbSession, tc, cfg), }, + { + Path: apiPathPrefix + "/sku/:id", + Method: http.MethodPatch, + Handler: apiHandler.NewUpdateSkuHandler(dbSession, scp), + }, + { + Path: apiPathPrefix + "/sku/:id", + Method: http.MethodDelete, + Handler: apiHandler.NewDeleteSkuHandler(dbSession, scp), + }, // Task endpoints (Flow). /rack/task/* and /task/* share get/cancel // handlers; list operations are exposed under /rack/{id}/task and // /tray/{id}/task. diff --git a/rest-api/api/pkg/api/routes_test.go b/rest-api/api/pkg/api/routes_test.go index 31d119691e..e4004d7679 100644 --- a/rest-api/api/pkg/api/routes_test.go +++ b/rest-api/api/pkg/api/routes_test.go @@ -72,7 +72,7 @@ func TestNewAPIRoutes(t *testing.T) { "network-security-group": 5, "machine-validation": 11, "dpu-extension-service": 7, - "sku": 2, + "sku": 5, "task": 2, "rule": 5, "run": 8, @@ -148,7 +148,14 @@ func TestNewAPIRoutes(t *testing.T) { assertRouteExists(t, got, http.MethodGet, ipxeTemplatePath) assertRouteExists(t, got, http.MethodGet, ipxeTemplatePath+"/:id") + skuPath := "/org/:orgName/" + cfg.GetAPIName() + "/sku" + assertRouteExists(t, got, http.MethodPost, skuPath) + assertRouteExists(t, got, http.MethodGet, skuPath) + assertRouteExists(t, got, http.MethodGet, skuPath+"/:id") + assertRouteExists(t, got, http.MethodPatch, skuPath+"/:id") + assertRouteExists(t, got, http.MethodDelete, skuPath+"/:id") runPath := "/org/:orgName/" + cfg.GetAPIName() + "/task/run" + assertRouteExists(t, got, http.MethodPost, runPath) assertRouteExists(t, got, http.MethodGet, runPath) assertRouteExists(t, got, http.MethodGet, runPath+"/:id") diff --git a/rest-api/db/pkg/db/model/operatingsystem.go b/rest-api/db/pkg/db/model/operatingsystem.go index 918bd29199..28e0b061a3 100644 --- a/rest-api/db/pkg/db/model/operatingsystem.go +++ b/rest-api/db/pkg/db/model/operatingsystem.go @@ -108,6 +108,11 @@ var ( corev1.OperatingSystemType_OS_TYPE_IPXE: OperatingSystemTypeIPXE, corev1.OperatingSystemType_OS_TYPE_TEMPLATED_IPXE: OperatingSystemTypeTemplatedIPXE, } + // OperatingSystemTypeToProtoMap maps model OS types to nico-core OS types. + OperatingSystemTypeToProtoMap = map[string]corev1.OperatingSystemType{ + OperatingSystemTypeIPXE: corev1.OperatingSystemType_OS_TYPE_IPXE, + OperatingSystemTypeTemplatedIPXE: corev1.OperatingSystemType_OS_TYPE_TEMPLATED_IPXE, + } // OperatingSystemStatusFromProtoMap maps nico-core tenant states to OperatingSystem status values. OperatingSystemStatusFromProtoMap = map[corev1.TenantState]string{ @@ -116,6 +121,16 @@ var ( corev1.TenantState_CONFIGURING: OperatingSystemStatusSyncing, corev1.TenantState_TERMINATING: OperatingSystemStatusDeleting, corev1.TenantState_FAILED: OperatingSystemStatusError, + corev1.TenantState_INVALID: OperatingSystemStatusPending, + } + // OperatingSystemStatusToProtoMap maps model statuses to nico-core tenant states. + OperatingSystemStatusToProtoMap = map[string]corev1.TenantState{ + OperatingSystemStatusPending: corev1.TenantState_INVALID, + OperatingSystemStatusProvisioning: corev1.TenantState_PROVISIONING, + OperatingSystemStatusReady: corev1.TenantState_READY, + OperatingSystemStatusError: corev1.TenantState_FAILED, + OperatingSystemStatusDeleting: corev1.TenantState_TERMINATING, + OperatingSystemStatusSyncing: corev1.TenantState_CONFIGURING, } ) @@ -252,13 +267,115 @@ func (os *OperatingSystem) GetSiteID() *uuid.UUID { return &os.ID } +// ToProto converts this OperatingSystem into its canonical nico-core proto. +// Provider-owned records omit tenant_organization_id; tenant-owned records +// carry their organization. REST-only ownership IDs and image fields have no +// representation in this proto. +func (os *OperatingSystem) ToProto() *corev1.OperatingSystem { + var tenantOrganizationID *string + if os.TenantID != nil && os.Org != "" { + tenantOrganizationID = &os.Org + } + + params := make([]*corev1.IpxeTemplateParameter, 0, len(os.IpxeTemplateParameters)) + for i := range os.IpxeTemplateParameters { + params = append(params, os.IpxeTemplateParameters[i].ToProto()) + } + artifacts := make([]*corev1.IpxeTemplateArtifact, 0, len(os.IpxeTemplateArtifacts)) + for i := range os.IpxeTemplateArtifacts { + artifacts = append(artifacts, os.IpxeTemplateArtifacts[i].ToProto()) + } + var templateID *corev1.IpxeTemplateId + if os.IpxeTemplateId != nil && *os.IpxeTemplateId != "" { + templateID = &corev1.IpxeTemplateId{Value: *os.IpxeTemplateId} + } + + return &corev1.OperatingSystem{ + Id: &corev1.OperatingSystemId{Value: os.ID.String()}, + Name: os.Name, + Description: os.Description, + TenantOrganizationId: tenantOrganizationID, + Type: OperatingSystemTypeToProtoMap[os.Type], + Status: OperatingSystemStatusToProtoMap[os.Status], + IsActive: os.IsActive, + AllowOverride: os.AllowOverride, + PhoneHomeEnabled: os.PhoneHomeEnabled, + UserData: os.UserData, + Created: os.Created.Format(time.RFC3339), + Updated: os.Updated.Format(time.RFC3339), + IpxeScript: os.IpxeScript, + IpxeTemplateId: templateID, + IpxeTemplateParameters: params, + IpxeTemplateArtifacts: artifacts, + IpxeTemplateDefinitionHash: os.IpxeTemplateDefinitionHash, + } +} + +// FromProto populates this OperatingSystem from its canonical nico-core proto. +// A nil proto is a no-op. REST-only ownership IDs, image fields, and audit +// fields are not represented by the proto and remain untouched; callers resolve +// tenant/provider ownership from tenant_organization_id and Site context. +func (os *OperatingSystem) FromProto(protoOS *corev1.OperatingSystem) { + if protoOS == nil { + return + } + if protoOS.Id != nil { + if id, err := uuid.Parse(protoOS.Id.Value); err == nil { + os.ID = id + } + } + + os.Name = protoOS.Name + os.Description = protoOS.Description + if protoOS.TenantOrganizationId != nil { + os.Org = protoOS.GetTenantOrganizationId() + } + os.Type = OperatingSystemTypeFromProtoMap[protoOS.Type] + os.Status = OperatingSystemStatusFromProtoMap[protoOS.Status] + if os.Status == "" { + os.Status = OperatingSystemStatusSyncing + } + os.IsActive = protoOS.IsActive + os.AllowOverride = protoOS.AllowOverride + os.PhoneHomeEnabled = protoOS.PhoneHomeEnabled + os.UserData = protoOS.UserData + os.IpxeScript = protoOS.IpxeScript + os.IpxeTemplateDefinitionHash = protoOS.IpxeTemplateDefinitionHash + + os.Created = time.Time{} + if created, err := time.Parse(time.RFC3339, protoOS.Created); err == nil { + os.Created = created + } + os.Updated = time.Time{} + if updated, err := time.Parse(time.RFC3339, protoOS.Updated); err == nil { + os.Updated = updated + } + + os.IpxeTemplateId = nil + if value := protoOS.GetIpxeTemplateId().GetValue(); value != "" { + os.IpxeTemplateId = &value + } + os.IpxeTemplateParameters = make([]OperatingSystemIpxeParameter, 0, len(protoOS.IpxeTemplateParameters)) + for _, protoParam := range protoOS.IpxeTemplateParameters { + var param OperatingSystemIpxeParameter + param.FromProto(protoParam) + os.IpxeTemplateParameters = append(os.IpxeTemplateParameters, param) + } + os.IpxeTemplateArtifacts = make([]OperatingSystemIpxeArtifact, 0, len(protoOS.IpxeTemplateArtifacts)) + for _, protoArtifact := range protoOS.IpxeTemplateArtifacts { + var artifact OperatingSystemIpxeArtifact + artifact.FromProto(protoArtifact) + os.IpxeTemplateArtifacts = append(os.IpxeTemplateArtifacts, artifact) + } +} + // ToImageAttributesProto builds the OsImageAttributes proto used by // both the create and update workflows. tenantOrg is the owning // tenant's organization id (not stored on the entity directly). // // The same proto shape is sent for both create and update flows, so // this entity-level method is the canonical entity-to-proto for OS -// image data; the request-shape ToProto methods on +// image data; the request-shape ToImageProto methods on // APIOperatingSystemCreateRequest and APIOperatingSystemUpdateRequest // layer on top of it without altering the wire fields. // @@ -283,15 +400,23 @@ func (os *OperatingSystem) ToImageAttributesProto(tenantOrg string) *corev1.OsIm } } -// ToDeletionRequestProto builds the workflow request that asks a Site +// ToImageDeletionRequestProto builds the workflow request that asks a Site // to delete this OS image. -func (os *OperatingSystem) ToDeletionRequestProto(tenantOrg string) *corev1.DeleteOsImageRequest { +func (os *OperatingSystem) ToImageDeletionRequestProto(tenantOrg string) *corev1.DeleteOsImageRequest { return &corev1.DeleteOsImageRequest{ Id: &corev1.UUID{Value: os.GetSiteID().String()}, TenantOrganizationId: tenantOrg, } } +// ToDeletionRequestProto builds the nico-core request for deleting an iPXE +// Operating System. This request has no API body, so it belongs on the entity. +func (os *OperatingSystem) ToDeletionRequestProto() *corev1.DeleteOperatingSystemRequest { + return &corev1.DeleteOperatingSystemRequest{ + Id: &corev1.OperatingSystemId{Value: os.ID.String()}, + } +} + // OperatingSystemCreateInput input parameters for Create method type OperatingSystemCreateInput struct { // ID optionally pre-specifies the primary key. When set (e.g. during inventory sync from @@ -326,53 +451,30 @@ type OperatingSystemCreateInput struct { CreatedBy uuid.UUID } -// FromProto fills the proto-derived definition fields of the receiver from a -// nico-core OperatingSystem proto: OS type, status, scalar flags, iPXE script / -// template reference, template parameters, artifacts and definition hash. -// -// Ownership and sync-context fields (ID, Org, InfrastructureProviderID, TenantID, -// CreatedBy and the image-* fields) are not carried on this proto and must be set -// by the caller after calling FromProto. A nil proto is a no-op. +// FromProto fills this create input through the canonical OperatingSystem +// entity conversion. Ownership IDs, CreatedBy, and image fields are supplied by +// the caller from REST/Site context. A nil proto is a no-op. func (in *OperatingSystemCreateInput) FromProto(protoOS *corev1.OperatingSystem) { if protoOS == nil { return } - in.Name = protoOS.Name - in.Description = protoOS.Description - in.UserData = protoOS.UserData - in.IpxeScript = protoOS.IpxeScript - in.AllowOverride = protoOS.AllowOverride - in.PhoneHomeEnabled = protoOS.PhoneHomeEnabled - in.IpxeOSHash = protoOS.IpxeTemplateDefinitionHash - - in.OsType = OperatingSystemTypeFromProtoMap[protoOS.Type] - - status := OperatingSystemStatusFromProtoMap[protoOS.Status] - if status == "" { - status = OperatingSystemStatusSyncing - } - in.Status = status - - // Only persist a template reference when non-empty; non-templated OS types - // carry no template. - if v := protoOS.GetIpxeTemplateId().GetValue(); v != "" { - in.IpxeTemplateId = &v - } - - in.IpxeTemplateParameters = make([]OperatingSystemIpxeParameter, 0, len(protoOS.IpxeTemplateParameters)) - for _, p := range protoOS.IpxeTemplateParameters { - var param OperatingSystemIpxeParameter - param.FromProto(p) - in.IpxeTemplateParameters = append(in.IpxeTemplateParameters, param) - } - - in.IpxeTemplateArtifacts = make([]OperatingSystemIpxeArtifact, 0, len(protoOS.IpxeTemplateArtifacts)) - for _, a := range protoOS.IpxeTemplateArtifacts { - var artifact OperatingSystemIpxeArtifact - artifact.FromProto(a) - in.IpxeTemplateArtifacts = append(in.IpxeTemplateArtifacts, artifact) - } + var os OperatingSystem + os.FromProto(protoOS) + in.ID = os.ID + in.Name = os.Name + in.Description = os.Description + in.Org = os.Org + in.OsType = os.Type + in.IpxeScript = os.IpxeScript + in.IpxeTemplateId = os.IpxeTemplateId + in.IpxeTemplateParameters = os.IpxeTemplateParameters + in.IpxeTemplateArtifacts = os.IpxeTemplateArtifacts + in.IpxeOSHash = os.IpxeTemplateDefinitionHash + in.UserData = os.UserData + in.AllowOverride = os.AllowOverride + in.PhoneHomeEnabled = os.PhoneHomeEnabled + in.Status = os.Status } // OperatingSystemUpdateInput input parameters for Update method @@ -605,7 +707,6 @@ func (ossd OperatingSystemSQLDAO) GetAll(ctx context.Context, tx *db.Tx, filter if filter.OperatingSystemIds != nil && len(filter.OperatingSystemIds) == 0 { return oss, 0, nil } - query := db.GetIDB(tx, ossd.dbSession).NewSelect().Model(&oss) if filter.Names != nil { query = query.Where("os.name IN (?)", bun.In(filter.Names)) @@ -615,13 +716,25 @@ func (ossd OperatingSystemSQLDAO) GetAll(ctx context.Context, tx *db.Tx, filter query = query.Where("os.org IN (?)", bun.In(filter.Orgs)) ossd.tracerSpan.SetAttribute(operatingSystemSQLDAOSpan, "filter.org", filter.Orgs) } - if filter.InfrastructureProviderID != nil { - query = query.Where("os.infrastructure_provider_id = ?", *filter.InfrastructureProviderID) - ossd.tracerSpan.SetAttribute(operatingSystemSQLDAOSpan, "infrastructure_provider_id", filter.InfrastructureProviderID.String()) - } - if filter.TenantIDs != nil { + hasTenants := len(filter.TenantIDs) > 0 + hasProvider := filter.InfrastructureProviderID != nil + + switch { + case hasTenants && hasProvider: + // Dual-role view: own tenant entries + own provider entries, no site restriction. + query = query.WhereGroup(" AND ", func(q *bun.SelectQuery) *bun.SelectQuery { + return q. + Where("os.tenant_id IN (?)", bun.In(filter.TenantIDs)). + WhereOr("os.infrastructure_provider_id = ?", *filter.InfrastructureProviderID) + }) + ossd.tracerSpan.SetAttribute(operatingSystemSQLDAOSpan, "tenant_or_provider", filter.TenantIDs) + case hasTenants: query = query.Where("os.tenant_id IN (?)", bun.In(filter.TenantIDs)) ossd.tracerSpan.SetAttribute(operatingSystemSQLDAOSpan, "tenant_id", filter.TenantIDs) + case hasProvider: + // Provider-only view: only provider-owned entries. + query = query.Where("os.infrastructure_provider_id = ?", *filter.InfrastructureProviderID) + ossd.tracerSpan.SetAttribute(operatingSystemSQLDAOSpan, "infrastructure_provider_id", filter.InfrastructureProviderID.String()) } if filter.OsTypes != nil { query = query.Where("os.type IN (?)", bun.In(filter.OsTypes)) diff --git a/rest-api/db/pkg/db/model/operatingsystem_ipxe_test.go b/rest-api/db/pkg/db/model/operatingsystem_ipxe_test.go index e9680717e6..1304b2affa 100644 --- a/rest-api/db/pkg/db/model/operatingsystem_ipxe_test.go +++ b/rest-api/db/pkg/db/model/operatingsystem_ipxe_test.go @@ -6,13 +6,78 @@ package model import ( "context" "testing" + "time" cutil "github.com/NVIDIA/infra-controller/rest-api/common/pkg/util" + corev1 "github.com/NVIDIA/infra-controller/rest-api/proto/core/gen/v1" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestOperatingSystem_ToProtoFromProto(t *testing.T) { + id := uuid.New() + tenantID := uuid.New() + created := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC) + updated := created.Add(time.Minute) + templateID := uuid.NewString() + authToken := "secret" + os := &OperatingSystem{ + ID: id, + Name: "templated-os", + Description: cutil.GetPtr("description"), + Org: "tenant-org", + TenantID: &tenantID, + Type: OperatingSystemTypeTemplatedIPXE, + Status: OperatingSystemStatusReady, + IsActive: true, + AllowOverride: true, + PhoneHomeEnabled: true, + UserData: cutil.GetPtr("user-data"), + Created: created, + Updated: updated, + IpxeTemplateId: &templateID, + IpxeTemplateParameters: []OperatingSystemIpxeParameter{ + {Name: "version", Value: "24.04"}, + }, + IpxeTemplateArtifacts: []OperatingSystemIpxeArtifact{ + { + Name: "kernel", + URL: "https://example.test/kernel", + AuthToken: &authToken, + CacheStrategy: OperatingSystemIpxeArtifactCacheStrategyCacheAsNeeded, + }, + }, + IpxeTemplateDefinitionHash: cutil.GetPtr("definition-hash"), + } + + protoOS := os.ToProto() + require.NotNil(t, protoOS) + assert.Equal(t, id.String(), protoOS.GetId().GetValue()) + assert.Equal(t, "tenant-org", protoOS.GetTenantOrganizationId()) + assert.Equal(t, corev1.OperatingSystemType_OS_TYPE_TEMPLATED_IPXE, protoOS.Type) + assert.Equal(t, corev1.TenantState_READY, protoOS.Status) + assert.Equal(t, templateID, protoOS.GetIpxeTemplateId().GetValue()) + require.Len(t, protoOS.IpxeTemplateParameters, 1) + require.Len(t, protoOS.IpxeTemplateArtifacts, 1) + + roundTripped := &OperatingSystem{TenantID: &tenantID} + roundTripped.FromProto(protoOS) + assert.Equal(t, id, roundTripped.ID) + assert.Equal(t, os.Name, roundTripped.Name) + assert.Equal(t, os.Description, roundTripped.Description) + assert.Equal(t, os.Org, roundTripped.Org) + assert.Equal(t, os.TenantID, roundTripped.TenantID, "REST ownership context should remain untouched") + assert.Equal(t, os.Type, roundTripped.Type) + assert.Equal(t, os.Status, roundTripped.Status) + assert.Equal(t, os.Created, roundTripped.Created) + assert.Equal(t, os.Updated, roundTripped.Updated) + assert.Equal(t, os.IpxeTemplateId, roundTripped.IpxeTemplateId) + assert.Equal(t, os.IpxeTemplateParameters, roundTripped.IpxeTemplateParameters) + assert.Equal(t, os.IpxeTemplateArtifacts, roundTripped.IpxeTemplateArtifacts) + assert.Equal(t, os.IpxeTemplateDefinitionHash, roundTripped.IpxeTemplateDefinitionHash) +} + // TestOperatingSystemSQLDAO_TemplatedIPXERoundTrip exercises the iPXE template definition // columns added for the Templated iPXE OS variant: create, read-back of the JSONB // parameter/artifact slices, artifact update, and iPXE definition clear. diff --git a/rest-api/db/pkg/db/model/operatingsystem_test.go b/rest-api/db/pkg/db/model/operatingsystem_test.go index 67c5193cf0..2011524586 100644 --- a/rest-api/db/pkg/db/model/operatingsystem_test.go +++ b/rest-api/db/pkg/db/model/operatingsystem_test.go @@ -77,10 +77,10 @@ func TestOperatingSystem_ToImageAttributesProto(t *testing.T) { assert.Equal(t, &rootFsLabel, got.RootfsLabel) } -func TestOperatingSystem_ToDeletionRequestProto(t *testing.T) { +func TestOperatingSystem_ToImageDeletionRequestProto(t *testing.T) { id := uuid.New() os := &OperatingSystem{ID: id} - got := os.ToDeletionRequestProto("org-1") + got := os.ToImageDeletionRequestProto("org-1") require.NotNil(t, got) require.NotNil(t, got.Id) assert.Equal(t, id.String(), got.Id.Value) @@ -2132,3 +2132,115 @@ func TestOperatingSystemSQLDAO_Delete(t *testing.T) { }) } } + +// TestOperatingSystemSQLDAO_GetAll_OwnershipFilters exercises the provider-only, +// tenant-only, and dual-role ownership filters. Cross-ownership visibility is +// composed by API handlers rather than by the DAO. +func TestOperatingSystemSQLDAO_GetAll_OwnershipFilters(t *testing.T) { + ctx := context.Background() + dbSession := testOperatingSystemInitDB(t) + defer dbSession.Close() + testOperatingSystemSetupSchema(t, dbSession) + + ip := testOperatingSystemBuildInfrastructureProvider(t, dbSession, "testIP") + tenant := testOperatingSystemBuildTenant(t, dbSession, "testTenant") + user := testOperatingSystemBuildUser(t, dbSession, "testUser") + siteX := TestBuildSite(t, dbSession, ip, "siteX", user) + siteY := TestBuildSite(t, dbSession, ip, "siteY", user) + otherIP := testOperatingSystemBuildInfrastructureProvider(t, dbSession, "otherIP") + siteZ := TestBuildSite(t, dbSession, otherIP, "siteZ", user) + + ossd := NewOperatingSystemDAO(dbSession) + ossaDAO := NewOperatingSystemSiteAssociationDAO(dbSession) + dummyUUID := uuid.New() + + buildOS := func(name string, providerID, tenantID *uuid.UUID) *OperatingSystem { + os, err := ossd.Create(ctx, nil, OperatingSystemCreateInput{ + Name: name, + Description: cutil.GetPtr("description"), + Org: "testOrg", + InfrastructureProviderID: providerID, + TenantID: tenantID, + ControllerOperatingSystemID: &dummyUUID, + Version: cutil.GetPtr("version"), + OsType: OperatingSystemTypeIPXE, + ImageURL: cutil.GetPtr("iPXE"), + IpxeScript: cutil.GetPtr("ipxeScript"), + UserData: cutil.GetPtr("userData"), + AllowOverride: true, + EnableBlockStorage: true, + PhoneHomeEnabled: false, + Status: OperatingSystemStatusPending, + CreatedBy: user.ID, + }) + require.NoError(t, err) + require.NotNil(t, os) + return os + } + + associate := func(osID, siteID uuid.UUID) { + ossa, err := ossaDAO.Create(ctx, nil, OperatingSystemSiteAssociationCreateInput{ + OperatingSystemID: osID, + SiteID: siteID, + Status: OperatingSystemSiteAssociationStatusSyncing, + CreatedBy: user.ID, + }) + require.NoError(t, err) + require.NotNil(t, ossa) + } + + // Provider-owned OSes. + provAtX := buildOS("prov-at-x", &ip.ID, nil) + associate(provAtX.ID, siteX.ID) + provAtY := buildOS("prov-at-y", &ip.ID, nil) + associate(provAtY.ID, siteY.ID) + buildOS("prov-no-site", &ip.ID, nil) + provAtZ := buildOS("other-prov-at-z", &otherIP.ID, nil) + associate(provAtZ.ID, siteZ.ID) + + // Tenant-owned OSes. + buildOS("tenant-1", nil, &tenant.ID) + buildOS("tenant-2", nil, &tenant.ID) + + tests := []struct { + desc string + providerID *uuid.UUID + tenantIDs []uuid.UUID + expectedNames []string + }{ + { + desc: "provider-only view returns all provider OSes", + providerID: &ip.ID, + expectedNames: []string{"prov-at-x", "prov-at-y", "prov-no-site"}, + }, + { + desc: "tenant-only view returns only tenant OSes", + tenantIDs: []uuid.UUID{tenant.ID}, + expectedNames: []string{"tenant-1", "tenant-2"}, + }, + { + desc: "dual-role view returns tenant and provider OSes", + providerID: &ip.ID, + tenantIDs: []uuid.UUID{tenant.ID}, + expectedNames: []string{"prov-at-x", "prov-at-y", "prov-no-site", "tenant-1", "tenant-2"}, + }, + } + for _, tc := range tests { + t.Run(tc.desc, func(t *testing.T) { + filter := OperatingSystemFilterInput{ + InfrastructureProviderID: tc.providerID, + TenantIDs: tc.tenantIDs, + } + page := paginator.PageInput{Limit: cutil.GetPtr(paginator.TotalLimit)} + got, total, err := ossd.GetAll(ctx, nil, filter, page, nil) + require.NoError(t, err) + gotNames := make([]string, len(got)) + for i, os := range got { + gotNames[i] = os.Name + } + assert.ElementsMatch(t, tc.expectedNames, gotNames) + assert.Equal(t, len(tc.expectedNames), total) + }) + } + +} diff --git a/rest-api/db/pkg/db/model/sku.go b/rest-api/db/pkg/db/model/sku.go index 5ff5226177..976f363f81 100644 --- a/rest-api/db/pkg/db/model/sku.go +++ b/rest-api/db/pkg/db/model/sku.go @@ -80,29 +80,26 @@ type SKU struct { ID string `bun:"id,pk"` SiteID uuid.UUID `bun:"site_id,type:uuid,notnull"` Site *Site `bun:"rel:belongs-to,join:site_id=id"` - DeviceType *string `bun:"device_type"` // NOTE: can be added once available in nico.proto + Description string `bun:"description,notnull,default:''"` + SchemaVersion uint32 `bun:"schema_version,notnull,default:0"` + DeviceType *string `bun:"device_type"` Components *SkuComponents `bun:"components,type:jsonb"` AssociatedMachineIds []string `bun:"associated_machines,type:text[],default:'{}'"` Created time.Time `bun:"created,nullzero,notnull,default:current_timestamp"` Updated time.Time `bun:"updated,nullzero,notnull,default:current_timestamp"` } -// ToProto converts this SKU into its workflow proto representation. -// Used as the canonical entity-to-proto conversion; SKU has no API -// Create/Update request shapes (the Site is the source of truth for -// SKU data, so the cloud API exposes read-only handlers), so this -// receiver is the only `ToProto` the model carries. +// ToProto converts the REST database projection into its Core SKU representation. // -// Fields that exist on the proto but not on the DB row -// (`Description`, the proto-level `Created` timestamp, `SchemaVersion`) -// are intentionally omitted — the DB does not carry the data to fill -// them, and no current caller depends on them. `SiteID` is on the -// model but not on the proto, so it is also dropped on the wire (the -// receiving side reconstructs it from context, mirroring `FromProto`). +// The Core-level Created timestamp is omitted because the REST projection does +// not store it. SiteID is also omitted because it is not carried by the Core SKU +// message; callers supply it separately to FromProto. func (sk *SKU) ToProto() *corev1.Sku { proto := &corev1.Sku{ - Id: sk.ID, - DeviceType: sk.DeviceType, + Id: sk.ID, + Description: &sk.Description, + SchemaVersion: sk.SchemaVersion, + DeviceType: sk.DeviceType, } if sk.Components != nil { proto.Components = sk.Components.SkuComponents @@ -117,13 +114,16 @@ func (sk *SKU) ToProto() *corev1.Sku { return proto } -// FromProto populates this SKU from a workflow proto reported by a Site. -// A nil proto is a no-op. This is the inverse of `ToProto`; `siteID` -// is supplied by the caller because it isn't carried on the proto. +// FromProto populates the REST database projection from a Core SKU returned by +// inventory synchronization or an immediate REST mutation. A nil proto is a +// no-op. This is the inverse of ToProto; siteID is supplied by the caller +// because it is not carried on the Core SKU message. // // Field-level contract: // - `sk.ID` is overwritten with `proto.Id` (callers pre-validate // non-empty IDs at the activity layer). +// - `Description` uses the protobuf default when absent, so a nil Core +// description clears any stale REST projection value. // - `Components` mirrors the proto: stays nil when `proto.Components` // is nil, otherwise wraps it, so the activity layer can distinguish // "not provided" from "explicitly set". @@ -137,6 +137,8 @@ func (sk *SKU) FromProto(proto *corev1.Sku, siteID uuid.UUID) { } sk.ID = proto.Id sk.SiteID = siteID + sk.Description = proto.GetDescription() + sk.SchemaVersion = proto.SchemaVersion sk.DeviceType = proto.DeviceType if proto.Components != nil { sk.Components = &SkuComponents{SkuComponents: proto.Components} @@ -158,8 +160,10 @@ func (sk *SKU) FromProto(proto *corev1.Sku, siteID uuid.UUID) { // SkuCreateInput input parameters for Create method type SkuCreateInput struct { - SkuID string // NICo is the source of truth: id must always be provided on creation. + SkuID string // Core is authoritative, so the ID must be provided when creating its REST projection. SiteID uuid.UUID + Description string + SchemaVersion uint32 Components *SkuComponents DeviceType *string AssociatedMachineIds []string @@ -168,6 +172,8 @@ type SkuCreateInput struct { // SkuUpdateInput input parameters for Update method type SkuUpdateInput struct { SkuID string + Description *string + SchemaVersion *uint32 Components *SkuComponents DeviceType *string AssociatedMachineIds []string @@ -236,6 +242,8 @@ func (ssd SkuSQLDAO) Create(ctx context.Context, tx *db.Tx, input SkuCreateInput sk := &SKU{ ID: input.SkuID, SiteID: input.SiteID, + Description: input.Description, + SchemaVersion: input.SchemaVersion, DeviceType: input.DeviceType, Components: input.Components, AssociatedMachineIds: input.AssociatedMachineIds, @@ -354,6 +362,16 @@ func (ssd SkuSQLDAO) Update(ctx context.Context, tx *db.Tx, input SkuUpdateInput sk := &SKU{ID: input.SkuID} updatedFields := []string{} + if input.Description != nil { + sk.Description = *input.Description + updatedFields = append(updatedFields, "description") + } + + if input.SchemaVersion != nil { + sk.SchemaVersion = *input.SchemaVersion + updatedFields = append(updatedFields, "schema_version") + } + if input.Components != nil { sk.Components = input.Components updatedFields = append(updatedFields, "components") diff --git a/rest-api/db/pkg/db/model/sku_test.go b/rest-api/db/pkg/db/model/sku_test.go index 74f4660e0c..6669f6a1f2 100644 --- a/rest-api/db/pkg/db/model/sku_test.go +++ b/rest-api/db/pkg/db/model/sku_test.go @@ -54,6 +54,8 @@ func TestSKU_ToProto(t *testing.T) { sk := &SKU{ ID: "sku-1", SiteID: siteID, + Description: "GPU compute SKU", + SchemaVersion: 5, DeviceType: &deviceType, Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}, AssociatedMachineIds: []string{"m-1", "m-2"}, @@ -61,6 +63,8 @@ func TestSKU_ToProto(t *testing.T) { proto := sk.ToProto() require.NotNil(t, proto) assert.Equal(t, "sku-1", proto.Id) + assert.Equal(t, "GPU compute SKU", proto.GetDescription()) + assert.Equal(t, uint32(5), proto.SchemaVersion) assert.Equal(t, &deviceType, proto.DeviceType) require.NotNil(t, proto.Components) require.Len(t, proto.AssociatedMachineIds, 2) @@ -97,9 +101,11 @@ func TestSKU_FromProto(t *testing.T) { t.Run("populates fields from proto", func(t *testing.T) { sk := &SKU{} sk.FromProto(&corev1.Sku{ - Id: "sku-1", - DeviceType: &deviceType, - Components: &corev1.SkuComponents{}, + Id: "sku-1", + Description: cutil.GetPtr("GPU compute SKU"), + SchemaVersion: 4, + DeviceType: &deviceType, + Components: &corev1.SkuComponents{}, AssociatedMachineIds: []*corev1.MachineId{ {Id: "m-1"}, {Id: ""}, // skipped @@ -108,11 +114,19 @@ func TestSKU_FromProto(t *testing.T) { }, siteID) assert.Equal(t, "sku-1", sk.ID) assert.Equal(t, siteID, sk.SiteID) + assert.Equal(t, "GPU compute SKU", sk.Description) + assert.Equal(t, uint32(4), sk.SchemaVersion) assert.Equal(t, &deviceType, sk.DeviceType) assert.Equal(t, []string{"m-1", "m-2"}, sk.AssociatedMachineIds) require.NotNil(t, sk.Components) }) + t.Run("nil description clears existing description", func(t *testing.T) { + sk := &SKU{Description: "stale description"} + sk.FromProto(&corev1.Sku{Id: "sku-1"}, siteID) + assert.Empty(t, sk.Description) + }) + t.Run("nil Components yields nil wrapper", func(t *testing.T) { sk := &SKU{Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}} sk.FromProto(&corev1.Sku{Id: "sku-1"}, siteID) @@ -153,7 +167,12 @@ func testSkuCreateSkus(ctx context.Context, t *testing.T, dbSession *db.Session, ids := []string{"sku-1", "sku-2", "sku-3"} for _, id := range ids { protoSku := &corev1.SkuComponents{} - sk, err := ssd.Create(ctx, nil, SkuCreateInput{SkuID: id, Components: &SkuComponents{SkuComponents: protoSku}, SiteID: siteId}) + sk, err := ssd.Create(ctx, nil, SkuCreateInput{ + SkuID: id, + SiteID: siteId, + Description: id + " description", + Components: &SkuComponents{SkuComponents: protoSku}, + }) require.NoError(t, err) require.NotNil(t, sk) created = append(created, *sk) @@ -185,13 +204,13 @@ func TestSkuSQLDAO_Create(t *testing.T) { }{ { desc: "create one", - inputs: []SkuCreateInput{{SkuID: "sku-1", Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}, SiteID: site.ID}}, + inputs: []SkuCreateInput{{SkuID: "sku-1", SiteID: site.ID, Description: "first SKU", SchemaVersion: 5, Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}}}, expectError: false, verifyChildSpanner: true, }, { desc: "create multiple", - inputs: []SkuCreateInput{{SkuID: "sku-2", Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}, SiteID: site.ID}, {SkuID: "sku-3", Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}, SiteID: site.ID}}, + inputs: []SkuCreateInput{{SkuID: "sku-2", SiteID: site.ID, Description: "second SKU", Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}}, {SkuID: "sku-3", SiteID: site.ID, Description: "third SKU", Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}}}, expectError: false, }, } @@ -203,6 +222,8 @@ func TestSkuSQLDAO_Create(t *testing.T) { if !tc.expectError { assert.NotNil(t, got) assert.Equal(t, input.SkuID, got.ID) + assert.Equal(t, input.SchemaVersion, got.SchemaVersion) + assert.Equal(t, input.Description, got.Description) if input.Components != nil { assert.NotNil(t, got.Components) } @@ -333,19 +354,51 @@ func TestSkuSQLDAO_Update(t *testing.T) { _, _, ctx = testCommonTraceProviderSetup(t, ctx) tests := []struct { - desc string - input SkuUpdateInput - check bool + desc string + input SkuUpdateInput + expectedDescription string + expectedVersion uint32 }{ - {desc: "update sku data", input: SkuUpdateInput{SkuID: created[0].ID, Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}}, check: true}, + { + desc: "update sku data", + input: SkuUpdateInput{ + SkuID: created[0].ID, + Description: cutil.GetPtr("updated description"), + SchemaVersion: cutil.GetPtr(uint32(5)), + Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}, + }, + expectedDescription: "updated description", + expectedVersion: 5, + }, + { + desc: "nil description preserves stored description", + input: SkuUpdateInput{ + SkuID: created[1].ID, + Description: nil, + Components: &SkuComponents{SkuComponents: &corev1.SkuComponents{}}, + }, + expectedDescription: created[1].Description, + expectedVersion: created[1].SchemaVersion, + }, + { + desc: "empty description clears stored description", + input: SkuUpdateInput{ + SkuID: created[2].ID, + Description: cutil.GetPtr(""), + }, + expectedDescription: "", + expectedVersion: created[2].SchemaVersion, + }, } for _, tc := range tests { t.Run(tc.desc, func(t *testing.T) { got, err := ssd.Update(ctx, nil, tc.input) require.NoError(t, err) - if tc.check { - assert.NotNil(t, got) - assert.Equal(t, tc.input.SkuID, got.ID) + require.NotNil(t, got) + assert.Equal(t, tc.input.SkuID, got.ID) + assert.Equal(t, tc.expectedDescription, got.Description) + assert.Equal(t, tc.expectedVersion, got.SchemaVersion) + if tc.input.Components != nil { assert.NotNil(t, got.Components) } // tracer diff --git a/rest-api/db/pkg/migrations/20260729020000_sku_description.go b/rest-api/db/pkg/migrations/20260729020000_sku_description.go new file mode 100644 index 0000000000..7b4dd21022 --- /dev/null +++ b/rest-api/db/pkg/migrations/20260729020000_sku_description.go @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package migrations + +import ( + "context" + "database/sql" + "fmt" + + "github.com/NVIDIA/infra-controller/rest-api/db/pkg/db/model" + "github.com/uptrace/bun" +) + +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + tx, terr := db.BeginTx(ctx, &sql.TxOptions{}) + if terr != nil { + handlePanic(terr, "failed to begin transaction") + } + + _, err := tx.NewAddColumn(). + Model((*model.SKU)(nil)). + IfNotExists(). + ColumnExpr("description TEXT NOT NULL DEFAULT ''"). + Exec(ctx) + handleError(tx, err) + + _, err = tx.NewAddColumn(). + Model((*model.SKU)(nil)). + IfNotExists(). + ColumnExpr("schema_version INTEGER NOT NULL DEFAULT 0"). + Exec(ctx) + handleError(tx, err) + + terr = tx.Commit() + if terr != nil { + handlePanic(terr, "failed to commit transaction") + } + + fmt.Print(" [up migration] Added 'description' and 'schema_version' columns to 'sku' table successfully. ") + return nil + }, func(ctx context.Context, db *bun.DB) error { + _, err := db.ExecContext(ctx, `ALTER TABLE sku DROP COLUMN IF EXISTS description, DROP COLUMN IF EXISTS schema_version`) + if err != nil { + return err + } + fmt.Print(" [down migration] Dropped 'description' and 'schema_version' columns from 'sku' table successfully. ") + return nil + }) +} diff --git a/rest-api/docs/index.html b/rest-api/docs/index.html index 93c3c9b032..0881217b7d 100644 --- a/rest-api/docs/index.html +++ b/rest-api/docs/index.html @@ -464,7 +464,7 @@ -
API docs by Redocly
API docs by Redocly
Typical API Call Flow for Tenant
https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteId}/status-history

Response samples

Content type
application/json
[
  • {
    }
]

BMC Credential

BMC Credential endpoints allow creating and updating BMC credentials across all Machines of a Site

Create Or Update BMC Credential

Create or update a site-wide or per-BMC root credential. Equivalent -to carbide-admin-cli credential add-bmc.

+to nico-admin-cli credential add-bmc.

User must have authorization role with PROVIDER_ADMIN suffix.

Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

@@ -2074,13 +2074,23 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Optional username; Core defaults to "root" for BMCRoot when omitted.

macAddress
string

BMC MAC address. Required for kind BMCRoot, ignored for SiteWideRoot.

-

Responses

Responses

Response Schema: application/json
siteId
required
string <uuid>

ID of the Site where the credential is stored.

+
kind
required
string
Enum: "SiteWideRoot" "BMCRoot"

Which BMC credential was stored.

+
username
string

Optional username; Core defaults to "root" for BMCRoot when omitted.

+
macAddress
string

BMC MAC address. Required for kind BMCRoot, ignored for SiteWideRoot.

+

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "SiteWideRoot",
  • "password": "string",
  • "username": "string",
  • "macAddress": "string"
}

Response samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "SiteWideRoot",
  • "username": "string",
  • "macAddress": "string"
}

UEFI Credential

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/credential/bmc

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "SiteWideRoot",
  • "password": "string",
  • "username": "string",
  • "macAddress": "string"
}

Response samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "SiteWideRoot",
  • "username": "string",
  • "macAddress": "string"
}

UEFI Credential

UEFI Credential endpoints allow creating site-default host and DPU UEFI credentials for a Site

Create UEFI Credential

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

The selected site-default UEFI credential already exists

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "Host",
  • "password": "string"
}

Response samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "Host"
}

Credential Rotation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/credential/uefi

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "Host",
  • "password": "string"
}

Response samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "kind": "Host"
}

Measured Boot Trusted Machine

Measured Boot Trusted Machine endpoints manage automatic measurement promotion approvals for Machines at a Site

+

Create Measured Boot Trusted Machine Approval

Approve a Machine, or all Machines using *, for automatic promotion of its next measured-boot report.

+

Org must have an Infrastructure Provider entity that owns the Site. User must have authorization role with PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
Request Body schema: application/json
required
siteId
required
string <uuid>

ID of the Site where the approval applies.

+
machineId
required
string

Machine UUID, or * to approve all Machines at the Site.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
string

Optional comma-separated PCR register selector. All registers are used when omitted.

+
comments
string

Optional operator comments about the approval.

+

Responses

Response Schema: application/json
approvalId
required
string <uuid>

Unique approval ID.

+
machineId
required
string

Machine UUID, or * when the approval applies to all Machines.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
required
string

Optional comma-separated PCR register selector.

+
comments
required
string

Optional operator comments about the approval.

+
created
required
string or null <date-time>

Time when the approval was created, or null when Core did not return a timestamp.

+

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "machineId": "string",
  • "approvalType": "Oneshot",
  • "pcrRegisters": "string",
  • "comments": "string"
}

Response samples

Content type
application/json
{
  • "approvalId": "23bbe807-dea1-4601-b208-07fd1aaad2b6",
  • "machineId": "string",
  • "approvalType": "Oneshot",
  • "pcrRegisters": "string",
  • "comments": "string",
  • "created": "2019-08-24T14:15:22Z"
}

Retrieve All Measured Boot Trusted Machine Approvals

Retrieve all measured-boot trusted Machine approvals for a Site.

+

Org must have an Infrastructure Provider entity that owns the Site. User must have authorization role with PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
query Parameters
siteId
required
string <uuid>

ID of the Site

+

Responses

Response Schema: application/json
Array
approvalId
required
string <uuid>

Unique approval ID.

+
machineId
required
string

Machine UUID, or * when the approval applies to all Machines.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
required
string

Optional comma-separated PCR register selector.

+
comments
required
string

Optional operator comments about the approval.

+
created
required
string or null <date-time>

Time when the approval was created, or null when Core did not return a timestamp.

+

Response samples

Content type
application/json
[
  • {
    }
]

Delete Measured Boot Trusted Machine Approval

Delete a measured-boot trusted Machine approval by approval ID or Machine ID.

+

Org must have an Infrastructure Provider entity that owns the Site. User must have authorization role with PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
id
required
string

Approval ID or Machine ID, as selected by the selector query parameter

+
query Parameters
siteId
required
string <uuid>

ID of the Site

+
selector
required
string
Enum: "ApprovalId" "MachineId"

Whether id identifies the approval or the Machine

+

Responses

Response Schema: application/json
approvalId
required
string <uuid>

Unique approval ID.

+
machineId
required
string

Machine UUID, or * when the approval applies to all Machines.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
required
string

Optional comma-separated PCR register selector.

+
comments
required
string

Optional operator comments about the approval.

+
created
required
string or null <date-time>

Time when the approval was created, or null when Core did not return a timestamp.

+

Response samples

Content type
application/json
{
  • "approvalId": "23bbe807-dea1-4601-b208-07fd1aaad2b6",
  • "machineId": "string",
  • "approvalType": "Oneshot",
  • "pcrRegisters": "string",
  • "comments": "string",
  • "created": "2019-08-24T14:15:22Z"
}

Measured Boot Trusted Profile

Measured Boot Trusted Profile endpoints manage automatic measurement promotion approvals for system profiles at a Site

+

Create Measured Boot Trusted Profile Approval

Approve a measured-boot system profile for automatic promotion of reports from matching Machines.

+

Org must have an Infrastructure Provider entity that owns the Site. User must have authorization role with PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
Request Body schema: application/json
required
siteId
required
string <uuid>

ID of the Site where the approval applies.

+
profileId
required
string <uuid>

ID of the measured-boot system profile.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
string

Optional comma-separated PCR register selector. All registers are used when omitted.

+
comments
string

Optional operator comments about the approval.

+

Responses

Response Schema: application/json
approvalId
required
string <uuid>

Unique approval ID.

+
profileId
required
string <uuid>

ID of the measured-boot system profile.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
required
string

Optional comma-separated PCR register selector.

+
comments
required
string

Optional operator comments about the approval.

+
created
required
string or null <date-time>

Time when the approval was created, or null when Core did not return a timestamp.

+

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "profileId": "faebe71b-2bf8-4bdb-9b67-258e4d6aa00a",
  • "approvalType": "Oneshot",
  • "pcrRegisters": "string",
  • "comments": "string"
}

Response samples

Content type
application/json
{
  • "approvalId": "23bbe807-dea1-4601-b208-07fd1aaad2b6",
  • "profileId": "faebe71b-2bf8-4bdb-9b67-258e4d6aa00a",
  • "approvalType": "Oneshot",
  • "pcrRegisters": "string",
  • "comments": "string",
  • "created": "2019-08-24T14:15:22Z"
}

Retrieve All Measured Boot Trusted Profile Approvals

Retrieve all measured-boot trusted Profile approvals for a Site.

+

Org must have an Infrastructure Provider entity that owns the Site. User must have authorization role with PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
query Parameters
siteId
required
string <uuid>

ID of the Site

+

Responses

Response Schema: application/json
Array
approvalId
required
string <uuid>

Unique approval ID.

+
profileId
required
string <uuid>

ID of the measured-boot system profile.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
required
string

Optional comma-separated PCR register selector.

+
comments
required
string

Optional operator comments about the approval.

+
created
required
string or null <date-time>

Time when the approval was created, or null when Core did not return a timestamp.

+

Response samples

Content type
application/json
[
  • {
    }
]

Delete Measured Boot Trusted Profile Approval

Delete a measured-boot trusted Profile approval by approval ID or Profile ID.

+

Org must have an Infrastructure Provider entity that owns the Site. User must have authorization role with PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
id
required
string <uuid>

Approval ID or Profile ID, as selected by the selector query parameter

+
query Parameters
siteId
required
string <uuid>

ID of the Site

+
selector
required
string
Enum: "ApprovalId" "ProfileId"

Whether id identifies the approval or the system profile

+

Responses

Response Schema: application/json
approvalId
required
string <uuid>

Unique approval ID.

+
profileId
required
string <uuid>

ID of the measured-boot system profile.

+
approvalType
required
string
Enum: "Oneshot" "Persist"

Whether the approval is consumed once or persists for future reports.

+
pcrRegisters
required
string

Optional comma-separated PCR register selector.

+
comments
required
string

Optional operator comments about the approval.

+
created
required
string or null <date-time>

Time when the approval was created, or null when Core did not return a timestamp.

+

Response samples

Content type
application/json
{
  • "approvalId": "23bbe807-dea1-4601-b208-07fd1aaad2b6",
  • "profileId": "faebe71b-2bf8-4bdb-9b67-258e4d6aa00a",
  • "approvalType": "Oneshot",
  • "pcrRegisters": "string",
  • "comments": "string",
  • "created": "2019-08-24T14:15:22Z"
}

Credential Rotation

Credential Rotation endpoints stage a site-wide credential rotation and report per-Site or per-device convergence

Rotate Credential

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "credentialType": "BMC",
  • "password": "string",
  • "reason": "string"
}

Response samples

Content type
application/json
{
  • "credentialType": "BMC",
  • "targetVersion": 0,
  • "started": "2019-08-24T14:15:22Z"
}

Get Credential Rotation Status

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/credential/rotation

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "credentialType": "BMC",
  • "password": "string",
  • "reason": "string"
}

Response samples

Content type
application/json
{
  • "credentialType": "BMC",
  • "targetVersion": 0,
  • "started": "2019-08-24T14:15:22Z"
}

Get Credential Rotation Status

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

No rotation record exists for the requested device

Response samples

Content type
application/json
{
  • "targetVersion": 0,
  • "converged": 0,
  • "pending": 0,
  • "quarantined": 0,
  • "quarantinedDeviceMacs": [
    ],
  • "started": "2019-08-24T14:15:22Z",
  • "complete": true,
  • "device": {
    }
}

Site Explorer

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/credential/rotation

Response samples

Content type
application/json
{
  • "targetVersion": 0,
  • "converged": 0,
  • "pending": 0,
  • "quarantined": 0,
  • "quarantinedDeviceMacs": [
    ],
  • "started": "2019-08-24T14:15:22Z",
  • "complete": true,
  • "device": {
    }
}

Site Explorer

Site Explorer discovers BMC endpoints on a Site and tracks their exploration state.

Trigger Site Explorer Endpoint Action

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Response when the API handler encounters an unexpected error

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "action": "ClearError",
  • "target": "All",
  • "endpointIds": [
    ]
}

Response samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "action": "ClearError",
  • "target": "All",
  • "endpointIds": [
    ]
}

Allocation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site-explorer/endpoint/action

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "action": "ClearError",
  • "target": "All",
  • "endpointIds": [
    ]
}

Response samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "action": "ClearError",
  • "target": "All",
  • "endpointIds": [
    ]
}

Allocation

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create Allocation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/allocation

Response samples

Content type
application/json
[
  • {
    }
]

Create Allocation

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "Echo Studios",
  • "description": "Echo Studios resource allocation in SJC4",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "allocationConstraints": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Echo Studios",
  • "description": "Echo Studios resource allocation in SJC4",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "allocationConstraints": [
    ]
}

Retrieve Allocation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/allocation

Request samples

Content type
application/json
{
  • "name": "Echo Studios",
  • "description": "Echo Studios resource allocation in SJC4",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "allocationConstraints": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Echo Studios",
  • "description": "Echo Studios resource allocation in SJC4",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "allocationConstraints": [
    ]
}

Retrieve Allocation

Retrieve Allocation by ID

@@ -2528,7 +2758,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Echo Studios",
  • "description": "Echo Studios resource allocation in SJC4",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "allocationConstraints": [
    ]
}

Delete Allocation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/allocation/{allocationId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Echo Studios",
  • "description": "Echo Studios resource allocation in SJC4",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "allocationConstraints": [
    ]
}

Delete Allocation

Delete an Allocation by ID.

@@ -2544,7 +2774,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Allocation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/allocation/{allocationId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Allocation

Update an existing Allocation

@@ -2620,7 +2850,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "Echo Studios Compute",
  • "description": "Echo Studios compute resource allocation in SJC4"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Echo Studios Compute",
  • "description": "Echo Studios compute resource allocation in SJC4",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "allocationConstraints": [
    ]
}

Update Allocation Constraint

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/allocation/{allocationId}

Request samples

Content type
application/json
{
  • "name": "Echo Studios Compute",
  • "description": "Echo Studios compute resource allocation in SJC4"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Echo Studios Compute",
  • "description": "Echo Studios compute resource allocation in SJC4",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "allocationConstraints": [
    ]
}

Update Allocation Constraint

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "constraintValue": 20
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "allocationId": "9ec871ce-3363-4de2-8f79-062881067628",
  • "resourceType": "InstanceType",
  • "resourceTypeId": "a59ee688-b5e5-4606-9891-f4a605edacd3",
  • "constraintType": "Reserved",
  • "constraintValue": 20,
  • "derivedResourceId": null,
  • "instanceType": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

VPC

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/allocation/{allocationId}/constraint/{allocationConstraintId}

Request samples

Content type
application/json
{
  • "constraintValue": 20
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "allocationId": "9ec871ce-3363-4de2-8f79-062881067628",
  • "resourceType": "InstanceType",
  • "resourceTypeId": "a59ee688-b5e5-4606-9891-f4a605edacd3",
  • "constraintType": "Reserved",
  • "constraintValue": 20,
  • "derivedResourceId": null,
  • "instanceType": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

VPC

VPC defines the networking isolation boundary for Tenant's Instances.

Retrieve all VPCs

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create VPC

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc

Response samples

Content type
application/json
[
  • {
    }
]

Create VPC

Create a VPC for the org.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix

@@ -2904,7 +3134,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc",
  • "description": "Virtual network for machines executing Spark jobs",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc",
  • "description": "Virtual network for machines executing Spark jobs",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve a VPC

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc

Request samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc",
  • "description": "Virtual network for machines executing Spark jobs",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc",
  • "description": "Virtual network for machines executing Spark jobs",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve a VPC

Retrieve a specific VPC by ID.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -2992,7 +3222,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc",
  • "description": "Virtual network for machines executing Spark jobs",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Ready",
  • "networkSecurityGroupId": "c602eb90-3039-11f0-997a-b38d4fc8389e",
  • "networkSecurityGroupPropagationDetails": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete a VPC

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc/{vpcId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc",
  • "description": "Virtual network for machines executing Spark jobs",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Ready",
  • "networkSecurityGroupId": "c602eb90-3039-11f0-997a-b38d4fc8389e",
  • "networkSecurityGroupPropagationDetails": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete a VPC

Delete a specific VPC by ID.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -3006,7 +3236,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update VPC

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc/{vpcId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update VPC

Update an existing VPC

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix

@@ -3104,7 +3334,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "spark-vpc-v1",
  • "description": "Virtual network for machines executing Spark jobs v1",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc-v1",
  • "description": "Virtual network for machines executing Spark jobs v1",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update VPC Virtualization

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc/{vpcId}

Request samples

Content type
application/json
{
  • "name": "spark-vpc-v1",
  • "description": "Virtual network for machines executing Spark jobs v1",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc-v1",
  • "description": "Virtual network for machines executing Spark jobs v1",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "ETHERNET_VIRTUALIZER",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update VPC Virtualization

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "networkVirtualizationType": "FNN"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc-v1",
  • "description": "Virtual network for machines executing Spark jobs v1",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "FNN",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

VPC Peering

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc/{vpcId}/virtualization

Request samples

Content type
application/json
{
  • "networkVirtualizationType": "FNN"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-vpc-v1",
  • "description": "Virtual network for machines executing Spark jobs v1",
  • "org": "xskkpgqpeakn",
  • "tenantId": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "controllerVpcId": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "networkVirtualizationType": "FNN",
  • "requestedVni": 12001,
  • "vni": 12001,
  • "nvLinkLogicalPartitionId": "dd887330-dbd3-45ce-b400-c42fc8e47315",
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

VPC Peering

VPC Peering allows Instances in one VPC to communicate with Instances in another VPC on the same Site.

Retrieve all VPC peerings

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create VPC peering

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-peering

Response samples

Content type
application/json
[
  • {
    }
]

Create VPC peering

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "vpc1Id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "vpc2Id": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "vpc1Id": "bafbaf4c-c730-48ae-8f5d-d7d251f3315a",
  • "vpc1": {
    },
  • "vpc2Id": "a7cd678e-fe62-4184-9f75-66ac903ae1c3",
  • "vpc2": {
    },
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "site": {
    },
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "tenant": {
    },
  • "isMultiTenant": true,
  • "status": "Pending",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve a VPC peering

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-peering

Request samples

Content type
application/json
{
  • "vpc1Id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "vpc2Id": "34f5c98e-f430-457b-a812-92637d0c6fd0",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "vpc1Id": "bafbaf4c-c730-48ae-8f5d-d7d251f3315a",
  • "vpc1": {
    },
  • "vpc2Id": "a7cd678e-fe62-4184-9f75-66ac903ae1c3",
  • "vpc2": {
    },
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "site": {
    },
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "tenant": {
    },
  • "isMultiTenant": true,
  • "status": "Pending",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve a VPC peering

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "vpc1Id": "bafbaf4c-c730-48ae-8f5d-d7d251f3315a",
  • "vpc1": {
    },
  • "vpc2Id": "a7cd678e-fe62-4184-9f75-66ac903ae1c3",
  • "vpc2": {
    },
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "site": {
    },
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "tenant": {
    },
  • "isMultiTenant": true,
  • "status": "Pending",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete a VPC peering

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-peering/{id}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "vpc1Id": "bafbaf4c-c730-48ae-8f5d-d7d251f3315a",
  • "vpc1": {
    },
  • "vpc2Id": "a7cd678e-fe62-4184-9f75-66ac903ae1c3",
  • "vpc2": {
    },
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "site": {
    },
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "tenant": {
    },
  • "isMultiTenant": true,
  • "status": "Pending",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete a VPC peering

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

VPC Prefix

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-peering/{id}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

VPC Prefix

VPC Prefix is a network prefix belonging to an IP Block allocated to a Tenant. Tenant can use VPC Prefixes to enable network connectivity between their Instances.

Only Sites that support Native Networking (FNN) offer VPC Prefix management.

@@ -3676,7 +3906,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create VPC Prefix

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-prefix

Response samples

Content type
application/json
[
  • {
    }
]

Create VPC Prefix

Create a VPC Prefix for the org.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -3740,7 +3970,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "east-vpc-traffic-net",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 20
}

Response samples

Content type
application/json
{
  • "id": "0c03ba01-d86b-4a57-a41e-cc359b380a6f",
  • "name": "east-vpc-traffic-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "prefix": "192.168.1.0/24",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 24,
  • "status": "Ready",
  • "usageStats": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve VPC Prefix

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-prefix

Request samples

Content type
application/json
{
  • "name": "east-vpc-traffic-net",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 20
}

Response samples

Content type
application/json
{
  • "id": "0c03ba01-d86b-4a57-a41e-cc359b380a6f",
  • "name": "east-vpc-traffic-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "prefix": "192.168.1.0/24",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 24,
  • "status": "Ready",
  • "usageStats": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve VPC Prefix

Retrieve a specific VPC Prefix

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -3802,7 +4032,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "0c03ba01-d86b-4a57-a41e-cc359b380a6f",
  • "name": "east-vpc-traffic-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "prefix": "192.168.1.0/24",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 24,
  • "status": "Ready",
  • "usageStats": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete VPC Prefix

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-prefix/{vpcPrefixId}

Response samples

Content type
application/json
{
  • "id": "0c03ba01-d86b-4a57-a41e-cc359b380a6f",
  • "name": "east-vpc-traffic-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "prefix": "192.168.1.0/24",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 24,
  • "status": "Ready",
  • "usageStats": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete VPC Prefix

Delete a specific VPC Prefix by ID.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -3816,7 +4046,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

Update VPC Prefix

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-prefix/{vpcPrefixId}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

Update VPC Prefix

Update an existing VPC Prefix

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -3876,7 +4106,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "east-vpc-traffic-net"
}

Response samples

Content type
application/json
{
  • "id": "0c03ba01-d86b-4a57-a41e-cc359b380a6f",
  • "name": "east-vpc-traffic-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "prefix": "192.168.1.0/24",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 24,
  • "status": "Ready",
  • "usageStats": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Subnet

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/vpc-prefix/{vpcPrefixId}

Request samples

Content type
application/json
{
  • "name": "east-vpc-traffic-net"
}

Response samples

Content type
application/json
{
  • "id": "0c03ba01-d86b-4a57-a41e-cc359b380a6f",
  • "name": "east-vpc-traffic-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "prefix": "192.168.1.0/24",
  • "ipBlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 24,
  • "status": "Ready",
  • "usageStats": {
    },
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Subnet

Subnet is a network prefix belonging to an IP Block allocated to a Tenant. Tenant can use Subnets to enable network connectivity between their Instances.

Subnets are used on Sites that do not support Native Networking (FNN).

@@ -3972,7 +4202,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create Subnet

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/subnet

Response samples

Content type
application/json
[
  • {
    }
]

Create Subnet

Create a Subnet for the org.

@@ -4058,7 +4288,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "spark-gpu-net",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 20
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-gpu-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "controllerNetworkSegmentId": null,
  • "ipv4Prefix": "202.168.16.0",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "ipv4Gateway": "202.168.0.1",
  • "ipv6Prefix": null,
  • "ipv6BlockId": null,
  • "ipv6Gateway": null,
  • "prefixLength": 20,
  • "routingType": "Public",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Subnet

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/subnet

Request samples

Content type
application/json
{
  • "name": "spark-gpu-net",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "prefixLength": 20
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-gpu-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "controllerNetworkSegmentId": null,
  • "ipv4Prefix": "202.168.16.0",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "ipv4Gateway": "202.168.0.1",
  • "ipv6Prefix": null,
  • "ipv6BlockId": null,
  • "ipv6Gateway": null,
  • "prefixLength": 20,
  • "routingType": "Public",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Subnet

Retrieve a specific Subnet

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -4136,7 +4366,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-gpu-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "controllerNetworkSegmentId": "abe7b0e8-67db-4e89-903e-fc4f2bd7f034",
  • "ipv4Prefix": "202.168.16.0",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "ipv4Gateway": "202.168.0.1",
  • "ipv6Prefix": null,
  • "ipv6BlockId": null,
  • "ipv6Gateway": null,
  • "prefixLength": 20,
  • "routingType": "Public",
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Subnet

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/subnet/{subnetId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-gpu-net",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "controllerNetworkSegmentId": "abe7b0e8-67db-4e89-903e-fc4f2bd7f034",
  • "ipv4Prefix": "202.168.16.0",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "ipv4Gateway": "202.168.0.1",
  • "ipv6Prefix": null,
  • "ipv6BlockId": null,
  • "ipv6Gateway": null,
  • "prefixLength": 20,
  • "routingType": "Public",
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Subnet

Delete a specific Subnet by ID.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -4150,7 +4380,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

Update Subnet

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/subnet/{subnetId}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

Update Subnet

Update an existing Subnet

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -4228,7 +4458,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "spark-gpu-subnet",
  • "description": "Subnet for dedicated GPU nodes"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-gpu-subnet",
  • "description": "Subnet for dedicated GPU nodes",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "controllerNetworkSegmentId": null,
  • "ipv4Prefix": "212.168.0.250",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "ipv4Gateway": "202.168.0.1",
  • "ipv6Prefix": null,
  • "ipv6BlockId": null,
  • "ipv6Gateway": null,
  • "prefixLength": 20,
  • "routingType": "Public",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Expected Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/subnet/{subnetId}

Request samples

Content type
application/json
{
  • "name": "spark-gpu-subnet",
  • "description": "Subnet for dedicated GPU nodes"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-gpu-subnet",
  • "description": "Subnet for dedicated GPU nodes",
  • "siteId": "ea144def-d68f-44c3-9485-4b103fa2686f",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "controllerNetworkSegmentId": null,
  • "ipv4Prefix": "212.168.0.250",
  • "ipv4BlockId": "8c1d1a06-90a2-4863-8ee1-6029265b9f0a",
  • "ipv4Gateway": "202.168.0.1",
  • "ipv6Prefix": null,
  • "ipv6BlockId": null,
  • "ipv6Gateway": null,
  • "prefixLength": 20,
  • "routingType": "Public",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Expected Machine

Expected Machine identifies a Machine that is expected to be discovered at a Site. Infrastructure Providers can pre-register Expected Machines using BMC credentials and serial numbers to help with Machine discovery and ingestion.

@@ -4302,6 +4532,10 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "defaultBmcUsername": "admin",
  • "defaultBmcPassword": "password123",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "skuId": "lenovo.sr650v2.cpu.1",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "rackId": "rack-01",
  • "isDpfEnabled": true,
  • "manufacturer": "Lenovo",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Machines

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-machine

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "defaultBmcUsername": "admin",
  • "defaultBmcPassword": "password123",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "skuId": "lenovo.sr650v2.cpu.1",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "rackId": "rack-01",
  • "isDpfEnabled": true,
  • "manufacturer": "Lenovo",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Machines

Retrieve all Expected Machines.

@@ -4424,6 +4658,10 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve Expected Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-machine

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve Expected Machine

Retrieve a specific Expected Machine by ID.

@@ -4522,6 +4760,10 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "skuId": "lenovo.sr650v2.cpu.1",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "rackId": "rack-01",
  • "isDpfEnabled": true,
  • "manufacturer": "Lenovo",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-machine/{expectedMachineId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "skuId": "lenovo.sr650v2.cpu.1",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "rackId": "rack-01",
  • "isDpfEnabled": true,
  • "manufacturer": "Lenovo",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Machine

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "defaultBmcUsername": "newadmin",
  • "defaultBmcPassword": "newpassword123",
  • "chassisSerialNumber": "CHASSIS-54321",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "skuId": "lenovo.sr650v2.cpu.1",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "rackId": "rack-01",
  • "isDpfEnabled": true,
  • "manufacturer": "Lenovo",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-machine/{expectedMachineId}

Request samples

Content type
application/json
{
  • "defaultBmcUsername": "newadmin",
  • "defaultBmcPassword": "newpassword123",
  • "chassisSerialNumber": "CHASSIS-54321",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "chassisSerialNumber": "CHASSIS-12345",
  • "fallbackDPUSerialNumbers": [
    ],
  • "skuId": "lenovo.sr650v2.cpu.1",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "rackId": "rack-01",
  • "isDpfEnabled": true,
  • "manufacturer": "Lenovo",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Machine

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Batch Create Expected Machines

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-machine/{expectedMachineId}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Batch Create Expected Machines

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Response samples

Content type
application/json
[
  • {
    }
]

Batch Update Expected Machines

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-machine/batch

Request samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Response samples

Content type
application/json
[
  • {
    }
]

Batch Update Expected Machines

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

One or more Expected Machines not found

Request samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Response samples

Content type
application/json
[
  • {
    }
]

Expected Power Shelf

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-machine/batch

Request samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Response samples

Content type
application/json
[
  • {
    }
]

Expected Power Shelf

Expected Power Shelf identifies a Power Shelf that is expected to be discovered at a Site. Infrastructure Providers can pre-register Expected Power Shelves using BMC credentials and serial numbers to help with Power Shelf discovery and ingestion.

@@ -5160,7 +5414,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "defaultBmcUsername": "admin",
  • "defaultBmcPassword": "password123",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "rackId": "rack-01",
  • "manufacturer": "Delta",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Power Shelves

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-power-shelf

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "defaultBmcUsername": "admin",
  • "defaultBmcPassword": "password123",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "rackId": "rack-01",
  • "manufacturer": "Delta",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Power Shelves

Retrieve all Expected Power Shelves.

@@ -5220,7 +5474,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve Expected Power Shelf

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-power-shelf

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve Expected Power Shelf

Retrieve a specific Expected Power Shelf by ID.

@@ -5274,7 +5528,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "rackId": "rack-01",
  • "manufacturer": "Delta",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Power Shelf

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-power-shelf/{expectedPowerShelfId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "rackId": "rack-01",
  • "manufacturer": "Delta",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Power Shelf

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "defaultBmcUsername": "newadmin",
  • "defaultBmcPassword": "newpassword123",
  • "shelfSerialNumber": "SHELF-54321",
  • "bmcIpAddress": "192.168.1.200",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "rackId": "rack-01",
  • "manufacturer": "Delta",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Power Shelf

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-power-shelf/{expectedPowerShelfId}

Request samples

Content type
application/json
{
  • "defaultBmcUsername": "newadmin",
  • "defaultBmcPassword": "newpassword123",
  • "shelfSerialNumber": "SHELF-54321",
  • "bmcIpAddress": "192.168.1.200",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "shelfSerialNumber": "SHELF-12345",
  • "bmcIpAddress": "192.168.1.100",
  • "rackId": "rack-01",
  • "manufacturer": "Delta",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Power Shelf

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Expected Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-power-shelf/{expectedPowerShelfId}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Expected Rack

Expected Rack identifies a Rack that is expected to be discovered at a Site. Infrastructure Providers can pre-register Expected Racks with an operator-supplied @@ -5446,7 +5700,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackId": "rack-01",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "rackId": "rack-01",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Racks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-rack

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackId": "rack-01",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "rackId": "rack-01",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Racks

Retrieve all Expected Racks.

@@ -5492,7 +5746,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Replace all Expected Racks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-rack

Response samples

Content type
application/json
[
  • {
    }
]

Replace all Expected Racks

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "expectedRacks": [
    ]
}

Response samples

Content type
application/json
[
  • {
    }
]

Delete all Expected Racks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-rack

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "expectedRacks": [
    ]
}

Response samples

Content type
application/json
[
  • {
    }
]

Delete all Expected Racks

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Retrieve Expected Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-rack/all

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Retrieve Expected Rack

Retrieve a specific Expected Rack by its id.

@@ -5606,7 +5860,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "rackId": "rack-01",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-rack/{id}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "rackId": "rack-01",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Rack

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "rackProfileId": "rp-standard-48u",
  • "name": "Rack 01 (updated)",
  • "description": "Production rack in row A, upgraded chassis",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "rackId": "rack-01",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-rack/{id}

Request samples

Content type
application/json
{
  • "rackProfileId": "rp-standard-48u",
  • "name": "Rack 01 (updated)",
  • "description": "Production rack in row A, upgraded chassis",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "rackId": "rack-01",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "rackProfileId": "rp-standard-42u",
  • "name": "Rack 01",
  • "description": "Production rack in row A",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Rack

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Expected Switch

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-rack/{id}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Expected Switch

Expected Switch identifies an NVLink Switch that is expected to be discovered at a Site. Infrastructure Providers can pre-register Expected Switches using BMC, NVOS credentials and serial numbers to help with NVLink Switch discovery and ingestion.

@@ -5774,7 +6028,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "defaultBmcUsername": "admin",
  • "defaultBmcPassword": "password123",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvOsUsername": "nvadmin",
  • "nvOsPassword": "nvpassword123",
  • "nvosMacAddresses": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvosMacAddresses": [
    ],
  • "rackId": "rack-01",
  • "manufacturer": "NVIDIA",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Switches

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-switch

Request samples

Content type
application/json
{
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "defaultBmcUsername": "admin",
  • "defaultBmcPassword": "password123",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvOsUsername": "nvadmin",
  • "nvOsPassword": "nvpassword123",
  • "nvosMacAddresses": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvosMacAddresses": [
    ],
  • "rackId": "rack-01",
  • "manufacturer": "NVIDIA",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Expected Switches

Retrieve all Expected Switches.

@@ -5836,7 +6090,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve Expected Switch

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-switch

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve Expected Switch

Retrieve a specific Expected Switch by ID.

@@ -5892,7 +6146,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvosMacAddresses": [
    ],
  • "rackId": "rack-01",
  • "manufacturer": "NVIDIA",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Switch

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-switch/{expectedSwitchId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvosMacAddresses": [
    ],
  • "rackId": "rack-01",
  • "manufacturer": "NVIDIA",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Expected Switch

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "defaultBmcUsername": "newadmin",
  • "defaultBmcPassword": "newpassword123",
  • "switchSerialNumber": "SWITCH-54321",
  • "nvOsUsername": "newnvadmin",
  • "nvOsPassword": "newnvpassword123",
  • "nvosMacAddresses": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvosMacAddresses": [
    ],
  • "rackId": "rack-01",
  • "manufacturer": "NVIDIA",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Switch

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-switch/{expectedSwitchId}

Request samples

Content type
application/json
{
  • "defaultBmcUsername": "newadmin",
  • "defaultBmcPassword": "newpassword123",
  • "switchSerialNumber": "SWITCH-54321",
  • "nvOsUsername": "newnvadmin",
  • "nvOsPassword": "newnvpassword123",
  • "nvosMacAddresses": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "siteId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "bmcMacAddress": "00:1A:2B:3C:4D:5E",
  • "bmcIpAddress": "192.168.1.100",
  • "switchSerialNumber": "SWITCH-12345",
  • "nvosMacAddresses": [
    ],
  • "rackId": "rack-01",
  • "manufacturer": "NVIDIA",
  • "labels": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Expected Switch

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

SKU

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/expected-switch/{expectedSwitchId}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

SKU

SKU (Stock Keeping Unit) defines one or more hardware configurations or Machine Bill of Materials (BOM).

-

SKUs are automatically derived from machine hardware characteristics and used to group similar machines. SKUs are read-only and managed by the system.

+

SKUs group similar machines by their hardware characteristics. Most SKUs are managed by the system, but Provider Admins can create, update, and delete SKUs for a Site.

Retrieve all SKUs

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve SKU

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sku

Response samples

Content type
application/json
[
  • {
    }
]

Create SKU

Create a SKU.

+

The Org must have an Infrastructure Provider that owns the Site. The user must have an authorization role with the PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
Request Body schema: application/json
required
siteId
required
string <uuid>

ID of the Site where the SKU will be created

+
id
required
string non-empty

Unique SKU identifier

+
description
string

Human-readable SKU description

+
deviceType
string or null

Optional device type identifier

+
required
object (SkuComponents)

Hardware components that make up a SKU

+
Array of objects (SkuCpu)

CPU components

+
Array
vendor
string

Vendor of the CPU

+
model
string

Model of the CPU

+
threadCount
integer <uint32>

Number of threads for the CPU

+
count
integer <uint32>

Number of CPUs present

+
Array of objects (SkuGpu)

GPU components

+
Array
vendor
string

Vendor of the GPU

+
model
string

Model of the GPU

+
totalMemory
string

Total memory of the GPU (e.g. "80GB HBM3")

+
count
integer <uint32>

Number of GPUs present

+
Array of objects (SkuMemory)

Memory components

+
Array
capacityMb
integer <uint32>

Capacity in megabytes

+
memoryType
string

Type of memory (e.g. "DDR4", "DDR5")

+
count
integer <uint32>

Number of memory modules present

+
Array of objects (SkuStorage)

Storage components

+
Array
model
string

Informational storage model. Starting with the 2.1 release, NICo does not use this field for storage matching or validation.

+
count
integer <uint32>

Number of storage devices present

+
minSizeMiB
integer or null <uint32>

Inclusive minimum size in MiB for each storage device. Null or omission means no lower bound. Used for SKU schema version 5 and later.

+
maxSizeMiB
integer or null <uint32>

Inclusive maximum size in MiB for each storage device. Null or omission means no upper bound. Used for SKU schema version 5 and later.

+
pciPatterns
Array of strings

Regular expressions matched against storage sysfs PCI paths. An empty or omitted list disables PCI location matching. Used for SKU schema version 5 and later.

+
object (SkuChassis)

Chassis component

+
vendor
string

Vendor of the chassis

+
model
string

Model of the chassis

+
architecture
string

Architecture of the chassis

+
Array of objects (SkuEthernetDevice)

Ethernet device components

+
Array
vendor
string

Vendor of the ethernet device

+
model
string

Model of the ethernet device

+
count
integer <uint32>

Number of ethernet devices present

+
isConnected
boolean

Whether the ethernet device is connected

+
Array of objects (SkuInfinibandDevice)

Infiniband device components

+
Array
vendor
string

Vendor of the infiniband device

+
model
string

Model of the infiniband device

+
count
integer <uint32>

Number of infiniband devices present

+
inactiveDevices
Array of integers <uint32> [ items <uint32 > ]

Zero-based indexes of inactive devices

+
Array of objects (SkuTpm)

TPM components

+
Array
vendor
string

Vendor of the TPM

+
version
string

Version of the TPM

+

Responses

Response Schema: application/json
id
string

Unique identifier for the SKU

+
siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

+
deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

+
associatedMachineIds
Array of strings

List of machine IDs associated with this SKU

+
object (SkuComponents)

Hardware components of this SKU

+
Array of objects (SkuCpu)

CPU components

+
Array
vendor
string

Vendor of the CPU

+
model
string

Model of the CPU

+
threadCount
integer <uint32>

Number of threads for the CPU

+
count
integer <uint32>

Number of CPUs present

+
Array of objects (SkuGpu)

GPU components

+
Array
vendor
string

Vendor of the GPU

+
model
string

Model of the GPU

+
totalMemory
string

Total memory of the GPU (e.g. "80GB HBM3")

+
count
integer <uint32>

Number of GPUs present

+
Array of objects (SkuMemory)

Memory components

+
Array
capacityMb
integer <uint32>

Capacity in megabytes

+
memoryType
string

Type of memory (e.g. "DDR4", "DDR5")

+
count
integer <uint32>

Number of memory modules present

+
Array of objects (SkuStorage)

Storage components

+
Array
vendor
string

Storage vendor used for schema version 4 matching. Read-only in REST mutation requests and preserved in responses for legacy SKUs. Schema version 5 does not use this field.

+
model
string

Informational storage model. Starting with the 2.1 release, NICo does not use this field for storage matching or validation.

+
capacityMb
integer <uint32>

Storage capacity in megabytes used for schema version 4 matching. Read-only in REST mutation requests and preserved in responses for legacy SKUs. Schema version 5 uses minSizeMiB and maxSizeMiB instead.

+
count
integer <uint32>

Number of storage devices present

+
minSizeMiB
integer or null <uint32>

Inclusive minimum size in MiB for each storage device. Null or omission means no lower bound. Used for SKU schema version 5 and later.

+
maxSizeMiB
integer or null <uint32>

Inclusive maximum size in MiB for each storage device. Null or omission means no upper bound. Used for SKU schema version 5 and later.

+
pciPatterns
Array of strings

Regular expressions matched against storage sysfs PCI paths. An empty or omitted list disables PCI location matching. Used for SKU schema version 5 and later.

+
object (SkuChassis)

Chassis component

+
vendor
string

Vendor of the chassis

+
model
string

Model of the chassis

+
architecture
string

Architecture of the chassis

+
Array of objects (SkuEthernetDevice)

Ethernet device components

+
Array
vendor
string

Vendor of the ethernet device

+
model
string

Model of the ethernet device

+
count
integer <uint32>

Number of ethernet devices present

+
isConnected
boolean

Whether the ethernet device is connected

+
Array of objects (SkuInfinibandDevice)

Infiniband device components

+
Array
vendor
string

Vendor of the infiniband device

+
model
string

Model of the infiniband device

+
count
integer <uint32>

Number of infiniband devices present

+
inactiveDevices
Array of integers <uint32> [ items <uint32 > ]

Zero-based indexes of inactive devices

+
Array of objects (SkuTpm)

TPM components

+
Array
vendor
string

Vendor of the TPM

+
version
string

Version of the TPM

+
created
string <date-time>

ISO 8601 datetime when the SKU was created

+
updated
string <date-time>

ISO 8601 datetime when the SKU was last updated

+

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "id": "dgx-h100",
  • "description": "DGX H100 compute node",
  • "deviceType": "gpu-server",
  • "components": {
    }
}

Response samples

Content type
application/json
{
  • "id": "lenovo.sr650v2.cpu.1",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "description": "Lenovo SR650 V2 compute node",
  • "schemaVersion": 5,
  • "deviceType": "gpu",
  • "associatedMachineIds": [
    ],
  • "components": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve SKU

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Unique identifier for the SKU

siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

associatedMachineIds
Array of strings
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Number of memory modules present

Array of objects (SkuStorage)

Storage components

-
Array
vendor
string

Vendor of the storage device

-
model
string

Model of the storage device

-
capacityMb
integer <uint32>

Capacity in megabytes

-
count
integer <uint32>
Array
vendor
string

Storage vendor used for schema version 4 matching. Read-only in REST mutation requests and preserved in responses for legacy SKUs. Schema version 5 does not use this field.

+
model
string

Informational storage model. Starting with the 2.1 release, NICo does not use this field for storage matching or validation.

+
capacityMb
integer <uint32>

Storage capacity in megabytes used for schema version 4 matching. Read-only in REST mutation requests and preserved in responses for legacy SKUs. Schema version 5 uses minSizeMiB and maxSizeMiB instead.

+
count
integer <uint32>

Number of storage devices present

+
minSizeMiB
integer or null <uint32>

Inclusive minimum size in MiB for each storage device. Null or omission means no lower bound. Used for SKU schema version 5 and later.

+
maxSizeMiB
integer or null <uint32>

Inclusive maximum size in MiB for each storage device. Null or omission means no upper bound. Used for SKU schema version 5 and later.

+
pciPatterns
Array of strings

Regular expressions matched against storage sysfs PCI paths. An empty or omitted list disables PCI location matching. Used for SKU schema version 5 and later.

object (SkuChassis)

Chassis component

vendor
string

Vendor of the chassis

-
model
string
model
string

Model of the chassis

+
architecture
string

Architecture of the chassis

Array of objects (SkuEthernetDevice)

Ethernet device components

Array
vendor
string

Vendor of the ethernet device

model
string

Model of the ethernet device

-
count
integer <uint32>
count
integer <uint32>

Number of ethernet devices present

+
isConnected
boolean

Whether the ethernet device is connected

Array of objects (SkuInfinibandDevice)

Infiniband device components

Array
vendor
string

Vendor of the infiniband device

model
string

Model of the infiniband device

-
count
integer <uint32>
count
integer <uint32>

Number of infiniband devices present

+
inactiveDevices
Array of integers <uint32> [ items <uint32 > ]

Zero-based indexes of inactive devices

Array of objects (SkuTpm)

TPM components

Array
vendor
string
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "lenovo.sr650v2.cpu.1",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "deviceType": "gpu",
  • "associatedMachineIds": [
    ],
  • "components": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

InfiniBand Partition

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sku/{skuId}

Response samples

Content type
application/json
{
  • "id": "lenovo.sr650v2.cpu.1",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "deviceType": "gpu",
  • "associatedMachineIds": [
    ],
  • "components": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete SKU

Delete an unused SKU. The Site is determined from the saved SKU record.

+

The Org must have an Infrastructure Provider that owns the SKU's Site. The user must have an authorization role with the PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
skuId
required
string

ID of the SKU

+

Responses

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Update SKU

Update selected mutable fields on a SKU. The Site is determined from the saved SKU record.

+

Fields omitted from the request retain their current values. If a user updates the components field, +the schema version will be updated to the current version (5).

+

The Org must have an Infrastructure Provider that owns the SKU's Site. The user must have an authorization role with the PROVIDER_ADMIN suffix.

+
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

+
skuId
required
string

ID of the SKU

+
Request Body schema: application/json
required
description
string

Replacement SKU description

+
deviceType
string

Replacement device type identifier

+
object (SkuComponents)

Hardware components that make up a SKU

+
Array of objects (SkuCpu)

CPU components

+
Array
vendor
string

Vendor of the CPU

+
model
string

Model of the CPU

+
threadCount
integer <uint32>

Number of threads for the CPU

+
count
integer <uint32>

Number of CPUs present

+
Array of objects (SkuGpu)

GPU components

+
Array
vendor
string

Vendor of the GPU

+
model
string

Model of the GPU

+
totalMemory
string

Total memory of the GPU (e.g. "80GB HBM3")

+
count
integer <uint32>

Number of GPUs present

+
Array of objects (SkuMemory)

Memory components

+
Array
capacityMb
integer <uint32>

Capacity in megabytes

+
memoryType
string

Type of memory (e.g. "DDR4", "DDR5")

+
count
integer <uint32>

Number of memory modules present

+
Array of objects (SkuStorage)

Storage components

+
Array
model
string

Informational storage model. Starting with the 2.1 release, NICo does not use this field for storage matching or validation.

+
count
integer <uint32>

Number of storage devices present

+
minSizeMiB
integer or null <uint32>

Inclusive minimum size in MiB for each storage device. Null or omission means no lower bound. Used for SKU schema version 5 and later.

+
maxSizeMiB
integer or null <uint32>

Inclusive maximum size in MiB for each storage device. Null or omission means no upper bound. Used for SKU schema version 5 and later.

+
pciPatterns
Array of strings

Regular expressions matched against storage sysfs PCI paths. An empty or omitted list disables PCI location matching. Used for SKU schema version 5 and later.

+
object (SkuChassis)

Chassis component

+
vendor
string

Vendor of the chassis

+
model
string

Model of the chassis

+
architecture
string

Architecture of the chassis

+
Array of objects (SkuEthernetDevice)

Ethernet device components

+
Array
vendor
string

Vendor of the ethernet device

+
model
string

Model of the ethernet device

+
count
integer <uint32>

Number of ethernet devices present

+
isConnected
boolean

Whether the ethernet device is connected

+
Array of objects (SkuInfinibandDevice)

Infiniband device components

+
Array
vendor
string

Vendor of the infiniband device

+
model
string

Model of the infiniband device

+
count
integer <uint32>

Number of infiniband devices present

+
inactiveDevices
Array of integers <uint32> [ items <uint32 > ]

Zero-based indexes of inactive devices

+
Array of objects (SkuTpm)

TPM components

+
Array
vendor
string

Vendor of the TPM

+
version
string

Version of the TPM

+

Responses

Response Schema: application/json
id
string

Unique identifier for the SKU

+
siteId
string <uuid>

ID of the Site this SKU belongs to

+
description
string

Human-readable SKU description

+
schemaVersion
integer <uint32>

Core SKU schema version when available

+
deviceType
string or null

Optional device type identifier (e.g. "gpu", "cpu", "storage")

+
associatedMachineIds
Array of strings

List of machine IDs associated with this SKU

+
object (SkuComponents)

Hardware components of this SKU

+
Array of objects (SkuCpu)

CPU components

+
Array
vendor
string

Vendor of the CPU

+
model
string

Model of the CPU

+
threadCount
integer <uint32>

Number of threads for the CPU

+
count
integer <uint32>

Number of CPUs present

+
Array of objects (SkuGpu)

GPU components

+
Array
vendor
string

Vendor of the GPU

+
model
string

Model of the GPU

+
totalMemory
string

Total memory of the GPU (e.g. "80GB HBM3")

+
count
integer <uint32>

Number of GPUs present

+
Array of objects (SkuMemory)

Memory components

+
Array
capacityMb
integer <uint32>

Capacity in megabytes

+
memoryType
string

Type of memory (e.g. "DDR4", "DDR5")

+
count
integer <uint32>

Number of memory modules present

+
Array of objects (SkuStorage)

Storage components

+
Array
vendor
string

Storage vendor used for schema version 4 matching. Read-only in REST mutation requests and preserved in responses for legacy SKUs. Schema version 5 does not use this field.

+
model
string

Informational storage model. Starting with the 2.1 release, NICo does not use this field for storage matching or validation.

+
capacityMb
integer <uint32>

Storage capacity in megabytes used for schema version 4 matching. Read-only in REST mutation requests and preserved in responses for legacy SKUs. Schema version 5 uses minSizeMiB and maxSizeMiB instead.

+
count
integer <uint32>

Number of storage devices present

+
minSizeMiB
integer or null <uint32>

Inclusive minimum size in MiB for each storage device. Null or omission means no lower bound. Used for SKU schema version 5 and later.

+
maxSizeMiB
integer or null <uint32>

Inclusive maximum size in MiB for each storage device. Null or omission means no upper bound. Used for SKU schema version 5 and later.

+
pciPatterns
Array of strings

Regular expressions matched against storage sysfs PCI paths. An empty or omitted list disables PCI location matching. Used for SKU schema version 5 and later.

+
object (SkuChassis)

Chassis component

+
vendor
string

Vendor of the chassis

+
model
string

Model of the chassis

+
architecture
string

Architecture of the chassis

+
Array of objects (SkuEthernetDevice)

Ethernet device components

+
Array
vendor
string

Vendor of the ethernet device

+
model
string

Model of the ethernet device

+
count
integer <uint32>

Number of ethernet devices present

+
isConnected
boolean

Whether the ethernet device is connected

+
Array of objects (SkuInfinibandDevice)

Infiniband device components

+
Array
vendor
string

Vendor of the infiniband device

+
model
string

Model of the infiniband device

+
count
integer <uint32>

Number of infiniband devices present

+
inactiveDevices
Array of integers <uint32> [ items <uint32 > ]

Zero-based indexes of inactive devices

+
Array of objects (SkuTpm)

TPM components

+
Array
vendor
string

Vendor of the TPM

+
version
string

Version of the TPM

+
created
string <date-time>

ISO 8601 datetime when the SKU was created

+
updated
string <date-time>

ISO 8601 datetime when the SKU was last updated

+

Request samples

Content type
application/json
{
  • "description": "DGX H100 compute node with InfiniBand",
  • "components": {
    }
}

Response samples

Content type
application/json
{
  • "id": "lenovo.sr650v2.cpu.1",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "description": "Lenovo SR650 V2 compute node",
  • "schemaVersion": 5,
  • "deviceType": "gpu",
  • "associatedMachineIds": [
    ],
  • "components": {
    },
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

InfiniBand Partition

InfiniBand (IB) is a high-performance, low-latency networking standard designed for interconnecting servers and storage in HPC (High-Performance Computing) and AI systems, utilizing RDMA (Remote Direct Memory Access) to reduce CPU overhead. InfiniBand Partitions are used to group Machines into logical partitions for network isolation and load distribution.

@@ -6248,7 +6932,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create InfiniBand Partition

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/infiniband-partition

Response samples

Content type
application/json
[
  • {
    }
]

Create InfiniBand Partition

Create an InfiniBand Partition for the org.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -6312,7 +6996,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "turbo-net",
  • "siteId": "69dae3c8-3554-4a1f-b391-858c6dc47fff",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "turbo-net",
  • "description": "InfiniBand Partition for model training Instances",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "controllerIBPartitionId": "0e60d064-3d38-4812-84d9-c3353bd96eaf",
  • "partitionKey": "0x1",
  • "partitionName": "turbo-net",
  • "serviceLevel": 5,
  • "rateLimit": 40,
  • "mtu": 4000,
  • "enableSharp": true,
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve InfiniBand Partition

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/infiniband-partition

Request samples

Content type
application/json
{
  • "name": "turbo-net",
  • "siteId": "69dae3c8-3554-4a1f-b391-858c6dc47fff",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "turbo-net",
  • "description": "InfiniBand Partition for model training Instances",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "controllerIBPartitionId": "0e60d064-3d38-4812-84d9-c3353bd96eaf",
  • "partitionKey": "0x1",
  • "partitionName": "turbo-net",
  • "serviceLevel": 5,
  • "rateLimit": 40,
  • "mtu": 4000,
  • "enableSharp": true,
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve InfiniBand Partition

Retrieve a specific InfiniBand Partition

@@ -6372,7 +7056,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "turbo-net",
  • "description": "InfiniBand Partition for model training Instances",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "controllerIBPartitionId": "0e60d064-3d38-4812-84d9-c3353bd96eaf",
  • "partitionKey": "0x1",
  • "partitionName": "turbo-net",
  • "serviceLevel": 5,
  • "rateLimit": 40,
  • "mtu": 4000,
  • "enableSharp": true,
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete InfiniBand Partition

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/infiniband-partition/{infiniBandPartitionId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "turbo-net",
  • "description": "InfiniBand Partition for model training Instances",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "controllerIBPartitionId": "0e60d064-3d38-4812-84d9-c3353bd96eaf",
  • "partitionKey": "0x1",
  • "partitionName": "turbo-net",
  • "serviceLevel": 5,
  • "rateLimit": 40,
  • "mtu": 4000,
  • "enableSharp": true,
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete InfiniBand Partition

Delete a specific InfiniBand Partition by ID.

@@ -6388,7 +7072,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update InfiniBand Partition

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/infiniband-partition/{infiniBandPartitionId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update InfiniBand Partition

Update an existing InfiniBand Partition

@@ -6454,7 +7138,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "turbo-net-v2",
  • "description": "Second version of the model training network",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "turbo-net-v2",
  • "description": "Second version of the model training network",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "controllerIBPartitionId": "0e60d064-3d38-4812-84d9-c3353bd96eaf",
  • "partitionKey": "0x1",
  • "partitionName": "turbo-net",
  • "serviceLevel": 5,
  • "rateLimit": 40,
  • "mtu": 4000,
  • "enableSharp": true,
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all InfiniBand Interfaces

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/infiniband-partition/{infiniBandPartitionId}

Request samples

Content type
application/json
{
  • "name": "turbo-net-v2",
  • "description": "Second version of the model training network",
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "turbo-net-v2",
  • "description": "Second version of the model training network",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "controllerIBPartitionId": "0e60d064-3d38-4812-84d9-c3353bd96eaf",
  • "partitionKey": "0x1",
  • "partitionName": "turbo-net",
  • "serviceLevel": 5,
  • "rateLimit": 40,
  • "mtu": 4000,
  • "enableSharp": true,
  • "labels": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all InfiniBand Interfaces

Get all InfiniBand Interfaces

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -6508,7 +7192,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Response samples

Content type
application/json
[
  • {
    }
]

Response samples

Content type
application/json
[
  • {
    }
]

Operating System

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/nvlink-interface

Response samples

Content type
application/json
[
  • {
    }
]

Operating System

Typical API Call Flow for Tenant
  • isCloudInit on create and update requests is deprecated and ignored; the response field is derived from whether userData is non-empty.
-

Retrieve all Operating Systems

List Operating Systems visible to the caller's Tenant.

-

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix. Only Operating Systems whose tenantId matches the caller's Tenant are returned.

+

Retrieve all Operating Systems

List Operating Systems visible to the caller.

+

User must have an authorization role with either the PROVIDER_ADMIN or TENANT_ADMIN suffix for the org.

+
    +
  • Provider admin: returns only Operating Systems owned by the org's Infrastructure Provider.
    • +
  • Tenant admin: returns Operating Systems owned by the caller's Tenant, plus Provider-owned Operating Systems that are associated with at least one Site the Tenant can access.
    • +
  • Dual-role (both Provider and Tenant): returns the union of the Tenant's and the Provider's Operating Systems.
    • +
Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

query Parameters
siteId
string
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
Example
[
  • {
    }
]

Create Operating System

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/operating-system

Response samples

Content type
application/json
Example
[
  • {
    }
]

Create Operating System

Create an Operating System for the org.

-

Either infrastructureProviderId or tenantId must be provided in request data. Both cannot be provided at the same time.

-

If infrastructureProviderId is provided in request data, then org must have an Infrastructure Provider entity and its ID should match the query parameter value. User must have authorization role with PROVIDER_ADMIN suffix.

-

If tenantId is provided in request data, then org must have a Tenant entity and its ID should match the query parameter value. User must have authorization role with TENANT_ADMIN suffix.

-

Only Tenants are allowed to create Operating System for MVP.

+

Ownership is derived from the caller's role:

+
    +
  • Tenant admin (TENANT_ADMIN): creates a Tenant-owned Operating System of any type (Image, iPXE, or Templated iPXE).
    • +
  • Provider admin (PROVIDER_ADMIN): may only create Provider-owned Operating Systems of type Templated iPXE.
    • +
+

The tenantId field in the request body is deprecated; if provided, it must match the org's Tenant.

Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

Request Body schema: application/json
name
required
string [ 2 .. 256 ] characters
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa fiNpIH bAoMjv">

Deprecated: Infrastructure Provider is now inferred from org membership.

tenantId
string or null <uuid>
Deprecated

Deprecated: Tenant is now inferred from org membership.

-
siteIds
Array of strings <uuid> [ items <uuid > ]

Target Sites for the Operating System. For image-based OS specify exactly one Site (more than one is not supported). For Templated iPXE OS at least one Site is required and the list is fixed at creation: it cannot be changed on update. Not applicable to raw iPXE OS.

+
siteIds
Array of strings <uuid> = 1 items [ items <uuid > ]

Target Site for the Operating System. For image-based and Templated iPXE Operating Systems exactly one Site is required, even though this field is an array. The list is fixed at creation and cannot be changed on update. Not applicable to raw iPXE OS.

ipxeScript
string or null
Deprecated

Deprecated: raw iPXE Operating Systems are superseded by Templated iPXE (ipxeTemplateId). iPXE script or URL, only applicable for iPXE-based OS. Cannot be specified if imageUrl is specified.

imageUrl
string or null <uri>
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "name": "ubuntu-official-22.04",
  • "description": "Official Ubuntu 22.04",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\n shell",
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "isCloudInit": true,
  • "phoneHomeEnabled": true,
  • "allowOverride": false
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "ubuntu-22.04",
  • "description": "Ubuntu 22.04",
  • "infrastructureProviderId": null,
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "type": "iPXE",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\n shell",
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "isCloudInit": true,
  • "phoneHomeEnabled": false,
  • "allowOverride": false,
  • "imageAuthToken": null,
  • "imageAuthType": null,
  • "imageDisk": null,
  • "imageSha": null,
  • "imageUrl": null,
  • "rootFsId": null,
  • "rootFsLabel": null,
  • "siteAssociations": [ ],
  • "isActive": true,
  • "deactivationNote": null,
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Operating System

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/operating-system

Request samples

Content type
application/json
Example
{
  • "name": "ubuntu-official-22.04",
  • "description": "Official Ubuntu 22.04",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\n shell",
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "isCloudInit": true,
  • "phoneHomeEnabled": true,
  • "allowOverride": false
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "ubuntu-22.04",
  • "description": "Ubuntu 22.04",
  • "infrastructureProviderId": null,
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "type": "iPXE",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\n shell",
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "isCloudInit": true,
  • "phoneHomeEnabled": false,
  • "allowOverride": false,
  • "imageAuthToken": null,
  • "imageAuthType": null,
  • "imageDisk": null,
  • "imageSha": null,
  • "imageUrl": null,
  • "rootFsId": null,
  • "rootFsLabel": null,
  • "siteAssociations": [ ],
  • "isActive": true,
  • "deactivationNote": null,
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Operating System

Get an Operating System by ID

If the Operating System has infrastructureProviderId set, then org must have an Infrastructure Provider entity and its ID should match the Operating System Infrastructure Provider ID. User must have authorization role with PROVIDER_ADMIN suffix.

If the Operating System has tenantId set, then org must have a Tenant entity and its ID should match the Operating System Tenant ID. User must have authorization role with TENANT_ADMIN suffix.

+

A Tenant admin may also read a Provider-owned Operating System when it is associated with at least one Site the Tenant can access.

Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

operatingSystemId
required
string
Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
Example
{
  • "id": "42b0f982-5c61-4d2f-a018-41ece61f4641",
  • "name": "debian-12-amd64",
  • "description": "Official Debian 12 for AMD/Intel",
  • "infrastructureProviderId": null,
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "type": "Image",
  • "imageUrl": "https://saimei.ftp.acc.umu.se/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
  • "imageSha": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
  • "imageAuthType": "Bearer",
  • "imageAuthToken": "acbd18db4cc2f85cedef654fccc4a4d8",
  • "imageDisk": "/dev/sda",
  • "rootFsId": "6c2ac315-3040-4728-94eb-b66d320206c1",
  • "rootFsLabel": null,
  • "ipxeScript": null,
  • "userData": null,
  • "isCloudInit": false,
  • "phoneHomeEnabled": false,
  • "allowOverride": false,
  • "siteAssociations": [
    ],
  • "isActive": true,
  • "deactivationNote": null,
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Operating System

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/operating-system/{operatingSystemId}

Response samples

Content type
application/json
Example
{
  • "id": "42b0f982-5c61-4d2f-a018-41ece61f4641",
  • "name": "debian-12-amd64",
  • "description": "Official Debian 12 for AMD/Intel",
  • "infrastructureProviderId": null,
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "type": "Image",
  • "imageUrl": "https://saimei.ftp.acc.umu.se/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
  • "imageSha": "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
  • "imageAuthType": "Bearer",
  • "imageAuthToken": "acbd18db4cc2f85cedef654fccc4a4d8",
  • "imageDisk": "/dev/sda",
  • "rootFsId": "6c2ac315-3040-4728-94eb-b66d320206c1",
  • "rootFsLabel": null,
  • "ipxeScript": null,
  • "userData": null,
  • "isCloudInit": false,
  • "phoneHomeEnabled": false,
  • "allowOverride": false,
  • "siteAssociations": [
    ],
  • "isActive": true,
  • "deactivationNote": null,
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Operating System

Delete an Operating System by ID

@@ -7240,7 +7940,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Operating System

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/operating-system/{operatingSystemId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Operating System

Update an Operating System by ID

@@ -7402,7 +8102,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "ubuntu-22.04-lts",
  • "description": "Ubuntu 22.04 LTS",
  • "allowOverride": true
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "ubuntu-22.04-lts",
  • "description": "Ubuntu 22.04 LTS",
  • "infrastructureProviderId": null,
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "type": "iPXE",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\n shell",
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "isCloudInit": true,
  • "allowOverride": true,
  • "phoneHomeEnabled": true,
  • "imageAuthToken": null,
  • "imageAuthType": null,
  • "imageDisk": null,
  • "imageSha": null,
  • "imageUrl": null,
  • "rootFsId": null,
  • "rootFsLabel": null,
  • "siteAssociations": [ ],
  • "isActive": true,
  • "deactivationNote": null,
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

iPXE Template

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/operating-system/{operatingSystemId}

Request samples

Content type
application/json
{
  • "name": "ubuntu-22.04-lts",
  • "description": "Ubuntu 22.04 LTS",
  • "allowOverride": true
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "ubuntu-22.04-lts",
  • "description": "Ubuntu 22.04 LTS",
  • "infrastructureProviderId": null,
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "type": "iPXE",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\n shell",
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "isCloudInit": true,
  • "allowOverride": true,
  • "phoneHomeEnabled": true,
  • "imageAuthToken": null,
  • "imageAuthType": null,
  • "imageDisk": null,
  • "imageSha": null,
  • "imageUrl": null,
  • "rootFsId": null,
  • "rootFsLabel": null,
  • "siteAssociations": [ ],
  • "isActive": true,
  • "deactivationNote": null,
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

iPXE Template

iPXE Templates are read-only definitions propagated from nico-core into the REST API. Each template has a stable core UUID and a globally unique name; per-site availability is tracked in IpxeTemplateSiteAssociation records rather than on the template row itself. List and retrieve operations return templates reported at sites the caller is authorized for (optionally filtered by siteId). Templated iPXE Operating Systems reference a template by ipxeTemplateId.

Get all iPXE templates

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve an iPXE template

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipxe-template

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve an iPXE template

Retrieve an iPXE template by its stable core ID. The caller must be authorized for at least one Site at which the template is available.

The Infrastructure Provider and Tenant are inferred from the org's membership. User must have authorization role with PROVIDER_ADMIN, PROVIDER_VIEWER, or TENANT_ADMIN suffix.

@@ -7470,7 +8170,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
  • "name": "ubuntu-autoinstall",
  • "template": "#!ipxe\nkernel ${kernel_url} initrd=initrd autoinstall\ninitrd ${initrd_url}\nboot\n",
  • "requiredParams": [
    ],
  • "reservedParams": [
    ],
  • "requiredArtifacts": [
    ],
  • "visibility": "Public",
  • "created": "2026-07-14T14:15:22Z",
  • "updated": "2026-07-14T14:15:22Z"
}

Instance Type

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipxe-template/{ipxeTemplateId}

Response samples

Content type
application/json
{
  • "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
  • "name": "ubuntu-autoinstall",
  • "template": "#!ipxe\nkernel ${kernel_url} initrd=initrd autoinstall\ninitrd ${initrd_url}\nboot\n",
  • "requiredParams": [
    ],
  • "reservedParams": [
    ],
  • "requiredArtifacts": [
    ],
  • "visibility": "Public",
  • "created": "2026-07-14T14:15:22Z",
  • "updated": "2026-07-14T14:15:22Z"
}

Instance Type

Instance Types allow grouping Machines into a pool defined by their capabilities. Providers can then allocate a portion of the Instance Type pool to a Tenant.

Retrieve all Instance Types

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
Example
[
  • {
    }
]

Create an Instance Type

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type

Response samples

Content type
application/json
Example
[
  • {
    }
]

Create an Instance Type

Create an Instance Type for Infrastructure Provider.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -7746,7 +8446,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "name": "x3.large",
  • "description": "Part of X family, the X3 Large features increased compute power",
  • "siteId": "8d97fa69-9199-49ff-bcf3-168c62d3874e",
  • "labels": {
    },
  • "machineCapabilities": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "x3.large",
  • "description": "Part of X family, the X3 Large features increased compute power",
  • "infrastructureProviderId": "5f2cc306-76e9-4fca-9186-950c9ef9a74e",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "labels": {
    },
  • "machineCapabilities": [
    ],
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve an Instance Type

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type

Request samples

Content type
application/json
Example
{
  • "name": "x3.large",
  • "description": "Part of X family, the X3 Large features increased compute power",
  • "siteId": "8d97fa69-9199-49ff-bcf3-168c62d3874e",
  • "labels": {
    },
  • "machineCapabilities": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "x3.large",
  • "description": "Part of X family, the X3 Large features increased compute power",
  • "infrastructureProviderId": "5f2cc306-76e9-4fca-9186-950c9ef9a74e",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "labels": {
    },
  • "machineCapabilities": [
    ],
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve an Instance Type

Get an Instance Type by ID.

@@ -7864,7 +8564,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "x3.large",
  • "description": "Part of X family, the X3 Large features increased compute power",
  • "infrastructureProviderId": "5f2cc306-76e9-4fca-9186-950c9ef9a74e",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "labels": {
    },
  • "machineCapabilities": [
    ],
  • "allocationStats": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Instance Type

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type/{instanceTypeId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "x3.large",
  • "description": "Part of X family, the X3 Large features increased compute power",
  • "infrastructureProviderId": "5f2cc306-76e9-4fca-9186-950c9ef9a74e",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "labels": {
    },
  • "machineCapabilities": [
    ],
  • "allocationStats": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Instance Type

Delete an Instance Type by ID.

Org must have an Infrastructure Provider entity that owns the Instance Type. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -7878,7 +8578,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Instance Type

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type/{instanceTypeId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Instance Type

Update an Instance Type by ID.

Org must have an Infrastructure Provider entity that owns the Instance Type. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -8020,7 +8720,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "description": "Updated version of the X3 Large family of machines",
  • "labels": {
    },
  • "machineCapabilities": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "x3.large",
  • "description": "Updated version of the X3 Large family of machines",
  • "infrastructureProviderId": "5f2cc306-76e9-4fca-9186-950c9ef9a74e",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "labels": {
    },
  • "machineCapabilities": [
    ],
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Machines/Instance Type associations

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type/{instanceTypeId}

Request samples

Content type
application/json
{
  • "description": "Updated version of the X3 Large family of machines",
  • "labels": {
    },
  • "machineCapabilities": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "x3.large",
  • "description": "Updated version of the X3 Large family of machines",
  • "infrastructureProviderId": "5f2cc306-76e9-4fca-9186-950c9ef9a74e",
  • "siteId": "72771e6a-6f5e-4de4-a5b9-1266c4197811",
  • "labels": {
    },
  • "machineCapabilities": [
    ],
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve all Machines/Instance Type associations

Get all Machines for a given Instance Type

Org must have an Infrastructure Provider entity that owns the Instance Type and the Machine. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -8066,7 +8766,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Create a Machine/Instance Type association

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type/{instanceTypeId}/machine

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Create a Machine/Instance Type association

Associate a Machine to an Instance Type

@@ -8110,7 +8810,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "machineIds": [
    ]
}

Response samples

Content type
application/json
[
  • {
    },
  • {
    },
  • {
    }
]

Delete a Machine/Instance Type association

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type/{instanceTypeId}/machine

Request samples

Content type
application/json
{
  • "machineIds": [
    ]
}

Response samples

Content type
application/json
[
  • {
    },
  • {
    },
  • {
    }
]

Delete a Machine/Instance Type association

Delete a Machine's association with an Instance Type.

@@ -8128,7 +8828,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Instance

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/type/{instanceTypeId}/machine/{machineAssociationId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Instance

Instance is a Machine provisioned with an Operating System by a Tenant and attached to one or more VPC Prefixes or Subnets.

Retrieve all Instances

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
Example
[
  • {
    }
]

Create an Instance

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance

Response samples

Content type
application/json
Example
[
  • {
    }
]

Create an Instance

Create an Instance for Tenant.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -8720,7 +9420,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when the Controller cannot allocate a requested resource because capacity is exhausted

Request samples

Content type
application/json
Example
{
  • "name": "spark-monitor-1",
  • "description": "Node for monitoring Spark",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\nshell",
  • "alwaysBootWithCustomIpxe": true,
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "labels": {
    },
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "sshKeyGroupIds": [
    ]
}

Response samples

Content type
application/json
Example
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-monitor-1",
  • "description": "Node for monitoring Spark",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "controllerInstanceId": null,
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\nshell",
  • "alwaysBootWithCustomIpxe": true,
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "labels": {
    },
  • "isUpdatePending": false,
  • "serialConsoleUrl": "ssh://user@nico.acme.com",
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [ ],
  • "sshKeyGroupIds": [
    ],
  • "sshKeyGroups": [
    ],
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Batch Create Instances

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance

Request samples

Content type
application/json
Example
{
  • "name": "spark-monitor-1",
  • "description": "Node for monitoring Spark",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\nshell",
  • "alwaysBootWithCustomIpxe": true,
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "labels": {
    },
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "sshKeyGroupIds": [
    ]
}

Response samples

Content type
application/json
Example
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-monitor-1",
  • "description": "Node for monitoring Spark",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "controllerInstanceId": null,
  • "ipxeScript": "#!ipxe\nkernel http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux initrd=initrd.gz\ninitrd http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz\nboot || imgfree\nshell",
  • "alwaysBootWithCustomIpxe": true,
  • "userData": "#cloud-config\nautoinstall:\n apt:\n geoip: true\n preserve_sources_list: false\n primary:\n - arches: [amd64, i386]\n uri: http://archive.ubuntu.com/ubuntu\n - arches: [default]\n uri: http://ports.ubuntu.com/ubuntu-ports",
  • "labels": {
    },
  • "isUpdatePending": false,
  • "serialConsoleUrl": "ssh://user@nico.acme.com",
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [ ],
  • "sshKeyGroupIds": [
    ],
  • "sshKeyGroups": [
    ],
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Batch Create Instances

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when the Controller cannot allocate a requested resource because capacity is exhausted

Request samples

Content type
application/json
Example
{
  • "namePrefix": "gpu-worker",
  • "count": 4,
  • "description": "GPU worker nodes for distributed training",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "topologyOptimized": true,
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "sshKeyGroupIds": [
    ]
}

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Retrieve Instance

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/batch

Request samples

Content type
application/json
Example
{
  • "namePrefix": "gpu-worker",
  • "count": 4,
  • "description": "GPU worker nodes for distributed training",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "topologyOptimized": true,
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "sshKeyGroupIds": [
    ]
}

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Retrieve Instance

Get an Instance by ID

Org must have a Tenant entity. Instance must belong to Tenant. User must have authorization role with TENANT_ADMIN suffix.

@@ -9290,7 +9990,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-monitor-1",
  • "description": "Node for monitoring Spark",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "controllerInstanceId": null,
  • "ipxeScript": null,
  • "alwaysBootWithCustomIpxe": false,
  • "userData": null,
  • "networkSecurityGroupId": "c602eb90-3039-11f0-997a-b38d4fc8389e",
  • "networkSecurityGroupPropagationDetails": {
    },
  • "networkSecurityGroupInherited": false,
  • "labels": {
    },
  • "isUpdatePending": false,
  • "serialConsoleUrl": "ssh://user@nico.acme.com",
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [
    ],
  • "sshKeyGroupIds": [
    ],
  • "sshKeyGroups": [
    ],
  • "tpmEkCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMxVENDQWJ5Z0F3SUJBZ0lVTEE1ZHFPK1E5OXZQM3VYRTRKcjBncVRtOW93d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0xqRUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQW9NQ2s1MmFXUnBZU0JEYjNKNw==",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Instance

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/{instanceId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-monitor-1",
  • "description": "Node for monitoring Spark",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "controllerInstanceId": null,
  • "ipxeScript": null,
  • "alwaysBootWithCustomIpxe": false,
  • "userData": null,
  • "networkSecurityGroupId": "c602eb90-3039-11f0-997a-b38d4fc8389e",
  • "networkSecurityGroupPropagationDetails": {
    },
  • "networkSecurityGroupInherited": false,
  • "labels": {
    },
  • "isUpdatePending": false,
  • "serialConsoleUrl": "ssh://user@nico.acme.com",
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [
    ],
  • "sshKeyGroupIds": [
    ],
  • "sshKeyGroups": [
    ],
  • "tpmEkCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMxVENDQWJ5Z0F3SUJBZ0lVTEE1ZHFPK1E5OXZQM3VYRTRKcjBncVRtOW93d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0xqRUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQW9NQ2s1MmFXUnBZU0JEYjNKNw==",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Instance

Delete an Instance by ID

Org must have a Tenant entity. Instance must belong to Tenant. User must have authorization role with TENANT_ADMIN suffix.

@@ -9316,7 +10016,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "machineHealthIssue": {
    },
  • "isRepairTenant": false
}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Instance

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/{instanceId}

Request samples

Content type
application/json
{
  • "machineHealthIssue": {
    },
  • "isRepairTenant": false
}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update Instance

Update an Instance by ID

Org must have a Tenant entity. Instance must belong to Tenant. User must have authorization role with TENANT_ADMIN suffix.

@@ -9642,7 +10342,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when the Controller cannot allocate a requested resource because capacity is exhausted

Request samples

Content type
application/json
{
  • "name": "spark-monitor-1",
  • "description": "Spark Monitor Node 1",
  • "triggerReboot": true,
  • "rebootWithCustomIpxe": true,
  • "applyUpdatesOnReboot": true,
  • "sshKeyGroupIds": [
    ],
  • "labels": {
    },
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-monitor-2",
  • "description": "Spark Monitor Node 1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "controllerInstanceId": "158fc2bc-f2fb-4e1f-a5a4-2211062d14df",
  • "ipxeScript": null,
  • "alwaysBootWithCustomIpxe": false,
  • "userData": null,
  • "labels": {
    },
  • "isUpdatePending": false,
  • "serialConsoleUrl": "ssh://user@nico.acme.com",
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [
    ],
  • "sshKeyGroupIds": [
    ],
  • "sshKeyGroups": [
    ],
  • "tpmEkCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMxVENDQWJ5Z0F3SUJBZ0lVTEE1ZHFPK1E5OXZQM3VYRTRKcjBncVRtOW93d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0xqRUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQW9NQ2s1MmFXUnBZU0JEYjNKNw==",
  • "status": "Rebooting",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Instance status history

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/{instanceId}

Request samples

Content type
application/json
{
  • "name": "spark-monitor-1",
  • "description": "Spark Monitor Node 1",
  • "triggerReboot": true,
  • "rebootWithCustomIpxe": true,
  • "applyUpdatesOnReboot": true,
  • "sshKeyGroupIds": [
    ],
  • "labels": {
    },
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "nvLinkInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "spark-monitor-2",
  • "description": "Spark Monitor Node 1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "41e36058-8403-4086-a9b8-39cb5bc9cb98",
  • "vpcId": "5e28ad7c-5fb7-46d6-a28a-fc0ba6fdc4a3",
  • "machineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "operatingSystemId": "eaeb86ee-c435-444e-9e01-8346f67f194b",
  • "controllerInstanceId": "158fc2bc-f2fb-4e1f-a5a4-2211062d14df",
  • "ipxeScript": null,
  • "alwaysBootWithCustomIpxe": false,
  • "userData": null,
  • "labels": {
    },
  • "isUpdatePending": false,
  • "serialConsoleUrl": "ssh://user@nico.acme.com",
  • "interfaces": [
    ],
  • "infinibandInterfaces": [
    ],
  • "dpuExtensionServiceDeployments": [
    ],
  • "sshKeyGroupIds": [
    ],
  • "sshKeyGroups": [
    ],
  • "tpmEkCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUMxVENDQWJ5Z0F3SUJBZ0lVTEE1ZHFPK1E5OXZQM3VYRTRKcjBncVRtOW93d0RRWUpLb1pJaHZjTkFRRUwKQlFBd0xqRUxNQWtHQTFVRUJoTUNWVk14RXpBUkJnTlZCQW9NQ2s1MmFXUnBZU0JEYjNKNw==",
  • "status": "Rebooting",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Instance status history

Get Instance status history

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -9672,7 +10372,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve all Interfaces

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/{instanceId}/status-history

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve all Interfaces

Get all Interfaces for an Instance

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -9734,7 +10434,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Retrieve all Instance InfiniBand Interfaces

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/{instanceId}/interface

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Retrieve all Instance InfiniBand Interfaces

Get all InfiniBand Interfaces for an Instance

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -9784,7 +10484,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/instance/{instanceId}/nvlink-interface

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Machine

Machine is a physical server that contains CPUs, GPUs, memory, storage, and networking hardware. Machines are the physical building blocks of a Site.

Retrieve all Machines

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Machine

Org must have either an Infrastructure Provider entity or a Tenant entity.

@@ -10254,7 +10954,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
Example
{
  • "id": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "2e016c02-2c67-48aa-b289-5d3ca6320c52",
  • "instanceId": "59bdaaff-3998-4fd9-a140-8749beeb605e",
  • "tenantId": "99819e6e-4017-4021-9edd-ea1bdf4dbd59",
  • "controllerMachineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "controllerMachineType": "x86_64",
  • "hwSkuDeviceType": "cpu",
  • "vendor": "Lenovo",
  • "productName": "ThinkSystem SR670 V2",
  • "serialNumber": "J1060ACR.D3KS2CS001G",
  • "machineCapabilities": [
    ],
  • "machineInterfaces": [
    ],
  • "maintenanceMessage": null,
  • "health": {
    },
  • "labels": {
    },
  • "status": "Ready",
  • "isUsableByTenant": true,
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}

Response samples

Content type
application/json
Example
{
  • "id": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "2e016c02-2c67-48aa-b289-5d3ca6320c52",
  • "instanceId": "59bdaaff-3998-4fd9-a140-8749beeb605e",
  • "tenantId": "99819e6e-4017-4021-9edd-ea1bdf4dbd59",
  • "controllerMachineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "controllerMachineType": "x86_64",
  • "hwSkuDeviceType": "cpu",
  • "vendor": "Lenovo",
  • "productName": "ThinkSystem SR670 V2",
  • "serialNumber": "J1060ACR.D3KS2CS001G",
  • "machineCapabilities": [
    ],
  • "machineInterfaces": [
    ],
  • "maintenanceMessage": null,
  • "health": {
    },
  • "labels": {
    },
  • "status": "Ready",
  • "isUsableByTenant": true,
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update Machine

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "instanceTypeId": "2e016c02-2c67-48aa-b289-5d3ca6320c52"
}

Response samples

Content type
application/json
{
  • "id": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "2e016c02-2c67-48aa-b289-5d3ca6320c52",
  • "instanceId": "59bdaaff-3998-4fd9-a140-8749beeb605e",
  • "tenantId": "99819e6e-4017-4021-9edd-ea1bdf4dbd59",
  • "controllerMachineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "controllerMachineType": "x86_64",
  • "hwSkuDeviceType": "cpu",
  • "vendor": "Lenovo",
  • "productName": "ThinkSystem SR670 V2",
  • "serialNumber": "J1060ACR.D3KS2CS001G",
  • "machineCapabilities": [
    ],
  • "machineInterfaces": [
    ],
  • "maintenanceMessage": null,
  • "health": {
    },
  • "labels": {
    },
  • "status": "Ready",
  • "isUsableByTenant": true,
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete a Machine from a Site

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}

Request samples

Content type
application/json
Example
{
  • "instanceTypeId": "2e016c02-2c67-48aa-b289-5d3ca6320c52"
}

Response samples

Content type
application/json
{
  • "id": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "instanceTypeId": "2e016c02-2c67-48aa-b289-5d3ca6320c52",
  • "instanceId": "59bdaaff-3998-4fd9-a140-8749beeb605e",
  • "tenantId": "99819e6e-4017-4021-9edd-ea1bdf4dbd59",
  • "controllerMachineId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "controllerMachineType": "x86_64",
  • "hwSkuDeviceType": "cpu",
  • "vendor": "Lenovo",
  • "productName": "ThinkSystem SR670 V2",
  • "serialNumber": "J1060ACR.D3KS2CS001G",
  • "machineCapabilities": [
    ],
  • "machineInterfaces": [
    ],
  • "maintenanceMessage": null,
  • "health": {
    },
  • "labels": {
    },
  • "status": "Ready",
  • "isUsableByTenant": true,
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete a Machine from a Site

Org must have an Infrastructure Provider entity. Machine must belong to the Provider. User must have authorization role with PROVIDER_ADMIN suffix. Machine must meet certain criteria to be eligible for deletion.

Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

@@ -10546,20 +11246,22 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Describes an error response for 500 Internal Server Error

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Machine power control

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Machine power control

Execute power control actions for a specific Machine.

+" class="sc-iJSMbW sc-cBEgGa fiNpIH ewCFMV">

Execute power control actions for a specific Machine. When an Instance is +attached to the Machine, acknowledgeAttachedInstance must be true.

Org must have an Infrastructure Provider entity and own the Site that the Machine belongs to. User must have authorization role with PROVIDER_ADMIN suffix.

Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

machineId
required
string

ID of the Machine

-
Request Body schema: application/json
required
action
required
string
Enum: "On" "GracefulShutdown" "ForceOff" "GracefulRestart" "ForceRestart" "ACPowercycle"

Power control action to apply.

+
Request Body schema: application/json
required
action
required
string
Enum: "On" "GracefulShutdown" "ForceOff" "GracefulRestart" "ForceRestart" "ACPowercycle"

Redfish power control action to apply to the Machine. ACPowercycle is not supported on Viking systems.

acknowledgeAttachedInstance
boolean

Acknowledges that an Instance is currently attached to the Machine and this action may disrupt Tenant workload on the Instance.

-

Responses

Responses

Request samples

Content type
application/json
Example
{
  • "action": "On"
}

Response samples

Content type
application/json
{
  • "message": "Power control accepted"
}

Retrieve Machine status history

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/power

Request samples

Content type
application/json
Example
{
  • "action": "On"
}

Response samples

Content type
application/json
{
  • "message": ""
}

Retrieve Machine status history

Org must have either an Infrastructure Provider entity or a Tenant entity.

@@ -10602,7 +11304,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve DPU Machines attached to a host Machine

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/status-history

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve DPU Machines attached to a host Machine

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Response when the API handler encounters an unexpected error

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve GPU stats for machines at a site

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/dpu

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve GPU stats for machines at a site

Returns GPU summary stats grouped by GPU name for machines at the specified site.

User must have authorization role with PROVIDER_ADMIN suffix. The specified site must belong to the Provider.

@@ -10808,7 +11510,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve machine instance type assignment summary for a site

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/gpu/stats

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve machine instance type assignment summary for a site

Returns machine counts grouped by assigned (has instance type) vs unassigned, broken down by status.

User must have authorization role with PROVIDER_ADMIN suffix. The specified site must belong to the Provider.

@@ -10854,7 +11556,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "assigned": {
    },
  • "unassigned": {
    }
}

Retrieve detailed per-instance-type machine stats for a site

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/instance-type/stats/summary

Response samples

Content type
application/json
{
  • "assigned": {
    },
  • "unassigned": {
    }
}

Retrieve detailed per-instance-type machine stats for a site

Returns machine stats for each instance type including allocation details and tenant breakdown.

User must have authorization role with PROVIDER_ADMIN suffix. The specified site must belong to the Provider.

@@ -10922,7 +11624,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve all Machine Capabilities

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/instance-type/stats

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve all Machine Capabilities

Get all distinct Machine Capabilities across all Machines

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -10982,20 +11684,26 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    },
  • {
    },
  • {
    },
  • {
    },
  • {
    },
  • {
    }
]

BMC Reset

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine-capability

Response samples

Content type
application/json
[
  • {
    },
  • {
    },
  • {
    },
  • {
    },
  • {
    },
  • {
    }
]

BMC Reset

BMC Reset allows resetting a Machine's BMC

-

Reset Machine BMC

Reset Machine BMC

Reset BMC of a specific Machine.

+" class="sc-iJSMbW sc-cBEgGa fiNpIH ewCFMV">

Reset the BMC of a specific Machine. When an Instance is attached to the +Machine, acknowledgeAttachedInstance must be true.

+

If useIpmiTool is true, the BMC reset request may be silently +ignored while the BMC is in lockdown mode.

For Infrastructure Providers: Org must have an Infrastructure Provider entity and own the Site that the Machine belongs to. User must have authorization role with PROVIDER_ADMIN suffix.

For Tenants: Org must have a Tenant with TargetedInstanceCreation capability enabled and Tenant Account with Machine's Provider. User must have authorization role with TENANT_ADMIN suffix.

Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

machineId
required
string

ID of the Machine

-
Request Body schema: application/json
required
useIpmiTool
boolean
Default: false

Reset the BMC via ipmitool instead of Redfish.

+
Request Body schema: application/json
required
useIpmiTool
boolean
Default: false

Reset the BMC via ipmitool instead of Redfish. The request may be silently ignored while the BMC is in lockdown mode.

acknowledgeAttachedInstance
boolean

Acknowledges that an Instance is currently attached to the Machine and this action may disrupt Tenant workload on the Instance.

Responses

Request samples

Content type
application/json
{
  • "useIpmiTool": true
}

Response samples

Content type
application/json
{
  • "message": "Machine BMC reset request was accepted"
}

DPU Reprovision

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/bmc/reset

Request samples

Content type
application/json
{
  • "useIpmiTool": true
}

Response samples

Content type
application/json
{
  • "message": "Machine BMC reset request was accepted"
}

DPU Reprovision

DPU Reprovision allows re-provisioning Machine's DPUs

-

Reprovision Machine DPUs

Reprovision Machine DPUs

Trigger DPU reprovisioning for a Machine. It may be necessary to create a HostUpdateInProgress health report for the Machine before DPU reprovisioning can be triggered.

+" class="sc-iJSMbW sc-cBEgGa fiNpIH ewCFMV">

Trigger DPU reprovisioning for a Machine. Before calling this operation, +create a Machine health report containing an alert whose ID is +HostUpdateInProgress and whose classifications include +PreventAllocations. When an Instance is attached to the Machine, +acknowledgeAttachedInstance must be true.

For Infrastructure Providers: Org must have an Infrastructure Provider entity and own the Site that the Machine belongs to. User must have authorization role with PROVIDER_ADMIN suffix.

For Tenants: Org must have a Tenant with TargetedInstanceCreation capability enabled and Tenant Account with Machine's Provider. User must have authorization role with TENANT_ADMIN suffix.

Authorizations:
JWTBearerToken
path Parameters
org
required
string

Name of the Org

machineId
required
string

ID of the Machine

-
Request Body schema: application/json
required
mode
required
string
Enum: "Set" "Clear" "Restart"

DPU reprovisioning mode.

-
updateFirmware
boolean
Default: false

Whether to update firmware during reprovisioning.

+
Request Body schema: application/json
required
mode
required
string
Enum: "Set" "Clear" "Restart"

Use Set to start reprovisioning, Clear to remove a pending request, or Restart to restart DPUs that already have a request. Restart accepts a host Machine ID only.

+
updateFirmware
boolean
Deprecated
Default: false

Deprecated compatibility field. Firmware is always verified and updated during reprovisioning.

acknowledgeAttachedInstance
boolean

Acknowledges that an Instance is currently attached to the Machine and this action may disrupt Tenant workload on the Instance.

Responses

Request samples

Content type
application/json
{
  • "mode": "Restart",
  • "updateFirmware": true
}

Response samples

Content type
application/json
{
  • "message": "DPU reprovisioning request was accepted"
}

Health Report

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/dpu/reprovision

Request samples

Content type
application/json
{
  • "mode": "Set",
  • "updateFirmware": false
}

Response samples

Content type
application/json
{
  • "message": "DPU reprovisioning request was accepted"
}

Health Report

Machine Health Report contains information about the health of a Machine including user enforced overrides

Retrieve all Machine health reports

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Response when the API handler encounters an unexpected error

Response samples

Content type
application/json
[
  • {
    }
]

Create or update Machine health report

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/health-report

Response samples

Content type
application/json
[
  • {
    }
]

Create or update Machine health report

Add or update health report override for a specific Machine.

@@ -11164,7 +11880,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Response when the API handler encounters an unexpected error

Request samples

Content type
application/json
{
  • "source": "overrides.sre",
  • "mode": "Merge",
  • "alerts": [
    ]
}

Response samples

Content type
application/json
{
  • "source": "overrides.sre",
  • "triggeredBy": "operator",
  • "observedAt": "2026-06-24T12:00:00Z",
  • "mode": "Merge",
  • "alerts": [
    ]
}

Delete Machine health report

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/health-report

Request samples

Content type
application/json
{
  • "source": "maintenance.dpu-reprovision",
  • "mode": "Merge",
  • "alerts": [
    ]
}

Response samples

Content type
application/json
{
  • "source": "maintenance.dpu-reprovision",
  • "triggeredBy": "operator",
  • "observedAt": "2026-06-24T12:00:00Z",
  • "mode": "Merge",
  • "alerts": [
    ]
}

Delete Machine health report

Remove a health report override for a specific Machine.

@@ -11188,7 +11904,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Response when the API handler encounters an unexpected error

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Machine Capability

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/machine/{machineId}/health-report/{source}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Machine Capability

Machine Capability defines the hardware capabilities of a Machine. Machine Capabilities can be used to group Machines into Instance Types.

Rack

Rack is a physical enclosure that contains a number of Machines. Racks are the physical building blocks of a Site.

@@ -11278,7 +11994,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Rack

Get a Rack by ID.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -11356,7 +12072,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "550e8400-e29b-41d4-a716-446655440000",
  • "name": "Rack-01",
  • "manufacturer": "Dell",
  • "model": "PowerEdge R750",
  • "serialNumber": "SN-RACK-001",
  • "description": "Primary compute rack",
  • "location": {
    },
  • "components": [
    ]
}

Validate Racks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/{id}

Response samples

Content type
application/json
{
  • "id": "550e8400-e29b-41d4-a716-446655440000",
  • "name": "Rack-01",
  • "manufacturer": "Dell",
  • "model": "PowerEdge R750",
  • "serialNumber": "SN-RACK-001",
  • "description": "Primary compute rack",
  • "location": {
    },
  • "components": [
    ]
}

Validate Racks

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
Example
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 10
}

Validate a Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/validation

Response samples

Content type
application/json
Example
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 10
}

Validate a Rack

Validate a Rack's components by comparing expected vs actual state.

@@ -11448,7 +12164,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
Example
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 5
}

Power control Racks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/{id}/validation

Response samples

Content type
application/json
Example
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 5
}

Power control Racks

Power control Racks with optional filters. If no filter is specified, targets all racks in the Site.

@@ -11488,7 +12204,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "off"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Power control a Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/power

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "off"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Power control a Rack

Power control a Rack identified by Rack UUID.

@@ -11538,7 +12254,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "on"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update Racks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/{id}/power

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "on"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update Racks

Update firmware on Racks with optional name filter. If no filter is specified, targets all racks in the Site.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -11576,7 +12292,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update a Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/firmware

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update a Rack

Update firmware on a Rack identified by Rack UUID.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -11662,7 +12378,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "version": "24.11.0"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Bring up Racks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/{id}/firmware

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "version": "24.11.0"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Bring up Racks

Bring up Racks with optional name filter. If no filter is specified, targets all racks in the Site.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -11698,7 +12414,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Bring up a Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/bringup

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Bring up a Rack

Bring up a Rack identified by Rack UUID.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -11730,7 +12446,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Retrieve all Tasks for a Rack

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/{id}/bringup

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Retrieve all Tasks for a Rack

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
[
  • {
    }
]

Tray

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/rack/{id}/task

Response samples

Content type
application/json
[
  • {
    }
]

Tray

Tray represents a component within a Rack.

Retrieve all Trays

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Tray

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Tray

Get a Tray by ID.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -11948,7 +12664,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "660e8400-e29b-41d4-a716-446655440001",
  • "componentId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "type": "compute",
  • "name": "compute-tray-1",
  • "manufacturer": "NVIDIA",
  • "model": "GB200",
  • "serialNumber": "TSN001",
  • "description": "Compute tray in slot 1",
  • "firmwareVersion": "2.1.0",
  • "powerState": "on",
  • "position": {
    },
  • "rackId": "550e8400-e29b-41d4-a716-446655440000"
}

Validate Trays

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/{id}

Response samples

Content type
application/json
{
  • "id": "660e8400-e29b-41d4-a716-446655440001",
  • "componentId": "fm100ht4v4mce2qstjnl8970nnj3ie6ecek4mtjn27pea4kre5gsa49jg0g",
  • "type": "compute",
  • "name": "compute-tray-1",
  • "manufacturer": "NVIDIA",
  • "model": "GB200",
  • "serialNumber": "TSN001",
  • "description": "Compute tray in slot 1",
  • "firmwareVersion": "2.1.0",
  • "powerState": "on",
  • "position": {
    },
  • "rackId": "550e8400-e29b-41d4-a716-446655440000"
}

Validate Trays

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 10
}

Validate a Tray

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/validation

Response samples

Content type
application/json
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 10
}

Validate a Tray

Validate a Tray by comparing expected vs actual state.

@@ -12050,7 +12766,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 5
}

Power control Trays

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/{id}/validation

Response samples

Content type
application/json
{
  • "diffs": [ ],
  • "totalDiffs": 0,
  • "missingCount": 0,
  • "unexpectedCount": 0,
  • "mismatchCount": 0,
  • "matchCount": 5
}

Power control Trays

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "on"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Power control a Tray

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/power

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "on"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Power control a Tray

Power control a Tray identified by Tray UUID.

@@ -12162,7 +12878,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "on"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update Trays

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/{id}/power

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "state": "on"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update Trays

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update a Tray

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/firmware

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Firmware update a Tray

Update firmware on a Tray identified by Tray UUID.

Org must have an Infrastructure Provider entity. User must have authorization role with PROVIDER_ADMIN suffix.

@@ -12360,7 +13076,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "version": "24.11.0"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Retrieve all Tasks for a Tray

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/{id}/firmware

Request samples

Content type
application/json
Example
{
  • "siteId": "550e8400-e29b-41d4-a716-446655440000",
  • "version": "24.11.0"
}

Response samples

Content type
application/json
{
  • "taskIds": [
    ]
}

Retrieve all Tasks for a Tray

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
[
  • {
    }
]

Task

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/tray/{id}/task

Response samples

Content type
application/json
[
  • {
    }
]

Task

Task represents an asynchronous, site-scoped operation (for example firmware update, power state change, or rack bring-up). Tasks are created when operations run against Racks, Trays, or other components. Endpoints in this tag retrieve or cancel a Task by ID; list Tasks for a Rack or Tray under the Rack and Tray tags.

Retrieve a Task

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "550e8400-e29b-41d4-a716-446655440000",
  • "status": "Running",
  • "description": "Power on rack components",
  • "message": "Processing 3 of 5 components"
}

Cancel a Task

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/{id}

Response samples

Content type
application/json
{
  • "id": "550e8400-e29b-41d4-a716-446655440000",
  • "status": "Running",
  • "description": "Power on rack components",
  • "message": "Processing 3 of 5 components"
}

Cancel a Task

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "siteId": "660e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "id": "550e8400-e29b-41d4-a716-446655440000",
  • "status": "Terminated",
  • "description": "Power on rack components",
  • "message": "Cancelled by user"
}

Rule

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/{id}/cancel

Request samples

Content type
application/json
{
  • "siteId": "660e8400-e29b-41d4-a716-446655440000"
}

Response samples

Content type
application/json
{
  • "id": "550e8400-e29b-41d4-a716-446655440000",
  • "status": "Terminated",
  • "description": "Power on rack components",
  • "message": "Cancelled by user"
}

Rule

Operation Rule defines, per Site, how a particular operation (for example PowerControl / power_on or FirmwareControl / upgrade) should be executed against a set of components: ordered execution stages, per-component-type concurrency, pre / main / post actions, timeouts, and retry policy. Rules are reusable templates owned by Flow; this tag exposes CRUD (POST, GET, PATCH, DELETE) over them.

Create an Operation Rule

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "ruleDefinition": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "ruleDefinition": {
    },
  • "isDefault": true,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

List Operation Rules

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/rule

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "ruleDefinition": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "ruleDefinition": {
    },
  • "isDefault": true,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

List Operation Rules

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve an Operation Rule

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/rule

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve an Operation Rule

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "ruleDefinition": {
    },
  • "isDefault": true,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update an Operation Rule

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/rule/{id}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "ruleDefinition": {
    },
  • "isDefault": true,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Update an Operation Rule

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "name": "string",
  • "description": "string",
  • "ruleDefinition": {
    }
}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Delete an Operation Rule

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/rule/{id}

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "name": "string",
  • "description": "string",
  • "ruleDefinition": {
    }
}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Delete an Operation Rule

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Task Run

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/rule/{id}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Task Run

A Task Run is a phased, policy-gated execution of one operation (currently firmware) across many Racks. A Task Run narrows a candidate set of Racks with an optional selector, divides the selected Racks into phases, and drives one execution target per Rack; each target in turn drives at most one Task. Safety gates pause the Task Run when failures exceed a threshold, and phase gates hold each phase until an operator advances it. This tag exposes creation, retrieval, target listing, and the pause / resume / advance / cancel lifecycle actions; drill into per-Rack execution detail via the Task tag using each target's taskId.

Create a Task Run

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "name": "string",
  • "description": "string",
  • "selector": {
    },
  • "options": {
    },
  • "operation": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Retrieve all Task Runs

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "name": "string",
  • "description": "string",
  • "selector": {
    },
  • "options": {
    },
  • "operation": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Retrieve all Task Runs

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Task Run

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run

Response samples

Content type
application/json
[
  • {
    }
]

Retrieve a Task Run

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Retrieve all Task Run Targets

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run/{id}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Retrieve all Task Run Targets

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
[
  • {
    }
]

Pause a Task Run

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run/{id}/target

Response samples

Content type
application/json
[
  • {
    }
]

Pause a Task Run

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Resume a Task Run

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run/{id}/pause

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Resume a Task Run

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Advance a Task Run to its next phase

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run/{id}/resume

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Advance a Task Run to its next phase

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "expectedPhaseIndex": 0
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Cancel a Task Run

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run/{id}/advance

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "expectedPhaseIndex": 0
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Cancel a Task Run

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "reason": "string"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Network Security Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/task/run/{id}/cancel

Request samples

Content type
application/json
{
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "reason": "string"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "string",
  • "description": "string",
  • "operationType": "PowerControl",
  • "operationCode": "string",
  • "status": "Unknown",
  • "statusReason": "Unknown",
  • "statusMessage": "string",
  • "totalPhases": 0,
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z",
  • "started": "2019-08-24T14:15:22Z",
  • "finished": "2019-08-24T14:15:22Z",
  • "stats": {
    }
}

Network Security Group

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Describes an error response for 501 Not Implemented

Response samples

Content type
application/json
[
  • {
    }
]

Create Network Security Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/network-security-group

Response samples

Content type
application/json
[
  • {
    }
]

Create Network Security Group

Create a Network Security Group for Tenant.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -13686,7 +14402,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Describes an error response for 501 Not Implemented

Request samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "188a8f32-0001-45cf-b243-f62720a22cc4",
  • "rules": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "56f1a3ed-3653-454f-b861-9136207be660",
  • "tenantId": "79595ebe-934f-4f19-bc74-c16aefd0c57a",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "rules": [
    ],
  • "labels": {
    },
  • "created": "2025-02-26T18:17:44.861317-05:00",
  • "updated": "2025-02-26T18:17:44.861317-05:00"
}

Retrieve Network Security Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/network-security-group

Request samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "188a8f32-0001-45cf-b243-f62720a22cc4",
  • "rules": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "56f1a3ed-3653-454f-b861-9136207be660",
  • "tenantId": "79595ebe-934f-4f19-bc74-c16aefd0c57a",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "rules": [
    ],
  • "labels": {
    },
  • "created": "2025-02-26T18:17:44.861317-05:00",
  • "updated": "2025-02-26T18:17:44.861317-05:00"
}

Retrieve Network Security Group

Get a Network Security Group by ID

Org must have a Tenant entity. Instance must belong to Tenant. User must have authorization role with TENANT_ADMIN suffix.

@@ -13772,7 +14488,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Describes an error response for 501 Not Implemented

Response samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "56f1a3ed-3653-454f-b861-9136207be660",
  • "tenantId": "79595ebe-934f-4f19-bc74-c16aefd0c57a",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "rules": [
    ],
  • "labels": {
    },
  • "created": "2025-02-26T18:17:44.861317-05:00",
  • "updated": "2025-02-26T18:17:44.861317-05:00"
}

Update Network Security Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/network-security-group/{networkSecurityGroupId}

Response samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "56f1a3ed-3653-454f-b861-9136207be660",
  • "tenantId": "79595ebe-934f-4f19-bc74-c16aefd0c57a",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "rules": [
    ],
  • "labels": {
    },
  • "created": "2025-02-26T18:17:44.861317-05:00",
  • "updated": "2025-02-26T18:17:44.861317-05:00"
}

Update Network Security Group

Update a Network Security Group by ID

@@ -13886,7 +14602,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Describes an error response for 501 Not Implemented

Request samples

Content type
application/json
{
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "rules": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "56f1a3ed-3653-454f-b861-9136207be660",
  • "tenantId": "79595ebe-934f-4f19-bc74-c16aefd0c57a",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "rules": [
    ],
  • "labels": {
    },
  • "created": "2025-02-26T18:17:44.861317-05:00",
  • "updated": "2025-02-26T18:17:44.861317-05:00"
}

Delete Network Security Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/network-security-group/{networkSecurityGroupId}

Request samples

Content type
application/json
{
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "rules": [
    ],
  • "labels": {
    }
}

Response samples

Content type
application/json
{
  • "id": "2a21cf79-ea5e-4d28-b585-2e78948fcefb",
  • "name": "Spark VPC Firewall",
  • "description": "Security policies for machines in Spark VPC",
  • "siteId": "56f1a3ed-3653-454f-b861-9136207be660",
  • "tenantId": "79595ebe-934f-4f19-bc74-c16aefd0c57a",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "rules": [
    ],
  • "labels": {
    },
  • "created": "2025-02-26T18:17:44.861317-05:00",
  • "updated": "2025-02-26T18:17:44.861317-05:00"
}

Delete Network Security Group

Delete a Network Security Group by ID

@@ -13912,7 +14628,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Describes an error response for 501 Not Implemented

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

IP Block

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/network-security-group/{networkSecurityGroupId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

IP Block

IP Block is a contiguous block of IP addresses defined by a prefix and prefix length.

It can be used by the Provider to describe the overlay network of a particular Site. Providers can also use Allocations to delegate portions of these IP Blocks to Tenants.

@@ -13998,7 +14714,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create IP Block

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipblock

Response samples

Content type
application/json
[
  • {
    }
]

Create IP Block

Create an IP block for the org.

@@ -14076,7 +14792,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "Public Network Overlay for Site SJC4",
  • "description": "This is the primary IP overlay for SJC4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "routingType": "Public",
  • "prefix": "202.168.1.0",
  • "prefixLength": 24,
  • "protocolVersion": "IPv4"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Public Network Overlay for Site SJC4",
  • "description": "This is the primary IP overlay for SJC4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "tenantId": null,
  • "routingType": "Public",
  • "prefix": "202.168.1.0",
  • "prefixLength": 24,
  • "protocolVersion": "IPv4",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve IP Block

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipblock

Request samples

Content type
application/json
{
  • "name": "Public Network Overlay for Site SJC4",
  • "description": "This is the primary IP overlay for SJC4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "routingType": "Public",
  • "prefix": "202.168.1.0",
  • "prefixLength": 24,
  • "protocolVersion": "IPv4"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Public Network Overlay for Site SJC4",
  • "description": "This is the primary IP overlay for SJC4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "tenantId": null,
  • "routingType": "Public",
  • "prefix": "202.168.1.0",
  • "prefixLength": 24,
  • "protocolVersion": "IPv4",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve IP Block

Retrieve an IP Block by ID.

User must have authorization role with PROVIDER_ADMIN or TENANT_ADMIN suffix.

@@ -14146,7 +14862,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Public Network Overlay for Site SJC4",
  • "description": "This is the primary IP overlay for SJC4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "tenantId": null,
  • "routingType": "Public",
  • "prefix": "192.168.20.0",
  • "prefixLength": 24,
  • "protocolVersion": "IPv4",
  • "usageStats": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete IP Block

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipblock/{ipBlockId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Public Network Overlay for Site SJC4",
  • "description": "This is the primary IP overlay for SJC4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "tenantId": null,
  • "routingType": "Public",
  • "prefix": "192.168.20.0",
  • "prefixLength": 24,
  • "protocolVersion": "IPv4",
  • "usageStats": {
    },
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete IP Block

Delete an IP block

@@ -14164,7 +14880,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update IP Block

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipblock/{ipBlockId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update IP Block

Update an existing IP Block

@@ -14234,7 +14950,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "Public Network Overlay for Site SJC-4",
  • "description": "This is the primary IP overlay for SJC-4. All IPs are publicly routable"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Public Network Overlay for Site SJC-4",
  • "description": "This is the primary IP overlay for SJC-4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "tenantId": null,
  • "routingType": "Public",
  • "prefix": "202.168.16.0",
  • "prefixLength": 20,
  • "protocolVersion": "IPv4",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve All Derived IP Blocks

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipblock/{ipBlockId}

Request samples

Content type
application/json
{
  • "name": "Public Network Overlay for Site SJC-4",
  • "description": "This is the primary IP overlay for SJC-4. All IPs are publicly routable"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "Public Network Overlay for Site SJC-4",
  • "description": "This is the primary IP overlay for SJC-4. All IPs are publicly routable",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "infrastructureProviderId": "e94bcfda-f6cb-42e4-80ec-516811e5abbf",
  • "tenantId": null,
  • "routingType": "Public",
  • "prefix": "202.168.16.0",
  • "prefixLength": 20,
  • "protocolVersion": "IPv4",
  • "status": "Pending",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve All Derived IP Blocks

Retrieve all child IP Blocks allocated to Tenants from a specific Provider super IP Block. When allocations are created from a super block, individual Tenant IP Blocks are created as a result.

@@ -14312,7 +15028,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

DPU Extension Service

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/ipblock/{ipBlockId}/derived

Response samples

Content type
application/json
[
  • {
    }
]

DPU Extension Service

DPU Extension Service allows users to run custom services in the DPUs of their Instances. Currently K8s pods are the only supported service type.

Retrieve all DPU Extension Services

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create DPU Extension Service

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/dpu-extension-service

Response samples

Content type
application/json
[
  • {
    }
]

Create DPU Extension Service

Create a DPU Extension Service for the current Tenant.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -14472,7 +15188,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "data": "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: busybox-deployment\\n labels:\\n app: busybox\\nspec:\\n replicas: 1 # You can adjust the number of desired replicas here\\n selector:\\n matchLabels:\\n app: busybox\\n template:\\n metadata:\\n labels:\\n app: busybox\\n spec:\\n containers:\\n - name: busybox-container\\n image: busybox:latest # You can specify a different BusyBox image tag\\n command: [\"sh\", \"-c\", \"echo \\'BusyBox container running\\' && sleep 3600\"]",
  • "credentials": {},
  • "observability": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "V1-T1761856992374052",
  • "versionInfo": {
    },
  • "activeVersions": [
    ],
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve DPU Extension Service

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/dpu-extension-service

Request samples

Content type
application/json
{
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "data": "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: busybox-deployment\\n labels:\\n app: busybox\\nspec:\\n replicas: 1 # You can adjust the number of desired replicas here\\n selector:\\n matchLabels:\\n app: busybox\\n template:\\n metadata:\\n labels:\\n app: busybox\\n spec:\\n containers:\\n - name: busybox-container\\n image: busybox:latest # You can specify a different BusyBox image tag\\n command: [\"sh\", \"-c\", \"echo \\'BusyBox container running\\' && sleep 3600\"]",
  • "credentials": {},
  • "observability": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "V1-T1761856992374052",
  • "versionInfo": {
    },
  • "activeVersions": [
    ],
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve DPU Extension Service

Retrieve a DPU Extension Service for the current Tenant by ID

@@ -14532,7 +15248,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "V1-T1761856992374052",
  • "versionInfo": {
    },
  • "activeVersions": [
    ],
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete DPU Extension Service

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/dpu-extension-service/{dpuExtensionServiceId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "V1-T1761856992374052",
  • "versionInfo": {
    },
  • "activeVersions": [
    ],
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete DPU Extension Service

Delete a specific DPU Extension Service by ID. All versions will be deleted.

@@ -14548,7 +15264,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

Update DPU Extension Service

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/dpu-extension-service/{dpuExtensionServiceId}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

Update DPU Extension Service

Update a specific DPU Extension Service.

@@ -14638,7 +15354,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Request samples

Content type
application/json
{
  • "name": "busybox-ha",
  • "data": "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: busybox-deployment\\n labels:\\n app: busybox\\nspec:\\n replicas: 3 # You can adjust the number of desired replicas here\\n selector:\\n matchLabels:\\n app: busybox\\n template:\\n metadata:\\n labels:\\n app: busybox\\n spec:\\n containers:\\n - name: busybox-container\\n image: busybox:latest # You can specify a different BusyBox image tag\\n command: [\"sh\", \"-c\", \"echo \\'BusyBox container running\\' && sleep 3600\"]",
  • "credentials": {},
  • "observability": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "V1-T1761856992374052",
  • "versionInfo": {
    },
  • "activeVersions": [
    ],
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve DPU Extension Service Version

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/dpu-extension-service/{dpuExtensionServiceId}

Request samples

Content type
application/json
{
  • "name": "busybox-ha",
  • "data": "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: busybox-deployment\\n labels:\\n app: busybox\\nspec:\\n replicas: 3 # You can adjust the number of desired replicas here\\n selector:\\n matchLabels:\\n app: busybox\\n template:\\n metadata:\\n labels:\\n app: busybox\\n spec:\\n containers:\\n - name: busybox-container\\n image: busybox:latest # You can specify a different BusyBox image tag\\n command: [\"sh\", \"-c\", \"echo \\'BusyBox container running\\' && sleep 3600\"]",
  • "credentials": {},
  • "observability": {
    }
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "busybox",
  • "description": "Single, multi-call executable that contains stripped-down versions of common Unix utilities",
  • "serviceType": "KubernetesPod",
  • "siteId": "60189e9c-7d12-438c-b9ca-6998d9c364b1",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "V1-T1761856992374052",
  • "versionInfo": {
    },
  • "activeVersions": [
    ],
  • "status": "Ready",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve DPU Extension Service Version

Retrieve details for a specific version of a DPU Extension Service.

@@ -14674,7 +15390,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when requested object is not found

Response samples

Content type
application/json
{
  • "version": "V1-T1761856992374052",
  • "data": "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: busybox-deployment\\n labels:\\n app: busybox\\nspec:\\n replicas: 1 # You can adjust the number of desired replicas here\\n selector:\\n matchLabels:\\n app: busybox\\n template:\\n metadata:\\n labels:\\n app: busybox\\n spec:\\n containers:\\n - name: busybox-container\\n image: busybox:latest # You can specify a different BusyBox image tag\\n command: [\"sh\", \"-c\", \"echo \\'BusyBox container running\\' && sleep 3600\"]",
  • "hasCredentials": true,
  • "created": "2019-08-24T14:15:22Z",
  • "observability": {
    }
}

Delete DPU Extension Service Version

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/dpu-extension-service/{dpuExtensionServiceId}/version/{version}

Response samples

Content type
application/json
{
  • "version": "V1-T1761856992374052",
  • "data": "apiVersion: apps/v1\\nkind: Deployment\\nmetadata:\\n name: busybox-deployment\\n labels:\\n app: busybox\\nspec:\\n replicas: 1 # You can adjust the number of desired replicas here\\n selector:\\n matchLabels:\\n app: busybox\\n template:\\n metadata:\\n labels:\\n app: busybox\\n spec:\\n containers:\\n - name: busybox-container\\n image: busybox:latest # You can specify a different BusyBox image tag\\n command: [\"sh\", \"-c\", \"echo \\'BusyBox container running\\' && sleep 3600\"]",
  • "hasCredentials": true,
  • "created": "2019-08-24T14:15:22Z",
  • "observability": {
    }
}

Delete DPU Extension Service Version

Delete a specific version of a DPU Extension Service.

@@ -14692,7 +15408,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

SSH Key Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/dpu-extension-service/{dpuExtensionServiceId}/version/{version}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "User is not allowed to perform this action",
  • "data": null
}

SSH Key Group

SSH Key Groups allow grouping several SSH Keys together so they can be synced to Sites and used to access the Serial Console of Instances.

Retrieve all SSH Key Groups

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create SSH Key Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkeygroup

Response samples

Content type
application/json
[
  • {
    }
]

Create SSH Key Group

Create an SSH Key Group for the current Tenant.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -14854,7 +15570,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "reno-integration-sre",
  • "description": "SRE access SSH keys for Reno Integration",
  • "siteIds": [
    ],
  • "sshKeyIds": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-integration-sre",
  • "description": "SRE access SSH keys for Reno Integration",
  • "org": "wdksahew1rqf",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75",
  • "siteAssociations": [
    ],
  • "sshKeys": [
    ],
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve an SSH Key Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkeygroup

Request samples

Content type
application/json
{
  • "name": "reno-integration-sre",
  • "description": "SRE access SSH keys for Reno Integration",
  • "siteIds": [
    ],
  • "sshKeyIds": [
    ]
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-integration-sre",
  • "description": "SRE access SSH keys for Reno Integration",
  • "org": "wdksahew1rqf",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75",
  • "siteAssociations": [
    ],
  • "sshKeys": [
    ],
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve an SSH Key Group

Retrieve an SSH Key Group for the current Tenant by ID

@@ -14924,7 +15640,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-integration-sre",
  • "description": "SRE access SSH keys for Reno Integration",
  • "org": "wdksahew1rqf",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75",
  • "siteAssociations": [
    ],
  • "sshKeys": [
    ],
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete an SSH Key Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkeygroup/{sshKeyGroupId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-integration-sre",
  • "description": "SRE access SSH keys for Reno Integration",
  • "org": "wdksahew1rqf",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75",
  • "siteAssociations": [
    ],
  • "sshKeys": [
    ],
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete an SSH Key Group

Delete a specific SSH key Group.

@@ -14940,7 +15656,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update an SSH Key Group

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkeygroup/{sshKeyGroupId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update an SSH Key Group

Update a specific SSH Key Group.

@@ -15024,7 +15740,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "reno-int-sre",
  • "description": "SRE access SSH keys for Reno Integration Site",
  • "siteIds": [
    ],
  • "sshKeyIds": [
    ],
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-int-sre",
  • "description": "SRE access SSH keys for Reno Integration Site",
  • "org": "wdksahew1rqf",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75",
  • "siteAssociations": [
    ],
  • "sshKeys": [
    ],
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

SSH Key

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkeygroup/{sshKeyGroupId}

Request samples

Content type
application/json
{
  • "name": "reno-int-sre",
  • "description": "SRE access SSH keys for Reno Integration Site",
  • "siteIds": [
    ],
  • "sshKeyIds": [
    ],
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-int-sre",
  • "description": "SRE access SSH keys for Reno Integration Site",
  • "org": "wdksahew1rqf",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "version": "fbc692b61ffef6fbfc38a3833f6b7e7ae508da75",
  • "siteAssociations": [
    ],
  • "sshKeys": [
    ],
  • "status": "Syncing",
  • "statusHistory": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

SSH Key

SSH Key is a public key that can be used to access the Serial Console of an Instance.

Retrieve all SSH Keys

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    }
]

Create SSH Key

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkey

Response samples

Content type
application/json
[
  • {
    }
]

Create SSH Key

Create an SSH Key for the current Tenant. If an SSH Key Group is specified, all Sites associated with the SSH Key Group must be online.

Org must have a Tenant entity. User must have authorization role with TENANT_ADMIN suffix.

@@ -15096,7 +15812,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "reno-sre-access",
  • "publicKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICip4hl6WjuVHs60PeikVUs0sWE/kPhk2D0rRHWsIuyL jdoe@test.com",
  • "sshKeyGroupId": "86ca8cab-b285-4c2d-9e00-25c88810dc2e"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-sre-access",
  • "org": "xskkpgqpeakn",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "fingerprint": "CaK2yoj5fDOhf1swM2kFyjQrd3bwZfDYlWnVjBHgveQ",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve an SSH key

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkey

Request samples

Content type
application/json
{
  • "name": "reno-sre-access",
  • "publicKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICip4hl6WjuVHs60PeikVUs0sWE/kPhk2D0rRHWsIuyL jdoe@test.com",
  • "sshKeyGroupId": "86ca8cab-b285-4c2d-9e00-25c88810dc2e"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-sre-access",
  • "org": "xskkpgqpeakn",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "fingerprint": "CaK2yoj5fDOhf1swM2kFyjQrd3bwZfDYlWnVjBHgveQ",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve an SSH key

Retrieve an SSH key for the current Tenant by ID

@@ -15124,7 +15840,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "staging-sre-access",
  • "org": "xskkpgqpeakn",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "fingerprint": "CaK2yoj5fDOhf1swM2kFyjQrd3bwZfDYlWnVjBHgveQ",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete an SSH Key

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkey/{sshKeyId}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "staging-sre-access",
  • "org": "xskkpgqpeakn",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "fingerprint": "CaK2yoj5fDOhf1swM2kFyjQrd3bwZfDYlWnVjBHgveQ",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete an SSH Key

Delete an SSH key for the current Tenant by ID.

@@ -15140,7 +15856,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update an SSH Key

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkey/{sshKeyId}

Response samples

Content type
application/json
{
  • "message": "Deletion request was accepted"
}

Update an SSH Key

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Request samples

Content type
application/json
{
  • "name": "reno-sre-access-v2"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-sre-access-v2",
  • "org": "xskkpgqpeakn",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "fingerprint": "CaK2yoj5fDOhf1swM2kFyjQrd3bwZfDYlWnVjBHgveQ",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

User

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/sshkey/{sshKeyId}

Request samples

Content type
application/json
{
  • "name": "reno-sre-access-v2"
}

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "name": "reno-sre-access-v2",
  • "org": "xskkpgqpeakn",
  • "tenantId": "f97df110-f4de-492e-8849-4a6af68026b0",
  • "fingerprint": "CaK2yoj5fDOhf1swM2kFyjQrd3bwZfDYlWnVjBHgveQ",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

User

User is a logical entity that identifies individuals operating on behalf of an organization.

Retrieve Current User

Retrieve details of the current user.

@@ -15200,7 +15916,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authenticated

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "email": "janed@nvidia.com",
  • "firstName": "Jane",
  • "lastName": "Doe",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Audit

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/user/current

Response samples

Content type
application/json
{
  • "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  • "email": "janed@nvidia.com",
  • "firstName": "Jane",
  • "lastName": "Doe",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Audit

Audit is a record of actions taken by users on the API.

Retrieve all Audit Log Entries

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Retrieve Audit Log Entry

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/audit

Response samples

Content type
application/json
[
  • {
    },
  • {
    }
]

Retrieve Audit Log Entry

Retrieve a specific Audit Log Entry by ID

User must have authorization role with PROVIDER_ADMIN or TENANT_ADMIN suffix

@@ -15320,7 +16036,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authorized to call an endpoint or retrieve/modify objects

Response samples

Content type
application/json
{
  • "id": "e313b3ca-c47a-4ec1-a79b-a147fad51a50",
  • "endpoint": "/v2/org/test-org-1/nico/ep",
  • "queryParams": "{\"test\":[\"1234\"]}",
  • "method": "POST",
  • "body": "{\"key1\":\"value1\"}",
  • "statusCode": 200,
  • "clientIP": "12.123.43.112",
  • "userID": "5d9fe319-14d4-40e3-8e5a-7d79e680d55b",
  • "user": {
    },
  • "orgName": "test-org-1",
  • "timestamp": "2024-12-04T21:06:33.849293-08:00",
  • "durationMs": 250,
  • "apiVersion": "0.1.91"
}

Metadata

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/audit/{auditEntryId}

Response samples

Content type
application/json
{
  • "id": "e313b3ca-c47a-4ec1-a79b-a147fad51a50",
  • "endpoint": "/v2/org/test-org-1/nico/ep",
  • "queryParams": "{\"test\":[\"1234\"]}",
  • "method": "POST",
  • "body": "{\"key1\":\"value1\"}",
  • "statusCode": 200,
  • "clientIP": "12.123.43.112",
  • "userID": "5d9fe319-14d4-40e3-8e5a-7d79e680d55b",
  • "user": {
    },
  • "orgName": "test-org-1",
  • "timestamp": "2024-12-04T21:06:33.849293-08:00",
  • "durationMs": 250,
  • "apiVersion": "0.1.91"
}

Metadata

Metadata describes various system-level attributes of the API service.

Retrieve metadata about the API server

Retrieve system metadata providing information about the API server

@@ -15336,7 +16052,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Error response when user is not authenticated

Response samples

Content type
application/json
{
  • "version": "0.1.24",
  • "buildTime": "2019-08-24T14:15:22Z"
}

Host Firmware Config

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/metadata

Response samples

Content type
application/json
{
  • "version": "0.1.24",
  • "buildTime": "2019-08-24T14:15:22Z"
}

Host Firmware Config

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Core gRPC API is unavailable, so the host firmware config request cannot be served.

Request samples

Content type
application/json
{
  • "siteId": "00000000-0000-0000-0000-000000000001",
  • "vendor": "Nvidia",
  • "model": "DGXH100",
  • "explicitStartNeeded": true,
  • "ordering": [
    ],
  • "components": [
    ]
}

Response samples

Content type
application/json
{
  • "vendor": "Nvidia",
  • "model": "DGXH100",
  • "explicitStartNeeded": true,
  • "ordering": [
    ],
  • "components": [
    ],
  • "created": "2025-08-24T14:15:22Z",
  • "updated": "2025-08-24T14:15:22Z"
}

Delete Host Firmware Config

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/firmware-config/host

Request samples

Content type
application/json
{
  • "siteId": "00000000-0000-0000-0000-000000000001",
  • "vendor": "Nvidia",
  • "model": "DGXH100",
  • "explicitStartNeeded": true,
  • "ordering": [
    ],
  • "components": [
    ]
}

Response samples

Content type
application/json
{
  • "vendor": "Nvidia",
  • "model": "DGXH100",
  • "explicitStartNeeded": true,
  • "ordering": [
    ],
  • "components": [
    ],
  • "created": "2025-08-24T14:15:22Z",
  • "updated": "2025-08-24T14:15:22Z"
}

Delete Host Firmware Config

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Core gRPC API is unavailable, so the host firmware config request cannot be served.

Request samples

Content type
application/json
{
  • "siteId": "00000000-0000-0000-0000-000000000001",
  • "vendor": "Nvidia",
  • "model": "DGXH100"
}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Tenant Identity

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/firmware-config/host

Request samples

Content type
application/json
{
  • "siteId": "00000000-0000-0000-0000-000000000001",
  • "vendor": "Nvidia",
  • "model": "DGXH100"
}

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Tenant Identity

Typical API Call Flow for Tenant be served.

Request samples

Content type
application/json
Example
{
  • "enabled": true,
  • "issuer": "string",
  • "defaultAudience": "string",
  • "allowedAudiences": [ ],
  • "tokenTtlSeconds": 1,
  • "subjectPrefix": "string",
  • "rotateKey": false
}

Response samples

Content type
application/json
{
  • "org": "string",
  • "enabled": true,
  • "issuer": "string",
  • "defaultAudience": "string",
  • "allowedAudiences": [
    ],
  • "tokenTtlSeconds": 0,
  • "subjectPrefix": "string",
  • "signingKeys": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Tenant Identity Configuration for current Org

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/tenant-identity/config

Request samples

Content type
application/json
Example
{
  • "enabled": true,
  • "issuer": "string",
  • "defaultAudience": "string",
  • "allowedAudiences": [ ],
  • "tokenTtlSeconds": 1,
  • "subjectPrefix": "string",
  • "rotateKey": false
}

Response samples

Content type
application/json
{
  • "org": "string",
  • "enabled": true,
  • "issuer": "string",
  • "defaultAudience": "string",
  • "allowedAudiences": [
    ],
  • "tokenTtlSeconds": 0,
  • "subjectPrefix": "string",
  • "signingKeys": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Tenant Identity Configuration for current Org

Typical API Call Flow for Tenant be served.

Response samples

Content type
application/json
{
  • "org": "string",
  • "enabled": true,
  • "issuer": "string",
  • "defaultAudience": "string",
  • "allowedAudiences": [
    ],
  • "tokenTtlSeconds": 0,
  • "subjectPrefix": "string",
  • "signingKeys": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Tenant Identity Configuration

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/tenant-identity/config

Response samples

Content type
application/json
{
  • "org": "string",
  • "enabled": true,
  • "issuer": "string",
  • "defaultAudience": "string",
  • "allowedAudiences": [
    ],
  • "tokenTtlSeconds": 0,
  • "subjectPrefix": "string",
  • "signingKeys": [
    ],
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Tenant Identity Configuration

Typical API Call Flow for Tenant be served.

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Create or Update Token Delegation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/tenant-identity/config

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Create or Update Token Delegation

Typical API Call Flow for Tenant be served.

Request samples

Content type
application/json
{
  • "tokenEndpoint": "http://example.com",
  • "clientSecretBasic": {
    },
  • "subjectTokenAudience": "string"
}

Response samples

Content type
application/json
{
  • "tokenEndpoint": "http://example.com",
  • "clientSecretBasic": {
    },
  • "subjectTokenAudience": "string",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Token Delegation for current Org

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/tenant-identity/token-delegation

Request samples

Content type
application/json
{
  • "tokenEndpoint": "http://example.com",
  • "clientSecretBasic": {
    },
  • "subjectTokenAudience": "string"
}

Response samples

Content type
application/json
{
  • "tokenEndpoint": "http://example.com",
  • "clientSecretBasic": {
    },
  • "subjectTokenAudience": "string",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Retrieve Token Delegation for current Org

Retrieve the registered token exchange callback for the tenant.

@@ -16034,7 +16750,7 @@

Typical API Call Flow for Tenant

be served.

Response samples

Content type
application/json
{
  • "tokenEndpoint": "http://example.com",
  • "clientSecretBasic": {
    },
  • "subjectTokenAudience": "string",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Token Delegation

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/tenant-identity/token-delegation

Response samples

Content type
application/json
{
  • "tokenEndpoint": "http://example.com",
  • "clientSecretBasic": {
    },
  • "subjectTokenAudience": "string",
  • "created": "2019-08-24T14:15:22Z",
  • "updated": "2019-08-24T14:15:22Z"
}

Delete Token Delegation

Delete the RFC 8693 token exchange callback for the tenant.

@@ -16062,7 +16778,7 @@

Typical API Call Flow for Tenant

be served.

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Retrieve OIDC JWKS for current Org

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/tenant-identity/token-delegation

Response samples

Content type
application/json
{
  • "source": "nico",
  • "message": "Error validating request data",
  • "data": {
    }
}

Retrieve OIDC JWKS for current Org

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Core gRPC API is unavailable.

Response samples

Content type
application/json
{
  • "keys": [
    ]
}

Retrieve OpenID Configuration for current Org

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/.well-known/jwks.json

Response samples

Content type
application/json
{
  • "keys": [
    ]
}

Retrieve OpenID Configuration for current Org

Typical API Call Flow for Tenant " class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Core gRPC API is unavailable.

Response samples

Content type
application/json
{
  • "issuer": "string",
  • "jwks_uri": "http://example.com",
  • "response_types_supported": [
    ],
  • "subject_types_supported": [
    ],
  • "id_token_signing_alg_values_supported": [
    ],
  • "spiffe_jwks_uri": "http://example.com"
}

Retrieve SPIFFE JWKS for current Org

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/.well-known/openid-configuration

Response samples

Content type
application/json
{
  • "issuer": "string",
  • "jwks_uri": "http://example.com",
  • "response_types_supported": [
    ],
  • "subject_types_supported": [
    ],
  • "id_token_signing_alg_values_supported": [
    ],
  • "spiffe_jwks_uri": "http://example.com"
}

Retrieve SPIFFE JWKS for current Org

SPIFFE trust-domain JWKS — same key material as the OIDC JWKS but with use: jwt-svid for SPIFFE-native verifiers. No authentication required.

Not-configured and malformed-body behavior matches the OIDC JWKS endpoint.

@@ -16178,7 +16894,7 @@

Typical API Call Flow for Tenant

" class="sc-iJSMbW sc-cBEgGa sc-ciCrSJ fiNpIH dNfUH dDDioG">

Core gRPC API is unavailable.

Response samples

Content type
application/json
{
  • "keys": [
    ]
}

Deprecations

https://nico-rest-api.nico.svc.cluster.local/v2/org/{org}/nico/site/{siteID}/.well-known/spiffe/jwks.json

Response samples

Content type
application/json
{
  • "keys": [
    ]
}

Deprecations

Typical API Call Flow for Tenant