diff --git a/common/library/module_utils/input_validation/common_utils/config.py b/common/library/module_utils/input_validation/common_utils/config.py index d1bb4b5b61..667da006ea 100644 --- a/common/library/module_utils/input_validation/common_utils/config.py +++ b/common/library/module_utils/input_validation/common_utils/config.py @@ -130,7 +130,7 @@ "intel_benchmarks": "2024.1.0", "ucx": "1.19.0", "openmpi": "5.0.8", - "csi_driver_powerscale": "v2.15.0", + "csi_driver_powerscale": "v2.16.0", "rocm": "6.3.1", "service_k8s": "1.35.1" } diff --git a/common/library/module_utils/input_validation/schema/credential_rules.json b/common/library/module_utils/input_validation/schema/credential_rules.json index ae0872671c..8fc1e6251a 100644 --- a/common/library/module_utils/input_validation/schema/credential_rules.json +++ b/common/library/module_utils/input_validation/schema/credential_rules.json @@ -161,5 +161,17 @@ "maxLength": 128, "pattern": "^.+$", "description": "Password for Dell OpenManage Enterprise (OME). Length must be between 1 and 128 characters." + }, + "ufm_username": { + "minLength": 1, + "maxLength": 64, + "pattern": "^[^\\\\'\";]+$", + "description": "Username for NVIDIA UFM (Unified Fabric Manager). Must not contain backslashes (\\), single quotes ('), double quotes (\"), or semicolons (;). Length must be between 1 and 64 characters." + }, + "ufm_password": { + "minLength": 1, + "maxLength": 128, + "pattern": "^.+$", + "description": "Password for NVIDIA UFM (Unified Fabric Manager). Length must be between 1 and 128 characters." } } diff --git a/common/library/module_utils/input_validation/schema/telemetry_config.json b/common/library/module_utils/input_validation/schema/telemetry_config.json index 28260bc42f..0f1b84d93e 100644 --- a/common/library/module_utils/input_validation/schema/telemetry_config.json +++ b/common/library/module_utils/input_validation/schema/telemetry_config.json @@ -94,9 +94,37 @@ } }, "required": ["metrics_enabled", "collection_targets"] + }, + "ufm": { + "type": "object", + "description": "NVIDIA UFM InfiniBand Fabric telemetry.", + "properties": { + "metrics_enabled": { + "type": "boolean", + "default": false, + "description": "Enable or disable UFM InfiniBand metrics collection." + }, + "logs_enabled": { + "type": "boolean", + "default": false, + "description": "Enable or disable UFM syslog logs collection." + }, + "collection_targets": { + "type": "array", + "items": { + "type": "string", + "enum": ["victoria_metrics", "victoria_logs"] + }, + "minItems": 1, + "uniqueItems": true, + "default": ["victoria_metrics"], + "description": "UFM supports victoria_metrics (metrics) and victoria_logs (logs)." + } + }, + "required": ["metrics_enabled", "collection_targets"] } }, - "required": ["idrac", "ldms", "dcgm", "powerscale"] + "required": ["idrac", "ldms", "dcgm", "powerscale", "ufm"] }, "telemetry_bridges": { "type": "object", @@ -398,7 +426,54 @@ } }, "required": ["otel_collector_storage_size", "csm_observability_values_file_path"] + }, + "ufm_configuration": { + "type": "object", + "description": "UFM telemetry detailed configurations.", + "properties": { + "ufm_endpoint": { + "type": "string", + "description": "UFM appliance IP address or hostname. Required when UFM telemetry is enabled (metrics_enabled or logs_enabled is true)." + }, + "ufm_metrics_port": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "default": 9001, + "description": "UFM Prometheus exporter port." + }, + "scrape_interval": { + "type": "string", + "pattern": "^[0-9]+[smh]$", + "default": "30s", + "description": "Prometheus scrape interval for UFM metrics." + }, + "scrape_timeout": { + "type": "string", + "pattern": "^[0-9]+[smh]$", + "default": "15s", + "description": "Prometheus scrape timeout (must be <= scrape_interval)." + }, + "tls_mode": { + "type": "string", + "enum": ["self_signed", "ca_signed"], + "default": "self_signed", + "description": "TLS mode for connecting to UFM Prometheus endpoint." + }, + "ufm_ca_cert_path": { + "type": "string", + "default": "", + "description": "Path to CA certificate file for UFM TLS verification." + }, + "auth_mode": { + "type": "string", + "enum": ["basic", "none"], + "default": "basic", + "description": "Authentication mode for UFM Prometheus endpoint." + } + }, + "required": [] } }, - "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations"] + "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations", "ufm_configuration"] } diff --git a/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py b/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py index 5f60311920..48f5e0d4e7 100644 --- a/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py @@ -328,6 +328,7 @@ def validate_telemetry_config( idrac_source = telemetry_sources.get("idrac", {}) ldms_source = telemetry_sources.get("ldms", {}) powerscale_source = telemetry_sources.get("powerscale", {}) + ufm_source = telemetry_sources.get("ufm", {}) idrac_telemetry_support = idrac_source.get("metrics_enabled", False) idrac_collection_targets = idrac_source.get("collection_targets", []) @@ -383,6 +384,17 @@ def validate_telemetry_config( list(invalid_powerscale_targets), f"Invalid collection targets for PowerScale. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_powerscale_targets}" )) + + # UFM: supports victoria_metrics and victoria_logs + ufm_targets = set(ufm_source.get("collection_targets", [])) + allowed_ufm_targets = {"victoria_metrics", "victoria_logs"} + invalid_ufm_targets = ufm_targets - allowed_ufm_targets + if invalid_ufm_targets: + errors.append(create_error_msg( + "telemetry_sources.ufm.collection_targets", + list(invalid_ufm_targets), + f"Invalid collection targets for UFM. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_ufm_targets}" + )) # ========================================================================= # Validate service cluster and slurm cluster @@ -784,4 +796,59 @@ def validate_telemetry_config( is_service_cluster_defined, config_paths, logger, errors ) + # ========================================================================= + # Validate UFM telemetry configuration + # ========================================================================= + ufm_metrics_enabled = ufm_source.get("metrics_enabled", False) + ufm_logs_enabled = ufm_source.get("logs_enabled", False) + ufm_detailed_config = data.get("ufm_configuration", {}) + + if ufm_metrics_enabled or ufm_logs_enabled: + # Check required UFM endpoint + ufm_endpoint = ufm_detailed_config.get("ufm_endpoint", "") + if not ufm_endpoint or (isinstance(ufm_endpoint, str) and ufm_endpoint.strip() == ""): + errors.append(create_error_msg( + "ufm_configuration.ufm_endpoint", + ufm_endpoint, + "ufm_endpoint is required when UFM telemetry is enabled. Provide the UFM appliance IP address or hostname." + )) + + # Validate UFM metrics port if metrics enabled + if ufm_metrics_enabled: + ufm_metrics_port = ufm_detailed_config.get("ufm_metrics_port", 9001) + if not isinstance(ufm_metrics_port, int) or ufm_metrics_port < 1 or ufm_metrics_port > 65535: + errors.append(create_error_msg( + "ufm_configuration.ufm_metrics_port", + ufm_metrics_port, + "ufm_metrics_port must be an integer between 1 and 65535." + )) + + # Validate TLS mode + tls_mode = ufm_detailed_config.get("tls_mode", "self_signed") + if tls_mode not in ["self_signed", "ca_signed"]: + errors.append(create_error_msg( + "ufm_configuration.tls_mode", + tls_mode, + "tls_mode must be 'self_signed' or 'ca_signed'." + )) + + # Validate CA certificate path when tls_mode is ca_signed + if tls_mode == "ca_signed": + ca_cert_path = ufm_detailed_config.get("ufm_ca_cert_path", "") + if not ca_cert_path or (isinstance(ca_cert_path, str) and ca_cert_path.strip() == ""): + errors.append(create_error_msg( + "ufm_configuration.ufm_ca_cert_path", + ca_cert_path, + "ufm_ca_cert_path is required when tls_mode is 'ca_signed'. Provide path to CA certificate file." + )) + + # Validate auth mode + auth_mode = ufm_detailed_config.get("auth_mode", "basic") + if auth_mode not in ["basic", "none"]: + errors.append(create_error_msg( + "ufm_configuration.auth_mode", + auth_mode, + "auth_mode must be 'basic' or 'none'." + )) + return errors diff --git a/common/library/modules/fetch_telemetry_status.py b/common/library/modules/fetch_telemetry_status.py index d6db54ab97..622d05846f 100644 --- a/common/library/modules/fetch_telemetry_status.py +++ b/common/library/modules/fetch_telemetry_status.py @@ -72,6 +72,13 @@ def main(): if telemetry_sources.get("idrac", {}).get("metrics_enabled", False): telemetry_status_list.append("idrac_telemetry") + # Check UFM telemetry + ufm_config = telemetry_sources.get("ufm", {}) + if ufm_config.get("metrics_enabled", False): + telemetry_status_list.append("ufm_telemetry") + if ufm_config.get("logs_enabled", False): + telemetry_status_list.append("ufm_logs") + module.exit_json( changed=False, telemetry_status_list=telemetry_status_list diff --git a/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json b/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json index 7470b45cf5..87d133328d 100644 --- a/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json +++ b/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json @@ -5,63 +5,63 @@ "package": "csi-powerscale", "url": "https://github.com/dell/csi-powerscale.git", "type": "git", - "version": "v2.15.0" + "version": "v2.16.0" }, { "package": "external-snapshotter", "url": "https://github.com/kubernetes-csi/external-snapshotter.git", "type": "git", - "version": "v8.3.0" + "version": "v8.4.0" }, { "package": "helm-charts", "url": "https://github.com/dell/helm-charts.git", "type": "git", - "version": "csi-isilon-2.15.0" + "version": "csi-isilon-2.16.0" }, { "package": "quay.io/dell/container-storage-modules/csi-isilon", - "tag": "v2.15.0", + "tag": "v2.16.0", "type": "image" }, { "package": "registry.k8s.io/sig-storage/csi-attacher", - "tag": "v4.9.0", + "tag": "v4.10.0", "type": "image" }, { "package": "registry.k8s.io/sig-storage/csi-provisioner", - "tag": "v5.3.0", + "tag": "v6.1.0", "type": "image" }, { "package": "registry.k8s.io/sig-storage/csi-snapshotter", - "tag": "v8.3.0", + "tag": "v8.4.0", "type": "image" }, { "package": "registry.k8s.io/sig-storage/csi-resizer", - "tag": "v1.14.0", + "tag": "v2.0.0", "type": "image" }, { "package": "registry.k8s.io/sig-storage/csi-node-driver-registrar", - "tag": "v2.14.0", + "tag": "v2.15.0", "type": "image" }, { "package": "registry.k8s.io/sig-storage/csi-external-health-monitor-controller", - "tag": "v0.15.0", + "tag": "v0.16.0", "type": "image" }, { "package": "quay.io/dell/container-storage-modules/dell-csi-replicator", - "tag": "v1.13.0", + "tag": "v1.14.0", "type": "image" }, { "package": "quay.io/dell/container-storage-modules/podmon", - "tag": "v1.14.0", + "tag": "v1.15.0", "type": "image" }, { @@ -71,12 +71,12 @@ }, { "package": "quay.io/dell/container-storage-modules/csi-metadata-retriever", - "tag": "v1.12.0", + "tag": "v1.13.0", "type": "image" }, { "package": "registry.k8s.io/sig-storage/snapshot-controller", - "tag": "v8.3.0", + "tag": "v8.4.0", "type": "image" }, { diff --git a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json index 749634d89f..36e83d7fb8 100644 --- a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json +++ b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json @@ -78,7 +78,7 @@ { "package": "prettytable==3.14.0", "type": "pip_module" }, { "package": "python3-3.12.9", "type": "rpm", "repo_name": "baseos" }, { "package": "git", "type": "rpm", "repo_name": "appstream"}, - { "package": "kubernetes==35.0.0", "type": "pip_module" }, + { "package": "kubernetes==33.1.0", "type": "pip_module" }, { "package": "PyMySQL==1.1.2", "type": "pip_module" } ] @@ -106,7 +106,7 @@ { "package": "prettytable==3.14.0", "type": "pip_module" }, { "package": "python3-3.12.9", "type": "rpm", "repo_name": "baseos" }, { "package": "git", "type": "rpm", "repo_name": "appstream"}, - { "package": "kubernetes==35.0.0", "type": "pip_module" }, + { "package": "kubernetes==33.1.0", "type": "pip_module" }, { "package": "PyMySQL==1.1.2", "type": "pip_module" } ] }, @@ -118,4 +118,4 @@ { "package": "quay.io/metallb/controller", "tag": "v0.15.3", "type": "image" } ] } -} \ No newline at end of file +} diff --git a/input/telemetry_config.yml b/input/telemetry_config.yml index a66c418ecb..17491767af 100644 --- a/input/telemetry_config.yml +++ b/input/telemetry_config.yml @@ -146,6 +146,29 @@ telemetry_sources: - "victoria_metrics" - "victoria_logs" + # -------------------------------------------------------------------------- + # UFM — NVIDIA UFM InfiniBand Fabric Telemetry + # -------------------------------------------------------------------------- + # Collects: IB port state, transmit/receive data, error counters, fabric topology + # Requires: NVIDIA UFM appliance with Prometheus exporter enabled (port 9090) + # Data path: UFM Prometheus Exporter → vmagent(shared) → victoria_metrics + # + # NOTE: Omnia does NOT deploy UFM itself. Omnia configures the existing + # vmagent to scrape the UFM Prometheus endpoint for IB fabric metrics. + ufm: + # Enable or disable UFM InfiniBand metrics collection + # Default: false + metrics_enabled: false + + # Enable or disable UFM syslog logs collection + # Default: false + logs_enabled: false + + # UFM uses vmagent(shared) for metrics and VLAgent for logs + collection_targets: + - "victoria_metrics" + - "victoria_logs" + # ============================================================================ # TELEMETRY BRIDGES (Data Routers) @@ -349,3 +372,54 @@ powerscale_configurations: # Required when powerscale_configurations.powerscale_telemetry_support: true # Reference: https://raw.githubusercontent.com/dell/helm-charts/refs/heads/release-v1.16.3/charts/karavi-observability/values.yaml csm_observability_values_file_path: "" + +# -------------------------------------------------------------------------- +# UFM Telemetry Configuration +# -------------------------------------------------------------------------- +# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances +# using Prometheus scraping. +# +# DATA PIPELINE: +# UFM Prometheus Exporter → vmagent(shared) → victoria_metrics +# +# NOTE: UFM does NOT use Vector bridges. It uses the shared vmagent instance +# for metrics collection. +ufm_configuration: + # UFM appliance IP address or hostname + # Required when telemetry_sources.ufm.metrics_enabled is true + # Example: "172.20.44.180" or "ufm.example.com" + ufm_endpoint: "" + + # UFM Prometheus exporter port + # Default: 9001 (UFM default Prometheus port) + ufm_metrics_port: 9001 + + # Prometheus scrape interval for UFM metrics + # Accepted values: Prometheus duration format (e.g., "15s", "30s", "1m") + # Default: "30s" + scrape_interval: "30s" + + # Prometheus scrape timeout (must be <= scrape_interval) + # Accepted values: Prometheus duration format (e.g., "10s", "15s") + # Default: "15s" + scrape_timeout: "15s" + + # TLS mode for connecting to UFM Prometheus endpoint + # Accepted values: "self_signed", "ca_signed" + # - self_signed: Skip TLS verification (insecure_skip_verify=true) + # - ca_signed: Use CA certificate for TLS verification + # Default: "self_signed" + tls_mode: "self_signed" + + # Path to CA certificate file for UFM TLS verification + # Required when tls_mode is "ca_signed" + # Must be a valid PEM-format certificate file + # Default: "" (empty — not used when tls_mode is "self_signed") + ufm_ca_cert_path: "" + + # Authentication mode for UFM Prometheus endpoint + # Accepted values: "basic", "none" + # - basic: Use ufm_username/ufm_password from omnia_config_credentials.yml + # - none: No authentication (UFM endpoint is open) + # Default: "basic" + auth_mode: "basic" diff --git a/prepare_oim/prepare_oim.yml b/prepare_oim/prepare_oim.yml index ab05158e0a..51f9dc1f0c 100644 --- a/prepare_oim/prepare_oim.yml +++ b/prepare_oim/prepare_oim.yml @@ -102,7 +102,9 @@ (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or - (telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool) + (telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool) or + (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or + (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool) - name: Check discovery configuration for OME block: diff --git a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml index 14ad44d29b..1a8c914f9b 100644 --- a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml +++ b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml @@ -40,15 +40,3 @@ --host-bucket={{ cluster_boot_ip }}:9000 changed_when: true when: storage_backend == 'minio' - -- name: Update the boot policy (S3/PowerScale) - ansible.builtin.command: > - s3cmd setpolicy {{ s3_work_dir }}/s3-public-read-boot.json s3://boot-images - changed_when: true - when: storage_backend == 'powerscale' - -- name: Update the efi policy (S3/PowerScale) - ansible.builtin.command: > - s3cmd setpolicy {{ s3_work_dir }}/s3-public-read-efi.json s3://efi - changed_when: true - when: storage_backend == 'powerscale' diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 index 02b1e0be8c..60b0a47616 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 @@ -182,7 +182,17 @@ runcmd: - /usr/local/bin/set-ssh.sh - /usr/local/bin/configure_vast_installation.sh -{# Mount-specific runcmd entries #} + # DOCA prerequisites - mount /cert and prepare for DOCA installation + - mkdir -p {{ client_mount_path }}/slurm/ssh + - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools + - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - mount -av + - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + # DOCA and IB configuration - now ready before vendor_data mounts + - bash /usr/local/bin/doca-install.sh || true + - bash /usr/local/bin/configure-ib-network.sh +{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %} {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %} - {{ cmd }} @@ -222,11 +232,8 @@ - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path}}/hpc_tools /hpc_tools nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf {% if login_compiler_node_present %} - /usr/local/bin/generate_install_uuid.sh @@ -256,8 +263,7 @@ - echo " Starting LDMS setup " | tee -a /var/log/ldms-cloudinit.log - /root/ldms_sampler.sh {% endif %} - - bash /usr/local/bin/doca-install.sh || true - - bash /usr/local/bin/configure-ib-network.sh + - /usr/local/bin/check_slurm_controller_status.sh - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }} - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }} diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 index c14e89a82a..a4b89e1efa 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 @@ -181,7 +181,17 @@ runcmd: - /usr/local/bin/set-ssh.sh - /usr/local/bin/configure_vast_installation.sh -{# Mount-specific runcmd entries #} + # DOCA prerequisites - mount /cert and prepare for DOCA installation + - mkdir -p {{ client_mount_path }}/slurm/ssh + - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools + - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - mount -av + - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + # DOCA and IB configuration - now ready before vendor_data mounts + - bash /usr/local/bin/doca-install.sh || true + - bash /usr/local/bin/configure-ib-network.sh +{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %} {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %} - {{ cmd }} @@ -221,12 +231,9 @@ - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path}}/hpc_tools /hpc_tools nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -a - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf {% if login_compiler_node_present %} - /usr/local/bin/generate_install_uuid.sh @@ -257,8 +264,6 @@ - /root/ldms_sampler.sh {% endif %} - - bash /usr/local/bin/doca-install.sh || true - - bash /usr/local/bin/configure-ib-network.sh - /usr/local/bin/check_slurm_controller_status.sh - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }} - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }} diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2 index 0fc7739e1a..ad767a2e59 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2 @@ -140,7 +140,17 @@ runcmd: - /usr/local/bin/set-ssh.sh - /usr/local/bin/configure_vast_installation.sh -{# Mount-specific runcmd entries #} + # DOCA prerequisites - mount /cert and prepare for DOCA installation + - mkdir -p {{ client_mount_path }}/slurm/ssh + - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools + - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - mount -av + - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + # DOCA and IB configuration - now ready before vendor_data mounts + - bash /usr/local/bin/doca-install.sh || true + - bash /usr/local/bin/configure-ib-network.sh +{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %} {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %} - {{ cmd }} @@ -172,6 +182,7 @@ # Ensure Slurm NFS root is mounted at client_mount_path (e.g. /share_omnia) - mkdir -p {{ client_mount_path }}/slurm/ssh - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools + # Additional Slurm login node mounts (excluding cert which was mounted earlier) - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab @@ -179,14 +190,9 @@ - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path}}/hpc_tools /hpc_tools nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf - - - bash /usr/local/bin/doca-install.sh || true - - bash /usr/local/bin/configure-ib-network.sh + - mount -av - /usr/local/bin/check_slurm_controller_status.sh - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }} - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }} @@ -251,8 +257,6 @@ - systemctl start slurmd - systemctl daemon-reexec - systemctl restart sshd - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf - mkdir -p /etc/containers/registries.conf.d - mv /tmp/apptainer_mirror.conf /etc/containers/registries.conf.d/apptainer_mirror.conf diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2 index 7571acf7c7..faa5c234b6 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2 @@ -139,7 +139,17 @@ runcmd: - /usr/local/bin/set-ssh.sh - /usr/local/bin/configure_vast_installation.sh -{# Mount-specific runcmd entries #} + # DOCA prerequisites - mount /cert and prepare for DOCA installation + - mkdir -p {{ client_mount_path }}/slurm/ssh + - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools + - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - mount -av + - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + # DOCA and IB configuration - now ready before vendor_data mounts + - bash /usr/local/bin/doca-install.sh || true + - bash /usr/local/bin/configure-ib-network.sh +{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %} {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %} - {{ cmd }} @@ -168,10 +178,7 @@ {% for pv_entry in cloud_init_groups_dict[functional_group_name].powervault_scripts | default([]) %} - bash /usr/local/bin/setup_iscsi_storage_{{ pv_entry.name }}.sh {% endfor %} - # Ensure Slurm NFS root is mounted at client_mount_path (e.g. /share_omnia) - - mkdir -p {{ client_mount_path }}/slurm/ssh - - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools - - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + # Additional Slurm login node mounts (excluding cert which was mounted earlier) - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm {{ slurm_slurmd_log_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool/slurmd {{ slurm_slurmd_spool_dir_effective }} nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d /etc/slurm/epilog.d nfs defaults,_netdev 0 0" >> /etc/fstab @@ -179,14 +186,9 @@ - echo "{{ trackfile_nfs_path }} /var/log/track nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path}}/hpc_tools /hpc_tools nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab - echo "{{ cloud_init_nfs_path }}/packages /var/lib/packages nfs defaults,_netdev 0 0" >> /etc/fstab - - chmod {{ file_mode }} /etc/fstab - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf - - bash /usr/local/bin/doca-install.sh || true - - bash /usr/local/bin/configure-ib-network.sh + - mount -av - /usr/local/bin/check_slurm_controller_status.sh - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }} - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }} @@ -252,8 +254,6 @@ - systemctl start slurmd - systemctl daemon-reexec - systemctl restart sshd - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf - mkdir -p /etc/containers/registries.conf.d - mv /tmp/apptainer_mirror.conf /etc/containers/registries.conf.d/apptainer_mirror.conf diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 index 378fb8fe63..9a5b615eee 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 @@ -923,8 +923,8 @@ echo "Snapshot Controller deployment failed (expected). Continuing..." } - echo "Updating Snapshot Controller image to v8.3.0..." - kubectl set image deployment/snapshot-controller snapshot-controller=registry.k8s.io/sig-storage/snapshot-controller:v8.3.0 -n kube-system >/dev/null 2>&1 || true + echo "Updating Snapshot Controller image to v8.4.0..." + kubectl set image deployment/snapshot-controller snapshot-controller=registry.k8s.io/sig-storage/snapshot-controller:v8.4.0 -n kube-system >/dev/null 2>&1 || true echo "Waiting for Snapshot Controller rollout to finish (timeout: 5 minutes)..." kubectl rollout status deployment/snapshot-controller -n kube-system --timeout=300s >/dev/null 2>&1 || { @@ -966,7 +966,7 @@ INSTALL_SCRIPT="/opt/omnia/csi-driver-powerscale/csi-powerscale/dell-csi-helm-installer/csi-install.sh" if [ -x "$INSTALL_SCRIPT" ]; then cd "$(dirname "$INSTALL_SCRIPT")" || true - ./csi-install.sh --namespace isilon --values /opt/omnia/csi-driver-powerscale/values.yaml & + ./csi-install.sh --namespace isilon --values /opt/omnia/csi-driver-powerscale/values.yaml --skip-verify & CSI_PID=$! echo "Waiting for CSI install script (PID $CSI_PID) to complete..." wait $CSI_PID diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2 index bde54bd9e7..2ee561109c 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2 @@ -276,7 +276,17 @@ runcmd: - /usr/local/bin/set-ssh.sh -{# Mount-specific runcmd entries #} + # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts + - mkdir -p {{ client_mount_path }}/slurm/ssh + - mkdir -p {{ slurm_ctld_log_dir_effective }} {{ slurmdbd_log_dir_effective }} {{ slurm_ctld_pid_dir_effective }} {{ slurmdbd_pid_dir_effective }} {{ slurm_state_save_location_effective }} {% if slurm_sched_log_dir_effective %}{{ slurm_sched_log_dir_effective }} {% endif %}/etc/slurm {{ home_dir }} /etc/my.cnf.d /etc/munge /var/lib/mysql /var/log/mariadb /cert /var/log/track /var/lib/packages /hpc_tools/container_images /hpc_tools/scripts + - echo "{{ cloud_init_nfs_path }}/cert /cert nfs defaults,_netdev 0 0" >> /etc/fstab + - mount -av + - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + # DOCA and IB configuration - now ready before vendor_data mounts + - bash /usr/local/bin/doca-install.sh || true + - bash /usr/local/bin/configure-ib-network.sh +{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %} {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %} - {{ cmd }} @@ -327,11 +337,6 @@ - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab - chmod {{ file_mode }} /etc/fstab - mount -av - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf - - - bash /usr/local/bin/doca-install.sh || true - - bash /usr/local/bin/configure-ib-network.sh - chown -R {{ slurm_user }}:{{ slurm_user }} {{ home_dir }} - chmod {{ file_mode_755 }} {{ home_dir }} - chown -R {{ slurm_user }}:{{ slurm_user }} /etc/slurm diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 index 92687f38e5..2d4b7ad001 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 @@ -394,7 +394,14 @@ runcmd: - /usr/local/bin/set-ssh.sh - /usr/local/bin/configure_vast_installation.sh -{# Mount-specific runcmd entries #} + # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts + - /usr/local/bin/configure_dirs_and_mounts.sh + - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + # DOCA and IB configuration - now ready before vendor_data mounts + - bash /usr/local/bin/doca-install.sh || true + - bash /usr/local/bin/configure-ib-network.sh +{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %} {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %} - {{ cmd }} @@ -427,11 +434,7 @@ {% if dcgm_support %} - /usr/local/bin/setup_dcgm.sh {% endif %} - - /usr/local/bin/configure_dirs_and_mounts.sh - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf - - bash /usr/local/bin/doca-install.sh || true - - bash /usr/local/bin/configure-ib-network.sh + {% if slurm_node_present %} - | set -e diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 index 5ada0be1b7..3cae337b69 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 @@ -395,10 +395,18 @@ {{ pv_entry.content | indent(12) }} {% endfor %} + runcmd: - /usr/local/bin/set-ssh.sh - /usr/local/bin/configure_vast_installation.sh -{# Mount-specific runcmd entries #} + # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts + - /usr/local/bin/configure_dirs_and_mounts.sh + - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + # DOCA and IB configuration - now ready before vendor_data mounts + - bash /usr/local/bin/doca-install.sh || true + - bash /usr/local/bin/configure-ib-network.sh +{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %} {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %} - {{ cmd }} @@ -431,12 +439,6 @@ {% if dcgm_support %} - /usr/local/bin/setup_dcgm.sh {% endif %} - # slurm user and group created in the users module - - /usr/local/bin/configure_dirs_and_mounts.sh - - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust - - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf - - bash /usr/local/bin/doca-install.sh || true - - bash /usr/local/bin/configure-ib-network.sh {% if slurm_node_present %} - | set -e @@ -494,3 +496,4 @@ - systemctl restart slurmd - echo "Cloud-Init has completed successfully." + diff --git a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 index db8a7cb9cc..f1ce4389ef 100644 --- a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 +++ b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 @@ -52,26 +52,21 @@ else dnf install -y doca-ofed fi -echo "Unloading RDMA kernel modules..." -rmmod bnxt_re || true -rmmod mlx5_ib || true -rmmod ib_uverbs || true -rmmod xpmem || true -rmmod ib_core || true -rmmod mlx5_core || true +# NOTE: Removed module unload/reload sequence - DKMS handles module installation automatically +# The modprobe commands were causing module dependency issues and are not needed +# DOCA DKMS installation automatically handles kernel module replacement -echo "Loading RDMA kernel modules..." -modprobe mlx5_core || true -modprobe mlx5_ib || true -modprobe ib_core || true -modprobe ib_uverbs || true -modprobe ib_umad || true -modprobe ib_cm || true -modprobe rdma_cm || true -modprobe rdma_ucm || true -modprobe xpmem || true -modprobe knem || true -modprobe ib_ipoib || true +# NOTE: Remove MLNX OFED kernel modules to preserve standard RDMA compatibility +# This ensures rpcrdma and other standard RDMA functionality works correctly +echo "Removing MLNX OFED kernel modules to ensure standard RDMA compatibility" +if dkms status mlnx-ofa_kernel/25.10 -k $(uname -r) >/dev/null 2>&1; then + dkms uninstall mlnx-ofa_kernel/25.10 -k $(uname -r) || true + dkms remove mlnx-ofa_kernel/25.10 --all || true + depmod -a + echo "MLNX OFED kernel modules removed successfully, standard kernel modules restored" +else + echo "MLNX OFED kernel modules not found, skipping removal" +fi if command -v firewall-cmd &>/dev/null; then echo "Adding firewall ports..." @@ -86,4 +81,5 @@ else fi echo "DOCA-OFED installation completed successfully." + diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml new file mode 100644 index 0000000000..700d5be132 --- /dev/null +++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml @@ -0,0 +1,148 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# Deploy UFM InfiniBand telemetry (Metrics Only) +# Gated by: telemetry_sources.ufm.metrics_enabled +# +# DATA PIPELINE: +# UFM Prometheus Exporter (HTTPS) → vmagent(shared) → victoria_metrics +# +# What Omnia does: +# - Validates UFM endpoint, port, TLS mode, auth mode, credentials +# - Creates K8s Secret with UFM credentials (and optional CA cert) +# - VMAgent scrape config + deployment mount are handled by Jinja2 templates +# - Deploys Grafana dashboard as ConfigMap (sidecar auto-discovery) +# +# What Omnia does NOT do: +# - Omnia does NOT deploy or configure UFM itself + +- name: Set UFM configuration facts + ansible.builtin.set_fact: + _ufm_config: "{{ telemetry_config.telemetry_sources.ufm }}" + _ufm_detailed_config: "{{ telemetry_config.ufm_configuration }}" + +# --- Validate UFM telemetry inputs --- +# Only run validation when UFM telemetry is enabled + +- name: Validate UFM endpoint is not empty + ansible.builtin.fail: + msg: "{{ ufm_endpoint_empty_msg }}" + when: + - _ufm_config.metrics_enabled | default(false) | bool + - _ufm_detailed_config.ufm_endpoint | default('') | trim == '' + +- name: Validate UFM metrics port range + ansible.builtin.fail: + msg: "{{ ufm_port_range_msg }}" + when: >- + (_ufm_config.metrics_enabled | default(false) | bool) and + ((_ufm_detailed_config.ufm_metrics_port | default(9090) | int < 1) or + (_ufm_detailed_config.ufm_metrics_port | default(9090) | int > 65535)) + +- name: Validate UFM TLS mode + ansible.builtin.fail: + msg: "{{ ufm_tls_mode_invalid_msg }}" + when: + - _ufm_config.metrics_enabled | default(false) | bool + - _ufm_detailed_config.tls_mode | default('self_signed') not in ['self_signed', 'ca_signed'] + +- name: Validate UFM auth mode + ansible.builtin.fail: + msg: "{{ ufm_auth_mode_invalid_msg }}" + when: + - _ufm_config.metrics_enabled | default(false) | bool + - _ufm_detailed_config.auth_mode | default('basic') not in ['basic', 'none'] + +- name: Validate UFM credentials when auth_mode is basic + ansible.builtin.fail: + msg: "{{ ufm_credentials_missing_msg }}" + when: + - _ufm_config.metrics_enabled | default(false) | bool + - _ufm_detailed_config.auth_mode | default('basic') == 'basic' + - >- + (hostvars['localhost']['ufm_username'] | default('') | trim == '') or + (hostvars['localhost']['ufm_password'] | default('') | trim == '') + +- name: Validate CA certificate file when tls_mode is ca_signed + when: + - _ufm_config.metrics_enabled | default(false) | bool + - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' + block: + - name: Check CA certificate file exists + ansible.builtin.stat: + path: "{{ _ufm_detailed_config.ufm_ca_cert_path }}" + register: _ufm_ca_cert_stat + delegate_to: localhost + + - name: Fail if CA certificate file not found + ansible.builtin.fail: + msg: "{{ ufm_ca_cert_not_found_msg }}" + when: >- + (_ufm_detailed_config.ufm_ca_cert_path | default('') | trim == '') or + (not _ufm_ca_cert_stat.stat.exists | default(false)) + +- name: Display validated UFM configuration + ansible.builtin.debug: + msg: + - "UFM endpoint: {{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}" + - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}" + - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}" + verbosity: 2 + when: _ufm_config.metrics_enabled | default(false) | bool + + +# --- Create UFM credentials Kubernetes Secret --- + +- name: Read CA certificate content + ansible.builtin.slurp: + src: "{{ _ufm_detailed_config.ufm_ca_cert_path }}" + register: _ufm_ca_cert_content + delegate_to: localhost + when: + - _ufm_config.metrics_enabled | default(false) | bool + - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' + - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' + +- name: Set CA certificate content for template processing + ansible.builtin.set_fact: + ufm_ca_cert_content: "{{ _ufm_ca_cert_content.content | b64decode }}" + when: + - _ufm_ca_cert_content is defined + - _ufm_ca_cert_content is not skipped + - _ufm_ca_cert_content.content is defined + +- name: Display UFM Secret creation status + ansible.builtin.debug: + msg: "{{ ufm_secret_created_msg }}" + verbosity: 2 + when: _ufm_config.metrics_enabled | default(false) | bool + + +# UFM telemetry resources are now deployed via Kustomize templates +# Secret and external-service ConfigMap are created through: +# - templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2 +# - templates/telemetry/ufm/ufm-external-service.yaml.j2 +# These are processed in generate_telemetry_deployments.yml and applied via kubectl apply -k + +- name: Display UFM integration summary + ansible.builtin.debug: + msg: + - "UFM telemetry integration complete" + - "UFM endpoint: {{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}" + - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}" + - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}" + - "Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}" + verbosity: 2 + when: _ufm_config.metrics_enabled | default(false) | bool diff --git a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml index ac4b6df5ad..0d97419187 100644 --- a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml +++ b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml @@ -50,6 +50,7 @@ powerscale_metrics_enabled: "{{ telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool }}" powerscale_log_enabled: "{{ telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool }}" dcgm_support: "{{ telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(true) | bool }}" + ufm_telemetry_support: "{{ telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool }}" cacheable: true - name: Map powerscale source + configurations to legacy powerscale_configurations @@ -68,7 +69,8 @@ cacheable: true when: >- 'victoria_metrics' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([])) or - 'victoria_metrics' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([])) + 'victoria_metrics' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([])) or + 'victoria_metrics' in (telemetry_config.telemetry_sources.ufm.collection_targets | default([])) - name: Check if any source targets victoria_logs ansible.builtin.set_fact: @@ -76,7 +78,8 @@ cacheable: true when: >- 'victoria_logs' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([])) or - 'victoria_logs' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([])) + 'victoria_logs' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([])) or + 'victoria_logs' in (telemetry_config.telemetry_sources.ufm.collection_targets | default([])) - name: Check if any source targets Kafka ansible.builtin.set_fact: @@ -138,4 +141,6 @@ victoria_metrics_support or victoria_logs_support or ldms_support or - kafka_support + kafka_support or + ufm_telemetry_support or + ufm_log_enabled diff --git a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml index ea25d10924..de726a140c 100644 --- a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml +++ b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml @@ -234,6 +234,20 @@ - victoria_logs_support | default(false) | bool tags: telemetry_deployment +- name: Populate UFM telemetry deployment configs + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ hostvars['localhost']['k8s_client_share_path'] }}/telemetry/deployments/{{ item.dest }}" + mode: "{{ hostvars['localhost']['file_permissions_644'] }}" + loop: + - src: 'telemetry/ufm/ufm-telemetry-secret.yaml.j2' + dest: 'ufm-telemetry-secret.yaml' + - src: 'telemetry/ufm/ufm-external-service.yaml.j2' + dest: 'ufm-external-service.yaml' + when: telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool + tags: telemetry_deployment + + - name: Deploy telemetry cleanup script ansible.builtin.template: src: 'telemetry/cleanup_telemetry.sh.j2' diff --git a/provision/roles/telemetry/tasks/main.yml b/provision/roles/telemetry/tasks/main.yml index ebcb424462..84c16c9941 100644 --- a/provision/roles/telemetry/tasks/main.yml +++ b/provision/roles/telemetry/tasks/main.yml @@ -44,6 +44,8 @@ (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or + (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or + (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool) or ldms_support | default(false) | bool block: - name: Set NFS info fact @@ -58,12 +60,18 @@ - telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool ansible.builtin.include_tasks: deploy_powerscale_metrics.yml + - name: Deploy UFM telemetry metrics + when: + - telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool + ansible.builtin.include_tasks: deploy_ufm_telemetry.yml + - name: Generate telemetry deployments (idrac/ldms/powerscale/vector) when: - >- (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or + (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool) or (telemetry_config.telemetry_bridges.vector_ome.logs_enabled | default(false) | bool) or ldms_support | default(false) | bool @@ -119,11 +127,7 @@ - name: Generate telemetry deployment script ansible.builtin.include_tasks: generate_telemetry_script.yml when: - - >- - (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or - (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or - (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or - ldms_support | default(false) | bool + - telemetry_enabled | default(false) | bool tags: - telemetry_deployment diff --git a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2 b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2 index 2dc831ba25..65ef9a3bfd 100644 --- a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2 @@ -19,4 +19,4 @@ metadata: type: Opaque stringData: cluster-id: "{{ cluster_id.stdout }}" -{% endif %} \ No newline at end of file +{% endif %} diff --git a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 index 5eb7f1ee90..c55f7a5f74 100644 --- a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 @@ -58,6 +58,11 @@ resources: - telemetry_cleaner_rbac.yaml - telemetry_pod_cleanup.yaml {% endif %} +{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) %} +# UFM InfiniBand Telemetry Resources + - ufm-telemetry-secret.yaml + - ufm-external-service.yaml +{% endif %} {% if telemetry_config.telemetry_bridges.vector_ldms.metrics_enabled | default(false) and ldms_support | default(false) and kafka_support | default(false) and victoria_metrics_support | default(false) %} # Vector-LDMS Bridge Resources (Kafka-to-VictoriaMetrics pipeline) - vector/vmagent-vector-deployment.yaml diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2 new file mode 100644 index 0000000000..d6ccf90b37 --- /dev/null +++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2 @@ -0,0 +1,44 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool %} +apiVersion: v1 +kind: Service +metadata: + name: ufm-external + namespace: {{ telemetry_namespace }} + labels: + app: ufm-external + app.kubernetes.io/managed-by: omnia + app.kubernetes.io/component: ufm-telemetry +spec: + ports: + - name: metrics + port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }} + clusterIP: None + +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: ufm-external + namespace: {{ telemetry_namespace }} +subsets: + - addresses: + - ip: "{{ telemetry_config.ufm_configuration.ufm_endpoint }}" + ports: + - name: metrics + port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }} +{% endif %} diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2 new file mode 100644 index 0000000000..a641f3d93f --- /dev/null +++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2 @@ -0,0 +1,58 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- + +{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool %} +{% set ufm_config = telemetry_config.telemetry_sources.ufm %} +{% set ufm_detailed_config = telemetry_config.ufm_configuration %} +{% set auth_mode = ufm_detailed_config.auth_mode | default('basic') %} +{% set tls_mode = ufm_detailed_config.tls_mode | default('self_signed') %} + +{% if auth_mode == 'basic' %} +apiVersion: v1 +kind: Secret +metadata: + name: ufm-telemetry-credentials + namespace: {{ telemetry_namespace }} + labels: + app.kubernetes.io/managed-by: omnia + app.kubernetes.io/component: ufm-telemetry +type: Opaque +{% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %} +data: + username: {{ (hostvars['localhost']['ufm_username'] | string) | b64encode }} + password: {{ (hostvars['localhost']['ufm_password'] | string) | b64encode }} + ca.crt: {{ (ufm_ca_cert_content | default('') | string) | b64encode }} +{% else %} +stringData: + username: "{{ hostvars['localhost']['ufm_username'] | default('') | string }}" + password: "{{ hostvars['localhost']['ufm_password'] | default('') | string }}" +{% endif %} +{% else %} +{% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %} +apiVersion: v1 +kind: Secret +metadata: + name: ufm-telemetry-credentials + namespace: {{ telemetry_namespace }} + labels: + app.kubernetes.io/managed-by: omnia + app.kubernetes.io/component: ufm-telemetry +type: Opaque +data: + ca.crt: {{ (ufm_ca_cert_content | default('') | string) | b64encode }} +{% endif %} +{% endif %} +{% endif %} diff --git a/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2 b/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2 index 74ed4a80a3..3c9196ff06 100644 --- a/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2 @@ -105,3 +105,58 @@ spec: - targetLabel: component replacement: volume-status {% endif %} + +--- +{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) %} +# VMServiceScrape for UFM InfiniBand Metrics +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + name: ufm-infiniband-metrics + namespace: {{ telemetry_namespace }} +spec: + jobLabel: ufm + selector: + matchLabels: + app: ufm-external + endpoints: + - port: metrics + interval: {{ telemetry_config.ufm_configuration.scrape_interval | default('30s') }} + scrapeTimeout: {{ telemetry_config.ufm_configuration.scrape_timeout | default('15s') }} +{% if telemetry_config.ufm_configuration.tls_mode | default('self_signed') == 'ca_signed' %} + scheme: https + tlsConfig: + caFile: /etc/ufm-credentials/ca.crt + insecureSkipVerify: false +{% elif telemetry_config.ufm_configuration.tls_mode | default('self_signed') == 'self_signed' %} + scheme: https + tlsConfig: + insecureSkipVerify: true +{% else %} + scheme: http +{% endif %} + path: /metrics + honorLabels: true +{% if telemetry_config.ufm_configuration.auth_mode | default('basic') == 'basic' %} + basicAuth: + username: + name: ufm-telemetry-credentials + key: username + password: + name: ufm-telemetry-credentials + key: password +{% endif %} + relabelings: + - action: replace + sourceLabels: [] + targetLabel: source + replacement: ufm + - action: replace + sourceLabels: [] + targetLabel: subsystem + replacement: infiniband + - action: replace + sourceLabels: [] + targetLabel: job + replacement: ufm-infiniband-metrics +{% endif %} diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml index 3ecc0fb0e0..829c74ef47 100644 --- a/provision/roles/telemetry/vars/main.yml +++ b/provision/roles/telemetry/vars/main.yml @@ -658,3 +658,61 @@ vector: limits: cpu: "500m" memory: "512Mi" + +# ============================================================================ +# UFM Telemetry Configuration +# ============================================================================ +# Usage: deploy_ufm_telemetry.yml +# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances +# using Prometheus scraping via the shared vmagent instance. + +# Kubernetes resource names +ufm_credentials_secret_name: "ufm-telemetry-credentials" +ufm_grafana_dashboard_configmap_name: "grafana-dashboard-ufm" + +# UFM telemetry defaults +ufm_deploy_grafana_dashboard: true +ufm_endpoint_test_timeout: 10 + +# Validation messages +ufm_endpoint_empty_msg: >- + UFM telemetry is enabled but ufm_endpoint is empty. + Set telemetry_sources.ufm.ufm_endpoint in telemetry_config.yml. + +ufm_credentials_missing_msg: >- + UFM auth_mode is 'basic' but ufm_username or ufm_password is not set. + Provide UFM credentials in omnia_config_credentials.yml. + +ufm_ca_cert_not_found_msg: >- + UFM tls_mode is 'ca_signed' but the CA certificate file was not found + at the path specified in ufm_ca_cert_path. Provide a valid PEM certificate file. + +ufm_port_range_msg: >- + UFM metrics port must be between 1 and 65535. + +ufm_tls_mode_invalid_msg: >- + UFM tls_mode must be 'self_signed' or 'ca_signed'. + +ufm_auth_mode_invalid_msg: >- + UFM auth_mode must be 'basic' or 'none'. + +ufm_endpoint_unreachable_msg: >- + WARNING: UFM endpoint is not reachable. UFM telemetry configuration will + proceed, but VMAgent will not be able to scrape metrics until UFM is accessible. + +ufm_secret_created_msg: >- + UFM credentials Kubernetes Secret created in namespace '{{ telemetry_namespace }}'. + +ufm_victoria_logs_validation_fail_msg: >- + UFM log collection requires 'victoria_logs' in + telemetry_sources.ufm.collection_targets. + + +ufm_integration_summary_msg: >- + UFM telemetry integration complete. + {% if _ufm_config.metrics_enabled | default(false) | bool %} + Metrics: https://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }} + TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }} + Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }} + Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }} + {% endif %} diff --git a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2 b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2 index bf231510ed..d9ffd938b4 100644 --- a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2 +++ b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2 @@ -54,3 +54,7 @@ gitlab_root_password: "" # OME discovery credentials ome_username: "" ome_password: "" + +# UFM telemetry credentials +ufm_username: "" +ufm_password: "" diff --git a/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml b/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml index 7f904b227a..8207abd29b 100644 --- a/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml +++ b/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml @@ -30,6 +30,7 @@ - >- service.key in software_names or service.key in ["provision", "prepare_oim", "local_repo", "idrac_telemetry", - "visualization", "build_aarch_image", "gitlab", "csi_driver_powerscale", "discovery"] + "visualization", "build_aarch_image", "gitlab", "csi_driver_powerscale", "discovery", + "ufm_telemetry"] - (omnia_run_tags | default([]) | difference(['all']) | length == 0) or service.key in (omnia_run_tags | default([])) diff --git a/utils/credential_utility/roles/update_config/vars/main.yml b/utils/credential_utility/roles/update_config/vars/main.yml index 6c60ad6110..8ebc0ea3be 100644 --- a/utils/credential_utility/roles/update_config/vars/main.yml +++ b/utils/credential_utility/roles/update_config/vars/main.yml @@ -118,3 +118,8 @@ omnia_credentials: - username: ome_username password: ome_password condition: "{{ enable_bmc_discovery | default(false) | bool }}" + ufm_telemetry: + conditional_mandatory: + - username: ufm_username + password: ufm_password + condition: "{{ ufm_telemetry_support | default(false) | bool }}" diff --git a/utils/credential_utility/roles/validation/tasks/pre_requisite.yml b/utils/credential_utility/roles/validation/tasks/pre_requisite.yml index e12c09aa28..f7b3f3bbea 100644 --- a/utils/credential_utility/roles/validation/tasks/pre_requisite.yml +++ b/utils/credential_utility/roles/validation/tasks/pre_requisite.yml @@ -48,3 +48,17 @@ when: - not result.skipped | default(false) - result.telemetry_status_list | length > 0 + +- name: Derive UFM telemetry support flag from telemetry_config.yml + when: "'ufm_telemetry' in (omnia_run_tags | default([]))" + block: + - name: Load telemetry_config.yml for UFM support flag + ansible.builtin.include_vars: + file: "{{ telemetry_config_path }}" + name: _telemetry_config + failed_when: false + + - name: Set ufm_telemetry_support flag + ansible.builtin.set_fact: + ufm_telemetry_support: "{{ _telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool }}" + when: _telemetry_config is defined and _telemetry_config.telemetry_sources is defined