From a1d0727b65a69855a88876f842c9ee143d79f00f Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Tue, 12 May 2026 20:38:09 +0530
Subject: [PATCH 01/32] ufm telemetry support [metrics,syslog]

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../schema/telemetry_config.json              |  80 +++++-
 .../validation_flows/telemetry_validation.py  |  67 +++++
 .../library/modules/fetch_telemetry_status.py |   7 +
 input/telemetry_config.yml                    |  74 ++++++
 ...ce_kube_control_plane_first_x86_64.yaml.j2 |   4 +-
 .../telemetry/tasks/deploy_ufm_telemetry.yml  | 234 ++++++++++++++++++
 .../tasks/derive_sink_support_flags.yml       |   7 +-
 .../tasks/generate_telemetry_deployments.yml  |  14 ++
 provision/roles/telemetry/tasks/main.yml      |   8 +-
 .../common/telemetry_secret_creation.yaml.j2  |   2 +-
 .../ufm/ufm-external-service.yaml.j2          |  46 ++++
 .../ufm/ufm-telemetry-secret.yaml.j2          |  58 +++++
 .../victoria-operator-vmscrape.yaml.j2        |  55 ++++
 provision/roles/telemetry/vars/main.yml       |  58 +++++
 14 files changed, 706 insertions(+), 8 deletions(-)
 create mode 100644 provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
 create mode 100644 provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
 create mode 100644 provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2

diff --git a/common/library/module_utils/input_validation/schema/telemetry_config.json b/common/library/module_utils/input_validation/schema/telemetry_config.json
index 28260bc42f..c0ce313832 100644
--- a/common/library/module_utils/input_validation/schema/telemetry_config.json
+++ b/common/library/module_utils/input_validation/schema/telemetry_config.json
@@ -94,9 +94,37 @@
             }
           },
           "required": ["metrics_enabled", "collection_targets"]
+        },
+        "ufm": {
+          "type": "object",
+          "description": "NVIDIA UFM InfiniBand Fabric telemetry.",
+          "properties": {
+            "metrics_enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "Enable or disable UFM InfiniBand metrics collection."
+            },
+            "logs_enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "Enable or disable UFM syslog logs collection."
+            },
+            "collection_targets": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": ["victoria_metrics", "victoria_logs"]
+              },
+              "minItems": 1,
+              "uniqueItems": true,
+              "default": ["victoria_metrics"],
+              "description": "UFM supports victoria_metrics (metrics) and victoria_logs (logs)."
+            }
+          },
+          "required": ["metrics_enabled", "collection_targets"]
         }
       },
-      "required": ["idrac", "ldms", "dcgm", "powerscale"]
+      "required": ["idrac", "ldms", "dcgm", "powerscale", "ufm"]
     },
     "telemetry_bridges": {
       "type": "object",
@@ -398,7 +426,55 @@
         }
       },
       "required": ["otel_collector_storage_size", "csm_observability_values_file_path"]
+    },
+    "ufm_configuration": {
+      "type": "object",
+      "description": "UFM telemetry detailed configurations.",
+      "properties": {
+        "ufm_endpoint": {
+          "type": "string",
+          "minLength": 1,
+          "description": "UFM appliance IP address or hostname."
+        },
+        "ufm_metrics_port": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 65535,
+          "default": 9001,
+          "description": "UFM Prometheus exporter port."
+        },
+        "scrape_interval": {
+          "type": "string",
+          "pattern": "^[0-9]+[smh]$",
+          "default": "30s",
+          "description": "Prometheus scrape interval for UFM metrics."
+        },
+        "scrape_timeout": {
+          "type": "string",
+          "pattern": "^[0-9]+[smh]$",
+          "default": "15s",
+          "description": "Prometheus scrape timeout (must be <= scrape_interval)."
+        },
+        "tls_mode": {
+          "type": "string",
+          "enum": ["self_signed", "ca_signed"],
+          "default": "self_signed",
+          "description": "TLS mode for connecting to UFM Prometheus endpoint."
+        },
+        "ufm_ca_cert_path": {
+          "type": "string",
+          "default": "",
+          "description": "Path to CA certificate file for UFM TLS verification."
+        },
+        "auth_mode": {
+          "type": "string",
+          "enum": ["basic", "none"],
+          "default": "basic",
+          "description": "Authentication mode for UFM Prometheus endpoint."
+        }
+      },
+      "required": ["ufm_endpoint"]
     }
   },
-  "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations"]
+  "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations", "ufm_configuration"]
 }
diff --git a/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py b/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py
index cd2ee50b4d..6b8299a86b 100644
--- a/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py
+++ b/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py
@@ -315,6 +315,7 @@ def validate_telemetry_config(
     idrac_source = telemetry_sources.get("idrac", {})
     ldms_source = telemetry_sources.get("ldms", {})
     powerscale_source = telemetry_sources.get("powerscale", {})
+    ufm_source = telemetry_sources.get("ufm", {})
 
     idrac_telemetry_support = idrac_source.get("metrics_enabled", False)
     idrac_collection_targets = idrac_source.get("collection_targets", [])
@@ -370,6 +371,17 @@ def validate_telemetry_config(
             list(invalid_powerscale_targets),
             f"Invalid collection targets for PowerScale. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_powerscale_targets}"
         ))
+    
+    # UFM: supports victoria_metrics and victoria_logs
+    ufm_targets = set(ufm_source.get("collection_targets", []))
+    allowed_ufm_targets = {"victoria_metrics", "victoria_logs"}
+    invalid_ufm_targets = ufm_targets - allowed_ufm_targets
+    if invalid_ufm_targets:
+        errors.append(create_error_msg(
+            "telemetry_sources.ufm.collection_targets",
+            list(invalid_ufm_targets),
+            f"Invalid collection targets for UFM. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_ufm_targets}"
+        ))
 
     # =========================================================================
     # Validate service cluster and slurm cluster
@@ -666,4 +678,59 @@ def validate_telemetry_config(
         is_service_cluster_defined, config_paths, logger, errors
     )
 
+    # =========================================================================
+    # Validate UFM telemetry configuration
+    # =========================================================================
+    ufm_metrics_enabled = ufm_source.get("metrics_enabled", False)
+    ufm_logs_enabled = ufm_source.get("logs_enabled", False)
+    ufm_detailed_config = data.get("ufm_configuration", {})
+    
+    if ufm_metrics_enabled or ufm_logs_enabled:
+        # Check required UFM endpoint
+        ufm_endpoint = ufm_detailed_config.get("ufm_endpoint", "")
+        if not ufm_endpoint or (isinstance(ufm_endpoint, str) and ufm_endpoint.strip() == ""):
+            errors.append(create_error_msg(
+                "ufm_configuration.ufm_endpoint",
+                ufm_endpoint,
+                "ufm_endpoint is required when UFM telemetry is enabled. Provide the UFM appliance IP address or hostname."
+            ))
+        
+        # Validate UFM metrics port if metrics enabled
+        if ufm_metrics_enabled:
+            ufm_metrics_port = ufm_detailed_config.get("ufm_metrics_port", 9001)
+            if not isinstance(ufm_metrics_port, int) or ufm_metrics_port < 1 or ufm_metrics_port > 65535:
+                errors.append(create_error_msg(
+                    "ufm_configuration.ufm_metrics_port",
+                    ufm_metrics_port,
+                    "ufm_metrics_port must be an integer between 1 and 65535."
+                ))
+        
+        # Validate TLS mode
+        tls_mode = ufm_detailed_config.get("tls_mode", "self_signed")
+        if tls_mode not in ["self_signed", "ca_signed"]:
+            errors.append(create_error_msg(
+                "ufm_configuration.tls_mode",
+                tls_mode,
+                "tls_mode must be 'self_signed' or 'ca_signed'."
+            ))
+        
+        # Validate CA certificate path when tls_mode is ca_signed
+        if tls_mode == "ca_signed":
+            ca_cert_path = ufm_detailed_config.get("ufm_ca_cert_path", "")
+            if not ca_cert_path or (isinstance(ca_cert_path, str) and ca_cert_path.strip() == ""):
+                errors.append(create_error_msg(
+                    "ufm_configuration.ufm_ca_cert_path",
+                    ca_cert_path,
+                    "ufm_ca_cert_path is required when tls_mode is 'ca_signed'. Provide path to CA certificate file."
+                ))
+        
+        # Validate auth mode
+        auth_mode = ufm_detailed_config.get("auth_mode", "basic")
+        if auth_mode not in ["basic", "none"]:
+            errors.append(create_error_msg(
+                "ufm_configuration.auth_mode",
+                auth_mode,
+                "auth_mode must be 'basic' or 'none'."
+            ))
+
     return errors
diff --git a/common/library/modules/fetch_telemetry_status.py b/common/library/modules/fetch_telemetry_status.py
index d6db54ab97..622d05846f 100644
--- a/common/library/modules/fetch_telemetry_status.py
+++ b/common/library/modules/fetch_telemetry_status.py
@@ -72,6 +72,13 @@ def main():
     if telemetry_sources.get("idrac", {}).get("metrics_enabled", False):
         telemetry_status_list.append("idrac_telemetry")
 
+    # Check UFM telemetry
+    ufm_config = telemetry_sources.get("ufm", {})
+    if ufm_config.get("metrics_enabled", False):
+        telemetry_status_list.append("ufm_telemetry")
+    if ufm_config.get("logs_enabled", False):
+        telemetry_status_list.append("ufm_logs")
+
     module.exit_json(
             changed=False,
             telemetry_status_list=telemetry_status_list
diff --git a/input/telemetry_config.yml b/input/telemetry_config.yml
index a66c418ecb..17491767af 100644
--- a/input/telemetry_config.yml
+++ b/input/telemetry_config.yml
@@ -146,6 +146,29 @@ telemetry_sources:
       - "victoria_metrics"
       - "victoria_logs"
 
+  # --------------------------------------------------------------------------
+  # UFM — NVIDIA UFM InfiniBand Fabric Telemetry
+  # --------------------------------------------------------------------------
+  # Collects: IB port state, transmit/receive data, error counters, fabric topology
+  # Requires: NVIDIA UFM appliance with Prometheus exporter enabled (port 9090)
+  # Data path: UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
+  #
+  # NOTE: Omnia does NOT deploy UFM itself. Omnia configures the existing
+  # vmagent to scrape the UFM Prometheus endpoint for IB fabric metrics.
+  ufm:
+    # Enable or disable UFM InfiniBand metrics collection
+    # Default: false
+    metrics_enabled: false
+
+    # Enable or disable UFM syslog logs collection
+    # Default: false
+    logs_enabled: false
+
+    # UFM uses vmagent(shared) for metrics and VLAgent for logs
+    collection_targets:
+      - "victoria_metrics"
+      - "victoria_logs"
+
 
 # ============================================================================
 # TELEMETRY BRIDGES (Data Routers)
@@ -349,3 +372,54 @@ powerscale_configurations:
   # Required when powerscale_configurations.powerscale_telemetry_support: true
   # Reference: https://raw.githubusercontent.com/dell/helm-charts/refs/heads/release-v1.16.3/charts/karavi-observability/values.yaml
   csm_observability_values_file_path: ""
+
+# --------------------------------------------------------------------------
+# UFM Telemetry Configuration
+# --------------------------------------------------------------------------
+# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances
+# using Prometheus scraping.
+#
+# DATA PIPELINE:
+#   UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
+#
+# NOTE: UFM does NOT use Vector bridges. It uses the shared vmagent instance
+#       for metrics collection.
+ufm_configuration:
+  # UFM appliance IP address or hostname
+  # Required when telemetry_sources.ufm.metrics_enabled is true
+  # Example: "172.20.44.180" or "ufm.example.com"
+  ufm_endpoint: ""
+
+  # UFM Prometheus exporter port
+  # Default: 9001 (UFM default Prometheus port)
+  ufm_metrics_port: 9001
+
+  # Prometheus scrape interval for UFM metrics
+  # Accepted values: Prometheus duration format (e.g., "15s", "30s", "1m")
+  # Default: "30s"
+  scrape_interval: "30s"
+
+  # Prometheus scrape timeout (must be <= scrape_interval)
+  # Accepted values: Prometheus duration format (e.g., "10s", "15s")
+  # Default: "15s"
+  scrape_timeout: "15s"
+
+  # TLS mode for connecting to UFM Prometheus endpoint
+  # Accepted values: "self_signed", "ca_signed"
+  # - self_signed: Skip TLS verification (insecure_skip_verify=true)
+  # - ca_signed: Use CA certificate for TLS verification
+  # Default: "self_signed"
+  tls_mode: "self_signed"
+
+  # Path to CA certificate file for UFM TLS verification
+  # Required when tls_mode is "ca_signed"
+  # Must be a valid PEM-format certificate file
+  # Default: "" (empty — not used when tls_mode is "self_signed")
+  ufm_ca_cert_path: ""
+
+  # Authentication mode for UFM Prometheus endpoint
+  # Accepted values: "basic", "none"
+  # - basic: Use ufm_username/ufm_password from omnia_config_credentials.yml
+  # - none: No authentication (UFM endpoint is open)
+  # Default: "basic"
+  auth_mode: "basic"
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
index fb549a10fa..e93ffed079 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
@@ -321,7 +321,7 @@
               ipAddressPools:
               - first-pool
 
-{% if telemetry_enabled %}
+{% if telemetry_enabled or ldms_support or ufm_telemetry_support%}
         - path: /root/telemetry.sh
           owner: root:root
           permissions: '0755'
@@ -1073,7 +1073,7 @@
 {% include 'powerscale/deploy_powerscale_telemetry.sh.j2' %}
 {% endif %}
 
-{% if telemetry_enabled %}
+{% if telemetry_enabled or ldms_support or ufm_telemetry_support%}
             echo "Applying Telemetry Kubernetes deployments"
             /root/telemetry.sh
 {% endif %}
diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
new file mode 100644
index 0000000000..be43e590cd
--- /dev/null
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -0,0 +1,234 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# Deploy UFM InfiniBand telemetry (Metrics Only)
+# Gated by: telemetry_sources.ufm.metrics_enabled
+#
+# DATA PIPELINE:
+#   UFM Prometheus Exporter (HTTPS) → vmagent(shared) → victoria_metrics
+#
+# What Omnia does:
+#   - Validates UFM endpoint, port, TLS mode, auth mode, credentials
+#   - Creates K8s Secret with UFM credentials (and optional CA cert)
+#   - VMAgent scrape config + deployment mount are handled by Jinja2 templates
+#   - Deploys Grafana dashboard as ConfigMap (sidecar auto-discovery)
+#
+# What Omnia does NOT do:
+#   - Omnia does NOT deploy or configure UFM itself
+
+- name: Set UFM configuration facts
+  ansible.builtin.set_fact:
+    _ufm_config: "{{ telemetry_config.telemetry_sources.ufm }}"
+    _ufm_detailed_config: "{{ telemetry_config.ufm_configuration }}"
+
+- name: Load UFM credentials from omnia_config_credentials.yml
+  block:
+    - name: Check if credentials file is encrypted
+      ansible.builtin.command: >-
+        cat "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
+      changed_when: false
+      register: _ufm_cred_file_content
+      delegate_to: localhost
+      no_log: true
+
+    - name: Decrypt credentials file if encrypted
+      ansible.builtin.command: >-
+        ansible-vault decrypt "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
+        --vault-password-file "{{ hostvars['localhost']['input_project_dir'] }}/.omnia_config_credentials_key"
+      when: "'$ANSIBLE_VAULT;' in _ufm_cred_file_content.stdout"
+      changed_when: false
+      delegate_to: localhost
+
+    - name: Include credentials and extract UFM credentials
+      ansible.builtin.include_vars:
+        file: "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
+        name: _omnia_creds
+      no_log: true
+
+    - name: Set UFM credentials as facts for template processing
+      ansible.builtin.set_fact:
+        ufm_username: "{{ _omnia_creds.ufm_username | default('') }}"
+        ufm_password: "{{ _omnia_creds.ufm_password | default('') }}"
+      no_log: true
+
+    - name: Set CA certificate content for template processing
+      ansible.builtin.set_fact:
+        ufm_ca_cert_content: "{{ _ufm_ca_cert_content.content | default('') }}"
+      when: 
+        - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
+        - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != ''
+
+    - name: Debug UFM credentials and file path
+      ansible.builtin.debug:
+        msg:
+          - "Credentials file path: {{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
+          - "UFM username found: {{ _omnia_creds.ufm_username is defined }}"
+          - "UFM password found: {{ _omnia_creds.ufm_password is defined }}"
+          - "UFM username length: {{ _omnia_creds.ufm_username | default('') | length }}"
+          - "UFM password length: {{ _omnia_creds.ufm_password | default('') | length }}"
+          - "Input project dir: {{ hostvars['localhost']['input_project_dir'] }}"
+
+    - name: Re-encrypt credentials file if it was encrypted
+      ansible.builtin.command: >-
+        ansible-vault encrypt "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
+        --vault-password-file "{{ hostvars['localhost']['input_project_dir'] }}/.omnia_config_credentials_key"
+      when: "'$ANSIBLE_VAULT;' in _ufm_cred_file_content.stdout"
+      changed_when: false
+      delegate_to: localhost
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+# --- Validate UFM telemetry inputs ---
+# Only run validation when UFM telemetry is enabled
+
+- name: Validate UFM endpoint is not empty
+  ansible.builtin.fail:
+    msg: "{{ ufm_endpoint_empty_msg }}"
+  when: 
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.ufm_endpoint | default('') | trim == ''
+
+- name: Validate UFM metrics port range
+  ansible.builtin.fail:
+    msg: "{{ ufm_port_range_msg }}"
+  when: >-
+    (_ufm_config.metrics_enabled | default(false) | bool) and
+    ((_ufm_detailed_config.ufm_metrics_port | default(9090) | int < 1) or
+    (_ufm_detailed_config.ufm_metrics_port | default(9090) | int > 65535))
+
+- name: Validate UFM TLS mode
+  ansible.builtin.fail:
+    msg: "{{ ufm_tls_mode_invalid_msg }}"
+  when: 
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.tls_mode | default('self_signed') not in ['self_signed', 'ca_signed']
+
+- name: Validate UFM auth mode
+  ansible.builtin.fail:
+    msg: "{{ ufm_auth_mode_invalid_msg }}"
+  when: 
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.auth_mode | default('basic') not in ['basic', 'none']
+
+
+- name: Validate UFM credentials when auth_mode is basic
+  ansible.builtin.fail:
+    msg: "{{ ufm_credentials_missing_msg }}"
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.auth_mode | default('basic') == 'basic'
+    - (ufm_username | default('') | trim == '') or (ufm_password | default('') | trim == '')
+
+- name: Validate CA certificate file when tls_mode is ca_signed
+  when: 
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
+  block:
+    - name: Check CA certificate file exists
+      ansible.builtin.stat:
+        path: "{{ _ufm_detailed_config.ufm_ca_cert_path }}"
+      register: _ufm_ca_cert_stat
+      delegate_to: localhost
+
+    - name: Fail if CA certificate file not found
+      ansible.builtin.fail:
+        msg: "{{ ufm_ca_cert_not_found_msg }}"
+      when: >-
+        (_ufm_detailed_config.ufm_ca_cert_path | default('') | trim == '') or
+        (not _ufm_ca_cert_stat.stat.exists | default(false))
+
+- name: Display validated UFM configuration
+  ansible.builtin.debug:
+    msg:
+      - "UFM endpoint: {{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}"
+      - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}"
+      - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
+    verbosity: 2
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+
+# --- Create UFM credentials Kubernetes Secret ---
+
+- name: Read CA certificate content
+  ansible.builtin.slurp:
+    src: "{{ _ufm_detailed_config.ufm_ca_cert_path }}"
+  register: _ufm_ca_cert_content
+  delegate_to: localhost
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
+    - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != ''
+
+
+- name: Debug UFM credentials before template processing
+  ansible.builtin.debug:
+    msg:
+      - "UFM username set: {{ ufm_username | default('') | length > 0 }}"
+      - "UFM password set: {{ ufm_password | default('') | length > 0 }}"
+      - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
+      - "Metrics enabled: {{ _ufm_config.metrics_enabled | default(false) }}"
+      - "Username length: {{ ufm_username | default('') | length }}"
+      - "Password length: {{ ufm_password | default('') | length }}"
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+- name: Display UFM Secret creation status
+  ansible.builtin.debug:
+    msg: "{{ ufm_secret_created_msg }}"
+    verbosity: 2
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+# --- Validate UFM endpoint connectivity (best-effort, warn only) ---
+
+- name: Test UFM endpoint reachability
+  ansible.builtin.uri:
+    url: "{{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9090) }}/metrics"
+    method: GET
+    validate_certs: false
+    timeout: "{{ ufm_endpoint_test_timeout }}"
+    status_code: [200, 401, 403]
+  register: _ufm_reachability
+  failed_when: false
+  delegate_to: localhost
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+- name: Display UFM endpoint reachability status
+  ansible.builtin.debug:
+    msg: >-
+      UFM endpoint {{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9090) }}
+      is {{ 'reachable' if _ufm_reachability.status | default(-1) | int in [200, 401, 403] else 'NOT reachable' }}
+      (HTTP {{ _ufm_reachability.status | default('N/A') }}).
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+- name: Warn if UFM endpoint is unreachable
+  ansible.builtin.debug:
+    msg: "{{ ufm_endpoint_unreachable_msg }}"
+  when: 
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_reachability.status | default(-1) | int not in [200, 401, 403]
+
+# UFM telemetry resources are now deployed via Kustomize templates
+# Secret and Grafana dashboard ConfigMap are created through:
+# - templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
+# - templates/telemetry/ufm/grafana-dashboard-ufm.yaml.j2
+# These are processed in generate_telemetry_deployments.yml and applied via kubectl apply -k
+
+- name: Display UFM integration summary
+  ansible.builtin.debug:
+    msg:
+      - "UFM telemetry integration complete"
+      - "Endpoint: {{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}"
+      - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}"
+      - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
+      - "Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}"
+  when: _ufm_config.metrics_enabled | default(false) | bool
diff --git a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
index 81370b221c..2e354f8c4e 100644
--- a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
+++ b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
@@ -48,6 +48,7 @@
     powerscale_metrics_enabled: "{{ telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool }}"
     powerscale_log_enabled: "{{ telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool }}"
     dcgm_support: "{{ telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(true) | bool }}"
+    ufm_telemetry_support: "{{ telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool }}"
 
 - name: Map powerscale source + configurations to legacy powerscale_configurations
   ansible.builtin.set_fact:
@@ -64,14 +65,16 @@
     victoria_metrics_support: true
   when: >-
     'victoria_metrics' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([])) or
-    'victoria_metrics' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([]))
+    'victoria_metrics' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([])) or
+    'victoria_metrics' in (telemetry_config.telemetry_sources.ufm.collection_targets | default([]))
 
 - name: Check if any source targets victoria_logs
   ansible.builtin.set_fact:
     victoria_logs_support: true
   when: >-
     'victoria_logs' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([])) or
-    'victoria_logs' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([]))
+    'victoria_logs' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([])) or
+    'victoria_logs' in (telemetry_config.telemetry_sources.ufm.collection_targets | default([]))
 
 - name: Check if any source targets Kafka
   ansible.builtin.set_fact:
diff --git a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
index ae1108b2a1..5da3644351 100644
--- a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
+++ b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
@@ -190,6 +190,20 @@
     - telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool
   tags: telemetry_deployment
 
+- name: Populate UFM telemetry deployment configs
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ hostvars['localhost']['k8s_client_share_path'] }}/telemetry/deployments/{{ item.dest }}"
+    mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
+  loop:
+    - src: 'telemetry/ufm/ufm-telemetry-secret.yaml.j2'
+      dest: 'ufm-telemetry-secret.yaml'
+    - src: 'telemetry/ufm/ufm-external-service.yaml.j2'
+      dest: 'ufm-external-service.yaml'
+  when: telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool
+  tags: telemetry_deployment
+
+
 - name: Deploy telemetry cleanup script
   ansible.builtin.template:
     src: 'telemetry/cleanup_telemetry.sh.j2'
diff --git a/provision/roles/telemetry/tasks/main.yml b/provision/roles/telemetry/tasks/main.yml
index 2cbe852288..9ef686e795 100644
--- a/provision/roles/telemetry/tasks/main.yml
+++ b/provision/roles/telemetry/tasks/main.yml
@@ -60,12 +60,18 @@
     - telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool
   ansible.builtin.include_tasks: deploy_powerscale_logs.yml
 
-- name: Generate telemetry deployments (idrac/ldms/powerscale)
+- name: Deploy UFM telemetry (metrics only)
+  when:
+    - telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool
+  ansible.builtin.include_tasks: deploy_ufm_telemetry.yml
+
+- name: Generate telemetry deployments (idrac/ldms/powerscale/ufm)
   when:
     - >-
       (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
       (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
       (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
+      (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
       ldms_support | default(false) | bool
   ansible.builtin.include_tasks: generate_telemetry_deployments.yml
 
diff --git a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2 b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
index 2dc831ba25..e054c5985c 100644
--- a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
@@ -18,5 +18,5 @@ metadata:
   namespace: "{{ telemetry_namespace }}"
 type: Opaque
 stringData:
-  cluster-id: "{{ cluster_id.stdout }}"
+  cluster-id: "{{ kafka.cluster_id }}"
 {% endif %}
\ No newline at end of file
diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
new file mode 100644
index 0000000000..6e958ab656
--- /dev/null
+++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
@@ -0,0 +1,46 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool %}
+apiVersion: v1
+kind: Service
+metadata:
+  name: ufm-external
+  namespace: {{ telemetry_namespace }}
+  labels:
+    app: ufm-external
+    app.kubernetes.io/managed-by: omnia
+    app.kubernetes.io/component: ufm-telemetry
+spec:
+  ports:
+    - name: metrics
+      port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }}
+      protocol: TCP
+  clusterIP: None
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: ufm-external
+  namespace: {{ telemetry_namespace }}
+subsets:
+  - addresses:
+      - ip: "{{ telemetry_config.ufm_configuration.ufm_endpoint }}"
+    ports:
+      - name: metrics
+        port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }}
+        protocol: TCP
+{% endif %}
diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
new file mode 100644
index 0000000000..7e1b901450
--- /dev/null
+++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
@@ -0,0 +1,58 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool %}
+{% set ufm_config = telemetry_config.telemetry_sources.ufm %}
+{% set ufm_detailed_config = telemetry_config.ufm_configuration %}
+{% set auth_mode = ufm_detailed_config.auth_mode | default('basic') %}
+{% set tls_mode = ufm_detailed_config.tls_mode | default('self_signed') %}
+
+{% if auth_mode == 'basic' %}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ufm-telemetry-credentials
+  namespace: {{ telemetry_namespace }}
+  labels:
+    app.kubernetes.io/managed-by: omnia
+    app.kubernetes.io/component: ufm-telemetry
+type: Opaque
+{% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %}
+data:
+  username: {{ (ufm_username | string) | b64encode }}
+  password: {{ (ufm_password | string) | b64encode }}
+  ca.crt: {{ (ufm_ca_cert_content | string) | b64encode }}
+{% else %}
+stringData:
+  username: "{{ ufm_username | string }}"
+  password: "{{ ufm_password | string }}"
+{% endif %}
+{% else %}
+{% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ufm-telemetry-credentials
+  namespace: {{ telemetry_namespace }}
+  labels:
+    app.kubernetes.io/managed-by: omnia
+    app.kubernetes.io/component: ufm-telemetry
+type: Opaque
+data:
+  ca.crt: {{ (ufm_ca_cert_content | string) | b64encode }}
+{% endif %}
+{% endif %}
+{% endif %}
diff --git a/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2 b/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2
index 74ed4a80a3..3c9196ff06 100644
--- a/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2
@@ -105,3 +105,58 @@ spec:
     - targetLabel: component
       replacement: volume-status
 {% endif %}
+
+---
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) %}
+# VMServiceScrape for UFM InfiniBand Metrics
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: ufm-infiniband-metrics
+  namespace: {{ telemetry_namespace }}
+spec:
+  jobLabel: ufm
+  selector:
+    matchLabels:
+      app: ufm-external
+  endpoints:
+    - port: metrics
+      interval: {{ telemetry_config.ufm_configuration.scrape_interval | default('30s') }}
+      scrapeTimeout: {{ telemetry_config.ufm_configuration.scrape_timeout | default('15s') }}
+{% if telemetry_config.ufm_configuration.tls_mode | default('self_signed') == 'ca_signed' %}
+      scheme: https
+      tlsConfig:
+        caFile: /etc/ufm-credentials/ca.crt
+        insecureSkipVerify: false
+{% elif telemetry_config.ufm_configuration.tls_mode | default('self_signed') == 'self_signed' %}
+      scheme: https
+      tlsConfig:
+        insecureSkipVerify: true
+{% else %}
+      scheme: http
+{% endif %}
+      path: /metrics
+      honorLabels: true
+{% if telemetry_config.ufm_configuration.auth_mode | default('basic') == 'basic' %}
+      basicAuth:
+        username:
+          name: ufm-telemetry-credentials
+          key: username
+        password:
+          name: ufm-telemetry-credentials
+          key: password
+{% endif %}
+      relabelings:
+        - action: replace
+          sourceLabels: []
+          targetLabel: source
+          replacement: ufm
+        - action: replace
+          sourceLabels: []
+          targetLabel: subsystem
+          replacement: infiniband
+        - action: replace
+          sourceLabels: []
+          targetLabel: job
+          replacement: ufm-infiniband-metrics
+{% endif %}
diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml
index 53202e31ae..8dd70e542a 100644
--- a/provision/roles/telemetry/vars/main.yml
+++ b/provision/roles/telemetry/vars/main.yml
@@ -647,3 +647,61 @@ vector:
       limits:
         cpu: "500m"
         memory: "512Mi"
+
+# ============================================================================
+# UFM Telemetry Configuration
+# ============================================================================
+# Usage: deploy_ufm_telemetry.yml
+# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances
+# using Prometheus scraping via the shared vmagent instance.
+
+# Kubernetes resource names
+ufm_credentials_secret_name: "ufm-telemetry-credentials"
+ufm_grafana_dashboard_configmap_name: "grafana-dashboard-ufm"
+
+# UFM telemetry defaults
+ufm_deploy_grafana_dashboard: true
+ufm_endpoint_test_timeout: 10
+
+# Validation messages
+ufm_endpoint_empty_msg: >-
+  UFM telemetry is enabled but ufm_endpoint is empty.
+  Set telemetry_sources.ufm.ufm_endpoint in telemetry_config.yml.
+
+ufm_credentials_missing_msg: >-
+  UFM auth_mode is 'basic' but ufm_username or ufm_password is not set.
+  Provide UFM credentials in omnia_config_credentials.yml.
+
+ufm_ca_cert_not_found_msg: >-
+  UFM tls_mode is 'ca_signed' but the CA certificate file was not found
+  at the path specified in ufm_ca_cert_path. Provide a valid PEM certificate file.
+
+ufm_port_range_msg: >-
+  UFM metrics port must be between 1 and 65535.
+
+ufm_tls_mode_invalid_msg: >-
+  UFM tls_mode must be 'self_signed' or 'ca_signed'.
+
+ufm_auth_mode_invalid_msg: >-
+  UFM auth_mode must be 'basic' or 'none'.
+
+ufm_endpoint_unreachable_msg: >-
+  WARNING: UFM endpoint is not reachable. UFM telemetry configuration will
+  proceed, but VMAgent will not be able to scrape metrics until UFM is accessible.
+
+ufm_secret_created_msg: >-
+  UFM credentials Kubernetes Secret created in namespace '{{ telemetry_namespace }}'.
+
+ufm_victoria_logs_validation_fail_msg: >-
+  UFM log collection requires 'victoria_logs' in
+  telemetry_sources.ufm.collection_targets.
+
+
+ufm_integration_summary_msg: >-
+  UFM telemetry integration complete.
+  {% if _ufm_config.metrics_enabled | default(false) | bool %}
+  Metrics: https://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}
+  TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}
+  Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}
+  Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}
+  {% endif %}

From cff026e113874a979013152f378a4f255547f610 Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Tue, 12 May 2026 22:51:23 +0530
Subject: [PATCH 02/32] ansible lint fix

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../telemetry/tasks/deploy_ufm_telemetry.yml  | 43 ++++---------------
 1 file changed, 8 insertions(+), 35 deletions(-)

diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
index be43e590cd..f5d2a641cc 100644
--- a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -34,6 +34,7 @@
     _ufm_detailed_config: "{{ telemetry_config.ufm_configuration }}"
 
 - name: Load UFM credentials from omnia_config_credentials.yml
+  when: _ufm_config.metrics_enabled | default(false) | bool
   block:
     - name: Check if credentials file is encrypted
       ansible.builtin.command: >-
@@ -66,7 +67,7 @@
     - name: Set CA certificate content for template processing
       ansible.builtin.set_fact:
         ufm_ca_cert_content: "{{ _ufm_ca_cert_content.content | default('') }}"
-      when: 
+      when:
         - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
         - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != ''
 
@@ -84,10 +85,11 @@
       ansible.builtin.command: >-
         ansible-vault encrypt "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
         --vault-password-file "{{ hostvars['localhost']['input_project_dir'] }}/.omnia_config_credentials_key"
-      when: "'$ANSIBLE_VAULT;' in _ufm_cred_file_content.stdout"
+      when: 
+        - "'$ANSIBLE_VAULT;' in _ufm_cred_file_content.stdout"
+        - _ufm_config.metrics_enabled | default(false) | bool
       changed_when: false
       delegate_to: localhost
-  when: _ufm_config.metrics_enabled | default(false) | bool
 
 # --- Validate UFM telemetry inputs ---
 # Only run validation when UFM telemetry is enabled
@@ -110,18 +112,17 @@
 - name: Validate UFM TLS mode
   ansible.builtin.fail:
     msg: "{{ ufm_tls_mode_invalid_msg }}"
-  when: 
+  when:
     - _ufm_config.metrics_enabled | default(false) | bool
     - _ufm_detailed_config.tls_mode | default('self_signed') not in ['self_signed', 'ca_signed']
 
 - name: Validate UFM auth mode
   ansible.builtin.fail:
     msg: "{{ ufm_auth_mode_invalid_msg }}"
-  when: 
+  when:
     - _ufm_config.metrics_enabled | default(false) | bool
     - _ufm_detailed_config.auth_mode | default('basic') not in ['basic', 'none']
 
-
 - name: Validate UFM credentials when auth_mode is basic
   ansible.builtin.fail:
     msg: "{{ ufm_credentials_missing_msg }}"
@@ -131,7 +132,7 @@
     - (ufm_username | default('') | trim == '') or (ufm_password | default('') | trim == '')
 
 - name: Validate CA certificate file when tls_mode is ca_signed
-  when: 
+  when:
     - _ufm_config.metrics_enabled | default(false) | bool
     - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
   block:
@@ -188,34 +189,6 @@
     verbosity: 2
   when: _ufm_config.metrics_enabled | default(false) | bool
 
-# --- Validate UFM endpoint connectivity (best-effort, warn only) ---
-
-- name: Test UFM endpoint reachability
-  ansible.builtin.uri:
-    url: "{{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9090) }}/metrics"
-    method: GET
-    validate_certs: false
-    timeout: "{{ ufm_endpoint_test_timeout }}"
-    status_code: [200, 401, 403]
-  register: _ufm_reachability
-  failed_when: false
-  delegate_to: localhost
-  when: _ufm_config.metrics_enabled | default(false) | bool
-
-- name: Display UFM endpoint reachability status
-  ansible.builtin.debug:
-    msg: >-
-      UFM endpoint {{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9090) }}
-      is {{ 'reachable' if _ufm_reachability.status | default(-1) | int in [200, 401, 403] else 'NOT reachable' }}
-      (HTTP {{ _ufm_reachability.status | default('N/A') }}).
-  when: _ufm_config.metrics_enabled | default(false) | bool
-
-- name: Warn if UFM endpoint is unreachable
-  ansible.builtin.debug:
-    msg: "{{ ufm_endpoint_unreachable_msg }}"
-  when: 
-    - _ufm_config.metrics_enabled | default(false) | bool
-    - _ufm_reachability.status | default(-1) | int not in [200, 401, 403]
 
 # UFM telemetry resources are now deployed via Kustomize templates
 # Secret and Grafana dashboard ConfigMap are created through:

From a22675f738da7e356dd6362c85354d0593952544 Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Tue, 12 May 2026 23:08:29 +0530
Subject: [PATCH 03/32] ansible lint fix

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
index f5d2a641cc..43fba16669 100644
--- a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -85,7 +85,7 @@
       ansible.builtin.command: >-
         ansible-vault encrypt "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
         --vault-password-file "{{ hostvars['localhost']['input_project_dir'] }}/.omnia_config_credentials_key"
-      when: 
+      when:
         - "'$ANSIBLE_VAULT;' in _ufm_cred_file_content.stdout"
         - _ufm_config.metrics_enabled | default(false) | bool
       changed_when: false
@@ -97,7 +97,7 @@
 - name: Validate UFM endpoint is not empty
   ansible.builtin.fail:
     msg: "{{ ufm_endpoint_empty_msg }}"
-  when: 
+  when:
     - _ufm_config.metrics_enabled | default(false) | bool
     - _ufm_detailed_config.ufm_endpoint | default('') | trim == ''
 
@@ -200,7 +200,8 @@
   ansible.builtin.debug:
     msg:
       - "UFM telemetry integration complete"
-      - "Endpoint: {{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}"
+      - >-
+        Endpoint: {{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}
       - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}"
       - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
       - "Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}"

From e7212090e948ef7cdecfb85bcad6fd1b7203a70b Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Tue, 12 May 2026 23:17:22 +0530
Subject: [PATCH 04/32] ansible lint fix

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
index 43fba16669..97246fb6fc 100644
--- a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -200,8 +200,7 @@
   ansible.builtin.debug:
     msg:
       - "UFM telemetry integration complete"
-      - >-
-        Endpoint: {{ 'https' if _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed' else 'http' }}://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}
+      - "UFM endpoint: {{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}"
       - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}"
       - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
       - "Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}"

From 7ce0525e75cd22cc4a703a49fbece7d3bde66b8a Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Wed, 13 May 2026 13:14:09 +0530
Subject: [PATCH 05/32] updated ufm config

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../telemetry/templates/telemetry/kustomization.yaml.j2      | 5 +++++
 .../templates/telemetry/ufm/ufm-external-service.yaml.j2     | 2 --
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
index 5eb7f1ee90..c55f7a5f74 100644
--- a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
@@ -58,6 +58,11 @@ resources:
   - telemetry_cleaner_rbac.yaml
   - telemetry_pod_cleanup.yaml
 {% endif %}
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) %}
+# UFM InfiniBand Telemetry Resources
+  - ufm-telemetry-secret.yaml
+  - ufm-external-service.yaml
+{% endif %}
 {% if telemetry_config.telemetry_bridges.vector_ldms.metrics_enabled | default(false) and ldms_support | default(false) and kafka_support | default(false) and victoria_metrics_support | default(false) %}
 # Vector-LDMS Bridge Resources (Kafka-to-VictoriaMetrics pipeline)
   - vector/vmagent-vector-deployment.yaml
diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
index 6e958ab656..d6ccf90b37 100644
--- a/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
@@ -27,7 +27,6 @@ spec:
   ports:
     - name: metrics
       port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }}
-      protocol: TCP
   clusterIP: None
 
 ---
@@ -42,5 +41,4 @@ subsets:
     ports:
       - name: metrics
         port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }}
-        protocol: TCP
 {% endif %}

From c28052e40037f2c877f56077aaf2a38057bb6fad Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Thu, 14 May 2026 04:58:58 +0000
Subject: [PATCH 06/32] doca script changes needed

---
 .../templates/doca-ofed/doca-install.sh.j2    | 40 +++++++++----------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
index db8a7cb9cc..f155565b3b 100644
--- a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
+++ b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
@@ -33,7 +33,7 @@ if rpm -q kernel-devel-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-devel package is already installed."
 else
     echo "kernel-devel package is not installed. Installing..."
-    dnf install -y kernel-devel-$(uname -r)
+    dnf install --setopt=sslverify=0  -y kernel-devel-$(uname -r)
 fi
 
 echo "Check if kernel-headers package is present"
@@ -41,7 +41,7 @@ if rpm -q kernel-headers-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-headers package is already installed."
 else
     echo "kernel-headers package is not installed. Installing..."
-    dnf install -y kernel-headers-$(uname -r)
+    dnf install --setopt=sslverify=0 -y kernel-headers-$(uname -r)
 fi
 
 echo "Installing doca-ofed..."
@@ -49,29 +49,24 @@ if rpm -q doca-ofed >/dev/null 2>&1; then
     echo "doca-ofed package is already installed."
 else
     echo "doca-ofed package is not installed. Installing..."
-    dnf install -y doca-ofed
+    dnf install --setopt=sslverify=0 -y doca-ofed
 fi
 
-echo "Unloading RDMA kernel modules..."
-rmmod bnxt_re || true
-rmmod mlx5_ib    || true
-rmmod ib_uverbs  || true
-rmmod xpmem      || true
-rmmod ib_core    || true
-rmmod mlx5_core  || true
+# NOTE: Removed module unload/reload sequence - DKMS handles module installation automatically
+# The modprobe commands were causing module dependency issues and are not needed
+# DOCA DKMS installation automatically handles kernel module replacement
 
-echo "Loading RDMA kernel modules..."
-modprobe mlx5_core || true
-modprobe mlx5_ib || true
-modprobe ib_core || true
-modprobe ib_uverbs || true
-modprobe ib_umad || true
-modprobe ib_cm || true
-modprobe rdma_cm || true
-modprobe rdma_ucm || true
-modprobe xpmem || true
-modprobe knem || true
-modprobe ib_ipoib || true
+# NOTE: Remove MLNX OFED kernel modules to preserve standard RDMA compatibility
+# This ensures rpcrdma and other standard RDMA functionality works correctly
+echo "Removing MLNX OFED kernel modules to ensure standard RDMA compatibility"
+if dkms status mlnx-ofa_kernel/25.10 -k $(uname -r) >/dev/null 2>&1; then
+    dkms uninstall mlnx-ofa_kernel/25.10 -k $(uname -r) || true
+    dkms remove mlnx-ofa_kernel/25.10 --all || true
+    depmod -a
+    echo "MLNX OFED kernel modules removed successfully, standard kernel modules restored"
+else
+    echo "MLNX OFED kernel modules not found, skipping removal"
+fi
 
 if command -v firewall-cmd &>/dev/null; then
     echo "Adding firewall ports..."
@@ -86,4 +81,5 @@ else
 fi
 
 echo "DOCA-OFED installation completed successfully."
+
  

From 4cbf778861591e289d2d67bcd88430c8876c2e16 Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Thu, 14 May 2026 10:28:58 +0000
Subject: [PATCH 07/32] removed ssl disabling

---
 .../configure_ochami/templates/doca-ofed/doca-install.sh.j2 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
index f155565b3b..f1ce4389ef 100644
--- a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
+++ b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
@@ -33,7 +33,7 @@ if rpm -q kernel-devel-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-devel package is already installed."
 else
     echo "kernel-devel package is not installed. Installing..."
-    dnf install --setopt=sslverify=0  -y kernel-devel-$(uname -r)
+    dnf install -y kernel-devel-$(uname -r)
 fi
 
 echo "Check if kernel-headers package is present"
@@ -41,7 +41,7 @@ if rpm -q kernel-headers-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-headers package is already installed."
 else
     echo "kernel-headers package is not installed. Installing..."
-    dnf install --setopt=sslverify=0 -y kernel-headers-$(uname -r)
+    dnf install -y kernel-headers-$(uname -r)
 fi
 
 echo "Installing doca-ofed..."
@@ -49,7 +49,7 @@ if rpm -q doca-ofed >/dev/null 2>&1; then
     echo "doca-ofed package is already installed."
 else
     echo "doca-ofed package is not installed. Installing..."
-    dnf install --setopt=sslverify=0 -y doca-ofed
+    dnf install -y doca-ofed
 fi
 
 # NOTE: Removed module unload/reload sequence - DKMS handles module installation automatically

From 7360eeb7d754b4c069981eed939db93c57ee66ce Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Thu, 14 May 2026 11:00:08 +0000
Subject: [PATCH 08/32] reordering

---
 .../ci-group-slurm_node_x86_64.yaml.j2          | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
index 5ada0be1b7..3cae337b69 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
@@ -395,10 +395,18 @@
             {{ pv_entry.content | indent(12) }}
 {% endfor %}
 
+
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts
+        - /usr/local/bin/configure_dirs_and_mounts.sh
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -431,12 +439,6 @@
 {% if dcgm_support %}
         - /usr/local/bin/setup_dcgm.sh
 {% endif %}
-        # slurm user and group created in the users module
-        - /usr/local/bin/configure_dirs_and_mounts.sh
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
 {% if slurm_node_present %}
         - |
           set -e
@@ -494,3 +496,4 @@
 
         - systemctl restart slurmd
         - echo "Cloud-Init has completed successfully."
+

From e13802789ead678efedc6369ff39e7490cad2540 Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Thu, 14 May 2026 17:20:57 +0530
Subject: [PATCH 09/32] added verbosity to the msg

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
index 97246fb6fc..68f4f6d554 100644
--- a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -204,4 +204,5 @@
       - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}"
       - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
       - "Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}"
+    verbosity: 2
   when: _ufm_config.metrics_enabled | default(false) | bool

From a1d514371dc3fd182711c5dd85e5daff46f2011f Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Thu, 14 May 2026 19:30:01 +0530
Subject: [PATCH 10/32] reference of deploy_powerscale_logs.yml removed

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 provision/roles/telemetry/tasks/main.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/provision/roles/telemetry/tasks/main.yml b/provision/roles/telemetry/tasks/main.yml
index 8a6c87dac5..261d3598ad 100644
--- a/provision/roles/telemetry/tasks/main.yml
+++ b/provision/roles/telemetry/tasks/main.yml
@@ -55,11 +55,6 @@
     - telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool
   ansible.builtin.include_tasks: deploy_powerscale_metrics.yml
 
-- name: Deploy PowerScale syslog log collection
-  when:
-    - telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool
-  ansible.builtin.include_tasks: deploy_powerscale_logs.yml
-
 - name: Deploy UFM telemetry (metrics only)
   when:
     - telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool

From 387d643e6cf1c528d04592bd58db82027604586b Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Thu, 14 May 2026 21:50:23 +0530
Subject: [PATCH 11/32] added ufm credential in config credentials utils

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../create_config/templates/omnia_credential.j2    |  7 +++++--
 .../roles/update_config/vars/main.yml              | 14 +++++++-------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2 b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
index bf231510ed..299f65943a 100644
--- a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
+++ b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
@@ -19,8 +19,7 @@ bmc_username: ""
 bmc_password: ""
 
 # Prepare_oim credentials
-s3_access_id: ""
-s3_secret_key: ""
+minio_s3_password: ""
 pulp_password: ""
 docker_username: ""
 docker_password: ""
@@ -54,3 +53,7 @@ gitlab_root_password: ""
 # OME discovery credentials
 ome_username: ""
 ome_password: ""
+
+# UFM telemetry credentials
+ufm_username: ""
+ufm_password: ""
diff --git a/utils/credential_utility/roles/update_config/vars/main.yml b/utils/credential_utility/roles/update_config/vars/main.yml
index 6c60ad6110..62392b9320 100644
--- a/utils/credential_utility/roles/update_config/vars/main.yml
+++ b/utils/credential_utility/roles/update_config/vars/main.yml
@@ -73,11 +73,8 @@ omnia_credentials:
       - { username: docker_username, password: docker_password }
     mandatory:
       - { password: pulp_password }
-      - { password: s3_secret_key }
+      - { password: minio_s3_password }
     conditional_mandatory:
-      # PowerScale S3 Access Key -- only prompted when storage provider is 'powerscale'
-      - username: s3_access_id
-        condition: "{{ s3_configurations.provider == 'powerscale' }}"
       - username: build_stream_auth_username
         password: build_stream_auth_password
         condition: "{{ enable_build_stream | default(false) | bool }}"
@@ -114,7 +111,10 @@ omnia_credentials:
     mandatory:
       - { password: ldms_sampler_password }
   discovery:
+    mandatory:
+      - { username: ome_username, password: ome_password }
+  ufm_telemetry:
     conditional_mandatory:
-      - username: ome_username
-        password: ome_password
-        condition: "{{ enable_bmc_discovery | default(false) | bool }}"
+      - username: ufm_username
+        password: ufm_password
+        condition: "{{ ufm_telemetry_support | default(false) | bool }}"

From 9c64cc82277e5c26a1394ed4a231bc71f8ca6b2e Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Thu, 14 May 2026 21:59:39 +0530
Subject: [PATCH 12/32] resolved merge conflicts

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../roles/create_config/templates/omnia_credential.j2 |  3 ++-
 .../roles/update_config/vars/main.yml                 | 11 ++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2 b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
index 299f65943a..d9ffd938b4 100644
--- a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
+++ b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
@@ -19,7 +19,8 @@ bmc_username: ""
 bmc_password: ""
 
 # Prepare_oim credentials
-minio_s3_password: ""
+s3_access_id: ""
+s3_secret_key: ""
 pulp_password: ""
 docker_username: ""
 docker_password: ""
diff --git a/utils/credential_utility/roles/update_config/vars/main.yml b/utils/credential_utility/roles/update_config/vars/main.yml
index 62392b9320..8ebc0ea3be 100644
--- a/utils/credential_utility/roles/update_config/vars/main.yml
+++ b/utils/credential_utility/roles/update_config/vars/main.yml
@@ -73,8 +73,11 @@ omnia_credentials:
       - { username: docker_username, password: docker_password }
     mandatory:
       - { password: pulp_password }
-      - { password: minio_s3_password }
+      - { password: s3_secret_key }
     conditional_mandatory:
+      # PowerScale S3 Access Key -- only prompted when storage provider is 'powerscale'
+      - username: s3_access_id
+        condition: "{{ s3_configurations.provider == 'powerscale' }}"
       - username: build_stream_auth_username
         password: build_stream_auth_password
         condition: "{{ enable_build_stream | default(false) | bool }}"
@@ -111,8 +114,10 @@ omnia_credentials:
     mandatory:
       - { password: ldms_sampler_password }
   discovery:
-    mandatory:
-      - { username: ome_username, password: ome_password }
+    conditional_mandatory:
+      - username: ome_username
+        password: ome_password
+        condition: "{{ enable_bmc_discovery | default(false) | bool }}"
   ufm_telemetry:
     conditional_mandatory:
       - username: ufm_username

From 7ea8e5aadc888c424e7bb3b7531b3abc89acacdb Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Thu, 14 May 2026 22:03:39 +0530
Subject: [PATCH 13/32] Removed debug msgs

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../roles/telemetry/tasks/deploy_ufm_telemetry.yml   | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
index 68f4f6d554..840e9981a6 100644
--- a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -171,18 +171,6 @@
     - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
     - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != ''
 
-
-- name: Debug UFM credentials before template processing
-  ansible.builtin.debug:
-    msg:
-      - "UFM username set: {{ ufm_username | default('') | length > 0 }}"
-      - "UFM password set: {{ ufm_password | default('') | length > 0 }}"
-      - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
-      - "Metrics enabled: {{ _ufm_config.metrics_enabled | default(false) }}"
-      - "Username length: {{ ufm_username | default('') | length }}"
-      - "Password length: {{ ufm_password | default('') | length }}"
-  when: _ufm_config.metrics_enabled | default(false) | bool
-
 - name: Display UFM Secret creation status
   ansible.builtin.debug:
     msg: "{{ ufm_secret_created_msg }}"

From d9e32dfff21cc5db7f3147d54c3a6680ed2df4a8 Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Fri, 15 May 2026 00:39:04 +0530
Subject: [PATCH 14/32] updated the ufm secrets flow

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../schema/credential_rules.json              | 12 +++
 .../telemetry/tasks/deploy_ufm_telemetry.yml  | 74 ++++---------------
 .../ufm/ufm-telemetry-secret.yaml.j2          | 12 +--
 .../update_config/tasks/fetch_credentials.yml |  3 +-
 .../roles/validation/tasks/pre_requisite.yml  | 14 ++++
 5 files changed, 47 insertions(+), 68 deletions(-)

diff --git a/common/library/module_utils/input_validation/schema/credential_rules.json b/common/library/module_utils/input_validation/schema/credential_rules.json
index ae0872671c..8fc1e6251a 100644
--- a/common/library/module_utils/input_validation/schema/credential_rules.json
+++ b/common/library/module_utils/input_validation/schema/credential_rules.json
@@ -161,5 +161,17 @@
         "maxLength": 128,
         "pattern": "^.+$",
         "description": "Password for Dell OpenManage Enterprise (OME). Length must be between 1 and 128 characters."
+      },
+      "ufm_username": {
+        "minLength": 1,
+        "maxLength": 64,
+        "pattern": "^[^\\\\'\";]+$",
+        "description": "Username for NVIDIA UFM (Unified Fabric Manager). Must not contain backslashes (\\), single quotes ('), double quotes (\"), or semicolons (;). Length must be between 1 and 64 characters."
+      },
+      "ufm_password": {
+        "minLength": 1,
+        "maxLength": 128,
+        "pattern": "^.+$",
+        "description": "Password for NVIDIA UFM (Unified Fabric Manager). Length must be between 1 and 128 characters."
       }
   }
diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
index 840e9981a6..700d5be132 100644
--- a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -33,64 +33,6 @@
     _ufm_config: "{{ telemetry_config.telemetry_sources.ufm }}"
     _ufm_detailed_config: "{{ telemetry_config.ufm_configuration }}"
 
-- name: Load UFM credentials from omnia_config_credentials.yml
-  when: _ufm_config.metrics_enabled | default(false) | bool
-  block:
-    - name: Check if credentials file is encrypted
-      ansible.builtin.command: >-
-        cat "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
-      changed_when: false
-      register: _ufm_cred_file_content
-      delegate_to: localhost
-      no_log: true
-
-    - name: Decrypt credentials file if encrypted
-      ansible.builtin.command: >-
-        ansible-vault decrypt "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
-        --vault-password-file "{{ hostvars['localhost']['input_project_dir'] }}/.omnia_config_credentials_key"
-      when: "'$ANSIBLE_VAULT;' in _ufm_cred_file_content.stdout"
-      changed_when: false
-      delegate_to: localhost
-
-    - name: Include credentials and extract UFM credentials
-      ansible.builtin.include_vars:
-        file: "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
-        name: _omnia_creds
-      no_log: true
-
-    - name: Set UFM credentials as facts for template processing
-      ansible.builtin.set_fact:
-        ufm_username: "{{ _omnia_creds.ufm_username | default('') }}"
-        ufm_password: "{{ _omnia_creds.ufm_password | default('') }}"
-      no_log: true
-
-    - name: Set CA certificate content for template processing
-      ansible.builtin.set_fact:
-        ufm_ca_cert_content: "{{ _ufm_ca_cert_content.content | default('') }}"
-      when:
-        - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
-        - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != ''
-
-    - name: Debug UFM credentials and file path
-      ansible.builtin.debug:
-        msg:
-          - "Credentials file path: {{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
-          - "UFM username found: {{ _omnia_creds.ufm_username is defined }}"
-          - "UFM password found: {{ _omnia_creds.ufm_password is defined }}"
-          - "UFM username length: {{ _omnia_creds.ufm_username | default('') | length }}"
-          - "UFM password length: {{ _omnia_creds.ufm_password | default('') | length }}"
-          - "Input project dir: {{ hostvars['localhost']['input_project_dir'] }}"
-
-    - name: Re-encrypt credentials file if it was encrypted
-      ansible.builtin.command: >-
-        ansible-vault encrypt "{{ hostvars['localhost']['input_project_dir'] }}/omnia_config_credentials.yml"
-        --vault-password-file "{{ hostvars['localhost']['input_project_dir'] }}/.omnia_config_credentials_key"
-      when:
-        - "'$ANSIBLE_VAULT;' in _ufm_cred_file_content.stdout"
-        - _ufm_config.metrics_enabled | default(false) | bool
-      changed_when: false
-      delegate_to: localhost
-
 # --- Validate UFM telemetry inputs ---
 # Only run validation when UFM telemetry is enabled
 
@@ -129,7 +71,9 @@
   when:
     - _ufm_config.metrics_enabled | default(false) | bool
     - _ufm_detailed_config.auth_mode | default('basic') == 'basic'
-    - (ufm_username | default('') | trim == '') or (ufm_password | default('') | trim == '')
+    - >-
+      (hostvars['localhost']['ufm_username'] | default('') | trim == '') or
+      (hostvars['localhost']['ufm_password'] | default('') | trim == '')
 
 - name: Validate CA certificate file when tls_mode is ca_signed
   when:
@@ -171,6 +115,14 @@
     - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
     - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != ''
 
+- name: Set CA certificate content for template processing
+  ansible.builtin.set_fact:
+    ufm_ca_cert_content: "{{ _ufm_ca_cert_content.content | b64decode }}"
+  when:
+    - _ufm_ca_cert_content is defined
+    - _ufm_ca_cert_content is not skipped
+    - _ufm_ca_cert_content.content is defined
+
 - name: Display UFM Secret creation status
   ansible.builtin.debug:
     msg: "{{ ufm_secret_created_msg }}"
@@ -179,9 +131,9 @@
 
 
 # UFM telemetry resources are now deployed via Kustomize templates
-# Secret and Grafana dashboard ConfigMap are created through:
+# Secret and external-service ConfigMap are created through:
 # - templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
-# - templates/telemetry/ufm/grafana-dashboard-ufm.yaml.j2
+# - templates/telemetry/ufm/ufm-external-service.yaml.j2
 # These are processed in generate_telemetry_deployments.yml and applied via kubectl apply -k
 
 - name: Display UFM integration summary
diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
index 7e1b901450..a641f3d93f 100644
--- a/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
@@ -32,13 +32,13 @@ metadata:
 type: Opaque
 {% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %}
 data:
-  username: {{ (ufm_username | string) | b64encode }}
-  password: {{ (ufm_password | string) | b64encode }}
-  ca.crt: {{ (ufm_ca_cert_content | string) | b64encode }}
+  username: {{ (hostvars['localhost']['ufm_username'] | string) | b64encode }}
+  password: {{ (hostvars['localhost']['ufm_password'] | string) | b64encode }}
+  ca.crt: {{ (ufm_ca_cert_content | default('') | string) | b64encode }}
 {% else %}
 stringData:
-  username: "{{ ufm_username | string }}"
-  password: "{{ ufm_password | string }}"
+  username: "{{ hostvars['localhost']['ufm_username'] | default('') | string }}"
+  password: "{{ hostvars['localhost']['ufm_password'] | default('') | string }}"
 {% endif %}
 {% else %}
 {% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %}
@@ -52,7 +52,7 @@ metadata:
     app.kubernetes.io/component: ufm-telemetry
 type: Opaque
 data:
-  ca.crt: {{ (ufm_ca_cert_content | string) | b64encode }}
+  ca.crt: {{ (ufm_ca_cert_content | default('') | string) | b64encode }}
 {% endif %}
 {% endif %}
 {% endif %}
diff --git a/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml b/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml
index 7f904b227a..8207abd29b 100644
--- a/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml
+++ b/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml
@@ -30,6 +30,7 @@
     - >-
       service.key in software_names or
       service.key in ["provision", "prepare_oim", "local_repo", "idrac_telemetry",
-      "visualization", "build_aarch_image", "gitlab", "csi_driver_powerscale", "discovery"]
+      "visualization", "build_aarch_image", "gitlab", "csi_driver_powerscale", "discovery",
+      "ufm_telemetry"]
     - (omnia_run_tags | default([]) | difference(['all']) | length == 0)
       or service.key in (omnia_run_tags | default([]))
diff --git a/utils/credential_utility/roles/validation/tasks/pre_requisite.yml b/utils/credential_utility/roles/validation/tasks/pre_requisite.yml
index e12c09aa28..f7b3f3bbea 100644
--- a/utils/credential_utility/roles/validation/tasks/pre_requisite.yml
+++ b/utils/credential_utility/roles/validation/tasks/pre_requisite.yml
@@ -48,3 +48,17 @@
   when:
     - not result.skipped | default(false)
     - result.telemetry_status_list | length > 0
+
+- name: Derive UFM telemetry support flag from telemetry_config.yml
+  when: "'ufm_telemetry' in (omnia_run_tags | default([]))"
+  block:
+    - name: Load telemetry_config.yml for UFM support flag
+      ansible.builtin.include_vars:
+        file: "{{ telemetry_config_path }}"
+        name: _telemetry_config
+      failed_when: false
+
+    - name: Set ufm_telemetry_support flag
+      ansible.builtin.set_fact:
+        ufm_telemetry_support: "{{ _telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool }}"
+      when: _telemetry_config is defined and _telemetry_config.telemetry_sources is defined

From 944c1d2f24ff382014f969568277daee8ade600a Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Fri, 15 May 2026 08:25:57 +0000
Subject: [PATCH 15/32] removed ssl disabling

Signed-off-by: Nagachandan-P <Nagachandan.p@dell.com>
---
 .../ci-group-slurm_node_x86_64.yaml.j2          | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
index 3cae337b69..5ada0be1b7 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
@@ -395,18 +395,10 @@
             {{ pv_entry.content | indent(12) }}
 {% endfor %}
 
-
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-        # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts
-        - /usr/local/bin/configure_dirs_and_mounts.sh
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
-        # DOCA and IB configuration - now ready before vendor_data mounts
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
-{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
+{# Mount-specific runcmd entries #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -439,6 +431,12 @@
 {% if dcgm_support %}
         - /usr/local/bin/setup_dcgm.sh
 {% endif %}
+        # slurm user and group created in the users module
+        - /usr/local/bin/configure_dirs_and_mounts.sh
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
 {% if slurm_node_present %}
         - |
           set -e
@@ -496,4 +494,3 @@
 
         - systemctl restart slurmd
         - echo "Cloud-Init has completed successfully."
-

From f6908a6a9a8ab276a454bbb00cfa56dc38c195e2 Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Fri, 15 May 2026 08:26:32 +0000
Subject: [PATCH 16/32] reordering

Signed-off-by: Nagachandan-P <Nagachandan.p@dell.com>
---
 .../ci-group-slurm_node_x86_64.yaml.j2          | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
index 5ada0be1b7..3cae337b69 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2
@@ -395,10 +395,18 @@
             {{ pv_entry.content | indent(12) }}
 {% endfor %}
 
+
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts
+        - /usr/local/bin/configure_dirs_and_mounts.sh
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -431,12 +439,6 @@
 {% if dcgm_support %}
         - /usr/local/bin/setup_dcgm.sh
 {% endif %}
-        # slurm user and group created in the users module
-        - /usr/local/bin/configure_dirs_and_mounts.sh
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
 {% if slurm_node_present %}
         - |
           set -e
@@ -494,3 +496,4 @@
 
         - systemctl restart slurmd
         - echo "Cloud-Init has completed successfully."
+

From 4bb7d4dcc0077219d56e7a2209cf18ae1f07a5cb Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Fri, 15 May 2026 08:29:38 +0000
Subject: [PATCH 17/32] doca script changes needed

Signed-off-by: Nagachandan-P <Nagachandan.p@dell.com>
---
 .../configure_ochami/templates/doca-ofed/doca-install.sh.j2 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
index f1ce4389ef..f155565b3b 100644
--- a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
+++ b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
@@ -33,7 +33,7 @@ if rpm -q kernel-devel-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-devel package is already installed."
 else
     echo "kernel-devel package is not installed. Installing..."
-    dnf install -y kernel-devel-$(uname -r)
+    dnf install --setopt=sslverify=0  -y kernel-devel-$(uname -r)
 fi
 
 echo "Check if kernel-headers package is present"
@@ -41,7 +41,7 @@ if rpm -q kernel-headers-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-headers package is already installed."
 else
     echo "kernel-headers package is not installed. Installing..."
-    dnf install -y kernel-headers-$(uname -r)
+    dnf install --setopt=sslverify=0 -y kernel-headers-$(uname -r)
 fi
 
 echo "Installing doca-ofed..."
@@ -49,7 +49,7 @@ if rpm -q doca-ofed >/dev/null 2>&1; then
     echo "doca-ofed package is already installed."
 else
     echo "doca-ofed package is not installed. Installing..."
-    dnf install -y doca-ofed
+    dnf install --setopt=sslverify=0 -y doca-ofed
 fi
 
 # NOTE: Removed module unload/reload sequence - DKMS handles module installation automatically

From 860831af35bb8de26f4190523013d85690066cb8 Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Fri, 15 May 2026 08:55:26 +0000
Subject: [PATCH 18/32] dnf command

---
 .../configure_ochami/templates/doca-ofed/doca-install.sh.j2 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
index f155565b3b..b467cc863a 100644
--- a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
+++ b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
@@ -33,7 +33,7 @@ if rpm -q kernel-devel-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-devel package is already installed."
 else
     echo "kernel-devel package is not installed. Installing..."
-    dnf install --setopt=sslverify=0  -y kernel-devel-$(uname -r)
+    dnf install  -y kernel-devel-$(uname -r)
 fi
 
 echo "Check if kernel-headers package is present"
@@ -41,7 +41,7 @@ if rpm -q kernel-headers-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-headers package is already installed."
 else
     echo "kernel-headers package is not installed. Installing..."
-    dnf install --setopt=sslverify=0 -y kernel-headers-$(uname -r)
+    dnf install -y kernel-headers-$(uname -r)
 fi
 
 echo "Installing doca-ofed..."
@@ -49,7 +49,7 @@ if rpm -q doca-ofed >/dev/null 2>&1; then
     echo "doca-ofed package is already installed."
 else
     echo "doca-ofed package is not installed. Installing..."
-    dnf install --setopt=sslverify=0 -y doca-ofed
+    dnf install -y doca-ofed
 fi
 
 # NOTE: Removed module unload/reload sequence - DKMS handles module installation automatically

From d747b10589c2626407c5b38132f6f42f75aff61b Mon Sep 17 00:00:00 2001
From: Nagachandan P <Nagachandan.p@dell.com>
Date: Fri, 15 May 2026 14:26:58 +0530
Subject: [PATCH 19/32] Update space in doca script

Signed-off-by: Nagachandan P <Nagachandan.p@dell.com>
---
 .../configure_ochami/templates/doca-ofed/doca-install.sh.j2     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
index b467cc863a..f1ce4389ef 100644
--- a/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
+++ b/provision/roles/configure_ochami/templates/doca-ofed/doca-install.sh.j2
@@ -33,7 +33,7 @@ if rpm -q kernel-devel-$(uname -r) >/dev/null 2>&1; then
     echo "kernel-devel package is already installed."
 else
     echo "kernel-devel package is not installed. Installing..."
-    dnf install  -y kernel-devel-$(uname -r)
+    dnf install -y kernel-devel-$(uname -r)
 fi
 
 echo "Check if kernel-headers package is present"

From 67a77c18af800420b30ba87082b0d755abb1e874 Mon Sep 17 00:00:00 2001
From: Nagachandan-P <Nagachandan.p@dell.com>
Date: Fri, 15 May 2026 11:15:54 +0000
Subject: [PATCH 20/32] for all roles

Signed-off-by: Nagachandan-P <Nagachandan.p@dell.com>
---
 ...-group-login_compiler_node_aarch64.yaml.j2 | 18 ++++++++-----
 ...i-group-login_compiler_node_x86_64.yaml.j2 | 17 +++++++-----
 .../ci-group-login_node_aarch64.yaml.j2       | 22 +++++++++-------
 .../ci-group-login_node_x86_64.yaml.j2        | 26 +++++++++----------
 ...ci-group-slurm_control_node_x86_64.yaml.j2 | 17 +++++++-----
 .../ci-group-slurm_node_aarch64.yaml.j2       | 15 ++++++-----
 6 files changed, 69 insertions(+), 46 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2
index 02b1e0be8c..60b0a47616 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2
@@ -182,7 +182,17 @@
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - mount /cert and prepare for DOCA installation
+        - mkdir -p {{ client_mount_path }}/slurm/ssh
+        - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
+        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - mount -av
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -222,11 +232,8 @@
         - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path}}/hpc_tools  /hpc_tools   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
 
 {% if login_compiler_node_present %}
         - /usr/local/bin/generate_install_uuid.sh
@@ -256,8 +263,7 @@
         - echo " Starting LDMS setup " | tee -a /var/log/ldms-cloudinit.log
         - /root/ldms_sampler.sh
 {% endif %}
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
+
         - /usr/local/bin/check_slurm_controller_status.sh
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }}
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }}
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2
index c14e89a82a..a4b89e1efa 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2
@@ -181,7 +181,17 @@
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - mount /cert and prepare for DOCA installation
+        - mkdir -p {{ client_mount_path }}/slurm/ssh
+        - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
+        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - mount -av
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -221,12 +231,9 @@
         - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path}}/hpc_tools  /hpc_tools   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -a
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
 
 {% if login_compiler_node_present %}
         - /usr/local/bin/generate_install_uuid.sh
@@ -257,8 +264,6 @@
         - /root/ldms_sampler.sh
 {% endif %}
 
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
         - /usr/local/bin/check_slurm_controller_status.sh
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }}
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }}
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2
index 0fc7739e1a..ad767a2e59 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_aarch64.yaml.j2
@@ -140,7 +140,17 @@
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - mount /cert and prepare for DOCA installation
+        - mkdir -p {{ client_mount_path }}/slurm/ssh
+        - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
+        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - mount -av
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -172,6 +182,7 @@
         # Ensure Slurm NFS root is mounted at client_mount_path (e.g. /share_omnia)
         - mkdir -p {{ client_mount_path }}/slurm/ssh
         - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
+        # Additional Slurm login node mounts (excluding cert which was mounted earlier)
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
@@ -179,14 +190,9 @@
         - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path}}/hpc_tools  /hpc_tools   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
-
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
+        - mount -av
         - /usr/local/bin/check_slurm_controller_status.sh
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }}
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }}
@@ -251,8 +257,6 @@
         - systemctl start slurmd
         - systemctl daemon-reexec
         - systemctl restart sshd
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
         - mkdir -p /etc/containers/registries.conf.d
         - mv /tmp/apptainer_mirror.conf /etc/containers/registries.conf.d/apptainer_mirror.conf
 
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2
index 7571acf7c7..faa5c234b6 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_node_x86_64.yaml.j2
@@ -139,7 +139,17 @@
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - mount /cert and prepare for DOCA installation
+        - mkdir -p {{ client_mount_path }}/slurm/ssh
+        - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
+        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - mount -av
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -168,10 +178,7 @@
 {% for pv_entry in cloud_init_groups_dict[functional_group_name].powervault_scripts | default([]) %}
         - bash /usr/local/bin/setup_iscsi_storage_{{ pv_entry.name }}.sh
 {% endfor %}
-        # Ensure Slurm NFS root is mounted at client_mount_path (e.g. /share_omnia)
-        - mkdir -p {{ client_mount_path }}/slurm/ssh
-        - mkdir -p {{ slurm_slurmd_log_dir_effective }} {{ slurm_slurmd_pid_dir_effective }} {{ slurm_slurmd_spool_dir_effective }} {{ slurm_epilog_dirs_all | join(' ') }} /etc/munge /cert /var/log/track /var/lib/packages /hpc_tools
-        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        # Additional Slurm login node mounts (excluding cert which was mounted earlier)
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/log/slurm  {{ slurm_slurmd_log_dir_effective }}   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/var/spool/slurmd      {{ slurm_slurmd_spool_dir_effective }}       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/$(hostname -s)/etc/slurm/epilog.d     /etc/slurm/epilog.d      nfs defaults,_netdev 0 0" >> /etc/fstab
@@ -179,14 +186,9 @@
         - echo "{{ trackfile_nfs_path }}    /var/log/track       nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path}}/hpc_tools  /hpc_tools   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
-        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
         - echo "{{ cloud_init_nfs_path }}/packages  /var/lib/packages   nfs defaults,_netdev 0 0" >> /etc/fstab
-
         - chmod {{ file_mode }} /etc/fstab
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
+        - mount -av
         - /usr/local/bin/check_slurm_controller_status.sh
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_log_dir_effective }}
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ slurm_slurmd_pid_dir_effective }}
@@ -252,8 +254,6 @@
         - systemctl start slurmd
         - systemctl daemon-reexec
         - systemctl restart sshd
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
         - mkdir -p /etc/containers/registries.conf.d
         - mv /tmp/apptainer_mirror.conf /etc/containers/registries.conf.d/apptainer_mirror.conf
 
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2
index bde54bd9e7..2ee561109c 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_control_node_x86_64.yaml.j2
@@ -276,7 +276,17 @@
 
       runcmd:
         - /usr/local/bin/set-ssh.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts
+        - mkdir -p {{ client_mount_path }}/slurm/ssh
+        - mkdir -p {{ slurm_ctld_log_dir_effective }} {{ slurmdbd_log_dir_effective }} {{ slurm_ctld_pid_dir_effective }} {{ slurmdbd_pid_dir_effective }} {{ slurm_state_save_location_effective }} {% if slurm_sched_log_dir_effective %}{{ slurm_sched_log_dir_effective }} {% endif %}/etc/slurm {{ home_dir }} /etc/my.cnf.d /etc/munge /var/lib/mysql /var/log/mariadb /cert /var/log/track /var/lib/packages /hpc_tools/container_images /hpc_tools/scripts
+        - echo "{{ cloud_init_nfs_path }}/cert  /cert   nfs defaults,_netdev 0 0" >> /etc/fstab
+        - mount -av
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -327,11 +337,6 @@
         - echo "{{ cloud_init_nfs_path }}/ssh {{ client_mount_path }}/slurm/ssh nfs defaults,_netdev 0 0" >> /etc/fstab
         - chmod {{ file_mode }} /etc/fstab
         - mount -av
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
-
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
         - chown -R {{ slurm_user }}:{{ slurm_user }} {{ home_dir }}
         - chmod {{ file_mode_755 }} {{ home_dir }}
         - chown -R {{ slurm_user }}:{{ slurm_user }} /etc/slurm
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2
index 92687f38e5..2d4b7ad001 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2
@@ -394,7 +394,14 @@
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - /usr/local/bin/configure_vast_installation.sh
-{# Mount-specific runcmd entries #}
+        # DOCA prerequisites - moved early to ensure RDMA is ready before vendor_data mounts
+        - /usr/local/bin/configure_dirs_and_mounts.sh
+        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
+        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
+        # DOCA and IB configuration - now ready before vendor_data mounts
+        - bash /usr/local/bin/doca-install.sh || true
+        - bash /usr/local/bin/configure-ib-network.sh
+{# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #}
 {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined %}
 {% for cmd in cloud_init_groups_dict[functional_group_name].runcmd %}
         - {{ cmd }}
@@ -427,11 +434,7 @@
 {% if dcgm_support %}
         - /usr/local/bin/setup_dcgm.sh
 {% endif %}
-        - /usr/local/bin/configure_dirs_and_mounts.sh
-        - cp /cert/pulp_webserver.crt /etc/pki/ca-trust/source/anchors && update-ca-trust
-        - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf
-        - bash /usr/local/bin/doca-install.sh || true
-        - bash /usr/local/bin/configure-ib-network.sh
+
 {% if slurm_node_present %}
         - |
           set -e

From d9eda41ff816925b6d1ff2c1bc7f59ae297194ff Mon Sep 17 00:00:00 2001
From: snarthan <narthan.s@dell.com>
Date: Fri, 15 May 2026 17:20:37 +0530
Subject: [PATCH 21/32] Update
 ci-group-service_kube_control_plane_first_x86_64.yaml.j2

Signed-off-by: snarthan <narthan.s@dell.com>
---
 .../ci-group-service_kube_control_plane_first_x86_64.yaml.j2  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
index 378fb8fe63..182777cf85 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
@@ -923,8 +923,8 @@
                       echo "Snapshot Controller deployment failed (expected). Continuing..."
                     }
 
-                    echo "Updating Snapshot Controller image to v8.3.0..."
-                    kubectl set image deployment/snapshot-controller snapshot-controller=registry.k8s.io/sig-storage/snapshot-controller:v8.3.0 -n kube-system >/dev/null 2>&1 || true
+                    echo "Updating Snapshot Controller image to v8.4.0..."
+                    kubectl set image deployment/snapshot-controller snapshot-controller=registry.k8s.io/sig-storage/snapshot-controller:v8.4.0 -n kube-system >/dev/null 2>&1 || true
 
                     echo "Waiting for Snapshot Controller rollout to finish (timeout: 5 minutes)..."
                     kubectl rollout status deployment/snapshot-controller -n kube-system --timeout=300s >/dev/null 2>&1 || {

From eef8b20a4427c692ac4e3c3077ca83de53616d4f Mon Sep 17 00:00:00 2001
From: snarthan <narthan.s@dell.com>
Date: Fri, 15 May 2026 17:22:06 +0530
Subject: [PATCH 22/32] Update config.py

Signed-off-by: snarthan <narthan.s@dell.com>
---
 .../module_utils/input_validation/common_utils/config.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/library/module_utils/input_validation/common_utils/config.py b/common/library/module_utils/input_validation/common_utils/config.py
index d1bb4b5b61..667da006ea 100644
--- a/common/library/module_utils/input_validation/common_utils/config.py
+++ b/common/library/module_utils/input_validation/common_utils/config.py
@@ -130,7 +130,7 @@
     "intel_benchmarks": "2024.1.0",
     "ucx": "1.19.0",
     "openmpi": "5.0.8",
-    "csi_driver_powerscale": "v2.15.0",
+    "csi_driver_powerscale": "v2.16.0",
     "rocm": "6.3.1",
     "service_k8s": "1.35.1"
 }

From 41d1c99226a5f21698a958331f944664ad25ab0a Mon Sep 17 00:00:00 2001
From: snarthan <narthan.s@dell.com>
Date: Fri, 15 May 2026 17:24:49 +0530
Subject: [PATCH 23/32] Update csi_driver_powerscale.json

Signed-off-by: snarthan <narthan.s@dell.com>
---
 .../rhel/10.0/csi_driver_powerscale.json      | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json b/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json
index 7470b45cf5..87d133328d 100644
--- a/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json
+++ b/input/config/x86_64/rhel/10.0/csi_driver_powerscale.json
@@ -5,63 +5,63 @@
         "package": "csi-powerscale",
         "url": "https://github.com/dell/csi-powerscale.git",
         "type": "git",
-        "version": "v2.15.0" 
+        "version": "v2.16.0"
       },
       {
         "package": "external-snapshotter",
         "url": "https://github.com/kubernetes-csi/external-snapshotter.git",
         "type": "git",
-        "version": "v8.3.0"
+        "version": "v8.4.0"
       },
       {
         "package": "helm-charts",
         "url": "https://github.com/dell/helm-charts.git",
         "type": "git",
-        "version": "csi-isilon-2.15.0"
+        "version": "csi-isilon-2.16.0"
       },
       {
         "package": "quay.io/dell/container-storage-modules/csi-isilon",
-        "tag": "v2.15.0",
+        "tag": "v2.16.0",
         "type": "image"
       },
       {
         "package": "registry.k8s.io/sig-storage/csi-attacher",
-        "tag": "v4.9.0", 
+        "tag": "v4.10.0",
         "type": "image"
       },
       {
         "package": "registry.k8s.io/sig-storage/csi-provisioner",
-        "tag": "v5.3.0", 
+        "tag": "v6.1.0",
         "type": "image"
       },
       {
         "package": "registry.k8s.io/sig-storage/csi-snapshotter",
-        "tag": "v8.3.0",
+        "tag": "v8.4.0",
         "type": "image"
       },
       {
         "package": "registry.k8s.io/sig-storage/csi-resizer",
-        "tag": "v1.14.0",
+        "tag": "v2.0.0",
         "type": "image"
       },
       {
         "package": "registry.k8s.io/sig-storage/csi-node-driver-registrar",
-        "tag": "v2.14.0",
+        "tag": "v2.15.0",
         "type": "image"
       },
       {
         "package": "registry.k8s.io/sig-storage/csi-external-health-monitor-controller",
-        "tag": "v0.15.0",
+        "tag": "v0.16.0",
         "type": "image"
      },
      {
       "package": "quay.io/dell/container-storage-modules/dell-csi-replicator",
-      "tag": "v1.13.0",
+      "tag": "v1.14.0",
       "type": "image"
     },
      {
       "package": "quay.io/dell/container-storage-modules/podmon",
-      "tag": "v1.14.0",
+      "tag": "v1.15.0",
       "type": "image"
      },
       {
@@ -71,12 +71,12 @@
      },
     {
       "package": "quay.io/dell/container-storage-modules/csi-metadata-retriever",
-      "tag": "v1.12.0",
+      "tag": "v1.13.0",
       "type": "image"
     },
     {
       "package": "registry.k8s.io/sig-storage/snapshot-controller",
-      "tag": "v8.3.0",
+      "tag": "v8.4.0",
       "type": "image"
     },
     {

From 0764fbb7965281da540cf516326298bec6ba28c2 Mon Sep 17 00:00:00 2001
From: Katakam Rakesh Naga Sai
 <125246792+Katakam-Rakesh@users.noreply.github.com>
Date: Fri, 15 May 2026 17:57:38 +0530
Subject: [PATCH 24/32] Merge pull request #4453 from Katakam-Rakesh/pub/q2_dev

fix for kubernetes pip version
---
 input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json
index 749634d89f..36e83d7fb8 100644
--- a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json
+++ b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json
@@ -78,7 +78,7 @@
       { "package": "prettytable==3.14.0", "type": "pip_module" },
       { "package": "python3-3.12.9", "type": "rpm", "repo_name": "baseos" },
       { "package": "git", "type": "rpm", "repo_name": "appstream"},
-      { "package": "kubernetes==35.0.0", "type": "pip_module" },
+      { "package": "kubernetes==33.1.0", "type": "pip_module" },
       { "package": "PyMySQL==1.1.2", "type": "pip_module" }
 
     ]
@@ -106,7 +106,7 @@
       { "package": "prettytable==3.14.0", "type": "pip_module" },
       { "package": "python3-3.12.9", "type": "rpm", "repo_name": "baseos" },
       { "package": "git", "type": "rpm", "repo_name": "appstream"},
-      { "package": "kubernetes==35.0.0", "type": "pip_module" },
+      { "package": "kubernetes==33.1.0", "type": "pip_module" },
       { "package": "PyMySQL==1.1.2", "type": "pip_module" }
     ]
   },
@@ -118,4 +118,4 @@
       { "package": "quay.io/metallb/controller", "tag": "v0.15.3", "type": "image" } 
     ]
   }
-}
\ No newline at end of file
+}

From ba4e12671a9e9c80287d16e4c80d556a37396bf6 Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Fri, 15 May 2026 18:11:34 +0530
Subject: [PATCH 25/32] Fixes for merge conflicts and ufm

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 prepare_oim/prepare_oim.yml                    |  4 +++-
 ...ice_kube_control_plane_first_x86_64.yaml.j2 | 18 ++----------------
 .../tasks/derive_sink_support_flags.yml        |  4 +++-
 provision/roles/telemetry/tasks/main.yml       |  8 +++-----
 4 files changed, 11 insertions(+), 23 deletions(-)

diff --git a/prepare_oim/prepare_oim.yml b/prepare_oim/prepare_oim.yml
index 4b5839eaa2..cd994a7d81 100644
--- a/prepare_oim/prepare_oim.yml
+++ b/prepare_oim/prepare_oim.yml
@@ -102,7 +102,9 @@
                   (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
                   (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
                   (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or
-                  (telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool)
+                  (telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool) or
+                  (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
+                  (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool)
 
         - name: Check discovery configuration for OME
           block:
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
index c5aab6ed48..6dc6677396 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
@@ -321,20 +321,6 @@
               ipAddressPools:
               - first-pool
 
-{% if telemetry_enabled or ldms_support or ufm_telemetry_support%}
-        - path: /root/telemetry.sh
-          owner: root:root
-          permissions: '0755'
-          content: |
-            {{ lookup('template', 'templates/telemetry/telemetry.sh.j2') | indent(12) }}
-{% endif %}
-{% for pv_entry in cloud_init_groups_dict[functional_group_name].powervault_scripts | default([]) %}
-        - path: /usr/local/bin/setup_iscsi_storage_{{ pv_entry.name }}.sh
-          permissions: '{{ file_mode_755 }}'
-          content: |
-            {{ pv_entry.content | indent(12) }}
-{% endfor %}
-
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - "systemctl enable chronyd"
@@ -1073,7 +1059,7 @@
 {% include 'powerscale/deploy_powerscale_telemetry.sh.j2' %}
 {% endif %}
 
-{% if telemetry_enabled or ldms_support or ufm_telemetry_support%}
+{% if telemetry_enabled %}
             echo "Applying Telemetry Kubernetes deployments"
             {{ k8s_client_mount_path }}/telemetry/telemetry.sh
 {% endif %}
@@ -1123,4 +1109,4 @@
             kubectl get pods --all-namespaces -o wide
             echo "Cloud-Init finished successfully after the reboot."
 
-          fi
+          fi
\ No newline at end of file
diff --git a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
index c2f62662d6..0d97419187 100644
--- a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
+++ b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
@@ -141,4 +141,6 @@
     victoria_metrics_support or
     victoria_logs_support or
     ldms_support or
-    kafka_support
+    kafka_support or
+    ufm_telemetry_support or
+    ufm_log_enabled
diff --git a/provision/roles/telemetry/tasks/main.yml b/provision/roles/telemetry/tasks/main.yml
index 261d3598ad..2d0500417f 100644
--- a/provision/roles/telemetry/tasks/main.yml
+++ b/provision/roles/telemetry/tasks/main.yml
@@ -41,6 +41,8 @@
       (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
       (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
       (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or
+      (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
+      (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool) or
       ldms_support | default(false) | bool
   block:
     - name: Set NFS info fact
@@ -122,11 +124,7 @@
 - name: Generate telemetry deployment script
   ansible.builtin.include_tasks: generate_telemetry_script.yml
   when:
-    - >-
-      (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
-      ldms_support | default(false) | bool
+    - telemetry_enabled | default(false) | bool
   tags:
     - telemetry_deployment
 

From 355a9f6d25b6aae4f243cf71134e1974f5d46820 Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Fri, 15 May 2026 20:11:13 +0530
Subject: [PATCH 26/32] merge conflicts resolved

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 provision/roles/telemetry/tasks/main.yml | 223 ++++++++++++-----------
 1 file changed, 113 insertions(+), 110 deletions(-)

diff --git a/provision/roles/telemetry/tasks/main.yml b/provision/roles/telemetry/tasks/main.yml
index 9e706e494f..f9213c6db7 100644
--- a/provision/roles/telemetry/tasks/main.yml
+++ b/provision/roles/telemetry/tasks/main.yml
@@ -18,117 +18,120 @@
     file: "{{ telemetry_config_file_path }}"
     name: telemetry_config
 
-- name: Read telemetry packages from software config
-  ansible.builtin.include_tasks: read_software_config.yml
-
 - name: Derive sink support flags from collection_targets
   ansible.builtin.include_tasks: derive_sink_support_flags.yml
 
-- name: Load service images from service_k8s_v<version>.json
-  ansible.builtin.include_tasks: load_service_images.yml
-
-- name: Check kube_vip reachability for validation
-  ansible.builtin.include_tasks: check_kube_vip_reachability.yml
-  when:
-    - victoria_metrics_support | default(false) | bool
-    - kube_vip is defined
-    - kube_vip | length > 0
-
-- name: Configure of k8s telemetry service prerequisites
-  when:
-    - >-
-      (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool) or
-      ldms_support | default(false) | bool
-  block:
-    - name: Set NFS info fact
-      ansible.builtin.set_fact:
-        oim_shared_path: "{{ hostvars['localhost']['oim_shared_path'] }}"
-
-    - name: Service cluster prerequisite
-      ansible.builtin.include_tasks: telemetry_prereq.yml
-
-- name: Deploy PowerScale telemetry metrics
-  when:
-    - telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool
-  ansible.builtin.include_tasks: deploy_powerscale_metrics.yml
-
-- name: Deploy UFM telemetry (metrics only)
-  when:
-    - telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool
-  ansible.builtin.include_tasks: deploy_ufm_telemetry.yml
-
-- name: Generate telemetry deployments (idrac/ldms/powerscale/vector)
-  when:
-    - >-
-      (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool) or
-      (telemetry_config.telemetry_bridges.vector_ome.logs_enabled | default(false) | bool) or
-      ldms_support | default(false) | bool
-  ansible.builtin.include_tasks: generate_telemetry_deployments.yml
-
-- name: Deploy Vector-LDMS bridge (Kafka-to-VictoriaMetrics pipeline)
-  ansible.builtin.include_tasks: deploy_vector_ldms.yml
-  when:
-    - telemetry_config.telemetry_bridges.vector_ldms.metrics_enabled | default(false) | bool
-    - victoria_metrics_support | default(false) | bool
-  tags:
-    - telemetry_deployment
-    - vector_ldms
-
-- name: Deploy Vector-OME bridge (Kafka-to-Victoria pipeline for OME data)
-  ansible.builtin.include_tasks: deploy_vector_ome.yml
-  when:
-    - telemetry_config.telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool or
-      telemetry_config.telemetry_bridges.vector_ome.logs_enabled | default(false) | bool
-    - kafka_support | default(false) | bool
-  tags:
-    - telemetry_deployment
-    - vector_ome
-
-- name: Configure of k8s telemetry service
-  when:
-    - telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool
+- name: Configure service_k8s telemetry services
+  when: hostvars['localhost']['service_k8s_support'] | default(false) | bool
   block:
-    - name: Validate idrac telemetry config
-      ansible.builtin.include_tasks: validate_idrac_inventory.yml
-
-    - name: Generate service cluster metadata
-      ansible.builtin.include_tasks: generate_service_cluster_metadata.yml
-
-- name: Include update_ldms_sampler.yml
-  ansible.builtin.include_tasks: update_ldms_sampler.yml
-  when: ldms_support
-
-- name: Update ldms agg configuration
-  ansible.builtin.include_tasks: update_ldms_agg_config.yml
-  when: ldms_support
-
-- name: Check if PXE mapping has changed since last run
-  ansible.builtin.include_tasks: check_pxe_changes.yml
-  when: ldms_support
-
-- name: Restart LDMS configs for node addition and deletion
-  ansible.builtin.include_tasks: restart_ldms_configs.yml
-  when:
-    - ldms_support
-    - pxe_changed | default(false) | bool
-
-- name: Generate telemetry deployment script
-  ansible.builtin.include_tasks: generate_telemetry_script.yml
-  when:
-    - telemetry_enabled | default(false) | bool
-  tags:
-    - telemetry_deployment
-
-- name: Apply telemetry configurations on upgrade
-  ansible.builtin.include_tasks: apply_telemetry_on_upgrade.yml
-  when:
-    - hostvars['localhost']['upgrade_enabled'] | default(false) | bool
+    - name: Read telemetry packages from software config
+      ansible.builtin.include_tasks: read_software_config.yml
+
+    - name: Load service images from service_k8s_v<version>.json
+      ansible.builtin.include_tasks: load_service_images.yml
+
+    - name: Check kube_vip reachability for validation
+      ansible.builtin.include_tasks: check_kube_vip_reachability.yml
+      when:
+        - victoria_metrics_support | default(false) | bool
+        - kube_vip is defined
+        - kube_vip | length > 0
+
+    - name: Configure of k8s telemetry service prerequisites
+      when:
+        - >-
+          (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool) or
+          ldms_support | default(false) | bool
+      block:
+        - name: Set NFS info fact
+          ansible.builtin.set_fact:
+            oim_shared_path: "{{ hostvars['localhost']['oim_shared_path'] }}"
+
+        - name: Service cluster prerequisite
+          ansible.builtin.include_tasks: telemetry_prereq.yml
+
+    - name: Deploy PowerScale telemetry metrics
+      when:
+        - telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool
+      ansible.builtin.include_tasks: deploy_powerscale_metrics.yml
+
+    - name: Deploy UFM telemetry metrics
+      when:
+        - telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool
+      ansible.builtin.include_tasks: deploy_ufm_metrics.yml
+
+    - name: Generate telemetry deployments (idrac/ldms/powerscale/vector)
+      when:
+        - >-
+          (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_bridges.vector_ome.logs_enabled | default(false) | bool) or
+          ldms_support | default(false) | bool
+      ansible.builtin.include_tasks: generate_telemetry_deployments.yml
+
+    - name: Deploy Vector-LDMS bridge (Kafka-to-VictoriaMetrics pipeline)
+      ansible.builtin.include_tasks: deploy_vector_ldms.yml
+      when:
+        - telemetry_config.telemetry_bridges.vector_ldms.metrics_enabled | default(false) | bool
+        - victoria_metrics_support | default(false) | bool
+      tags:
+        - telemetry_deployment
+        - vector_ldms
+
+    - name: Deploy Vector-OME bridge (Kafka-to-Victoria pipeline for OME data)
+      ansible.builtin.include_tasks: deploy_vector_ome.yml
+      when:
+        - telemetry_config.telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool or
+          telemetry_config.telemetry_bridges.vector_ome.logs_enabled | default(false) | bool
+        - kafka_support | default(false) | bool
+      tags:
+        - telemetry_deployment
+        - vector_ome
+
+    - name: Configure of k8s telemetry service
+      when:
+        - telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool
+      block:
+        - name: Validate idrac telemetry config
+          ansible.builtin.include_tasks: validate_idrac_inventory.yml
+
+        - name: Generate service cluster metadata
+          ansible.builtin.include_tasks: generate_service_cluster_metadata.yml
+
+    - name: Include update_ldms_sampler.yml
+      ansible.builtin.include_tasks: update_ldms_sampler.yml
+      when: ldms_support
+
+    - name: Update ldms agg configuration
+      ansible.builtin.include_tasks: update_ldms_agg_config.yml
+      when: ldms_support
+
+    - name: Check if PXE mapping has changed since last run
+      ansible.builtin.include_tasks: check_pxe_changes.yml
+      when: ldms_support
+
+    - name: Restart LDMS configs for node addition and deletion
+      ansible.builtin.include_tasks: restart_ldms_configs.yml
+      when:
+        - ldms_support
+        - pxe_changed | default(false) | bool
+
+    - name: Generate telemetry deployment script
+      ansible.builtin.include_tasks: generate_telemetry_script.yml
+      when:
+        - telemetry_enabled | default(false) | bool
+      tags:
+        - telemetry_deployment
+
+    - name: Apply telemetry configurations on upgrade
+      ansible.builtin.include_tasks: apply_telemetry_on_upgrade.yml
+      when:
+        - hostvars['localhost']['upgrade_enabled'] | default(false) | bool
\ No newline at end of file

From 6deafdfc765a476d68792774fbc0987f2d0fb26f Mon Sep 17 00:00:00 2001
From: Abhishek S A <abhishek.sa3@dell.com>
Date: Fri, 15 May 2026 22:42:20 +0530
Subject: [PATCH 27/32] Update telemetry_secret_creation.yaml.j2

Signed-off-by: Abhishek S A <abhishek.sa3@dell.com>
---
 .../telemetry/common/telemetry_secret_creation.yaml.j2        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2 b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
index e054c5985c..65ef9a3bfd 100644
--- a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
@@ -18,5 +18,5 @@ metadata:
   namespace: "{{ telemetry_namespace }}"
 type: Opaque
 stringData:
-  cluster-id: "{{ kafka.cluster_id }}"
-{% endif %}
\ No newline at end of file
+  cluster-id: "{{ cluster_id.stdout }}"
+{% endif %}

From 3c9b30881e804fd2514912acdedebd49b7160a0c Mon Sep 17 00:00:00 2001
From: Abhishek S A <abhishek.sa3@dell.com>
Date: Fri, 15 May 2026 22:45:30 +0530
Subject: [PATCH 28/32] Update main.yml

Signed-off-by: Abhishek S A <abhishek.sa3@dell.com>
---
 provision/roles/telemetry/tasks/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/provision/roles/telemetry/tasks/main.yml b/provision/roles/telemetry/tasks/main.yml
index 860998b6b3..84c16c9941 100644
--- a/provision/roles/telemetry/tasks/main.yml
+++ b/provision/roles/telemetry/tasks/main.yml
@@ -63,7 +63,7 @@
     - name: Deploy UFM telemetry metrics
       when:
         - telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool
-      ansible.builtin.include_tasks: deploy_ufm_metrics.yml
+      ansible.builtin.include_tasks: deploy_ufm_telemetry.yml
 
     - name: Generate telemetry deployments (idrac/ldms/powerscale/vector)
       when:

From 9adea7c3ea4759bf6b2e4e50d7f4c8e396ec1985 Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Fri, 15 May 2026 22:50:10 +0530
Subject: [PATCH 29/32] ufm telemetry support [metrics,syslog] (#4405)

* ufm telemetry support [metrics,syslog]

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* ansible lint fix

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* ansible lint fix

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* ansible lint fix

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* updated ufm config

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* added verbosity to the msg

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* reference of deploy_powerscale_logs.yml removed

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* added ufm credential in config credentials utils

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* resolved merge conflicts

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* Removed debug msgs

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* updated the ufm secrets flow

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* Fixes for merge conflicts and ufm

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* merge conflicts resolved

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>

* Update telemetry_secret_creation.yaml.j2

Signed-off-by: Abhishek S A <abhishek.sa3@dell.com>

* Update main.yml

Signed-off-by: Abhishek S A <abhishek.sa3@dell.com>

---------

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
Signed-off-by: Abhishek S A <abhishek.sa3@dell.com>
Co-authored-by: Abhishek S A <abhishek.sa3@dell.com>
---
 .../schema/credential_rules.json              |  12 ++
 .../schema/telemetry_config.json              |  80 +++++++++-
 .../validation_flows/telemetry_validation.py  |  67 ++++++++
 .../library/modules/fetch_telemetry_status.py |   7 +
 input/telemetry_config.yml                    |  74 +++++++++
 prepare_oim/prepare_oim.yml                   |   4 +-
 ...ce_kube_control_plane_first_x86_64.yaml.j2 |   2 +-
 .../telemetry/tasks/deploy_ufm_telemetry.yml  | 148 ++++++++++++++++++
 .../tasks/derive_sink_support_flags.yml       |  11 +-
 .../tasks/generate_telemetry_deployments.yml  |  14 ++
 provision/roles/telemetry/tasks/main.yml      |  14 +-
 .../common/telemetry_secret_creation.yaml.j2  |   2 +-
 .../templates/telemetry/kustomization.yaml.j2 |   5 +
 .../ufm/ufm-external-service.yaml.j2          |  44 ++++++
 .../ufm/ufm-telemetry-secret.yaml.j2          |  58 +++++++
 .../victoria-operator-vmscrape.yaml.j2        |  55 +++++++
 provision/roles/telemetry/vars/main.yml       |  58 +++++++
 .../templates/omnia_credential.j2             |   4 +
 .../update_config/tasks/fetch_credentials.yml |   3 +-
 .../roles/update_config/vars/main.yml         |   5 +
 .../roles/validation/tasks/pre_requisite.yml  |  14 ++
 21 files changed, 667 insertions(+), 14 deletions(-)
 create mode 100644 provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
 create mode 100644 provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
 create mode 100644 provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2

diff --git a/common/library/module_utils/input_validation/schema/credential_rules.json b/common/library/module_utils/input_validation/schema/credential_rules.json
index ae0872671c..8fc1e6251a 100644
--- a/common/library/module_utils/input_validation/schema/credential_rules.json
+++ b/common/library/module_utils/input_validation/schema/credential_rules.json
@@ -161,5 +161,17 @@
         "maxLength": 128,
         "pattern": "^.+$",
         "description": "Password for Dell OpenManage Enterprise (OME). Length must be between 1 and 128 characters."
+      },
+      "ufm_username": {
+        "minLength": 1,
+        "maxLength": 64,
+        "pattern": "^[^\\\\'\";]+$",
+        "description": "Username for NVIDIA UFM (Unified Fabric Manager). Must not contain backslashes (\\), single quotes ('), double quotes (\"), or semicolons (;). Length must be between 1 and 64 characters."
+      },
+      "ufm_password": {
+        "minLength": 1,
+        "maxLength": 128,
+        "pattern": "^.+$",
+        "description": "Password for NVIDIA UFM (Unified Fabric Manager). Length must be between 1 and 128 characters."
       }
   }
diff --git a/common/library/module_utils/input_validation/schema/telemetry_config.json b/common/library/module_utils/input_validation/schema/telemetry_config.json
index 28260bc42f..c0ce313832 100644
--- a/common/library/module_utils/input_validation/schema/telemetry_config.json
+++ b/common/library/module_utils/input_validation/schema/telemetry_config.json
@@ -94,9 +94,37 @@
             }
           },
           "required": ["metrics_enabled", "collection_targets"]
+        },
+        "ufm": {
+          "type": "object",
+          "description": "NVIDIA UFM InfiniBand Fabric telemetry.",
+          "properties": {
+            "metrics_enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "Enable or disable UFM InfiniBand metrics collection."
+            },
+            "logs_enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "Enable or disable UFM syslog logs collection."
+            },
+            "collection_targets": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": ["victoria_metrics", "victoria_logs"]
+              },
+              "minItems": 1,
+              "uniqueItems": true,
+              "default": ["victoria_metrics"],
+              "description": "UFM supports victoria_metrics (metrics) and victoria_logs (logs)."
+            }
+          },
+          "required": ["metrics_enabled", "collection_targets"]
         }
       },
-      "required": ["idrac", "ldms", "dcgm", "powerscale"]
+      "required": ["idrac", "ldms", "dcgm", "powerscale", "ufm"]
     },
     "telemetry_bridges": {
       "type": "object",
@@ -398,7 +426,55 @@
         }
       },
       "required": ["otel_collector_storage_size", "csm_observability_values_file_path"]
+    },
+    "ufm_configuration": {
+      "type": "object",
+      "description": "UFM telemetry detailed configurations.",
+      "properties": {
+        "ufm_endpoint": {
+          "type": "string",
+          "minLength": 1,
+          "description": "UFM appliance IP address or hostname."
+        },
+        "ufm_metrics_port": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 65535,
+          "default": 9001,
+          "description": "UFM Prometheus exporter port."
+        },
+        "scrape_interval": {
+          "type": "string",
+          "pattern": "^[0-9]+[smh]$",
+          "default": "30s",
+          "description": "Prometheus scrape interval for UFM metrics."
+        },
+        "scrape_timeout": {
+          "type": "string",
+          "pattern": "^[0-9]+[smh]$",
+          "default": "15s",
+          "description": "Prometheus scrape timeout (must be <= scrape_interval)."
+        },
+        "tls_mode": {
+          "type": "string",
+          "enum": ["self_signed", "ca_signed"],
+          "default": "self_signed",
+          "description": "TLS mode for connecting to UFM Prometheus endpoint."
+        },
+        "ufm_ca_cert_path": {
+          "type": "string",
+          "default": "",
+          "description": "Path to CA certificate file for UFM TLS verification."
+        },
+        "auth_mode": {
+          "type": "string",
+          "enum": ["basic", "none"],
+          "default": "basic",
+          "description": "Authentication mode for UFM Prometheus endpoint."
+        }
+      },
+      "required": ["ufm_endpoint"]
     }
   },
-  "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations"]
+  "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations", "ufm_configuration"]
 }
diff --git a/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py b/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py
index 5f60311920..48f5e0d4e7 100644
--- a/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py
+++ b/common/library/module_utils/input_validation/validation_flows/telemetry_validation.py
@@ -328,6 +328,7 @@ def validate_telemetry_config(
     idrac_source = telemetry_sources.get("idrac", {})
     ldms_source = telemetry_sources.get("ldms", {})
     powerscale_source = telemetry_sources.get("powerscale", {})
+    ufm_source = telemetry_sources.get("ufm", {})
 
     idrac_telemetry_support = idrac_source.get("metrics_enabled", False)
     idrac_collection_targets = idrac_source.get("collection_targets", [])
@@ -383,6 +384,17 @@ def validate_telemetry_config(
             list(invalid_powerscale_targets),
             f"Invalid collection targets for PowerScale. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_powerscale_targets}"
         ))
+    
+    # UFM: supports victoria_metrics and victoria_logs
+    ufm_targets = set(ufm_source.get("collection_targets", []))
+    allowed_ufm_targets = {"victoria_metrics", "victoria_logs"}
+    invalid_ufm_targets = ufm_targets - allowed_ufm_targets
+    if invalid_ufm_targets:
+        errors.append(create_error_msg(
+            "telemetry_sources.ufm.collection_targets",
+            list(invalid_ufm_targets),
+            f"Invalid collection targets for UFM. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_ufm_targets}"
+        ))
 
     # =========================================================================
     # Validate service cluster and slurm cluster
@@ -784,4 +796,59 @@ def validate_telemetry_config(
         is_service_cluster_defined, config_paths, logger, errors
     )
 
+    # =========================================================================
+    # Validate UFM telemetry configuration
+    # =========================================================================
+    ufm_metrics_enabled = ufm_source.get("metrics_enabled", False)
+    ufm_logs_enabled = ufm_source.get("logs_enabled", False)
+    ufm_detailed_config = data.get("ufm_configuration", {})
+    
+    if ufm_metrics_enabled or ufm_logs_enabled:
+        # Check required UFM endpoint
+        ufm_endpoint = ufm_detailed_config.get("ufm_endpoint", "")
+        if not ufm_endpoint or (isinstance(ufm_endpoint, str) and ufm_endpoint.strip() == ""):
+            errors.append(create_error_msg(
+                "ufm_configuration.ufm_endpoint",
+                ufm_endpoint,
+                "ufm_endpoint is required when UFM telemetry is enabled. Provide the UFM appliance IP address or hostname."
+            ))
+        
+        # Validate UFM metrics port if metrics enabled
+        if ufm_metrics_enabled:
+            ufm_metrics_port = ufm_detailed_config.get("ufm_metrics_port", 9001)
+            if not isinstance(ufm_metrics_port, int) or ufm_metrics_port < 1 or ufm_metrics_port > 65535:
+                errors.append(create_error_msg(
+                    "ufm_configuration.ufm_metrics_port",
+                    ufm_metrics_port,
+                    "ufm_metrics_port must be an integer between 1 and 65535."
+                ))
+        
+        # Validate TLS mode
+        tls_mode = ufm_detailed_config.get("tls_mode", "self_signed")
+        if tls_mode not in ["self_signed", "ca_signed"]:
+            errors.append(create_error_msg(
+                "ufm_configuration.tls_mode",
+                tls_mode,
+                "tls_mode must be 'self_signed' or 'ca_signed'."
+            ))
+        
+        # Validate CA certificate path when tls_mode is ca_signed
+        if tls_mode == "ca_signed":
+            ca_cert_path = ufm_detailed_config.get("ufm_ca_cert_path", "")
+            if not ca_cert_path or (isinstance(ca_cert_path, str) and ca_cert_path.strip() == ""):
+                errors.append(create_error_msg(
+                    "ufm_configuration.ufm_ca_cert_path",
+                    ca_cert_path,
+                    "ufm_ca_cert_path is required when tls_mode is 'ca_signed'. Provide path to CA certificate file."
+                ))
+        
+        # Validate auth mode
+        auth_mode = ufm_detailed_config.get("auth_mode", "basic")
+        if auth_mode not in ["basic", "none"]:
+            errors.append(create_error_msg(
+                "ufm_configuration.auth_mode",
+                auth_mode,
+                "auth_mode must be 'basic' or 'none'."
+            ))
+
     return errors
diff --git a/common/library/modules/fetch_telemetry_status.py b/common/library/modules/fetch_telemetry_status.py
index d6db54ab97..622d05846f 100644
--- a/common/library/modules/fetch_telemetry_status.py
+++ b/common/library/modules/fetch_telemetry_status.py
@@ -72,6 +72,13 @@ def main():
     if telemetry_sources.get("idrac", {}).get("metrics_enabled", False):
         telemetry_status_list.append("idrac_telemetry")
 
+    # Check UFM telemetry
+    ufm_config = telemetry_sources.get("ufm", {})
+    if ufm_config.get("metrics_enabled", False):
+        telemetry_status_list.append("ufm_telemetry")
+    if ufm_config.get("logs_enabled", False):
+        telemetry_status_list.append("ufm_logs")
+
     module.exit_json(
             changed=False,
             telemetry_status_list=telemetry_status_list
diff --git a/input/telemetry_config.yml b/input/telemetry_config.yml
index a66c418ecb..17491767af 100644
--- a/input/telemetry_config.yml
+++ b/input/telemetry_config.yml
@@ -146,6 +146,29 @@ telemetry_sources:
       - "victoria_metrics"
       - "victoria_logs"
 
+  # --------------------------------------------------------------------------
+  # UFM — NVIDIA UFM InfiniBand Fabric Telemetry
+  # --------------------------------------------------------------------------
+  # Collects: IB port state, transmit/receive data, error counters, fabric topology
+  # Requires: NVIDIA UFM appliance with Prometheus exporter enabled (port 9090)
+  # Data path: UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
+  #
+  # NOTE: Omnia does NOT deploy UFM itself. Omnia configures the existing
+  # vmagent to scrape the UFM Prometheus endpoint for IB fabric metrics.
+  ufm:
+    # Enable or disable UFM InfiniBand metrics collection
+    # Default: false
+    metrics_enabled: false
+
+    # Enable or disable UFM syslog logs collection
+    # Default: false
+    logs_enabled: false
+
+    # UFM uses vmagent(shared) for metrics and VLAgent for logs
+    collection_targets:
+      - "victoria_metrics"
+      - "victoria_logs"
+
 
 # ============================================================================
 # TELEMETRY BRIDGES (Data Routers)
@@ -349,3 +372,54 @@ powerscale_configurations:
   # Required when powerscale_configurations.powerscale_telemetry_support: true
   # Reference: https://raw.githubusercontent.com/dell/helm-charts/refs/heads/release-v1.16.3/charts/karavi-observability/values.yaml
   csm_observability_values_file_path: ""
+
+# --------------------------------------------------------------------------
+# UFM Telemetry Configuration
+# --------------------------------------------------------------------------
+# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances
+# using Prometheus scraping.
+#
+# DATA PIPELINE:
+#   UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
+#
+# NOTE: UFM does NOT use Vector bridges. It uses the shared vmagent instance
+#       for metrics collection.
+ufm_configuration:
+  # UFM appliance IP address or hostname
+  # Required when telemetry_sources.ufm.metrics_enabled is true
+  # Example: "172.20.44.180" or "ufm.example.com"
+  ufm_endpoint: ""
+
+  # UFM Prometheus exporter port
+  # Default: 9001 (UFM default Prometheus port)
+  ufm_metrics_port: 9001
+
+  # Prometheus scrape interval for UFM metrics
+  # Accepted values: Prometheus duration format (e.g., "15s", "30s", "1m")
+  # Default: "30s"
+  scrape_interval: "30s"
+
+  # Prometheus scrape timeout (must be <= scrape_interval)
+  # Accepted values: Prometheus duration format (e.g., "10s", "15s")
+  # Default: "15s"
+  scrape_timeout: "15s"
+
+  # TLS mode for connecting to UFM Prometheus endpoint
+  # Accepted values: "self_signed", "ca_signed"
+  # - self_signed: Skip TLS verification (insecure_skip_verify=true)
+  # - ca_signed: Use CA certificate for TLS verification
+  # Default: "self_signed"
+  tls_mode: "self_signed"
+
+  # Path to CA certificate file for UFM TLS verification
+  # Required when tls_mode is "ca_signed"
+  # Must be a valid PEM-format certificate file
+  # Default: "" (empty — not used when tls_mode is "self_signed")
+  ufm_ca_cert_path: ""
+
+  # Authentication mode for UFM Prometheus endpoint
+  # Accepted values: "basic", "none"
+  # - basic: Use ufm_username/ufm_password from omnia_config_credentials.yml
+  # - none: No authentication (UFM endpoint is open)
+  # Default: "basic"
+  auth_mode: "basic"
diff --git a/prepare_oim/prepare_oim.yml b/prepare_oim/prepare_oim.yml
index ab05158e0a..51f9dc1f0c 100644
--- a/prepare_oim/prepare_oim.yml
+++ b/prepare_oim/prepare_oim.yml
@@ -102,7 +102,9 @@
                   (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
                   (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
                   (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or
-                  (telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool)
+                  (telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool) or
+                  (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
+                  (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool)
 
         - name: Check discovery configuration for OME
           block:
diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
index 182777cf85..574a040e3e 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
@@ -1109,4 +1109,4 @@
             kubectl get pods --all-namespaces -o wide
             echo "Cloud-Init finished successfully after the reboot."
 
-          fi
+          fi
\ No newline at end of file
diff --git a/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
new file mode 100644
index 0000000000..700d5be132
--- /dev/null
+++ b/provision/roles/telemetry/tasks/deploy_ufm_telemetry.yml
@@ -0,0 +1,148 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+# Deploy UFM InfiniBand telemetry (Metrics Only)
+# Gated by: telemetry_sources.ufm.metrics_enabled
+#
+# DATA PIPELINE:
+#   UFM Prometheus Exporter (HTTPS) → vmagent(shared) → victoria_metrics
+#
+# What Omnia does:
+#   - Validates UFM endpoint, port, TLS mode, auth mode, credentials
+#   - Creates K8s Secret with UFM credentials (and optional CA cert)
+#   - VMAgent scrape config + deployment mount are handled by Jinja2 templates
+#   - Deploys Grafana dashboard as ConfigMap (sidecar auto-discovery)
+#
+# What Omnia does NOT do:
+#   - Omnia does NOT deploy or configure UFM itself
+
+- name: Set UFM configuration facts
+  ansible.builtin.set_fact:
+    _ufm_config: "{{ telemetry_config.telemetry_sources.ufm }}"
+    _ufm_detailed_config: "{{ telemetry_config.ufm_configuration }}"
+
+# --- Validate UFM telemetry inputs ---
+# Only run validation when UFM telemetry is enabled
+
+- name: Validate UFM endpoint is not empty
+  ansible.builtin.fail:
+    msg: "{{ ufm_endpoint_empty_msg }}"
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.ufm_endpoint | default('') | trim == ''
+
+- name: Validate UFM metrics port range
+  ansible.builtin.fail:
+    msg: "{{ ufm_port_range_msg }}"
+  when: >-
+    (_ufm_config.metrics_enabled | default(false) | bool) and
+    ((_ufm_detailed_config.ufm_metrics_port | default(9090) | int < 1) or
+    (_ufm_detailed_config.ufm_metrics_port | default(9090) | int > 65535))
+
+- name: Validate UFM TLS mode
+  ansible.builtin.fail:
+    msg: "{{ ufm_tls_mode_invalid_msg }}"
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.tls_mode | default('self_signed') not in ['self_signed', 'ca_signed']
+
+- name: Validate UFM auth mode
+  ansible.builtin.fail:
+    msg: "{{ ufm_auth_mode_invalid_msg }}"
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.auth_mode | default('basic') not in ['basic', 'none']
+
+- name: Validate UFM credentials when auth_mode is basic
+  ansible.builtin.fail:
+    msg: "{{ ufm_credentials_missing_msg }}"
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.auth_mode | default('basic') == 'basic'
+    - >-
+      (hostvars['localhost']['ufm_username'] | default('') | trim == '') or
+      (hostvars['localhost']['ufm_password'] | default('') | trim == '')
+
+- name: Validate CA certificate file when tls_mode is ca_signed
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
+  block:
+    - name: Check CA certificate file exists
+      ansible.builtin.stat:
+        path: "{{ _ufm_detailed_config.ufm_ca_cert_path }}"
+      register: _ufm_ca_cert_stat
+      delegate_to: localhost
+
+    - name: Fail if CA certificate file not found
+      ansible.builtin.fail:
+        msg: "{{ ufm_ca_cert_not_found_msg }}"
+      when: >-
+        (_ufm_detailed_config.ufm_ca_cert_path | default('') | trim == '') or
+        (not _ufm_ca_cert_stat.stat.exists | default(false))
+
+- name: Display validated UFM configuration
+  ansible.builtin.debug:
+    msg:
+      - "UFM endpoint: {{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}"
+      - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}"
+      - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
+    verbosity: 2
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+
+# --- Create UFM credentials Kubernetes Secret ---
+
+- name: Read CA certificate content
+  ansible.builtin.slurp:
+    src: "{{ _ufm_detailed_config.ufm_ca_cert_path }}"
+  register: _ufm_ca_cert_content
+  delegate_to: localhost
+  when:
+    - _ufm_config.metrics_enabled | default(false) | bool
+    - _ufm_detailed_config.tls_mode | default('self_signed') == 'ca_signed'
+    - _ufm_detailed_config.ufm_ca_cert_path | default('') | trim != ''
+
+- name: Set CA certificate content for template processing
+  ansible.builtin.set_fact:
+    ufm_ca_cert_content: "{{ _ufm_ca_cert_content.content | b64decode }}"
+  when:
+    - _ufm_ca_cert_content is defined
+    - _ufm_ca_cert_content is not skipped
+    - _ufm_ca_cert_content.content is defined
+
+- name: Display UFM Secret creation status
+  ansible.builtin.debug:
+    msg: "{{ ufm_secret_created_msg }}"
+    verbosity: 2
+  when: _ufm_config.metrics_enabled | default(false) | bool
+
+
+# UFM telemetry resources are now deployed via Kustomize templates
+# Secret and external-service ConfigMap are created through:
+# - templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
+# - templates/telemetry/ufm/ufm-external-service.yaml.j2
+# These are processed in generate_telemetry_deployments.yml and applied via kubectl apply -k
+
+- name: Display UFM integration summary
+  ansible.builtin.debug:
+    msg:
+      - "UFM telemetry integration complete"
+      - "UFM endpoint: {{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}"
+      - "TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}"
+      - "Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}"
+      - "Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}"
+    verbosity: 2
+  when: _ufm_config.metrics_enabled | default(false) | bool
diff --git a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
index ac4b6df5ad..0d97419187 100644
--- a/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
+++ b/provision/roles/telemetry/tasks/derive_sink_support_flags.yml
@@ -50,6 +50,7 @@
     powerscale_metrics_enabled: "{{ telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool }}"
     powerscale_log_enabled: "{{ telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool }}"
     dcgm_support: "{{ telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(true) | bool }}"
+    ufm_telemetry_support: "{{ telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool }}"
     cacheable: true
 
 - name: Map powerscale source + configurations to legacy powerscale_configurations
@@ -68,7 +69,8 @@
     cacheable: true
   when: >-
     'victoria_metrics' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([])) or
-    'victoria_metrics' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([]))
+    'victoria_metrics' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([])) or
+    'victoria_metrics' in (telemetry_config.telemetry_sources.ufm.collection_targets | default([]))
 
 - name: Check if any source targets victoria_logs
   ansible.builtin.set_fact:
@@ -76,7 +78,8 @@
     cacheable: true
   when: >-
     'victoria_logs' in (telemetry_config.telemetry_sources.powerscale.collection_targets | default([])) or
-    'victoria_logs' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([]))
+    'victoria_logs' in (telemetry_config.telemetry_sources.idrac.collection_targets | default([])) or
+    'victoria_logs' in (telemetry_config.telemetry_sources.ufm.collection_targets | default([]))
 
 - name: Check if any source targets Kafka
   ansible.builtin.set_fact:
@@ -138,4 +141,6 @@
     victoria_metrics_support or
     victoria_logs_support or
     ldms_support or
-    kafka_support
+    kafka_support or
+    ufm_telemetry_support or
+    ufm_log_enabled
diff --git a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
index ea25d10924..de726a140c 100644
--- a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
+++ b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml
@@ -234,6 +234,20 @@
         - victoria_logs_support | default(false) | bool
       tags: telemetry_deployment
 
+- name: Populate UFM telemetry deployment configs
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ hostvars['localhost']['k8s_client_share_path'] }}/telemetry/deployments/{{ item.dest }}"
+    mode: "{{ hostvars['localhost']['file_permissions_644'] }}"
+  loop:
+    - src: 'telemetry/ufm/ufm-telemetry-secret.yaml.j2'
+      dest: 'ufm-telemetry-secret.yaml'
+    - src: 'telemetry/ufm/ufm-external-service.yaml.j2'
+      dest: 'ufm-external-service.yaml'
+  when: telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool
+  tags: telemetry_deployment
+
+
 - name: Deploy telemetry cleanup script
   ansible.builtin.template:
     src: 'telemetry/cleanup_telemetry.sh.j2'
diff --git a/provision/roles/telemetry/tasks/main.yml b/provision/roles/telemetry/tasks/main.yml
index ebcb424462..84c16c9941 100644
--- a/provision/roles/telemetry/tasks/main.yml
+++ b/provision/roles/telemetry/tasks/main.yml
@@ -44,6 +44,8 @@
           (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
           (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
           (telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool) or
           ldms_support | default(false) | bool
       block:
         - name: Set NFS info fact
@@ -58,12 +60,18 @@
         - telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool
       ansible.builtin.include_tasks: deploy_powerscale_metrics.yml
 
+    - name: Deploy UFM telemetry metrics
+      when:
+        - telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool
+      ansible.builtin.include_tasks: deploy_ufm_telemetry.yml
+
     - name: Generate telemetry deployments (idrac/ldms/powerscale/vector)
       when:
         - >-
           (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
           (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
           (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
+          (telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
           (telemetry_config.telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool) or
           (telemetry_config.telemetry_bridges.vector_ome.logs_enabled | default(false) | bool) or
           ldms_support | default(false) | bool
@@ -119,11 +127,7 @@
     - name: Generate telemetry deployment script
       ansible.builtin.include_tasks: generate_telemetry_script.yml
       when:
-        - >-
-          (telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool) or
-          (telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
-          (telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
-          ldms_support | default(false) | bool
+        - telemetry_enabled | default(false) | bool
       tags:
         - telemetry_deployment
 
diff --git a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2 b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
index 2dc831ba25..65ef9a3bfd 100644
--- a/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/common/telemetry_secret_creation.yaml.j2
@@ -19,4 +19,4 @@ metadata:
 type: Opaque
 stringData:
   cluster-id: "{{ cluster_id.stdout }}"
-{% endif %}
\ No newline at end of file
+{% endif %}
diff --git a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
index 5eb7f1ee90..c55f7a5f74 100644
--- a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2
@@ -58,6 +58,11 @@ resources:
   - telemetry_cleaner_rbac.yaml
   - telemetry_pod_cleanup.yaml
 {% endif %}
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) %}
+# UFM InfiniBand Telemetry Resources
+  - ufm-telemetry-secret.yaml
+  - ufm-external-service.yaml
+{% endif %}
 {% if telemetry_config.telemetry_bridges.vector_ldms.metrics_enabled | default(false) and ldms_support | default(false) and kafka_support | default(false) and victoria_metrics_support | default(false) %}
 # Vector-LDMS Bridge Resources (Kafka-to-VictoriaMetrics pipeline)
   - vector/vmagent-vector-deployment.yaml
diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
new file mode 100644
index 0000000000..d6ccf90b37
--- /dev/null
+++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-external-service.yaml.j2
@@ -0,0 +1,44 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool %}
+apiVersion: v1
+kind: Service
+metadata:
+  name: ufm-external
+  namespace: {{ telemetry_namespace }}
+  labels:
+    app: ufm-external
+    app.kubernetes.io/managed-by: omnia
+    app.kubernetes.io/component: ufm-telemetry
+spec:
+  ports:
+    - name: metrics
+      port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }}
+  clusterIP: None
+
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: ufm-external
+  namespace: {{ telemetry_namespace }}
+subsets:
+  - addresses:
+      - ip: "{{ telemetry_config.ufm_configuration.ufm_endpoint }}"
+    ports:
+      - name: metrics
+        port: {{ telemetry_config.ufm_configuration.ufm_metrics_port | default(9001) }}
+{% endif %}
diff --git a/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2 b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
new file mode 100644
index 0000000000..a641f3d93f
--- /dev/null
+++ b/provision/roles/telemetry/templates/telemetry/ufm/ufm-telemetry-secret.yaml.j2
@@ -0,0 +1,58 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool %}
+{% set ufm_config = telemetry_config.telemetry_sources.ufm %}
+{% set ufm_detailed_config = telemetry_config.ufm_configuration %}
+{% set auth_mode = ufm_detailed_config.auth_mode | default('basic') %}
+{% set tls_mode = ufm_detailed_config.tls_mode | default('self_signed') %}
+
+{% if auth_mode == 'basic' %}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ufm-telemetry-credentials
+  namespace: {{ telemetry_namespace }}
+  labels:
+    app.kubernetes.io/managed-by: omnia
+    app.kubernetes.io/component: ufm-telemetry
+type: Opaque
+{% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %}
+data:
+  username: {{ (hostvars['localhost']['ufm_username'] | string) | b64encode }}
+  password: {{ (hostvars['localhost']['ufm_password'] | string) | b64encode }}
+  ca.crt: {{ (ufm_ca_cert_content | default('') | string) | b64encode }}
+{% else %}
+stringData:
+  username: "{{ hostvars['localhost']['ufm_username'] | default('') | string }}"
+  password: "{{ hostvars['localhost']['ufm_password'] | default('') | string }}"
+{% endif %}
+{% else %}
+{% if tls_mode == 'ca_signed' and ufm_detailed_config.ufm_ca_cert_path | default('') | trim != '' %}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ufm-telemetry-credentials
+  namespace: {{ telemetry_namespace }}
+  labels:
+    app.kubernetes.io/managed-by: omnia
+    app.kubernetes.io/component: ufm-telemetry
+type: Opaque
+data:
+  ca.crt: {{ (ufm_ca_cert_content | default('') | string) | b64encode }}
+{% endif %}
+{% endif %}
+{% endif %}
diff --git a/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2 b/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2
index 74ed4a80a3..3c9196ff06 100644
--- a/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2
+++ b/provision/roles/telemetry/templates/telemetry/victoria/victoria-operator-vmscrape.yaml.j2
@@ -105,3 +105,58 @@ spec:
     - targetLabel: component
       replacement: volume-status
 {% endif %}
+
+---
+{% if telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) %}
+# VMServiceScrape for UFM InfiniBand Metrics
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: ufm-infiniband-metrics
+  namespace: {{ telemetry_namespace }}
+spec:
+  jobLabel: ufm
+  selector:
+    matchLabels:
+      app: ufm-external
+  endpoints:
+    - port: metrics
+      interval: {{ telemetry_config.ufm_configuration.scrape_interval | default('30s') }}
+      scrapeTimeout: {{ telemetry_config.ufm_configuration.scrape_timeout | default('15s') }}
+{% if telemetry_config.ufm_configuration.tls_mode | default('self_signed') == 'ca_signed' %}
+      scheme: https
+      tlsConfig:
+        caFile: /etc/ufm-credentials/ca.crt
+        insecureSkipVerify: false
+{% elif telemetry_config.ufm_configuration.tls_mode | default('self_signed') == 'self_signed' %}
+      scheme: https
+      tlsConfig:
+        insecureSkipVerify: true
+{% else %}
+      scheme: http
+{% endif %}
+      path: /metrics
+      honorLabels: true
+{% if telemetry_config.ufm_configuration.auth_mode | default('basic') == 'basic' %}
+      basicAuth:
+        username:
+          name: ufm-telemetry-credentials
+          key: username
+        password:
+          name: ufm-telemetry-credentials
+          key: password
+{% endif %}
+      relabelings:
+        - action: replace
+          sourceLabels: []
+          targetLabel: source
+          replacement: ufm
+        - action: replace
+          sourceLabels: []
+          targetLabel: subsystem
+          replacement: infiniband
+        - action: replace
+          sourceLabels: []
+          targetLabel: job
+          replacement: ufm-infiniband-metrics
+{% endif %}
diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml
index 3ecc0fb0e0..829c74ef47 100644
--- a/provision/roles/telemetry/vars/main.yml
+++ b/provision/roles/telemetry/vars/main.yml
@@ -658,3 +658,61 @@ vector:
       limits:
         cpu: "500m"
         memory: "512Mi"
+
+# ============================================================================
+# UFM Telemetry Configuration
+# ============================================================================
+# Usage: deploy_ufm_telemetry.yml
+# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances
+# using Prometheus scraping via the shared vmagent instance.
+
+# Kubernetes resource names
+ufm_credentials_secret_name: "ufm-telemetry-credentials"
+ufm_grafana_dashboard_configmap_name: "grafana-dashboard-ufm"
+
+# UFM telemetry defaults
+ufm_deploy_grafana_dashboard: true
+ufm_endpoint_test_timeout: 10
+
+# Validation messages
+ufm_endpoint_empty_msg: >-
+  UFM telemetry is enabled but ufm_endpoint is empty.
+  Set telemetry_sources.ufm.ufm_endpoint in telemetry_config.yml.
+
+ufm_credentials_missing_msg: >-
+  UFM auth_mode is 'basic' but ufm_username or ufm_password is not set.
+  Provide UFM credentials in omnia_config_credentials.yml.
+
+ufm_ca_cert_not_found_msg: >-
+  UFM tls_mode is 'ca_signed' but the CA certificate file was not found
+  at the path specified in ufm_ca_cert_path. Provide a valid PEM certificate file.
+
+ufm_port_range_msg: >-
+  UFM metrics port must be between 1 and 65535.
+
+ufm_tls_mode_invalid_msg: >-
+  UFM tls_mode must be 'self_signed' or 'ca_signed'.
+
+ufm_auth_mode_invalid_msg: >-
+  UFM auth_mode must be 'basic' or 'none'.
+
+ufm_endpoint_unreachable_msg: >-
+  WARNING: UFM endpoint is not reachable. UFM telemetry configuration will
+  proceed, but VMAgent will not be able to scrape metrics until UFM is accessible.
+
+ufm_secret_created_msg: >-
+  UFM credentials Kubernetes Secret created in namespace '{{ telemetry_namespace }}'.
+
+ufm_victoria_logs_validation_fail_msg: >-
+  UFM log collection requires 'victoria_logs' in
+  telemetry_sources.ufm.collection_targets.
+
+
+ufm_integration_summary_msg: >-
+  UFM telemetry integration complete.
+  {% if _ufm_config.metrics_enabled | default(false) | bool %}
+  Metrics: https://{{ _ufm_detailed_config.ufm_endpoint }}:{{ _ufm_detailed_config.ufm_metrics_port | default(9001) }}
+  TLS mode: {{ _ufm_detailed_config.tls_mode | default('self_signed') }}
+  Auth mode: {{ _ufm_detailed_config.auth_mode | default('basic') }}
+  Scrape interval: {{ _ufm_detailed_config.scrape_interval | default('30s') }}
+  {% endif %}
diff --git a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2 b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
index bf231510ed..d9ffd938b4 100644
--- a/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
+++ b/utils/credential_utility/roles/create_config/templates/omnia_credential.j2
@@ -54,3 +54,7 @@ gitlab_root_password: ""
 # OME discovery credentials
 ome_username: ""
 ome_password: ""
+
+# UFM telemetry credentials
+ufm_username: ""
+ufm_password: ""
diff --git a/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml b/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml
index 7f904b227a..8207abd29b 100644
--- a/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml
+++ b/utils/credential_utility/roles/update_config/tasks/fetch_credentials.yml
@@ -30,6 +30,7 @@
     - >-
       service.key in software_names or
       service.key in ["provision", "prepare_oim", "local_repo", "idrac_telemetry",
-      "visualization", "build_aarch_image", "gitlab", "csi_driver_powerscale", "discovery"]
+      "visualization", "build_aarch_image", "gitlab", "csi_driver_powerscale", "discovery",
+      "ufm_telemetry"]
     - (omnia_run_tags | default([]) | difference(['all']) | length == 0)
       or service.key in (omnia_run_tags | default([]))
diff --git a/utils/credential_utility/roles/update_config/vars/main.yml b/utils/credential_utility/roles/update_config/vars/main.yml
index 6c60ad6110..8ebc0ea3be 100644
--- a/utils/credential_utility/roles/update_config/vars/main.yml
+++ b/utils/credential_utility/roles/update_config/vars/main.yml
@@ -118,3 +118,8 @@ omnia_credentials:
       - username: ome_username
         password: ome_password
         condition: "{{ enable_bmc_discovery | default(false) | bool }}"
+  ufm_telemetry:
+    conditional_mandatory:
+      - username: ufm_username
+        password: ufm_password
+        condition: "{{ ufm_telemetry_support | default(false) | bool }}"
diff --git a/utils/credential_utility/roles/validation/tasks/pre_requisite.yml b/utils/credential_utility/roles/validation/tasks/pre_requisite.yml
index e12c09aa28..f7b3f3bbea 100644
--- a/utils/credential_utility/roles/validation/tasks/pre_requisite.yml
+++ b/utils/credential_utility/roles/validation/tasks/pre_requisite.yml
@@ -48,3 +48,17 @@
   when:
     - not result.skipped | default(false)
     - result.telemetry_status_list | length > 0
+
+- name: Derive UFM telemetry support flag from telemetry_config.yml
+  when: "'ufm_telemetry' in (omnia_run_tags | default([]))"
+  block:
+    - name: Load telemetry_config.yml for UFM support flag
+      ansible.builtin.include_vars:
+        file: "{{ telemetry_config_path }}"
+        name: _telemetry_config
+      failed_when: false
+
+    - name: Set ufm_telemetry_support flag
+      ansible.builtin.set_fact:
+        ufm_telemetry_support: "{{ _telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool }}"
+      when: _telemetry_config is defined and _telemetry_config.telemetry_sources is defined

From 8c4dd31be49eaa8c2b3545750e7308e9bde0a03e Mon Sep 17 00:00:00 2001
From: pullan1 <sudha.pullalaravu@dell.com>
Date: Sat, 16 May 2026 20:18:40 +0530
Subject: [PATCH 30/32] UFM telemetry validation fix

Signed-off-by: pullan1 <sudha.pullalaravu@dell.com>
---
 .../input_validation/schema/telemetry_config.json            | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/common/library/module_utils/input_validation/schema/telemetry_config.json b/common/library/module_utils/input_validation/schema/telemetry_config.json
index c0ce313832..0f1b84d93e 100644
--- a/common/library/module_utils/input_validation/schema/telemetry_config.json
+++ b/common/library/module_utils/input_validation/schema/telemetry_config.json
@@ -433,8 +433,7 @@
       "properties": {
         "ufm_endpoint": {
           "type": "string",
-          "minLength": 1,
-          "description": "UFM appliance IP address or hostname."
+          "description": "UFM appliance IP address or hostname. Required when UFM telemetry is enabled (metrics_enabled or logs_enabled is true)."
         },
         "ufm_metrics_port": {
           "type": "integer",
@@ -473,7 +472,7 @@
           "description": "Authentication mode for UFM Prometheus endpoint."
         }
       },
-      "required": ["ufm_endpoint"]
+      "required": []
     }
   },
   "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations", "ufm_configuration"]

From e297ec358c7133c254b5eee124fb384d38b21441 Mon Sep 17 00:00:00 2001
From: snarthan <narthan.s@dell.com>
Date: Mon, 18 May 2026 17:35:46 +0530
Subject: [PATCH 31/32] Update
 ci-group-service_kube_control_plane_first_x86_64.yaml.j2 (#4464)

Signed-off-by: snarthan <narthan.s@dell.com>
---
 .../ci-group-service_kube_control_plane_first_x86_64.yaml.j2  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
index 574a040e3e..9a5b615eee 100644
--- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
+++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2
@@ -966,7 +966,7 @@
                   INSTALL_SCRIPT="/opt/omnia/csi-driver-powerscale/csi-powerscale/dell-csi-helm-installer/csi-install.sh"
                   if [ -x "$INSTALL_SCRIPT" ]; then
                     cd "$(dirname "$INSTALL_SCRIPT")" || true
-                    ./csi-install.sh --namespace isilon --values /opt/omnia/csi-driver-powerscale/values.yaml &
+                    ./csi-install.sh --namespace isilon --values /opt/omnia/csi-driver-powerscale/values.yaml --skip-verify &
                     CSI_PID=$!
                     echo "Waiting for CSI install script (PID $CSI_PID) to complete..."
                     wait $CSI_PID
@@ -1109,4 +1109,4 @@
             kubectl get pods --all-namespaces -o wide
             echo "Cloud-Init finished successfully after the reboot."
 
-          fi
\ No newline at end of file
+          fi

From 67e489a5c1e5e0880448358181ab0ba2d3a11f97 Mon Sep 17 00:00:00 2001
From: SOWJANYAJAGADISH123 <Sowjanya.Jagadish@dell.com>
Date: Mon, 18 May 2026 18:30:00 +0530
Subject: [PATCH 32/32] Update policy_update.yml (#4467)

Signed-off-by: SOWJANYAJAGADISH123 <sowjanya.jagadish@dell.com>
---
 .../openchami/tasks/configs/policy_update.yml        | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml
index 14ad44d29b..1a8c914f9b 100644
--- a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml
+++ b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/policy_update.yml
@@ -40,15 +40,3 @@
     --host-bucket={{ cluster_boot_ip }}:9000
   changed_when: true
   when: storage_backend == 'minio'
-
-- name: Update the boot policy (S3/PowerScale)
-  ansible.builtin.command: >
-    s3cmd setpolicy {{ s3_work_dir }}/s3-public-read-boot.json s3://boot-images
-  changed_when: true
-  when: storage_backend == 'powerscale'
-
-- name: Update the efi policy (S3/PowerScale)
-  ansible.builtin.command: >
-    s3cmd setpolicy {{ s3_work_dir }}/s3-public-read-efi.json s3://efi
-  changed_when: true
-  when: storage_backend == 'powerscale'