diff --git a/common/library/modules/bulk_discover_node_specs.py b/common/library/modules/bulk_discover_node_specs.py new file mode 100644 index 0000000000..26a473b59b --- /dev/null +++ b/common/library/modules/bulk_discover_node_specs.py @@ -0,0 +1,507 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/usr/bin/python +# pylint: disable=import-error,no-name-in-module,line-too-long,too-many-locals +""" +Ansible module to bulk-discover hardware specs from iDRAC Redfish API +across multiple nodes in parallel. + +Supports two discovery modes: + + Heterogeneous (per-node): Each compute node is individually queried + via its iDRAC. Use the ``nodes`` parameter. + + Homogeneous (per-group): For each hardware group one sample node is + queried and the discovered specs are replicated to every node in + that group. Use the ``groups`` parameter. + +Exactly one of ``nodes`` or ``groups`` must be provided. + +Designed for HPC clusters with 500-2000 nodes where serial Ansible +URI loops are prohibitively slow (50-80 min at 1000 nodes serial +vs ~4 min at 20 parallel threads). + +GPU fallback detection via PCIe device enumeration is included. + +Usage in playbook (heterogeneous): + bulk_discover_node_specs: + nodes: "{{ cmpt_list }}" + bmc_ip_map: "{{ bmc_ip_map }}" + bmc_username: "{{ bmc_username }}" + bmc_password: "{{ bmc_password }}" + max_parallel: 20 + connect_timeout: 60 + defaults: + real_memory: 864 + corespersocket: 72 + threadspercore: 1 + register: bulk_discovery + +Usage in playbook (homogeneous group): + bulk_discover_node_specs: + groups: "{{ sample_idrac_groups }}" + bmc_ip_map: "{{ bmc_ip_map }}" + bmc_username: "{{ bmc_username }}" + bmc_password: "{{ bmc_password }}" + max_parallel: 20 + connect_timeout: 60 + defaults: + real_memory: 864 + corespersocket: 72 + threadspercore: 1 + sockets: 2 + register: bulk_discovery +""" +import json +import re +import ssl +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from ansible.module_utils.basic import AnsibleModule + + +# ─── Redfish API helpers ─────────────────────────────────────────────────── + +def _create_ssl_context(): + """Create an unverified SSL context for iDRAC self-signed certs.""" + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + +def _redfish_get(bmc_ip, path, username, password, timeout): + """Perform a Redfish GET request and return parsed JSON. + + Returns (success, json_data_or_error_string). + """ + url = f"https://{bmc_ip}{path}" + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + "OData-Version": "4.0", + } + + # Build basic auth header + import base64 + credentials = base64.b64encode(f"{username}:{password}".encode()).decode() + headers["Authorization"] = f"Basic {credentials}" + + req = urllib.request.Request(url, headers=headers, method="GET") + ctx = _create_ssl_context() + + try: + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + data = json.loads(resp.read().decode()) + return (True, data) + except urllib.error.HTTPError as exc: + return (False, f"HTTP {exc.code}: {exc.reason}") + except urllib.error.URLError as exc: + return (False, f"URL error: {exc.reason}") + except Exception as exc: # pylint: disable=broad-except + return (False, str(exc)) + + +# ─── GPU detection ───────────────────────────────────────────────────────── + +def _detect_gpus_from_processors(proc_members): + """Extract NVIDIA GPUs from Processor members (primary detection).""" + gpus = [] + for member in proc_members: + if member.get("ProcessorType") != "GPU": + continue + manufacturer = member.get("Manufacturer", "") + if re.search(r"nvidia", manufacturer, re.IGNORECASE): + gpus.append(member) + return gpus + + +def _detect_gpus_from_pcie(bmc_ip, username, password, timeout): + """Fallback GPU detection via PCIe device enumeration. + + Queries PCIeDevices collection, then individual devices for ClassCode + and manufacturer matching. + """ + gpus = [] + + # Get PCIe device list + ok, data = _redfish_get( + bmc_ip, + "/redfish/v1/Chassis/System.Embedded.1/PCIeDevices", + username, password, timeout, + ) + if not ok or "Members" not in data: + return gpus + + device_urls = [ + m.get("@odata.id", "") for m in data.get("Members", []) + if m.get("@odata.id") + ] + + # Query each PCIe device for GPU identification + for dev_url in device_urls: + ok, dev_data = _redfish_get(bmc_ip, dev_url, username, password, timeout) + if not ok: + continue + + class_code = dev_data.get("ClassCode", "") + vendor_id = dev_data.get("VendorId", "") + manufacturer = dev_data.get("Manufacturer", "") + name = dev_data.get("Name", "") + + # ClassCode 0x0300 = VGA controller, 0x0302 = 3D controller + if class_code in ("0x0300", "0x0302") and vendor_id: + gpus.append(dev_data) + elif (re.search(r"nvidia", manufacturer, re.IGNORECASE) + and re.search(r"GPU|RTX|TESLA|A100|H100|L40|GB", name, re.IGNORECASE)): + gpus.append(dev_data) + + return gpus + + +# ─── Per-node discovery ──────────────────────────────────────────────────── + +def _discover_single_node(hostname, bmc_ip, username, password, + timeout, defaults): + """Discover hardware specs for a single node via iDRAC Redfish. + + Returns (hostname, node_params_dict, gpu_list, error_string_or_None). + """ + default_memory = defaults.get("real_memory", 864) + default_cores = defaults.get("corespersocket", 72) + default_threads = defaults.get("threadspercore", 1) + + # ── Step 1: Read Processors ── + ok, proc_data = _redfish_get( + bmc_ip, + "/redfish/v1/Systems/System.Embedded.1/Processors?$expand=*($levels=1)", + username, password, timeout, + ) + + if ok: + members = proc_data.get("Members", []) + cpus = [m for m in members if m.get("ProcessorType") == "CPU"] + gpus = _detect_gpus_from_processors(members) + else: + cpus = [] + gpus = [] + + # ── Step 2: GPU fallback via PCIe devices ── + if not gpus: + gpus = _detect_gpus_from_pcie(bmc_ip, username, password, timeout) + + # ── Step 3: Read System info for memory ── + ok, sys_data = _redfish_get( + bmc_ip, + "/redfish/v1/Systems/System.Embedded.1", + username, password, timeout, + ) + + if ok: + total_gib = (sys_data + .get("MemorySummary", {}) + .get("TotalSystemMemoryGiB", default_memory)) + total_mb = int(total_gib * 1024) + real_memory = int(total_mb * 0.90) + else: + real_memory = default_memory + + # ── Step 4: Build node_params ── + sockets = max(len(cpus), 1) + if cpus: + cores_per_socket = cpus[0].get("TotalEnabledCores", default_cores) + total_threads = cpus[0].get("TotalThreads", default_threads) + total_cores = cpus[0].get("TotalCores", 1) + threads_per_core = total_threads // max(total_cores, 1) + else: + cores_per_socket = default_cores + threads_per_core = default_threads + + node_params = { + "NodeName": hostname, + "Sockets": sockets, + "CoresPerSocket": cores_per_socket, + "ThreadsPerCore": threads_per_core, + "RealMemory": real_memory, + } + + if gpus: + node_params["Gres"] = f"gpu:{len(gpus)}" + + return (hostname, node_params, gpus, None) + + +# ─── Per-group discovery (homogeneous) ──────────────────────────────────── + +def _discover_group(group_name, group_nodes, bmc_ip_map, username, + password, timeout, defaults): + """Discover hardware specs for a homogeneous group via iDRAC Redfish. + + Tries each node in the group sequentially until one iDRAC responds, + then replicates the discovered specs to all nodes in the group. + + Returns (group_name, node_params_list, gpu_dict, sample_node_or_None, + failed_bool). + """ + default_memory = defaults.get("real_memory", 864) + default_cores = defaults.get("corespersocket", 72) + default_threads = defaults.get("threadspercore", 1) + default_sockets = defaults.get("sockets", 2) + + # Try each node until one responds + sample_hostname = None + sample_params = None + sample_gpus = [] + + for hostname in group_nodes: + bmc_ip = bmc_ip_map.get(hostname) + if not bmc_ip: + continue + _, params, gpus, error = _discover_single_node( + hostname, bmc_ip, username, password, timeout, defaults, + ) + if error is None: + sample_hostname = hostname + sample_params = params + sample_gpus = gpus + break + + # Replicate discovered (or default) specs to all nodes in group + node_params_list = [] + gpu_dict = {} + + if sample_params: + for hostname in group_nodes: + entry = dict(sample_params) + entry["NodeName"] = hostname + node_params_list.append(entry) + if sample_gpus: + gpu_dict[hostname] = sample_gpus + else: + # All iDRACs in group failed — use defaults + for hostname in group_nodes: + node_params_list.append({ + "NodeName": hostname, + "Sockets": default_sockets, + "CoresPerSocket": default_cores, + "ThreadsPerCore": default_threads, + "RealMemory": default_memory, + }) + + return (group_name, node_params_list, gpu_dict, sample_hostname, + sample_params is None) + + +# ─── Main module ──────────────────────────────────────────────────────────── + +def run_module(): + """Ansible module entry point.""" + module_args = { + "nodes": { + "type": "list", "required": False, "default": None, + "elements": "str", + }, + "groups": { + "type": "dict", "required": False, "default": None, + }, + "bmc_ip_map": {"type": "dict", "required": True}, + "bmc_username": {"type": "str", "required": True, "no_log": True}, + "bmc_password": {"type": "str", "required": True, "no_log": True}, + "max_parallel": { + "type": "int", "required": False, "default": 20, + }, + "connect_timeout": { + "type": "int", "required": False, "default": 60, + }, + "defaults": { + "type": "dict", "required": False, "default": {}, + }, + } + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True, + mutually_exclusive=[("nodes", "groups")], + required_one_of=[("nodes", "groups")], + ) + + nodes = module.params["nodes"] + groups = module.params["groups"] + bmc_ip_map = module.params["bmc_ip_map"] + bmc_username = module.params["bmc_username"] + bmc_password = module.params["bmc_password"] + max_parallel = module.params["max_parallel"] + connect_timeout = module.params["connect_timeout"] + defaults = module.params["defaults"] + + # Dispatch to per-node or per-group discovery + if groups is not None: + _run_group_discovery( + module, groups, bmc_ip_map, bmc_username, bmc_password, + max_parallel, connect_timeout, defaults, + ) + else: + _run_node_discovery( + module, nodes or [], bmc_ip_map, bmc_username, bmc_password, + max_parallel, connect_timeout, defaults, + ) + + +def _run_node_discovery(module, nodes, bmc_ip_map, bmc_username, + bmc_password, max_parallel, connect_timeout, + defaults): + """Heterogeneous mode: discover every node individually in parallel.""" + result = { + "changed": False, + "node_params": [], + "gpu_params": {}, + "failed_nodes": [], + "total_nodes": len(nodes), + "discovered_count": 0, + } + + if module.check_mode or not nodes: + module.exit_json(**result) + + # Validate that all nodes have BMC IPs + missing_bmc = [n for n in nodes if n not in bmc_ip_map] + if missing_bmc: + module.warn( + f"No BMC IP found for {len(missing_bmc)} node(s): " + f"{', '.join(missing_bmc[:10])}" + f"{'...' if len(missing_bmc) > 10 else ''}" + ) + + # Parallel iDRAC discovery + workers = min(max_parallel, len(nodes)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {} + for hostname in nodes: + bmc_ip = bmc_ip_map.get(hostname) + if not bmc_ip: + result["failed_nodes"].append(hostname) + continue + future = pool.submit( + _discover_single_node, + hostname, bmc_ip, bmc_username, bmc_password, + connect_timeout, defaults, + ) + futures[future] = hostname + + for future in as_completed(futures): + hostname = futures[future] + try: + _, node_params, gpus, error = future.result() + if error: + result["failed_nodes"].append(hostname) + module.warn( + f"iDRAC discovery failed for {hostname}: {error}" + ) + else: + result["node_params"].append(node_params) + if gpus: + result["gpu_params"][hostname] = gpus + result["discovered_count"] += 1 + except Exception as exc: # pylint: disable=broad-except + result["failed_nodes"].append(hostname) + module.warn( + f"iDRAC discovery exception for {hostname}: {exc}" + ) + + if result["failed_nodes"]: + module.warn( + f"iDRAC discovery failed for {len(result['failed_nodes'])} of " + f"{len(nodes)} node(s)" + ) + + module.exit_json(**result) + + +def _run_group_discovery(module, groups, bmc_ip_map, bmc_username, + bmc_password, max_parallel, connect_timeout, + defaults): + """Homogeneous mode: discover one sample per group in parallel.""" + all_nodes = [h for hosts in groups.values() for h in hosts] + result = { + "changed": False, + "node_params": [], + "gpu_params": {}, + "failed_nodes": [], + "failed_groups": [], + "total_nodes": len(all_nodes), + "total_groups": len(groups), + "discovered_count": 0, + "group_sample_nodes": {}, + } + + if module.check_mode or not groups: + module.exit_json(**result) + + # Parallel group discovery — one thread per group + workers = min(max_parallel, len(groups)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = {} + for group_name, group_nodes in groups.items(): + future = pool.submit( + _discover_group, + group_name, group_nodes, bmc_ip_map, + bmc_username, bmc_password, connect_timeout, defaults, + ) + futures[future] = group_name + + for future in as_completed(futures): + group_name = futures[future] + try: + (_, node_params_list, gpu_dict, sample_node, + failed) = future.result() + result["node_params"].extend(node_params_list) + result["gpu_params"].update(gpu_dict) + if sample_node: + result["group_sample_nodes"][group_name] = sample_node + if failed: + result["failed_groups"].append(group_name) + module.warn( + f"All iDRACs failed for group '{group_name}' " + f"({len(groups[group_name])} nodes) — " + f"using defaults" + ) + else: + result["discovered_count"] += len( + groups[group_name] + ) + except Exception as exc: # pylint: disable=broad-except + result["failed_groups"].append(group_name) + module.warn( + f"Group discovery exception for '{group_name}': " + f"{exc}" + ) + + if result["failed_groups"]: + module.warn( + f"iDRAC group discovery failed for " + f"{len(result['failed_groups'])} of " + f"{len(groups)} group(s)" + ) + + module.exit_json(**result) + + +def main(): + """Module entry point.""" + run_module() + + +if __name__ == "__main__": + main() diff --git a/common/library/modules/bulk_update_hosts.py b/common/library/modules/bulk_update_hosts.py new file mode 100644 index 0000000000..9aea837f2c --- /dev/null +++ b/common/library/modules/bulk_update_hosts.py @@ -0,0 +1,238 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/usr/bin/python +# pylint: disable=import-error,no-name-in-module,line-too-long,too-many-locals +""" +Ansible module to bulk-update /etc/hosts across multiple remote hosts +in parallel via SSH. + +Designed for HPC clusters with 500-2000 nodes where serial Ansible +loops are prohibitively slow. This module is responsible ONLY for +/etc/hosts management. Munge key status is handled separately in +the calling playbook tasks. + +Usage in playbook: + bulk_update_hosts: + hosts: "{{ reachable_hosts }}" + ip_name_map: "{{ ip_name_map }}" + ssh_key_path: "/root/.ssh/oim_rsa" + nodes_to_remove: [] + ssh_max_parallel: 20 + ssh_connect_timeout: 10 + register: bulk_update_result +""" +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +from ansible.module_utils.basic import AnsibleModule + + +# ─── Hosts-file content generation ────────────────────────────────────────── + +def _build_hosts_block(ip_name_map): + """Build the OMNIA MANAGED HOSTS block content. + + Returns a string containing the full managed block including markers. + Sorted by hostname for deterministic output. + """ + lines = ["# BEGIN OMNIA MANAGED HOSTS"] + for hostname in sorted(ip_name_map.keys()): + ip_addr = ip_name_map[hostname] + lines.append(f"{ip_addr} {hostname}") + lines.append("# END OMNIA MANAGED HOSTS") + return "\n".join(lines) + + +def _build_cleanup_sed(ip_name_map, nodes_to_remove): + """Build sed commands to remove stale /etc/hosts entries. + + Cleans up: + - Old managed block markers and content + - Unmanaged entries matching Omnia node IPs or hostnames + - Entries for explicitly removed nodes + """ + all_ips = list(ip_name_map.values()) + all_names = list(ip_name_map.keys()) + list(nodes_to_remove or []) + + cmds = [ + # Remove old managed block markers and content (idempotent) + "sed -i '/^# BEGIN OMNIA MANAGED HOSTS$/,/^# END OMNIA MANAGED HOSTS$/d' /etc/hosts 2>/dev/null || true", + ] + if all_ips: + ip_pattern = "|".join(_regex_escape(ip) for ip in all_ips) + cmds.append( + f"sed -i -E '/^({ip_pattern})[[:space:]]/d' /etc/hosts 2>/dev/null || true" + ) + if all_names: + name_pattern = "|".join(all_names) + cmds.append( + f"sed -i -E '/[[:space:]]({name_pattern})$/d' /etc/hosts 2>/dev/null || true" + ) + return "\n".join(cmds) + + +def _regex_escape(text): + """Escape regex special characters in a string for sed patterns.""" + special = r'\.^$*+?{}[]|()' + return "".join(f"\\{c}" if c in special else c for c in text) + + +# ─── Remote execution ────────────────────────────────────────────────────── + +def _ssh_run(host, script, ssh_key_path, ssh_connect_timeout): + """Execute a script on a remote host via SSH. + + Returns (host, success, stdout, stderr). + """ + cmd = [ + "ssh", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", f"ConnectTimeout={ssh_connect_timeout}", + "-o", "BatchMode=yes", + "-o", "LogLevel=ERROR", + "-i", ssh_key_path, + f"root@{host}", + "bash -s", + ] + try: + result = subprocess.run( + cmd, + input=script, + capture_output=True, + text=True, + timeout=ssh_connect_timeout + 30, + check=False, + ) + return (host, result.returncode == 0, result.stdout, result.stderr) + except subprocess.TimeoutExpired: + return (host, False, "", f"SSH timeout after {ssh_connect_timeout + 30}s") + except Exception as exc: # pylint: disable=broad-except + return (host, False, "", str(exc)) + + +def _update_single_host(host, cleanup_sed, hosts_block, ssh_key_path, + ssh_connect_timeout): + """Update /etc/hosts on a single remote host. + + Steps: + 1. Run sed cleanup to remove stale entries + 2. Append the new managed hosts block + """ + script = f"""set -o pipefail +{cleanup_sed} +cat >> /etc/hosts << 'OMNIA_HOSTS_EOF' +{hosts_block} +OMNIA_HOSTS_EOF +""" + return _ssh_run(host, script, ssh_key_path, ssh_connect_timeout) + + +# ─── Main module ──────────────────────────────────────────────────────────── + +def run_module(): + """Ansible module entry point.""" + module_args = { + "hosts": {"type": "list", "required": True, "elements": "str"}, + "ip_name_map": {"type": "dict", "required": True}, + "ssh_key_path": {"type": "str", "required": True}, + "nodes_to_remove": { + "type": "list", "required": False, "default": [], + "elements": "str", + }, + "ssh_max_parallel": { + "type": "int", "required": False, "default": 20, + }, + "ssh_connect_timeout": { + "type": "int", "required": False, "default": 10, + }, + } + + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) + + hosts = module.params["hosts"] + ip_name_map = module.params["ip_name_map"] + ssh_key_path = module.params["ssh_key_path"] + nodes_to_remove = module.params["nodes_to_remove"] + ssh_max_parallel = module.params["ssh_max_parallel"] + ssh_connect_timeout = module.params["ssh_connect_timeout"] + + result = { + "changed": False, + "hosts_updated": [], + "hosts_failed": [], + "total_hosts": len(hosts), + "per_host_results": {}, + } + + if module.check_mode: + result["changed"] = len(hosts) > 0 + module.exit_json(**result) + + if not hosts: + module.exit_json(**result) + + # Build content once (same for every node) + hosts_block = _build_hosts_block(ip_name_map) + cleanup_sed = _build_cleanup_sed(ip_name_map, nodes_to_remove) + + # Parallel SSH execution + with ThreadPoolExecutor(max_workers=min(ssh_max_parallel, len(hosts))) as pool: + futures = { + pool.submit( + _update_single_host, + host, cleanup_sed, hosts_block, + ssh_key_path, ssh_connect_timeout, + ): host + for host in hosts + } + + for future in as_completed(futures): + host = futures[future] + try: + _, success, stdout, stderr = future.result() + result["per_host_results"][host] = { + "success": success, + "stdout": stdout.strip() if stdout else "", + "stderr": stderr.strip() if stderr else "", + } + if success: + result["hosts_updated"].append(host) + else: + result["hosts_failed"].append(host) + except Exception as exc: # pylint: disable=broad-except + result["per_host_results"][host] = { + "success": False, + "stdout": "", + "stderr": str(exc), + } + result["hosts_failed"].append(host) + + result["changed"] = len(result["hosts_updated"]) > 0 + + if result["hosts_failed"]: + module.warn( + f"Failed to update /etc/hosts on {len(result['hosts_failed'])} host(s): " + f"{', '.join(result['hosts_failed'])}" + ) + + module.exit_json(**result) + + +def main(): + """Module entry point.""" + run_module() + + +if __name__ == "__main__": + main() diff --git a/input/iso_config.yml b/input/iso_config.yml index c255ddc70a..4e5e99e14b 100644 --- a/input/iso_config.yml +++ b/input/iso_config.yml @@ -22,6 +22,15 @@ # and customize as needed. # ───────────────────────────────────────────────────────────────────────── +# Target node BMC/iDRAC IP address (required). +target_bmc_ip: "" + +# Hostname for the installed node (required). +hostname: "nid101" + +# Target node OS IP address to set (required). +target_node_ip: "172.16.107.201" + # Path to the pre-placed RHEL 10.x AArch64 Server with GUI full DVD ISO. # The ISO must be accessible on the OIM node. iso_source_path: "/opt/omnia/RHEL-10.0-aarch64-dvd1.iso" @@ -66,12 +75,15 @@ rebuild_iso: false # Default: false # force_reinstall: false +# Embed kickstart file in ISO instead of hosting on NFS share +# embed_kickstart: true + # Target install disk (e.g., "sda", "nvme0n1"). When set, Anaconda will # only use this disk and ignore all others. Leave empty to auto-detect. # install_disk: "" # Network configuration for the installed node -# These override values derived from network_spec.yml and PXE mapping. +network_device: "enP6s3f0np0" # Network device name (e.g., "eno1", "eth0"). # netmask: "255.255.255.0" # gateway: "" # dns: "" diff --git a/provision/roles/provision_validations/tasks/update_hosts.yml b/provision/roles/provision_validations/tasks/update_hosts.yml index 3d6391413f..70205df0f2 100644 --- a/provision/roles/provision_validations/tasks/update_hosts.yml +++ b/provision/roles/provision_validations/tasks/update_hosts.yml @@ -13,27 +13,18 @@ # limitations under the License. --- -- name: Ensure 127.0.0.1 localhost entry exists +# Container-mount safe: lineinfile/blockinfile use atomic rename() which fails +# on overlay/bind-mounted filesystems (EXDEV). Use shell redirect instead. +- name: Write hosts file (container-mount safe) ansible.builtin.shell: | set -o pipefail - grep -qxF '127.0.0.1 localhost.localdomain localhost' {{ hosts_file_path }} || echo '127.0.0.1 localhost.localdomain localhost' >> {{ hosts_file_path }} + cat > {{ hosts_file_path }} << 'HOSTS_EOF' + 127.0.0.1 localhost.localdomain localhost + # BEGIN OMNIA MANAGED HOSTS + {% for item in read_mapping_file.dict | dict2items | sort(attribute='value.HOSTNAME') %} + {{ item.value.ADMIN_IP }} {{ item.value.HOSTNAME }} + {% endfor %} + # END OMNIA MANAGED HOSTS + HOSTS_EOF + chmod 0644 {{ hosts_file_path }} changed_when: true - -- name: Update /etc/hosts with PXE mapping hostnames - block: - - name: Remove stale entries for IPs and hostnames that are being updated - ansible.builtin.shell: | - set -o pipefail - grep -v '^{{ item.value.ADMIN_IP }}\s' {{ hosts_file_path }} | \ - grep -v '\s{{ item.value.HOSTNAME }}$' > {{ hosts_file_path }}.tmp - cat {{ hosts_file_path }}.tmp > {{ hosts_file_path }} - rm -f {{ hosts_file_path }}.tmp - changed_when: true - loop: "{{ read_mapping_file.dict | dict2items }}" - - - name: Add hosts file entry for cluster - ansible.builtin.shell: | - set -o pipefail - echo '{{ item.value.ADMIN_IP }} {{ item.value.HOSTNAME }}' >> {{ hosts_file_path }} - changed_when: true - loop: "{{ read_mapping_file.dict | dict2items }}" diff --git a/provision/roles/slurm_config/defaults/main.yml b/provision/roles/slurm_config/defaults/main.yml index 9413138884..ea777873b0 100644 --- a/provision/roles/slurm_config/defaults/main.yml +++ b/provision/roles/slurm_config/defaults/main.yml @@ -12,6 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. --- +# ─── Bulk SSH parallelism (bulk_update_hosts module) ─── +# Maximum number of concurrent SSH connections for parallel host updates. +# Conservative default for production HPC clusters to avoid SSH connection +# storms and sshd MaxStartups throttling. Typical sshd MaxStartups is 10:30:100. +# Increase cautiously if sshd is tuned (e.g., MaxStartups 100:30:1000). +# Recommended range: 10-50 for production, up to 100 for tuned clusters. +bulk_ssh_max_parallel: 20 + +# SSH connect timeout in seconds for each remote host connection. +# Applied to both reachability checks (wait_for) and bulk module SSH calls. +# 10s is reasonable for well-connected HPC networks; increase for WAN/VPN. +bulk_ssh_connect_timeout: 10 + +# ─── Bulk iDRAC parallelism (bulk_discover_node_specs module) ─── +# Maximum number of concurrent iDRAC Redfish connections for parallel +# hardware discovery. iDRAC firmware supports limited concurrent sessions +# (~4-6 per BMC). With 20 parallel across 1000 nodes each iDRAC sees at +# most 1 concurrent request (safe). Do not exceed 50 without validating +# iDRAC firmware tolerance. +bulk_idrac_max_parallel: 20 + +# Redfish API connect timeout in seconds per iDRAC request. +# 60s accommodates slower iDRAC firmware. Reduce to 30s for responsive BMCs. +bulk_idrac_connect_timeout: 60 + slurm_db_port_default: 3306 slurm_db_type_default: mariadb slurm_db_username_default: root diff --git a/provision/roles/slurm_config/tasks/check_ctld_running.yml b/provision/roles/slurm_config/tasks/check_ctld_running.yml index a6380573ad..1bc9416620 100644 --- a/provision/roles/slurm_config/tasks/check_ctld_running.yml +++ b/provision/roles/slurm_config/tasks/check_ctld_running.yml @@ -15,8 +15,8 @@ - name: Check if remote host is reachable via SSH ansible.builtin.wait_for: host: "{{ ctld }}" - port: 22 # TODO: make it configurable - timeout: 10 + port: 22 + timeout: "{{ bulk_ssh_connect_timeout }}" state: started delegate_to: localhost register: ssh_check @@ -54,7 +54,7 @@ ansible.builtin.wait_for: host: "{{ host }}" port: 22 - timeout: 10 + timeout: "{{ bulk_ssh_connect_timeout }}" state: started delegate_to: localhost loop: "{{ ip_name_map.values() | list }}" @@ -68,11 +68,24 @@ ansible.builtin.set_fact: reachable_hosts: "{{ ip_map_ssh_check.results | rejectattr('failed', 'true') | map(attribute='host') | list }}" - - name: Update basics on reachable_hosts - ansible.builtin.include_tasks: update_hosts_munge.yml - loop: "{{ reachable_hosts }}" - loop_control: - loop_var: slurmhost_ip + # /etc/hosts update — skipped entirely when CoreDNS is enabled + - name: Bulk update /etc/hosts on all reachable hosts + bulk_update_hosts: + hosts: "{{ reachable_hosts }}" + ip_name_map: "{{ ip_name_map }}" + ssh_key_path: "{{ ssh_private_key_path }}" + nodes_to_remove: "{{ nodes_in_normal_not_in_cmpt | default([]) }}" + ssh_max_parallel: "{{ bulk_ssh_max_parallel }}" + ssh_connect_timeout: "{{ bulk_ssh_connect_timeout }}" + register: bulk_update_result + when: not (dns_enabled | default(false) | bool) + + - name: Warn about failed hosts + ansible.builtin.debug: + msg: "Failed to update /etc/hosts on {{ bulk_update_result.hosts_failed | length }} host(s): {{ bulk_update_result.hosts_failed | join(', ') }}" + when: + - bulk_update_result is defined + - bulk_update_result.hosts_failed | default([]) | length > 0 - name: Trigger the scontrol reconfigure ansible.builtin.command: scontrol reconfigure diff --git a/provision/roles/slurm_config/tasks/confs.yml b/provision/roles/slurm_config/tasks/confs.yml index 7b188c3fe3..bed23d9b60 100644 --- a/provision/roles/slurm_config/tasks/confs.yml +++ b/provision/roles/slurm_config/tasks/confs.yml @@ -111,27 +111,74 @@ - name_hardware_group_map | length > 0 - name_hardware_group_map.get(item, '') not in groups_with_specs -- name: Process homogeneous groups with user specs (no iDRAC) - ansible.builtin.include_tasks: read_node_homogeneous.yml - loop: "{{ homogeneous_nodes }}" +- name: Bulk discover homogeneous groups without specs (parallel group iDRAC) + bulk_discover_node_specs: + groups: "{{ sample_idrac_groups }}" + bmc_ip_map: "{{ bmc_ip_map }}" + bmc_username: "{{ bmc_username }}" + bmc_password: "{{ bmc_password }}" + max_parallel: "{{ bulk_idrac_max_parallel }}" + connect_timeout: "{{ bulk_idrac_connect_timeout }}" + defaults: + real_memory: "{{ default_real_memory }}" + corespersocket: "{{ default_corespersocket }}" + threadspercore: "{{ default_threadspercore }}" + sockets: "{{ default_sockets }}" + register: bulk_discovery when: - discovery_mode == 'homogeneous' - - homogeneous_nodes | length > 0 + - sample_idrac_groups | default({}) | length > 0 -- name: Process homogeneous groups without specs (group iDRAC) - ansible.builtin.include_tasks: read_node_idrac_group.yml - loop: "{{ sample_idrac_groups | dict2items }}" - loop_control: - loop_var: group_item +- name: Set node_params from bulk group discovery + ansible.builtin.set_fact: + node_params: "{{ node_params + bulk_discovery.node_params }}" + gpu_params: "{{ gpu_params | default({}) | combine(bulk_discovery.gpu_params) }}" when: - discovery_mode == 'homogeneous' - - sample_idrac_groups | default({}) | length > 0 + - bulk_discovery is defined + - bulk_discovery is not skipped -- name: Process heterogeneous nodes (individual iDRAC) - ansible.builtin.include_tasks: read_node_idrac.yml - loop: "{{ cmpt_list }}" +- name: Warn about failed group discoveries + ansible.builtin.debug: + msg: >- + iDRAC group discovery failed for {{ bulk_discovery.failed_groups | length }} + group(s): {{ bulk_discovery.failed_groups | join(', ') }} — using defaults + when: + - discovery_mode == 'homogeneous' + - bulk_discovery is defined + - bulk_discovery is not skipped + - bulk_discovery.failed_groups | default([]) | length > 0 + +- name: Bulk discover node hardware specs via iDRAC (parallel per-node) + bulk_discover_node_specs: + nodes: "{{ cmpt_list }}" + bmc_ip_map: "{{ bmc_ip_map }}" + bmc_username: "{{ bmc_username }}" + bmc_password: "{{ bmc_password }}" + max_parallel: "{{ bulk_idrac_max_parallel }}" + connect_timeout: "{{ bulk_idrac_connect_timeout }}" + defaults: + real_memory: "{{ default_real_memory }}" + corespersocket: "{{ default_corespersocket }}" + threadspercore: "{{ default_threadspercore }}" + register: bulk_discovery + when: discovery_mode == 'heterogeneous' + +- name: Set node_params from bulk per-node discovery + ansible.builtin.set_fact: + node_params: "{{ node_params + bulk_discovery.node_params }}" + gpu_params: "{{ gpu_params | default({}) | combine(bulk_discovery.gpu_params) }}" + when: + - discovery_mode == 'heterogeneous' + - bulk_discovery is not skipped + +- name: Warn about failed iDRAC discoveries + ansible.builtin.debug: + msg: "iDRAC discovery failed for {{ bulk_discovery.failed_nodes | length }} node(s): {{ bulk_discovery.failed_nodes | join(', ') }}" when: - discovery_mode == 'heterogeneous' + - bulk_discovery is not skipped + - bulk_discovery.failed_nodes | default([]) | length > 0 - name: DEBUG - Show final node_params before building slurm.conf ansible.builtin.debug: diff --git a/provision/roles/slurm_config/tasks/create_slurm_dir.yml b/provision/roles/slurm_config/tasks/create_slurm_dir.yml index 0864bd2b6a..ed9c832e3a 100644 --- a/provision/roles/slurm_config/tasks/create_slurm_dir.yml +++ b/provision/roles/slurm_config/tasks/create_slurm_dir.yml @@ -170,6 +170,21 @@ (compiler_login_list | default([])) + (login_list | default([])) }}" +- name: Display munge key restart warnings + ansible.builtin.debug: + msg: >- + Munge key updated on NFS for {{ item.item }} ({{ ip_name_map[item.item] | default(item.item) }}). + Please restart munge service on {{ ip_name_map[item.item] | default(item.item) }} + followed by the dependent slurm services. + verbosity: 1 + loop: "{{ munge_key_copy.results | default([]) }}" + loop_control: + label: "{{ item.item }}" + when: + - item.changed | default(false) + - restart_slurm_services + no_log: "{{ _no_log }}" + - name: Conf merge and write using slurm_conf module ansible.builtin.include_tasks: confs.yml diff --git a/provision/roles/slurm_config/tasks/read_node_idrac.yml b/provision/roles/slurm_config/tasks/read_node_idrac.yml deleted file mode 100644 index 0d0206d3ae..0000000000 --- a/provision/roles/slurm_config/tasks/read_node_idrac.yml +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -# TODO: RealMemory -- name: Read Processor NodeParams - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[item] }}/redfish/v1/Systems/System.Embedded.1/Processors?$expand=*($levels=1)" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: proc_info - failed_when: false - -- name: Get CPU Processors list - ansible.builtin.set_fact: - cpus: "{{ proc_info.json.Members | default([]) | selectattr('ProcessorType', 'equalto', 'CPU') | list }}" - gpus: "{{ proc_info.json.Members | default([]) - | selectattr('ProcessorType', 'equalto', 'GPU') - | selectattr('Manufacturer', 'defined') - | selectattr('Manufacturer', 'search', '(?i)nvidia') | list }}" # TODO: other GPUs also - -- name: Fallback - Read PCIe Devices for GPU detection (when no GPUs found via Processors) - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[item] }}/redfish/v1/Chassis/System.Embedded.1/PCIeDevices" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: pcie_devices - failed_when: false - when: gpus | length == 0 - -- name: Debug - Show PCIe devices structure - ansible.builtin.debug: - var: pcie_devices.json.Members - when: gpus | length == 0 and pcie_devices.json.Members is defined - -- name: Fallback - Extract PCIe device URLs - ansible.builtin.set_fact: - pcie_device_urls: "{{ pcie_devices.json.Members | default([]) | json_query('[*].\"@odata.id\"') }}" - when: gpus | length == 0 - -- name: Fallback - Get PCIe Device details for GPU detection - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[item.0] }}{{ item.1 }}" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: pcie_device_details - with_nested: - - ["{{ item }}"] - - "{{ pcie_device_urls | default([]) }}" - loop_control: - label: "{{ item.1 }}" - failed_when: false - when: gpus | length == 0 and pcie_device_urls is defined and pcie_device_urls | length > 0 - -- name: Fallback - Detect GPUs from PCIe devices - ansible.builtin.set_fact: - fallback_gpus: "{{ pcie_device_details.results | default([]) - | selectattr('json', 'defined') - | map(attribute='json') - | selectattr('ClassCode', 'defined') - | selectattr('VendorId', 'defined') - | selectattr('ClassCode', 'equalto', '0x0300') | list }}" - when: gpus | length == 0 - -- name: Fallback - Detect GPUs from PCIe devices (additional criteria) - ansible.builtin.set_fact: - fallback_gpus_additional: "{{ pcie_device_details.results | default([]) - | selectattr('json', 'defined') - | map(attribute='json') - | selectattr('ClassCode', 'defined') - | selectattr('VendorId', 'defined') - | selectattr('ClassCode', 'equalto', '0x0302') | list }}" - when: gpus | length == 0 and fallback_gpus | default([]) | length == 0 - -- name: Fallback - Detect GPUs from Manufacturer/Name (NVIDIA only) - ansible.builtin.set_fact: - fallback_gpus_manufacturer: "{{ pcie_device_details.results | default([]) - | selectattr('json', 'defined') - | map(attribute='json') - | selectattr('Manufacturer', 'defined') - | selectattr('Name', 'defined') - | selectattr('Manufacturer', 'search', '(?i)NVIDIA') - | selectattr('Name', 'search', '(?i)GPU|RTX|TESLA|A100|H100|L40|GB') | list }}" - when: gpus | length == 0 and fallback_gpus | default([]) | length == 0 and fallback_gpus_additional | default([]) | length == 0 - -- name: Fallback - Update GPUs list if PCIe detection found GPUs - ansible.builtin.set_fact: - gpus: "{{ (fallback_gpus | default([])) or (fallback_gpus_additional | default([])) or (fallback_gpus_manufacturer | default([])) }}" - when: gpus | length == 0 - - -- name: Read Memory NodeParams - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[item] }}/redfish/v1/Systems/System.Embedded.1" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: mem_info - failed_when: false - -- name: Calculate total memory in MB (GiB → MB) - ansible.builtin.set_fact: - total_memory_mb: "{{ (mem_info.json.MemorySummary.TotalSystemMemoryGiB | default(default_real_memory)) * 1024 | int }}" - -- name: Calculate 90% of real memory - ansible.builtin.set_fact: - real_memory: "{{ ((total_memory_mb | float) * 0.90) | int }}" - -- name: Calculate proc facts - ansible.builtin.set_fact: - proc_params: "{{ {'NodeName': item} | combine({'Sockets': (1 if (cpus | length == 0) else (cpus | length))}) - | combine({'CoresPerSocket': (cpus[0].TotalEnabledCores | default(default_corespersocket))}) - | combine({'ThreadsPerCore': ((cpus[0].TotalThreads | default(default_threadspercore)) // (cpus[0].TotalCores | default(1)))}) - | combine({'RealMemory': real_memory | default(default_real_memory) }) - | combine( - {'Gres': 'gpu:' ~ (gpus | default([1]) | length | string)} - if (gpus | default([])) - else {} - ) }}" - -- name: Add to Nodeparam dict - ansible.builtin.set_fact: - node_params: "{{ (node_params | default([])) + [proc_params] }}" - gpu_params: "{{ gpu_params | default({}) | combine({item: gpus} if gpus else {}) }}" diff --git a/provision/roles/slurm_config/tasks/read_node_idrac_group.yml b/provision/roles/slurm_config/tasks/read_node_idrac_group.yml deleted file mode 100644 index dd6582923d..0000000000 --- a/provision/roles/slurm_config/tasks/read_node_idrac_group.yml +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Initialize group discovery variables - ansible.builtin.set_fact: - group_name: "{{ group_item.key }}" - group_nodes: "{{ group_item.value }}" - discovered_specs: {} - responsive_node: "" - -- name: Try each node in group until iDRAC responds - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[item] }}/redfish/v1/Systems/System.Embedded.1" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - loop: "{{ group_nodes }}" - when: responsive_node == "" - register: idrac_results - -- name: Set responsive node from successful iDRAC call - ansible.builtin.set_fact: - responsive_node: "{{ item.item }}" - loop: "{{ idrac_results.results }}" - when: - - responsive_node == "" - - item.status == 200 - -- name: Read Processor information from responsive node - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[responsive_node] }}/redfish/v1/Systems/System.Embedded.1/Processors?$expand=*($levels=1)" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: proc_info - when: responsive_node != "" - -- name: Read Memory information from responsive node - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[responsive_node] }}/redfish/v1/Systems/System.Embedded.1" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: mem_info - when: responsive_node != "" - -- name: Extract CPU and GPU information - ansible.builtin.set_fact: - cpus: "{{ proc_info.json.Members | default([]) | selectattr('ProcessorType', 'equalto', 'CPU') | list }}" - gpus: "{{ proc_info.json.Members | default([]) - | selectattr('ProcessorType', 'equalto', 'GPU') - | selectattr('Manufacturer', 'search', '(?i)nvidia') | list }}" - when: responsive_node != "" - -- name: Fallback - Get PCIe devices for GPU detection - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[responsive_node] }}/redfish/v1/Systems/System.Embedded.1/PCIeDevices" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: pcie_devices - failed_when: false - when: responsive_node != "" and gpus | length == 0 - -- name: Fallback - Extract PCIe device URLs - ansible.builtin.set_fact: - pcie_device_urls: "{{ pcie_devices.json.Members | default([]) | json_query('[*].\"@odata.id\"') }}" - when: responsive_node != "" and gpus | length == 0 - -- name: Fallback - Get PCIe Device details for GPU detection - ansible.builtin.uri: - url: "https://{{ bmc_ip_map[responsive_node] }}{{ item }}" - user: "{{ bmc_username }}" - password: "{{ bmc_password }}" - method: GET - force_basic_auth: true - validate_certs: false - return_content: true - body_format: json - timeout: 60 - headers: - Accept: "application/json" - Content-Type: "application/json" - OData-Version: "4.0" - status_code: - - 200 - register: pcie_device_details - loop: "{{ pcie_device_urls | default([]) }}" - loop_control: - label: "{{ item }}" - failed_when: false - when: responsive_node != "" and gpus | length == 0 and pcie_device_urls is defined and pcie_device_urls | length > 0 - -- name: Fallback - Detect GPUs from PCIe devices - ansible.builtin.set_fact: - fallback_gpus: "{{ pcie_device_details.results | default([]) - | selectattr('json', 'defined') - | map(attribute='json') - | selectattr('ClassCode', 'defined') - | selectattr('VendorId', 'defined') - | selectattr('ClassCode', 'equalto', '0x0300') | list }}" - when: responsive_node != "" and gpus | length == 0 - -- name: Fallback - Detect GPUs from PCIe devices (additional criteria) - ansible.builtin.set_fact: - fallback_gpus_additional: "{{ pcie_device_details.results | default([]) - | selectattr('json', 'defined') - | map(attribute='json') - | selectattr('ClassCode', 'defined') - | selectattr('VendorId', 'defined') - | selectattr('ClassCode', 'equalto', '0x0302') | list }}" - when: responsive_node != "" and gpus | length == 0 and fallback_gpus | default([]) | length == 0 - -- name: Fallback - Detect GPUs from Manufacturer/Name (NVIDIA only) - ansible.builtin.set_fact: - fallback_gpus_manufacturer: "{{ pcie_device_details.results | default([]) - | selectattr('json', 'defined') - | map(attribute='json') - | selectattr('Manufacturer', 'defined') - | selectattr('Name', 'defined') - | selectattr('Manufacturer', 'search', '(?i)NVIDIA') - | selectattr('Name', 'search', '(?i)GPU|RTX|TESLA|A100|H100|L40|GB') | list }}" - when: responsive_node != "" and gpus | length == 0 and fallback_gpus | default([]) | length == 0 and fallback_gpus_additional | default([]) | length == 0 - -- name: Fallback - Update GPUs list if PCIe detection found GPUs - ansible.builtin.set_fact: - gpus: "{{ (fallback_gpus | default([])) or (fallback_gpus_additional | default([])) or (fallback_gpus_manufacturer | default([])) }}" - when: responsive_node != "" and gpus | length == 0 - -- name: Calculate total memory in MB (GiB → MB) - ansible.builtin.set_fact: - total_memory_mb: "{{ (mem_info.json.MemorySummary.TotalSystemMemoryGiB | default(default_real_memory)) * 1024 | int }}" - when: responsive_node != "" - -- name: Calculate 90% of real memory - ansible.builtin.set_fact: - real_memory: "{{ ((total_memory_mb | float) * 0.90) | int }}" - when: responsive_node != "" - -- name: Build discovered hardware specs - ansible.builtin.set_fact: - discovered_specs: "{{ { - 'sockets': (1 if (cpus | length == 0) else (cpus | length)), - 'cores_per_socket': (cpus[0].TotalEnabledCores | default(default_corespersocket)), - 'threads_per_core': ((cpus[0].TotalThreads | default(default_threadspercore)) // (cpus[0].TotalCores | default(1))), - 'real_memory': real_memory | default(default_real_memory), - 'gres': ('gpu:' ~ (gpus | default([1]) | length | string) if (gpus | default([])) else '') - } }}" - when: responsive_node != "" - -- name: Apply discovered specs to all nodes in group - ansible.builtin.set_fact: - node_params: >- - {{ - node_params + [ - {'NodeName': hostname, - 'Sockets': discovered_specs.sockets, - 'CoresPerSocket': discovered_specs.cores_per_socket, - 'ThreadsPerCore': discovered_specs.threads_per_core, - 'RealMemory': discovered_specs.real_memory} - | combine({'Gres': discovered_specs.gres} if discovered_specs.gres != '' else {}) - ] - }} - loop: "{{ group_nodes }}" - loop_control: - loop_var: hostname - when: discovered_specs | length > 0 - -- name: Use default values for all nodes in group (iDRAC failed) - ansible.builtin.set_fact: - node_params: >- - {{ - node_params + [{ - 'NodeName': hostname, - 'Sockets': default_corespersocket, - 'CoresPerSocket': default_corespersocket, - 'ThreadsPerCore': default_threadspercore, - 'RealMemory': default_real_memory - }] - }} - loop: "{{ group_nodes }}" - loop_control: - loop_var: hostname - when: discovered_specs | length == 0 diff --git a/provision/roles/slurm_config/tasks/remove_node.yml b/provision/roles/slurm_config/tasks/remove_node.yml index ae0867d8b4..4a80eb9c89 100644 --- a/provision/roles/slurm_config/tasks/remove_node.yml +++ b/provision/roles/slurm_config/tasks/remove_node.yml @@ -97,13 +97,10 @@ - updated_merged_results is defined # TODO: This should be done by provision_validation role - idempotency -- name: Remove entries from /etc/hosts +- name: Remove entries for removed nodes from /etc/hosts ansible.builtin.shell: | set -o pipefail - grep -Ev '[[:space:]]{{ item }}$' {{ hosts_file_path }} > {{ hosts_file_path }}.tmp - cat {{ hosts_file_path }}.tmp > {{ hosts_file_path }} - rm -f {{ hosts_file_path }}.tmp - loop: "{{ nodes_in_normal_not_in_cmpt }}" + sed -i -E '/[[:space:]]({{ nodes_in_normal_not_in_cmpt | join("|") }})$/d' {{ hosts_file_path }} when: - nodes_in_normal_not_in_cmpt is defined - nodes_in_normal_not_in_cmpt | length > 0 diff --git a/provision/roles/slurm_config/tasks/update_hosts_munge.yml b/provision/roles/slurm_config/tasks/update_hosts_munge.yml deleted file mode 100644 index 783d821edd..0000000000 --- a/provision/roles/slurm_config/tasks/update_hosts_munge.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Edit /etc/hosts file (skipped when CoreDNS is enabled) - ignore_unreachable: true - delegate_to: "{{ slurmhost_ip }}" - when: not (dns_enabled | default(false) | bool) - block: - - name: Remove deleted nodes if any hostname exists in /etc/hosts - ansible.builtin.lineinfile: - path: "/etc/hosts" - regexp: '(\b{{ node_to_remove }}\b)' - state: absent - loop: "{{ nodes_in_normal_not_in_cmpt }}" - loop_control: - loop_var: node_to_remove - when: - - nodes_in_normal_not_in_cmpt is defined - - nodes_in_normal_not_in_cmpt | length > 0 - - - name: Remove existing /etc/hosts entries containing the IP or hostname - ansible.builtin.lineinfile: - path: "/etc/hosts" - regexp: '(\b{{ host_entry.value }}\b|\b{{ host_entry.key }}\b)' - state: absent - loop: "{{ ip_name_map | dict2items | list }}" - loop_control: - loop_var: host_entry - - - name: Add correct /etc/hosts entry for controller hostname and IP - ansible.builtin.lineinfile: - path: "/etc/hosts" - line: "{{ host_entry.value }} {{ host_entry.key }}" - state: present - mode: '0644' - create: true - loop: "{{ ip_name_map | dict2items | list }}" - loop_control: - loop_var: host_entry - rescue: - - name: Print error if editing /etc/hosts fails - ansible.builtin.debug: - msg: "Failed to edit /etc/hosts file on {{ slurmhost_ip }}" - -- name: Get munge changes - ansible.builtin.set_fact: - munge_key_changed: "{{ munge_key_copy.results | default([]) | rekey_on_member('item') }}" - when: munge_key_copy is defined - -- name: Block when munge key changed - ansible.builtin.debug: - msg: "Munge key updates detected on NFS for {{ slurmhost_ip }}\n. - Please restart munge service on {{ slurmhost_ip }} followed by the dependent slurm services on them\n." - when: - - munge_key_changed is defined - - munge_key_changed[name_ip_map[slurmhost_ip]]['changed'] | default(false) - - restart_slurm_services - no_log: "{{ _no_log }}" - ignore_unreachable: true diff --git a/utils/install_os/install_os.yml b/utils/install_os/install_os.yml index fb516c5e4c..275d701ab8 100644 --- a/utils/install_os/install_os.yml +++ b/utils/install_os/install_os.yml @@ -49,14 +49,15 @@ # - ks_static_ip: static IP for the installed node # # Optional variables: -# - iso_source_checksum: SHA-256 checksum for source ISO verification -# - iso_target_directory: output directory for repacked ISO (default: /opt/omnia/iso_output) -# - kickstart_file: user-provided kickstart file path -# - kickstart_template: Jinja2 template name (default: rhel10) -# - ks_install_disk: target install disk (default: auto-detect) -# - force_reinstall: proceed even if target is reachable (default: false) -# - silent_install: suppress all interactive prompts (default: false) -# - rebuild_iso: force ISO rebuild when it exists (default: false) +# - iso_source_checksum: SHA-256 checksum for source ISO verification +# - iso_target_directory: output directory for repacked ISO (default: /opt/omnia/iso_output) +# - kickstart_file: user-provided kickstart file path +# - kickstart_template: Jinja2 template name (default: rhel10) +# - ks_install_disk: target install disk (default: auto-detect) +# - force_reinstall: proceed even if target is reachable (default: false) +# - silent_install: suppress all interactive prompts (default: false) +# - rebuild_iso: force ISO rebuild when it exists (default: false) +# - embed_kickstart: embed kickstart in ISO instead of NFS (default: true) # ───────────────────────────────────────────────────────────────────────── # ───────────────────────────────────────────────────────────────────────── @@ -254,6 +255,7 @@ Install Disk : {{ ks_install_disk | default('auto-detect') }} Source ISO : {{ iso_source_path | default('unknown') }} Kickstart : {{ kickstart_file if (kickstart_file | default('')) | length > 0 else (kickstart_template | default('rhel10') + ' template') }} + Delivery Method : {{ 'Embedded in ISO' if (embed_kickstart | default(true) | bool) else 'Network share (NFS)' }} ============================================================== Press ENTER to continue, or Ctrl+C then 'A' to abort. diff --git a/utils/install_os/roles/iso_creation/tasks/build_embedded_iso.yml b/utils/install_os/roles/iso_creation/tasks/build_embedded_iso.yml new file mode 100644 index 0000000000..fce90a7643 --- /dev/null +++ b/utils/install_os/roles/iso_creation/tasks/build_embedded_iso.yml @@ -0,0 +1,82 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# Build custom ISO with embedded kickstart file +# This creates a self-contained ISO with kickstart embedded + +- name: Render GRUB2 configuration with embedded kickstart reference + ansible.builtin.template: + src: grub_embedded.cfg.j2 + dest: "{{ temp_grub_cfg }}" + mode: "0644" + +- name: Remove existing custom ISO if rebuilding + ansible.builtin.file: + path: "{{ repacked_iso_path }}" + state: absent + when: rebuild_iso | bool + +- name: Build custom ISO using xorriso (embed grub.cfg and kickstart) + ansible.builtin.command: >- + xorriso -indev {{ iso_source_path }} + -outdev {{ repacked_iso_path }} + -map {{ temp_grub_cfg }} /EFI/BOOT/grub.cfg + -map {{ iso_target_directory }}/{{ nfs_kickstart_filename }} /{{ nfs_kickstart_filename }} + -boot_image any replay + changed_when: true + register: iso_build_result + +- name: Remove temporary grub.cfg + ansible.builtin.file: + path: "{{ temp_grub_cfg }}" + state: absent + +- name: Implant MD5 checksum into custom ISO + ansible.builtin.command: "implantisomd5 {{ repacked_iso_path }}" + changed_when: true + failed_when: false + +- name: Compute SHA-256 checksum of custom ISO + ansible.builtin.stat: + path: "{{ repacked_iso_path }}" + checksum_algorithm: sha256 + register: custom_iso_stat_final + +- name: Generate install manifest + ansible.builtin.copy: + content: | + --- + # Install OS Manifest + # Generated by utils/install_os (Embedded kickstart approach) + source_iso: "{{ iso_source_path }}" + source_iso_checksum: "{{ iso_source_checksum | default('not provided') }}" + custom_iso: "{{ repacked_iso_path }}" + custom_iso_checksum: "{{ custom_iso_stat_final.stat.checksum }}" + custom_iso_size: "{{ custom_iso_stat_final.stat.size }}" + kickstart_location: "embedded:{{ nfs_kickstart_filename }}" + kickstart_template: "{{ kickstart_template | default('user-provided') }}" + build_method: "xorriso_streaming_embedded_kickstart" + timestamp: "{{ ansible_date_time.iso8601 | default(lookup('pipe', 'date -u +%Y-%m-%dT%H:%M:%SZ')) }}" + dest: "{{ manifest_path }}" + mode: "0644" + +- name: Log ISO build success + ansible.builtin.debug: + msg: >- + Custom ISO built successfully using xorriso streaming (embedded kickstart). + Output: {{ repacked_iso_path }} + (SHA-256: {{ custom_iso_stat_final.stat.checksum }}). + Kickstart: embedded in ISO as {{ nfs_kickstart_filename }}. + Manifest: {{ manifest_path }} diff --git a/utils/install_os/roles/iso_creation/tasks/main.yml b/utils/install_os/roles/iso_creation/tasks/main.yml index 60aff1486e..baee6f7b80 100644 --- a/utils/install_os/roles/iso_creation/tasks/main.yml +++ b/utils/install_os/roles/iso_creation/tasks/main.yml @@ -33,6 +33,7 @@ when: - custom_iso_stat.stat.exists - not (silent_install | bool) + - not (embed_kickstart | default(true) | bool) - name: Set rebuild flag from user input ansible.builtin.set_fact: @@ -63,6 +64,14 @@ msg: "Kickstart file created at {{ iso_target_directory }}/{{ nfs_kickstart_filename }}" # Step 3: Build custom ISO (if missing or rebuild requested) +- name: Build custom ISO with embedded kickstart + ansible.builtin.include_tasks: build_embedded_iso.yml + when: + - (not custom_iso_stat.stat.exists or (rebuild_iso | bool)) + - embed_kickstart | default(true) | bool + - name: Build custom ISO with NFS kickstart reference ansible.builtin.include_tasks: build_nfs_iso.yml - when: not custom_iso_stat.stat.exists or (rebuild_iso | bool) + when: + - (not custom_iso_stat.stat.exists or (rebuild_iso | bool)) + - not (embed_kickstart | default(true) | bool) diff --git a/utils/install_os/roles/iso_creation/templates/grub_embedded.cfg.j2 b/utils/install_os/roles/iso_creation/templates/grub_embedded.cfg.j2 new file mode 100644 index 0000000000..7a43d339c5 --- /dev/null +++ b/utils/install_os/roles/iso_creation/templates/grub_embedded.cfg.j2 @@ -0,0 +1,69 @@ +set default="0" + +function load_video { + insmod efi_gop + insmod efi_uga + insmod video_bochs + insmod video_cirrus + insmod all_video +} + +load_video +set gfxpayload=keep +insmod gzio +insmod part_gpt +insmod ext2 + +set timeout=10 + +### BEGIN /etc/grub.d/10_linux ### +menuentry 'Omnia - Install Operating System - {{ os_arch }} (Unattended - Embedded Kickstart)' --class red --class gnu-linux --class gnu --class os { +{% if os_arch == 'x86_64' %} + linuxefi /images/pxeboot/vmlinuz inst.stage2=cdrom inst.ks=cdrom:/{{ nfs_kickstart_filename }} quiet + initrdefi /images/pxeboot/initrd.img +{% else %} + linux /images/pxeboot/vmlinuz inst.stage2=cdrom inst.ks=cdrom:/{{ nfs_kickstart_filename }} quiet + initrd /images/pxeboot/initrd.img +{% endif %} +} + +menuentry 'Omnia - Install Operating System - {{ os_arch }} (Interactive)' --class red --class gnu-linux --class gnu --class os { +{% if os_arch == 'x86_64' %} + linuxefi /images/pxeboot/vmlinuz inst.stage2=cdrom quiet + initrdefi /images/pxeboot/initrd.img +{% else %} + linux /images/pxeboot/vmlinuz inst.stage2=cdrom quiet + initrd /images/pxeboot/initrd.img +{% endif %} +} + +menuentry 'Omnia - Test this media & install - {{ os_arch }}' --class red --class gnu-linux --class gnu --class os { +{% if os_arch == 'x86_64' %} + linuxefi /images/pxeboot/vmlinuz inst.stage2=cdrom rd.live.check quiet + initrdefi /images/pxeboot/initrd.img +{% else %} + linux /images/pxeboot/vmlinuz inst.stage2=cdrom rd.live.check quiet + initrd /images/pxeboot/initrd.img +{% endif %} +} + +submenu 'Troubleshooting -->' { + menuentry 'Omnia - Install Operating System in basic graphics mode - {{ os_arch }}' --class red --class gnu-linux --class gnu --class os { +{% if os_arch == 'x86_64' %} + linuxefi /images/pxeboot/vmlinuz inst.stage2=cdrom nomodeset quiet + initrdefi /images/pxeboot/initrd.img +{% else %} + linux /images/pxeboot/vmlinuz inst.stage2=cdrom nomodeset quiet + initrd /images/pxeboot/initrd.img +{% endif %} + } + menuentry 'Omnia - Rescue installed system - {{ os_arch }}' --class red --class gnu-linux --class gnu --class os { +{% if os_arch == 'x86_64' %} + linuxefi /images/pxeboot/vmlinuz inst.stage2=cdrom inst.rescue quiet + initrdefi /images/pxeboot/initrd.img +{% else %} + linux /images/pxeboot/vmlinuz inst.stage2=cdrom inst.rescue quiet + initrd /images/pxeboot/initrd.img +{% endif %} + } +} diff --git a/utils/install_os/roles/iso_creation/templates/rhel10.ks.j2 b/utils/install_os/roles/iso_creation/templates/rhel10.ks.j2 index 2827f32157..22069fae94 100644 --- a/utils/install_os/roles/iso_creation/templates/rhel10.ks.j2 +++ b/utils/install_os/roles/iso_creation/templates/rhel10.ks.j2 @@ -2,6 +2,8 @@ # Kickstart for Omnia admin node # Generated by Omnia utils/install_os +{% set install_disk = ks_install_disk if (ks_install_disk is defined and ks_install_disk | length > 0) else 'sda' %} + # Pre-install — clean disk signatures to prevent probe errors %pre --log=/tmp/ks-pre.log # Stop active RAID/LVM @@ -19,10 +21,11 @@ keyboard --vckeymap=us --xlayouts='us' timezone {{ ks_timezone | default('UTC') }} --utc # Network +{% set network_device = ks_network_device if (ks_network_device is defined and ks_network_device | length > 0) else 'link' %} {% if ks_dns is defined and ks_dns | length > 0 %} -network --bootproto=static --device=link --ip={{ ks_static_ip }} --netmask={{ ks_netmask }} --gateway={{ ks_gateway }} --nameserver={{ ks_dns }} --hostname={{ ks_hostname | default('admin-node') }} --activate --onboot=yes +network --bootproto=static --device={{ network_device }} --ip={{ ks_static_ip }} --netmask={{ ks_netmask }} --gateway={{ ks_gateway }} --nameserver={{ ks_dns }} --hostname={{ ks_hostname | default('admin-node') }} --activate --onboot=yes {% else %} -network --bootproto=static --device=link --ip={{ ks_static_ip }} --netmask={{ ks_netmask }}{% if ks_gateway is defined and ks_gateway | length > 0 %} --gateway={{ ks_gateway }}{% endif %} --hostname={{ ks_hostname | default('admin-node') }} --activate --onboot=yes +network --bootproto=static --device={{ network_device }} --ip={{ ks_static_ip }} --netmask={{ ks_netmask }}{% if ks_gateway is defined and ks_gateway | length > 0 %} --gateway={{ ks_gateway }}{% endif %} --hostname={{ ks_hostname | default('admin-node') }} --activate --onboot=yes {% endif %} # Authentication @@ -32,12 +35,12 @@ sshkey --username=root "{{ ks_ssh_public_key }}" {% endif %} # Bootloader -bootloader --append="crashkernel=auto" --boot-drive={{ ks_install_disk | default('sda') }} +bootloader --append="crashkernel=auto" --boot-drive={{ install_disk }} # Disk partitioning -ignoredisk --only-use={{ ks_install_disk | default('sda') }} +ignoredisk --only-use={{ install_disk }} zerombr -clearpart --all --initlabel +clearpart --drives={{ install_disk }} --all --initlabel autopart # System diff --git a/utils/install_os/roles/iso_creation/vars/main.yml b/utils/install_os/roles/iso_creation/vars/main.yml index 6cd4845ad3..b2213176aa 100644 --- a/utils/install_os/roles/iso_creation/vars/main.yml +++ b/utils/install_os/roles/iso_creation/vars/main.yml @@ -39,6 +39,9 @@ iso_volume_id: "" silent_install: false # Set rebuild_iso=true to force rebuilding the custom ISO (used by prompt or extra-var) rebuild_iso: false +# Embed kickstart file in ISO instead of hosting on NFS share +# Default: true +embed_kickstart: true # Manifest file manifest_path: "{{ iso_target_directory }}/install_manifest.yml" @@ -53,7 +56,7 @@ ks_gateway: "" ks_dns: "" ks_network_device: "" ks_timezone: "UTC" -ks_install_disk: "" +ks_install_disk: "sda" # Error messages kickstart_file_not_found_msg: >- diff --git a/utils/install_os_arm_node/roles/fetch_arm_params/tasks/main.yml b/utils/install_os_arm_node/roles/fetch_arm_params/tasks/main.yml index b1f42398f6..29ed8d184c 100644 --- a/utils/install_os_arm_node/roles/fetch_arm_params/tasks/main.yml +++ b/utils/install_os_arm_node/roles/fetch_arm_params/tasks/main.yml @@ -13,31 +13,6 @@ # limitations under the License. --- -- name: Read PXE mapping file - ansible.builtin.read_csv: - path: "{{ pxe_mapping_file_path }}" - register: pxe_mapping_data - -- name: Find os_aarch64 node in PXE mapping - ansible.builtin.set_fact: - arm_node_entry: >- - {{ pxe_mapping_data.list - | selectattr('FUNCTIONAL_GROUP_NAME', 'search', 'os_aarch64') - | list - | first - | default( - pxe_mapping_data.list - | selectattr('HOSTNAME', 'search', 'os_aarch64') - | list - | first - | default(None) - ) }} - -- name: Fail if no os_aarch64 node found - ansible.builtin.fail: - msg: "{{ no_aarch64_node_msg }}" - when: arm_node_entry is none or arm_node_entry | length == 0 - - name: Read OIM SSH public key ansible.builtin.slurp: src: "{{ oim_ssh_key_path }}" @@ -81,8 +56,8 @@ kickstart_file: "{{ iso_config.kickstart_file | default('') }}" kickstart_template: "{{ iso_config.kickstart_template | default(default_kickstart_template) }}" ks_ssh_public_key: "{{ oim_ssh_key_content.content | b64decode | trim }}" - ks_hostname: "{{ arm_node_entry.HOSTNAME | default('os-aarch64') }}" - ks_static_ip: "{{ arm_node_entry.ADMIN_IP | default('') }}" + ks_hostname: "{{ iso_config.hostname }}" + ks_static_ip: "{{ iso_config.target_node_ip }}" ks_netmask: "{{ iso_config.netmask | default('255.255.255.0') }}" ks_gateway: "{{ iso_config.gateway | default('') }}" ks_dns: "{{ iso_config.dns | default('') }}" @@ -90,8 +65,8 @@ ks_timezone: "{{ iso_config.timezone | default('UTC') }}" ks_install_disk: "{{ iso_config.install_disk | default('') }}" # iDRAC parameters - target_bmc_ip: "{{ arm_node_entry.BMC_IP }}" - target_node_ip: "{{ arm_node_entry.ADMIN_IP | default('') }}" + target_bmc_ip: "{{ iso_config.target_bmc_ip }}" + target_node_ip: "{{ iso_config.target_node_ip }}" force_reinstall: "{{ iso_config.force_reinstall | default(false) | bool }}" # NFS share path (server:/path) — user-provided or auto-detected from mount iso_nfs_share: "{{ user_nfs_share_path if user_nfs_share_path | length > 0 else nfs_mount_result.stdout | default('') }}" @@ -109,6 +84,7 @@ # Interactive behavior silent_install: "{{ iso_config.silent_install | default(false) | bool }}" rebuild_iso: "{{ iso_config.rebuild_iso | default(false) | bool }}" + embed_kickstart: "{{ iso_config.embed_kickstart | default(true) | bool }}" cacheable: true - name: Log fetched ARM parameters diff --git a/utils/install_os_arm_node/roles/fetch_arm_params/vars/main.yml b/utils/install_os_arm_node/roles/fetch_arm_params/vars/main.yml index 0a558652f6..8c1e86a7aa 100644 --- a/utils/install_os_arm_node/roles/fetch_arm_params/vars/main.yml +++ b/utils/install_os_arm_node/roles/fetch_arm_params/vars/main.yml @@ -20,11 +20,6 @@ password_hash_algorithm: "sha512" default_network_device: "" # Error and info messages -no_aarch64_node_msg: >- - No os_aarch64 node found in PXE mapping file. - Ensure the PXE mapping file contains an entry with FUNCTIONAL_GROUP_NAME or HOSTNAME - matching 'os_aarch64'. - provision_password_not_set_msg: >- provision_password is not set or empty. Please run the credential utility to set it. diff --git a/utils/install_os_arm_node/roles/validate_arm_config/tasks/main.yml b/utils/install_os_arm_node/roles/validate_arm_config/tasks/main.yml index 563b892f1b..54470f7723 100644 --- a/utils/install_os_arm_node/roles/validate_arm_config/tasks/main.yml +++ b/utils/install_os_arm_node/roles/validate_arm_config/tasks/main.yml @@ -13,30 +13,14 @@ # limitations under the License. --- -- name: Verify PXE mapping file exists - ansible.builtin.stat: - path: "{{ pxe_mapping_file_path }}" - register: pxe_mapping_stat - -- name: Fail if PXE mapping file not found +- name: Validate required iso_config fields ansible.builtin.fail: - msg: "{{ pxe_mapping_not_found_msg }}" - when: not pxe_mapping_stat.stat.exists - -- name: Verify network_spec.yml exists - ansible.builtin.stat: - path: "{{ network_spec_file }}" - register: network_spec_stat - -- name: Fail if network_spec.yml not found - ansible.builtin.fail: - msg: "{{ network_spec_not_found_msg }}" - when: not network_spec_stat.stat.exists - -- name: Load network_spec.yml - ansible.builtin.include_vars: - file: "{{ network_spec_file }}" - name: network_spec + msg: "{{ item.msg }}" + when: item.value | length == 0 + loop: + - { value: "{{ iso_config.target_bmc_ip }}", msg: "'target_bmc_ip' is required in iso_config.yml" } + - { value: "{{ iso_config.hostname }}", msg: "'hostname' is required in iso_config.yml" } + - { value: "{{ iso_config.target_node_ip }}", msg: "'target_node_ip' is required in iso_config.yml" } - name: Verify OIM SSH public key exists ansible.builtin.stat: @@ -55,4 +39,4 @@ - name: Log ARM config validation success ansible.builtin.debug: - msg: "{{ arm_config_success_msg }}" + msg: "ARM configuration validation passed. BMC: {{ iso_config.target_bmc_ip }}, Node IP: {{ iso_config.target_node_ip }}" diff --git a/utils/install_os_arm_node/roles/validate_arm_config/vars/main.yml b/utils/install_os_arm_node/roles/validate_arm_config/vars/main.yml index 9bb9cebfae..a3b6277715 100644 --- a/utils/install_os_arm_node/roles/validate_arm_config/vars/main.yml +++ b/utils/install_os_arm_node/roles/validate_arm_config/vars/main.yml @@ -13,10 +13,6 @@ # limitations under the License. --- -# Configuration file paths -pxe_mapping_file_path: "{{ input_project_dir | default('/opt/omnia/input/project_default') }}/pxe_mapping_file.csv" -network_spec_file: "{{ input_project_dir | default('/opt/omnia/input/project_default') }}/network_spec.yml" - # OIM SSH key oim_ssh_key_path: "/root/.ssh/id_rsa.pub" @@ -26,28 +22,9 @@ default_iso_target_directory: "/opt/omnia/iso_output" default_kickstart_template: "rhel10" # Error messages -iso_config_not_found_msg: >- - iso_config.yml not found at '{{ iso_config_file }}'. - Please create the configuration file or provide a valid path via iso_config_path parameter. -pxe_mapping_not_found_msg: >- - PXE mapping file not found at '{{ pxe_mapping_file_path }}'. - The PXE mapping file is required to determine the os_aarch64 node's BMC IP - and admin IP. Please create the PXE mapping file and re-run the playbook. -network_spec_not_found_msg: >- - network_spec.yml not found at '{{ network_spec_file }}'. - The network specification is required for static IP configuration. - Please ensure network_spec.yml exists and re-run the playbook. oim_ssh_key_not_found_msg: >- OIM SSH public key not found at '{{ oim_ssh_key_path }}'. Generate an SSH key pair with 'ssh-keygen -t rsa' and re-run the playbook. -no_aarch64_node_msg: >- - No os_aarch64 node found in the PXE mapping file. - Please add an entry with FUNCTIONAL_GROUP_NAME containing 'os_aarch64' - or HOSTNAME matching 'os_aarch64' to the PXE mapping file. provision_password_missing_msg: >- provision_password is not set. Please run the credential utility (utils/credential_utility/get_config_credentials.yml) to set the provision_password. -arm_config_success_msg: >- - ARM configuration validation passed. - PXE mapping: {{ pxe_mapping_file_path }}. - Network spec: {{ network_spec_file }}.