From 7e9df0c2405afe0079c605a58ce396f094e970d3 Mon Sep 17 00:00:00 2001 From: Abhishek S A Date: Wed, 5 Aug 2026 16:43:41 +0530 Subject: [PATCH 01/22] Update ansible-lint.yml Signed-off-by: Abhishek S A Signed-off-by: Abhishek S A --- .github/workflows/ansible-lint.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index aea0698715..58fe64e4d5 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -6,11 +6,7 @@ on: - main - staging - release_1.7.1 - - pub/build_stream - - pub/q2_dev - - pub/telemetry - - pub/q2_upgrade - - pub/q2_ansible + - pub/omnia_2.2.0.0_fix jobs: build: From c5e86713b517f6e6a50bacb8c2ecf9a9e8b19265 Mon Sep 17 00:00:00 2001 From: Abhishek S A Date: Wed, 5 Aug 2026 16:44:20 +0530 Subject: [PATCH 02/22] Update pylint.yml Signed-off-by: Abhishek S A Signed-off-by: Abhishek S A --- .github/workflows/pylint.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 3aaded93be..f7351452e5 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -6,11 +6,7 @@ on: - main - staging - release_1.7.1 - - pub/build_stream - - pub/q2_dev - - pub/telemetry - - pub/q2_upgrade - - pub/q2_ansible + - pub/omnia_2.2.0.0_fix jobs: build: From 9afe38d366554a7f05ad50f3b307a07683cbd9d1 Mon Sep 17 00:00:00 2001 From: Rajeshkumar-s2 Date: Wed, 5 Aug 2026 17:14:27 +0530 Subject: [PATCH 03/22] Update gitlab_runner_image: v18.8.0 -> v19.2.0 (#4941) Update gitlab_runner_helper_image_version: v18.8.0 -> v19.2.0 Confirmed fixes: CVE-2026-34986 (HIGH) in golang.org/x/net CVE-2026-33186 (CRITICAL) in google.golang.org/grpc Additional Go dependency CVEs expected to be resolved by the v19.2.0 base image and module updates. Signed-off-by: Rajeshkumar S --- gitlab/roles/hosted_gitlab/vars/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitlab/roles/hosted_gitlab/vars/main.yml b/gitlab/roles/hosted_gitlab/vars/main.yml index 693efa38f2..a5797d9e15 100644 --- a/gitlab/roles/hosted_gitlab/vars/main.yml +++ b/gitlab/roles/hosted_gitlab/vars/main.yml @@ -78,10 +78,10 @@ gitlab_initial_root_password_path: "/etc/gitlab/initial_root_password" gitlab_root_token_file_path: "/root/.gitlab_root_token" # Runner container -gitlab_runner_image: "docker.io/gitlab/gitlab-runner:v18.8.0" +gitlab_runner_image: "docker.io/gitlab/gitlab-runner:v19.2.0" gitlab_runner_default_image: "docker.io/library/alpine:3.23.3" gitlab_runner_helper_image_registry: "registry.gitlab.com/gitlab-org/gitlab-runner/gitlab-runner-helper" -gitlab_runner_helper_image_version: "v18.8.0" +gitlab_runner_helper_image_version: "v19.2.0" gitlab_runner_container_name: "gitlab-runner" gitlab_restart_policy: "always" gitlab_runner_description: "Omnia Hosted Runner" From 010a975b93d2470a84de93a746b7f9c4d8b65f4b Mon Sep 17 00:00:00 2001 From: pullan1 Date: Thu, 6 Aug 2026 11:46:57 +0530 Subject: [PATCH 04/22] pulp upgraded from 3.113 to 3.114.2 version Signed-off-by: pullan1 --- common/library/modules/pulp_pgsql_backup_restore.py | 2 +- common/vars/image_vars.yml | 2 +- prepare_oim/roles/deploy_containers/pulp/vars/main.yml | 2 +- rollback/roles/rollback_pulp/tasks/main.yml | 4 ++-- .../roles/rollback_pulp/tasks/rollback_pulp_container.yml | 2 +- rollback/roles/rollback_pulp/vars/main.yml | 4 ++-- upgrade/playbooks/upgrade_oim.yml | 2 +- upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml | 2 +- upgrade/roles/upgrade_pulp/tasks/main.yml | 6 +++--- upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml | 2 +- upgrade/roles/upgrade_pulp/vars/main.yml | 6 +++--- upgrade/upgrade.yml | 2 +- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/common/library/modules/pulp_pgsql_backup_restore.py b/common/library/modules/pulp_pgsql_backup_restore.py index b35aeccdc0..a5c51158f9 100644 --- a/common/library/modules/pulp_pgsql_backup_restore.py +++ b/common/library/modules/pulp_pgsql_backup_restore.py @@ -20,7 +20,7 @@ Handles the PostgreSQL version incompatibility between Pulp versions: - Pulp 3.80 uses PostgreSQL 12/13 - - Pulp 3.113 uses PostgreSQL 16 + - Pulp 3.114.2 uses PostgreSQL 16 Backup (action=backup): - Validates source PostgreSQL data exists diff --git a/common/vars/image_vars.yml b/common/vars/image_vars.yml index 97c07d9a75..e72566de1c 100644 --- a/common/vars/image_vars.yml +++ b/common/vars/image_vars.yml @@ -16,7 +16,7 @@ # Usage: get_container_image_list.yml container_tag: "latest" squid_tag: "6.6-24.04_beta" -pulp_tag: "3.113" +pulp_tag: "3.114.2" mysql_tag: "9.3.0" prometheus_tag: "v3.4.1" activemq_tag: "5.19.7" diff --git a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml index f5935b79fb..95b7afcdbe 100644 --- a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml @@ -25,7 +25,7 @@ device_name: "/dev/fuse:/dev/fuse:rwm" pulp_container_name: "pulp" pulp_protocol_https: true # Tag is fixed for the Pulp container image as of 10-06-2025 -pulp_image: "docker.io/pulp/pulp:3.113" +pulp_image: "docker.io/pulp/pulp:3.114.2" # Usage: deployment_prereq.yml - pull image retries pull_image_retries: 5 diff --git a/rollback/roles/rollback_pulp/tasks/main.yml b/rollback/roles/rollback_pulp/tasks/main.yml index ba34c235d2..748c603c34 100644 --- a/rollback/roles/rollback_pulp/tasks/main.yml +++ b/rollback/roles/rollback_pulp/tasks/main.yml @@ -16,7 +16,7 @@ # ============================================================================ # rollback_pulp — Main Orchestration # ============================================================================ -# Reverse the Pulp upgrade (3.113 → 3.80) by: +# Reverse the Pulp upgrade (3.114.2 → 3.80) by: # 1. Resolving admin NIC IP for health checks # 2. Pre-rollback checks (verify Pulp is deployed, backup exists) # 3. Stopping the current Pulp container @@ -29,7 +29,7 @@ # # IMPORTANT: PostgreSQL data must be restored because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions # # No idempotency check — rollback always executes regardless of current diff --git a/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml b/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml index cb74c969aa..f613674d3c 100644 --- a/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml +++ b/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml @@ -14,7 +14,7 @@ --- # ============================================================================ -# Pulp Container Rollback (3.113 → 3.80) +# Pulp Container Rollback (3.114.2 → 3.80) # # Steps: # 1. Pull the v2.1 Pulp image (3.80) diff --git a/rollback/roles/rollback_pulp/vars/main.yml b/rollback/roles/rollback_pulp/vars/main.yml index 21ff282c46..87926ad40a 100644 --- a/rollback/roles/rollback_pulp/vars/main.yml +++ b/rollback/roles/rollback_pulp/vars/main.yml @@ -16,7 +16,7 @@ # ============================================================================ # rollback_pulp — Variables # ============================================================================ -# Rollback target: Pulp 3.80 (from Pulp 3.113) +# Rollback target: Pulp 3.80 (from Pulp 3.114.2) # Reverses the upgrade performed by upgrade_pulp role. # ============================================================================ @@ -74,7 +74,7 @@ pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql" # PostgreSQL backup paths # PostgreSQL data must be restored during rollback because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp" pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql" diff --git a/upgrade/playbooks/upgrade_oim.yml b/upgrade/playbooks/upgrade_oim.yml index 65ff749392..2d46cbcc71 100644 --- a/upgrade/playbooks/upgrade_oim.yml +++ b/upgrade/playbooks/upgrade_oim.yml @@ -21,7 +21,7 @@ # # Flow: # 1. Pre-flight: read manifest, check idempotency -# 2. Phase 1: Pulp container upgrade (3.80 → 3.113) +# 2. Phase 1: Pulp container upgrade (3.80 → 3.114.2) # 3. Phase 2: OpenCHAMI container upgrade (pg_dump, deployment-recipes, # image pull, ordered restart, DB migration, validation) # 4. Mark OIM as completed in manifest diff --git a/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml b/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml index 2b19607817..3408ce26d2 100644 --- a/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml +++ b/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml @@ -18,7 +18,7 @@ # # PostgreSQL data must be backed up before upgrade because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions # # Without this backup, rollback is not possible. diff --git a/upgrade/roles/upgrade_pulp/tasks/main.yml b/upgrade/roles/upgrade_pulp/tasks/main.yml index 4ec5803ca2..771211b5a8 100644 --- a/upgrade/roles/upgrade_pulp/tasks/main.yml +++ b/upgrade/roles/upgrade_pulp/tasks/main.yml @@ -16,13 +16,13 @@ # =========================================================================== # Upgrade Pulp Role - Main Entry Point # -# This role upgrades the Pulp container from 3.80 to 3.113. +# This role upgrades the Pulp container from 3.80 to 3.114.2. # Pulp stores all data in persistent volumes mounted from shared storage, # so data is preserved during the upgrade. # # IMPORTANT: PostgreSQL data must be backed up before upgrade because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions # Without the backup, rollback is not possible. # @@ -30,7 +30,7 @@ # 1. Resolve admin NIC IP for health check endpoints # 2. Pre-upgrade health check (verify Pulp is deployed and accessible) # 3. Backup PostgreSQL data (required for rollback) -# 4. Pull the new Pulp image (3.113) +# 4. Pull the new Pulp image (3.114.2) # 5. Stop the Pulp container # 6. Update the quadlet file with the new image tag # 7. Reload systemd daemon and restart container diff --git a/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml index 937558189f..f91084efe7 100644 --- a/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml +++ b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml @@ -14,7 +14,7 @@ --- # =========================================================================== -# Upgrade Pulp Container (3.80 → 3.113) +# Upgrade Pulp Container (3.80 → 3.114.2) # # Steps: # 1. Pull new Pulp image diff --git a/upgrade/roles/upgrade_pulp/vars/main.yml b/upgrade/roles/upgrade_pulp/vars/main.yml index 722272dde5..66fc0629da 100644 --- a/upgrade/roles/upgrade_pulp/vars/main.yml +++ b/upgrade/roles/upgrade_pulp/vars/main.yml @@ -14,7 +14,7 @@ --- # =========================================================================== -# Pulp Container Upgrade Configuration (3.80 → 3.113) +# Pulp Container Upgrade Configuration (3.80 → 3.114.2) # =========================================================================== # File permissions @@ -23,7 +23,7 @@ file_permissions_644: "0644" # Pulp container settings pulp_container_name: "pulp" -pulp_target_tag: "3.113" +pulp_target_tag: "3.114.2" pulp_target_image: "docker.io/pulp/pulp:{{ pulp_target_tag }}" # Pulp quadlet file path @@ -60,7 +60,7 @@ pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql" # Backup settings # PostgreSQL data must be backed up before upgrade because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp" pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql" diff --git a/upgrade/upgrade.yml b/upgrade/upgrade.yml index e2c27ad10b..1d90679129 100644 --- a/upgrade/upgrade.yml +++ b/upgrade/upgrade.yml @@ -453,7 +453,7 @@ ── Omnia Upgrade Execution Plan (in order) ────────────────── 1. oim → Upgrade OpenCHAMI control-plane containers - and Pulp container (3.80 → 3.113) on the OIM + and Pulp container (3.80 → 3.114.2) on the OIM 2. build_stream → SKIPPED (not enabled in build_stream_config.yml) 3. local_repo → Synchronize Omnia 2.2 packages into the local Pulp repository for cluster nodes From cb6dfeeefb586a272b0feae9ad18ec3721428470 Mon Sep 17 00:00:00 2001 From: Sayuri Kamble <68416000+SAYUK09@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:20:27 +0530 Subject: [PATCH 05/22] Migrate to wolfi auth, buildstream and core containers (#4940) * fix: replace ipcalc with ansible.utils.ipaddr filter * fix: make update-ca-trust and CA trust paths OS-aware for Wolfi compatibility * fix: ca directory issue * Configured pulp as fallback to download RHEL packages * Updated version * Reverted core tag change --- .../library/module_utils/local_repo/config.py | 14 +- .../module_utils/local_repo/download_rpm.py | 277 ++++++++++++------ .../tasks/create_metadata.yml | 14 + .../tasks/process_rpm_repo.yml | 7 + .../roles/validation/tasks/prerequisites.yml | 7 + .../deploy_containers/auth/vars/main.yml | 2 +- .../build_stream/vars/main.yml | 2 +- .../deploy_containers/openchami/vars/main.yml | 2 +- .../pulp/tasks/create_pulp_config_https.yml | 78 ++++- .../deploy_containers/pulp/vars/main.yml | 1 - .../tasks/validate_network_spec.yml | 20 +- .../tasks/create_k8s_config_nfs.yml | 23 +- provision/roles/k8s_config/vars/main.yml | 1 - 13 files changed, 349 insertions(+), 99 deletions(-) diff --git a/common/library/module_utils/local_repo/config.py b/common/library/module_utils/local_repo/config.py index 4bce5e64ca..5ae64cff51 100644 --- a/common/library/module_utils/local_repo/config.py +++ b/common/library/module_utils/local_repo/config.py @@ -108,6 +108,7 @@ "x86_64": ["dnf", "info", "--quiet"], "aarch64": ["dnf", "info", "--quiet", "--forcearch=aarch64"] } +PULP_RPM_PACKAGES_API = "/pulp/api/v3/content/rpm/packages/" # ---------------------------- # Cleanup File Types @@ -156,7 +157,18 @@ ISO_TIMEOUT_MIN = 45 # minutes TASK_POLL_INTERVAL = 10 # seconds FILE_URI = "/pulp/api/v3/content/file/files/" -PULP_SSL_CA_CERT = "/etc/pki/ca-trust/source/anchors/pulp_webserver.crt" + +import os + +def _get_ca_cert_path(): + """Return CA cert path based on OS. Fedora/RHEL vs Wolfi/Debian.""" + rhel_path = "/etc/pki/ca-trust/source/anchors/pulp_webserver.crt" + wolfi_path = "/usr/local/share/ca-certificates/pulp_webserver.crt" + if os.path.exists("/etc/pki/ca-trust/source/anchors"): + return rhel_path + return wolfi_path + +PULP_SSL_CA_CERT = _get_ca_cert_path() # ---------------------------- # Used by download_image.py # ---------------------------- diff --git a/common/library/module_utils/local_repo/download_rpm.py b/common/library/module_utils/local_repo/download_rpm.py index e6fe9603a2..1af2c4ea9d 100644 --- a/common/library/module_utils/local_repo/download_rpm.py +++ b/common/library/module_utils/local_repo/download_rpm.py @@ -18,17 +18,91 @@ import subprocess import os import glob +import json import shutil from pathlib import Path from ansible.module_utils.local_repo.config import ( DNF_COMMANDS, - DNF_INFO_COMMANDS + DNF_INFO_COMMANDS, + PULP_RPM_PACKAGES_API ) from multiprocessing import Lock from ansible.module_utils.local_repo.parse_and_download import write_status_to_file, _prefix_repo_name_with_arch file_lock = Lock() + +def _is_dnf_available(): + """Check if dnf binary is available on the system.""" + return shutil.which('dnf') is not None + + +def _pulp_cmd(cmd_string, logger=None): + """Run a pulp CLI command and return parsed JSON output, or None on failure.""" + result = subprocess.run(cmd_string, shell=True, check=False, capture_output=True, text=True) + if result.returncode != 0: + if logger: + logger.debug(f"Pulp command failed: {cmd_string}, stderr: {result.stderr}") + return None + try: + return json.loads(result.stdout) + except (json.JSONDecodeError, ValueError): + return None + + +def _pulp_get_repo_version(repo_name, logger): + """Get latest_version_href for a Pulp RPM repository.""" + data = _pulp_cmd(f"pulp rpm repository show --name {repo_name}", logger) + if isinstance(data, dict): + return data.get("latest_version_href", "") + return "" + + +def _pulp_find_package(pkg_name, repo_name, logger): + """Find RPM package in a Pulp repository. Returns package info dict or None.""" + version_href = _pulp_get_repo_version(repo_name, logger) + if not version_href: + return None + api_url = f"{PULP_RPM_PACKAGES_API}?name={pkg_name}&repository_version={version_href}&limit=1" + data = _pulp_cmd(f"pulp show --href '{api_url}'", logger) + if isinstance(data, dict) and data.get("count", 0) > 0: + return data.get("results", [None])[0] + if isinstance(data, list) and len(data) > 0: + return data[0] + return None + + +def _pulp_validate_package(pkg_name, repo_name, logger): + """Check if package exists in Pulp repo (replaces dnf info).""" + return _pulp_find_package(pkg_name, repo_name, logger) is not None + + +def _pulp_download_rpm(pkg_name, repo_name, rpm_directory, logger): + """Download a single RPM from Pulp distribution (replaces dnf download).""" + pkg_info = _pulp_find_package(pkg_name, repo_name, logger) + if not pkg_info: + logger.error(f"Package '{pkg_name}' not found in Pulp repo '{repo_name}'") + return False + location_href = pkg_info.get("location_href", "") + if not location_href: + logger.error(f"No location_href for package '{pkg_name}'") + return False + dist_data = _pulp_cmd(f"pulp rpm distribution show --name {repo_name}", logger) + if not isinstance(dist_data, dict) or not dist_data.get("base_url"): + logger.error(f"Could not get distribution URL for '{repo_name}'") + return False + base_url = dist_data["base_url"].rstrip("/") + download_url = f"{base_url}/{location_href}" + logger.info(f"Downloading '{pkg_name}' from {download_url}") + dl_result = subprocess.run( + ["wget", "-c", "-q", "--no-check-certificate", "-P", rpm_directory, download_url], + check=False, capture_output=True, text=True + ) + if dl_result.returncode == 0 or _check_rpm_downloaded(rpm_directory, pkg_name): + return True + logger.error(f"wget failed for '{pkg_name}': {dl_result.stderr}") + return False + def _check_rpm_downloaded(rpm_directory, pkg_name): """ Check if an RPM file for the given package exists in the directory. @@ -100,86 +174,107 @@ def process_rpm(package, repo_store_path, status_file_path, cluster_os_type, arch_key = "x86_64" if arc.lower() in ("x86_64") else "aarch64" - # First try to download all at once - dnf_download_command = ( - DNF_COMMANDS[arch_key] - + [f"--destdir={rpm_directory}"] - + rpm_list - ) - - result = subprocess.run( - dnf_download_command, - check=False, - capture_output=True, - text=True - ) - logger.info(f"Return code {result.returncode}") - logger.debug(f"STDOUT:\n{result.stdout}") - logger.debug(f"STDERR:\n{result.stderr}") - - stdout_lines = result.stdout.splitlines() - stderr_lines = result.stderr.splitlines() - downloaded = [] failed = [] - # Detect successes/failures from combined run - # Use filesystem check instead of parsing output (works with both DNF4 and DNF5) - for pkg in rpm_list: - # Get repo_name for this specific RPM from mapping - pkg_repo_name = repo_mapping.get(pkg, "") + if _is_dnf_available(): + # First try to download all at once + dnf_download_command = ( + DNF_COMMANDS[arch_key] + + [f"--destdir={rpm_directory}"] + + rpm_list + ) - # Check if package was downloaded by looking for the RPM file - pkg_downloaded = _check_rpm_downloaded(rpm_directory, pkg) - - # Also check for "No match for argument" or "No package" errors in stderr - pkg_not_found = False - for line in stderr_lines: - if pkg in line and ("No match for argument" in line or - "No package" in line or - "not found" in line.lower()): - pkg_not_found = True - break - - if pkg_downloaded and not pkg_not_found: - downloaded.append(pkg) - write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) - logger.info(f"Package '{pkg}' downloaded successfully.") - else: - failed.append(pkg) - if pkg_not_found: - logger.warning(f"Package '{pkg}' not found in configured repositories") - - # Retry failed ones individually - if failed: - logger.warning(f"Retrying failed packages individually: {failed}") - for pkg in failed[:]: - cmd = DNF_COMMANDS[arch_key] + [f'--destdir={rpm_directory}', pkg] - retry_res = subprocess.run(cmd, check=False, capture_output=True, text=True) + result = subprocess.run( + dnf_download_command, + check=False, + capture_output=True, + text=True + ) + logger.info(f"Return code {result.returncode}") + logger.debug(f"STDOUT:\n{result.stdout}") + logger.debug(f"STDERR:\n{result.stderr}") + + stdout_lines = result.stdout.splitlines() + stderr_lines = result.stderr.splitlines() + + # Detect successes/failures from combined run + # Use filesystem check instead of parsing output (works with both DNF4 and DNF5) + for pkg in rpm_list: # Get repo_name for this specific RPM from mapping pkg_repo_name = repo_mapping.get(pkg, "") - # Check for package not found errors - retry_stderr = retry_res.stderr.lower() - pkg_invalid = any(err in retry_stderr for err in [ - "no match for argument", - "no package", - "not found", - "unable to find a match" - ]) - - # Check if RPM file exists after retry (works with both DNF4 and DNF5) - if retry_res.returncode == 0 and _check_rpm_downloaded(rpm_directory, pkg): + # Check if package was downloaded by looking for the RPM file + pkg_downloaded = _check_rpm_downloaded(rpm_directory, pkg) + + # Also check for "No match for argument" or "No package" errors in stderr + pkg_not_found = False + for line in stderr_lines: + if pkg in line and ("No match for argument" in line or + "No package" in line or + "not found" in line.lower()): + pkg_not_found = True + break + + if pkg_downloaded and not pkg_not_found: downloaded.append(pkg) - failed.remove(pkg) write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) - logger.info(f"Package '{pkg}' downloaded successfully on retry.") + logger.info(f"Package '{pkg}' downloaded successfully.") else: - write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, pkg_repo_name) - if pkg_invalid: - logger.error(f"Package '{pkg}' does not exist in configured repositories.") + failed.append(pkg) + if pkg_not_found: + logger.warning(f"Package '{pkg}' not found in configured repositories") + + # Retry failed ones individually + if failed: + logger.warning(f"Retrying failed packages individually: {failed}") + for pkg in failed[:]: + cmd = DNF_COMMANDS[arch_key] + [f'--destdir={rpm_directory}', pkg] + retry_res = subprocess.run(cmd, check=False, capture_output=True, text=True) + # Get repo_name for this specific RPM from mapping + pkg_repo_name = repo_mapping.get(pkg, "") + + # Check for package not found errors + retry_stderr = retry_res.stderr.lower() + pkg_invalid = any(err in retry_stderr for err in [ + "no match for argument", + "no package", + "not found", + "unable to find a match" + ]) + + # Check if RPM file exists after retry (works with both DNF4 and DNF5) + if retry_res.returncode == 0 and _check_rpm_downloaded(rpm_directory, pkg): + downloaded.append(pkg) + failed.remove(pkg) + write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) + logger.info(f"Package '{pkg}' downloaded successfully on retry.") + else: + write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, pkg_repo_name) + if pkg_invalid: + logger.error(f"Package '{pkg}' does not exist in configured repositories.") + else: + logger.error(f"Package '{pkg}' still failed after retry.") + + else: + # Pulp-based download (dnf not available, e.g. Wolfi container) + logger.info("dnf not available, using Pulp CLI for RPM download") + for pkg in rpm_list: + pkg_repo_name = repo_mapping.get(pkg, "") + if pkg_repo_name: + prefixed_repo_name = _prefix_repo_name_with_arch(pkg_repo_name, status_file_path, logger) + if _pulp_download_rpm(pkg, prefixed_repo_name, rpm_directory, logger): + downloaded.append(pkg) + write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) + logger.info(f"Package '{pkg}' downloaded successfully via Pulp.") else: - logger.error(f"Package '{pkg}' still failed after retry.") + failed.append(pkg) + write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, pkg_repo_name) + logger.error(f"Package '{pkg}' download failed via Pulp.") + else: + failed.append(pkg) + logger.warning(f"No repo_name mapping for package '{pkg}', cannot download via Pulp") + write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, "") # Determine final status if not failed: @@ -191,35 +286,47 @@ def process_rpm(package, repo_store_path, status_file_path, cluster_os_type, else: logger.info("RPM won't be downloaded when repo_config is partial or never") - logger.info("Validating package availability using dnf info...") arch_key = "x86_64" if arc.lower() in ("x86_64") else "aarch64" valid_packages = [] invalid_packages = [] + use_dnf = _is_dnf_available() + + if use_dnf: + logger.info("Validating package availability using dnf info...") + else: + logger.info("dnf not available, validating package availability using Pulp CLI...") for pkg in package["rpm_list"]: # Get repo_name for this specific RPM from mapping pkg_repo_name = repo_mapping.get(pkg, "") - # Validate package using dnf info with specific repo only - if pkg_repo_name: - # Apply architecture prefixing if needed - prefixed_repo_name = _prefix_repo_name_with_arch(pkg_repo_name, status_file_path, logger) + if not pkg_repo_name: + # Skip validation if no specific repo is defined + logger.warning(f"No repo_name defined for package '{pkg}', skipping validation") + continue + + # Apply architecture prefixing if needed + prefixed_repo_name = _prefix_repo_name_with_arch(pkg_repo_name, status_file_path, logger) + + if use_dnf: + # Validate package using dnf info with specific repo dnf_info_command = DNF_INFO_COMMANDS[arch_key] + [ f"--repo={prefixed_repo_name}", # Search specific repo from JSON pkg ] + result = subprocess.run( + dnf_info_command, + check=False, + capture_output=True, + text=True + ) + pkg_exists = result.returncode == 0 else: - # Skip validation if no specific repo is defined - logger.warning(f"No repo_name defined for package '{pkg}', skipping validation") - continue - result = subprocess.run( - dnf_info_command, - check=False, - capture_output=True, - text=True - ) - if result.returncode == 0: + # Validate package using Pulp CLI + pkg_exists = _pulp_validate_package(pkg, prefixed_repo_name, logger) + + if pkg_exists: # Package exists and is available valid_packages.append(pkg) write_status_to_file( diff --git a/local_repo/roles/parse_and_download/tasks/create_metadata.yml b/local_repo/roles/parse_and_download/tasks/create_metadata.yml index c994f225dc..54eaea6cc9 100644 --- a/local_repo/roles/parse_and_download/tasks/create_metadata.yml +++ b/local_repo/roles/parse_and_download/tasks/create_metadata.yml @@ -15,6 +15,13 @@ - name: Clean DNF cache ansible.builtin.command: dnf clean all changed_when: false + when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] + +- name: Clean apk cache + ansible.builtin.command: apk cache clean + changed_when: false + failed_when: false + when: ansible_facts['pkg_mgr'] | default('') == 'apk' - name: Remove pulp.repo if exists ansible.builtin.file: @@ -25,6 +32,13 @@ ansible.builtin.dnf: update_cache: true changed_when: false + when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] + +- name: Regenerate apk package index + community.general.apk: + update_cache: true + changed_when: false + when: ansible_facts['pkg_mgr'] | default('') == 'apk' - name: Check for data folder existence ansible.builtin.stat: diff --git a/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml b/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml index 166e6f8784..78de31f50e 100644 --- a/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml +++ b/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml @@ -43,3 +43,10 @@ ansible.builtin.dnf: update_cache: true changed_when: false + when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] + +- name: Regenerate apk package index + community.general.apk: + update_cache: true + changed_when: false + when: ansible_facts['pkg_mgr'] | default('') == 'apk' diff --git a/local_repo/roles/validation/tasks/prerequisites.yml b/local_repo/roles/validation/tasks/prerequisites.yml index a969bce84e..de6c5ffac8 100644 --- a/local_repo/roles/validation/tasks/prerequisites.yml +++ b/local_repo/roles/validation/tasks/prerequisites.yml @@ -25,3 +25,10 @@ loop: - "{{ ['max_parallel_downloads', dnf_max_parallel] }}" - "{{ ['strict', 'False'] }}" + when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] + +- name: Refresh apk package index + community.general.apk: + update_cache: true + changed_when: false + when: ansible_facts['pkg_mgr'] | default('') == 'apk' diff --git a/prepare_oim/roles/deploy_containers/auth/vars/main.yml b/prepare_oim/roles/deploy_containers/auth/vars/main.yml index fcf2defe4f..55bf2e5ece 100644 --- a/prepare_oim/roles/deploy_containers/auth/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/auth/vars/main.yml @@ -23,7 +23,7 @@ openldap_ports: - 636 wait_time: 10 auth_service_image_name: omnia_auth -auth_service_image_tag: "1.1" +auth_service_image_tag: "1.2" auth_service_registry: "docker.io/dellhpcomniaaisolution" auth_service_container_name: omnia_auth auth_service_image_pull_fail_msg: diff --git a/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml b/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml index 896b876c6e..baedf69322 100644 --- a/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml @@ -33,7 +33,7 @@ admin_ip: "{{ hostvars['localhost']['admin_nic_ip'] | default('localhost') }}" # Build Stream Image (Docker Hub) build_stream_dockerhub_registry: "docker.io/dellhpcomniaaisolution" build_stream_image_name: "{{ build_stream_dockerhub_registry }}/omnia_build_stream" -build_stream_image_tag: "1.1" +build_stream_image_tag: "1.2" # Ports & Logs build_stream_port: "{{ hostvars['localhost']['build_stream_port'] }}" diff --git a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml index dfdf99a745..8ab0d92a33 100644 --- a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml @@ -45,7 +45,7 @@ openchami_bss_tag: "v1.32.2" openchami_cloud_init_tag: "v1.3.0" openchami_coresmd_tag: "v0.4.3" # Third-party image tags for OpenCHAMI -minio_release_tag: "RELEASE.2026-04-17T00-00-00Z" +minio_release_tag: "RELEASE.2026-06-18T00-00-00Z" postgres_tag: "11.5-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" diff --git a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml index a548bf6ed6..b1a7468652 100644 --- a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml +++ b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml @@ -108,16 +108,92 @@ state: file loop: "{{ cert_items.values() }}" + - name: Detect CA trust anchors directory + ansible.builtin.set_fact: + ca_trust_anchors_dir: >- + {{ '/etc/pki/ca-trust/source/anchors' + if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] + else '/usr/local/share/ca-certificates' }} + when: ca_trust_anchors_dir is not defined + + - name: Set anchors path + ansible.builtin.set_fact: + ca_trust_path: "{{ ca_trust_anchors_dir }}/pulp_webserver.crt" + when: ca_trust_path is not defined + + - name: Ensure CA trust directory exists + ansible.builtin.file: + path: "{{ ca_trust_anchors_dir }}" + state: directory + mode: '0755' + + - name: Copy Pulp crt to container trust + ansible.builtin.copy: + src: "{{ pulp_cert_src }}" + dest: "{{ ca_trust_path }}" + mode: "{{ logs_dir_permission }}" + + - name: Change group ownership of SSL certificate and key + ansible.builtin.file: + path: "{{ item }}" + group: pulp + state: file + loop: "{{ cert_items.values() }}" + + # ── CA Trust: OS-aware cert installation ──────────────────────────────── + # Fedora/RHEL: use update-ca-trust (native command exists) + # Wolfi/Other: directly append cert to system CA bundle (no command exists) + + - name: Detect CA trust anchors directory + ansible.builtin.set_fact: + ca_trust_anchors_dir: >- + {{ '/etc/pki/ca-trust/source/anchors' + if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] + else '/usr/local/share/ca-certificates' }} + when: ca_trust_anchors_dir is not defined + + - name: Set anchors path + ansible.builtin.set_fact: + ca_trust_path: "{{ ca_trust_anchors_dir }}/pulp_webserver.crt" + when: ca_trust_path is not defined + + - name: Ensure CA trust directory exists + ansible.builtin.file: + path: "{{ ca_trust_anchors_dir }}" + state: directory + mode: '0755' + - name: Copy Pulp crt to container trust ansible.builtin.copy: src: "{{ pulp_cert_src }}" dest: "{{ ca_trust_path }}" mode: "{{ logs_dir_permission }}" - - name: Add Pulp Certificate to TrustStore + - name: Add Pulp Certificate to TrustStore (RHEL/Fedora) ansible.builtin.command: cmd: update-ca-trust extract changed_when: false + when: ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] + + - name: Add Pulp Certificate to TrustStore (Wolfi/Other) + ansible.builtin.shell: | + cat "{{ ca_trust_path }}" >> /etc/ssl/certs/ca-certificates.crt + changed_when: false + when: ansible_facts['os_family'] | default('') not in ['RedHat', 'Fedora'] + + # ── End CA Trust ──────────────────────────────────────────────────────── + + - name: Create a track file + ansible.builtin.file: + path: "{{ track_file_path }}" + state: touch + mode: "{{ logs_dir_permission }}" + + - name: Record current timestamp in track file + ansible.builtin.copy: + dest: "{{ track_file_path }}" + content: "Timestamp: {{ ansible_date_time.iso8601 }}" + mode: "{{ logs_dir_permission }}" - name: Create a track file ansible.builtin.file: diff --git a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml index 95b7afcdbe..bd2cf25d91 100644 --- a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml @@ -138,7 +138,6 @@ generate_cert_cmd: > -addext {{ cert_san }} pulp_cert_src: "/opt/omnia/pulp/settings/certs/pulp_webserver.crt" -ca_trust_path: "/etc/pki/ca-trust/source/anchors/" # Usage: reload_pulp_nginx.yml nginx_reload_cmd: "nginx -s reload" diff --git a/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml b/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml index 622f633d61..3779f7767f 100644 --- a/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml +++ b/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml @@ -79,14 +79,22 @@ success_msg: "{{ admin_nic_ip_success_msg }}" when: fetch_oim_hostname.stdout in oim_hostname -- name: Compute network address using ipcalc - ansible.builtin.command: "/usr/bin/ipcalc -n {{ admin_nic_ip }}/{{ admin_netmask_bits }}" - register: network_address_output - changed_when: false +# - name: Compute network address using ipcalc +# ansible.builtin.command: "/usr/bin/ipcalc -n {{ admin_nic_ip }}/{{ admin_netmask_bits }}" +# register: network_address_output +# changed_when: false + +# - name: Extract network address +# ansible.builtin.set_fact: +# admin_net_addr: "{{ network_address_output.stdout.split('=')[1] }}" + +- name: Compute network address + ansible.builtin.set_fact: + network_address: "{{ (admin_nic_ip + '/' + (admin_netmask_bits | string)) | ansible.utils.ipaddr('network') }}" -- name: Extract network address +- name: Set admin_net_addr from computed network address ansible.builtin.set_fact: - admin_net_addr: "{{ network_address_output.stdout.split('=')[1] }}" + admin_net_addr: "{{ network_address }}" - name: Initialize network_interface_type ansible.builtin.set_fact: diff --git a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml index 5ca3d4101b..f95df3668a 100644 --- a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml @@ -240,6 +240,19 @@ nfs_subdir_external_provisioner_pkg: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'tarball') | selectattr('package', 'search', 'nfs-subdir-external-provisioner') | map(attribute='package') | join }}" # noqa: yaml[line-length] whereabouts_pkg: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'git') | selectattr('package', 'search', 'whereabouts') | map(attribute='package') | join }}" # noqa: yaml[line-length] +- name: Detect CA trust anchors directory + ansible.builtin.set_fact: + ca_trust_anchors_dir: >- + {{ '/etc/pki/ca-trust/source/anchors' + if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] + else '/usr/local/share/ca-certificates' }} + when: ca_trust_anchors_dir is not defined + +- name: Set anchors path + ansible.builtin.set_fact: + anchors_path: "{{ ca_trust_anchors_dir }}/pulp_webserver.crt" + when: anchors_path is not defined + - name: Copy pulp webserver certificate to target host ansible.builtin.copy: src: "{{ pulp_webserver_cert_path }}" @@ -247,8 +260,16 @@ mode: "{{ file_mode }}" become: true +- name: Detect CA trust update command + ansible.builtin.set_fact: + ca_trust_cmd: >- + {{ 'update-ca-trust' + if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] + else 'update-ca-certificates' }} + when: ca_trust_cmd is not defined + - name: Update CA trust on target host - ansible.builtin.command: update-ca-trust + ansible.builtin.command: "{{ ca_trust_cmd }}" register: update_ca changed_when: false diff --git a/provision/roles/k8s_config/vars/main.yml b/provision/roles/k8s_config/vars/main.yml index c949564d70..baa9583d29 100644 --- a/provision/roles/k8s_config/vars/main.yml +++ b/provision/roles/k8s_config/vars/main.yml @@ -27,7 +27,6 @@ whereabouts_git_url: "{{ offline_git_path }}/{{ whereabouts_pkg }}/{{ whereabout file_mode: "0644" ha_config_file: "{{ input_project_dir }}/high_availability_config.yml" pulp_webserver_cert_path: "/opt/omnia/pulp/settings/certs/pulp_webserver.crt" -anchors_path: "/etc/pki/ca-trust/source/anchors/pulp_webserver.crt" # Usage: create_node_dir.yml nodes_yaml: "{{ hostvars['localhost']['oim_shared_path'] }}/omnia/openchami/workdir/nodes/nodes.yaml" From 352f57498df057371f79b7f29c95f59310fc19aa Mon Sep 17 00:00:00 2001 From: Kratika Patidar Date: Thu, 6 Aug 2026 18:30:45 +0530 Subject: [PATCH 06/22] security vulnerability fix for activeMQ, mysql, victoria metrics (#4946) * security vulnerability fix for activeMQ, mysql, victoria metrics Signed-off-by: Kratika_Patidar * catalog update with latest version of packages Signed-off-by: Kratika_Patidar --------- Signed-off-by: Kratika_Patidar --- common/vars/image_vars.yml | 4 +-- examples/catalog/catalog_rhel.json | 28 +++++++++---------- .../catalog_rhel_with_nfs_provisioner.json | 28 +++++++++---------- examples/catalog/catalog_rhel_x86_64.json | 28 +++++++++---------- .../x86_64/rhel/10.0/service_k8s_v1.35.1.json | 14 +++++----- .../idrac_telemetry/activemq-config.yaml.j2 | 2 -- provision/roles/telemetry/vars/main.yml | 18 ++++++------ 7 files changed, 60 insertions(+), 62 deletions(-) diff --git a/common/vars/image_vars.yml b/common/vars/image_vars.yml index e72566de1c..a7ad163749 100644 --- a/common/vars/image_vars.yml +++ b/common/vars/image_vars.yml @@ -17,9 +17,9 @@ container_tag: "latest" squid_tag: "6.6-24.04_beta" pulp_tag: "3.114.2" -mysql_tag: "9.3.0" +mysql_tag: "9.7.2" prometheus_tag: "v3.4.1" -activemq_tag: "5.19.7" +activemq_tag: "6.3.0" grafana_image_tag: "12.0.1" loki_image_tag: "3.5.1" promtail_image_tag: "3.5.1" diff --git a/examples/catalog/catalog_rhel.json b/examples/catalog/catalog_rhel.json index d5f5c5dd70..d040e7fe01 100644 --- a/examples/catalog/catalog_rhel.json +++ b/examples/catalog/catalog_rhel.json @@ -609,8 +609,8 @@ "x86_64" ], "Type": "image", - "Tag": "5.19.7", - "Version": "5.19.7" + "Tag": "6.3.0", + "Version": "6.3.0" }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", @@ -759,8 +759,8 @@ "x86_64" ], "Type": "image", - "Tag": "9.3.0", - "Version": "9.3.0" + "Tag": "9.7.2", + "Version": "9.7.2" }, "docker.io/library/python": { "Name": "docker.io/library/python", @@ -864,8 +864,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vlagent": { "Name": "docker.io/victoriametrics/vlagent", @@ -894,8 +894,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vminsert": { "Name": "docker.io/victoriametrics/vminsert", @@ -909,8 +909,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmselect": { "Name": "docker.io/victoriametrics/vmselect", @@ -924,8 +924,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmstorage": { "Name": "docker.io/victoriametrics/vmstorage", @@ -939,8 +939,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "firewalld": { "Name": "firewalld", diff --git a/examples/catalog/catalog_rhel_with_nfs_provisioner.json b/examples/catalog/catalog_rhel_with_nfs_provisioner.json index 665438cf9a..be0cb11ada 100644 --- a/examples/catalog/catalog_rhel_with_nfs_provisioner.json +++ b/examples/catalog/catalog_rhel_with_nfs_provisioner.json @@ -578,8 +578,8 @@ "x86_64" ], "Type": "image", - "Tag": "5.19.7", - "Version": "5.19.7" + "Tag": "6.3.0", + "Version": "6.3.0" }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", @@ -728,8 +728,8 @@ "x86_64" ], "Type": "image", - "Tag": "9.3.0", - "Version": "9.3.0" + "Tag": "9.7.2", + "Version": "9.7.2" }, "docker.io/library/python": { "Name": "docker.io/library/python", @@ -833,8 +833,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vlagent": { "Name": "docker.io/victoriametrics/vlagent", @@ -863,8 +863,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vminsert": { "Name": "docker.io/victoriametrics/vminsert", @@ -878,8 +878,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmselect": { "Name": "docker.io/victoriametrics/vmselect", @@ -893,8 +893,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmstorage": { "Name": "docker.io/victoriametrics/vmstorage", @@ -908,8 +908,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "firewalld": { "Name": "firewalld", diff --git a/examples/catalog/catalog_rhel_x86_64.json b/examples/catalog/catalog_rhel_x86_64.json index 9f34bfc706..72a354c8c3 100644 --- a/examples/catalog/catalog_rhel_x86_64.json +++ b/examples/catalog/catalog_rhel_x86_64.json @@ -595,8 +595,8 @@ "x86_64" ], "Type": "image", - "Tag": "5.19.7", - "Version": "5.19.7" + "Tag": "6.3.0", + "Version": "6.3.0" }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", @@ -745,8 +745,8 @@ "x86_64" ], "Type": "image", - "Tag": "9.3.0", - "Version": "9.3.0" + "Tag": "9.7.2", + "Version": "9.7.2" }, "docker.io/library/python": { "Name": "docker.io/library/python", @@ -850,8 +850,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vlagent": { "Name": "docker.io/victoriametrics/vlagent", @@ -880,8 +880,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vminsert": { "Name": "docker.io/victoriametrics/vminsert", @@ -895,8 +895,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmselect": { "Name": "docker.io/victoriametrics/vmselect", @@ -910,8 +910,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmstorage": { "Name": "docker.io/victoriametrics/vmstorage", @@ -925,8 +925,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "firewalld": { "Name": "firewalld", diff --git a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json index b120abd367..1c5bb469f7 100644 --- a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json +++ b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json @@ -13,17 +13,17 @@ { "package": "kubelet-1.35.1", "type": "rpm", "repo_name": "kubernetes-v1-35"}, { "package": "container-selinux", "type": "rpm", "repo_name": "appstream"}, { "package": "cri-o-1.35.1", "type": "rpm", "repo_name": "cri-o-v1-35"}, - { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.128.0" }, - { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.128.0" }, - { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.128.0-cluster" }, - { "package": "docker.io/victoriametrics/vminsert", "type": "image", "tag": "v1.128.0-cluster" }, - { "package": "docker.io/victoriametrics/vmselect", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.149.0" }, + { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.149.0" }, + { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.149.0-cluster" }, + { "package": "docker.io/victoriametrics/vminsert", "type": "image", "tag": "v1.149.0-cluster" }, + { "package": "docker.io/victoriametrics/vmselect", "type": "image", "tag": "v1.149.0-cluster" }, { "package": "docker.io/victoriametrics/victoria-logs", "type": "image", "tag": "v1.50.0" }, { "package": "docker.io/victoriametrics/vlagent", "type": "image", "tag": "v1.50.0" }, { "package": "docker.io/alpine/kubectl", "tag": "1.35.1", "type": "image" }, { "package": "docker.io/curlimages/curl", "type": "image", "tag": "8.17.0" }, - { "package": "docker.io/apache/activemq", "type": "image", "tag": "5.19.7" }, - { "package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0" }, + { "package": "docker.io/apache/activemq", "type": "image", "tag": "6.3.0" }, + { "package": "docker.io/library/mysql", "type": "image", "tag": "9.7.2" }, { "package": "docker.io/library/python", "type": "image", "tag": "3.12-slim" }, { "package": "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "type": "image", "tag": "1.3" }, { "package": "docker.io/dellhpcomniaaisolution/kafkapump", "type": "image", "tag": "1.3" }, diff --git a/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 index 5538a366e6..fa1420c944 100644 --- a/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 @@ -96,6 +96,4 @@ data: - - diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml index 1be2a599e7..8ecf03ba6e 100644 --- a/provision/roles/telemetry/vars/main.yml +++ b/provision/roles/telemetry/vars/main.yml @@ -53,7 +53,7 @@ idrac_telemetry_receiver_image: "{{ telemetry_images['dellhpcomniaaisolution/idr kafkapump_image: "{{ telemetry_images['dellhpcomniaaisolution/kafkapump'] | default('docker.io/dellhpcomniaaisolution/kafkapump:1.2') }}" victoriapump_image: "{{ telemetry_images['dellhpcomniaaisolution/victoriapump'] | default('docker.io/dellhpcomniaaisolution/victoriapump:1.2') }}" -activemq_image: "{{ telemetry_images['apache/activemq'] | default('docker.io/apache/activemq:5.19.7') }}" +activemq_image: "{{ telemetry_images['apache/activemq'] | default('docker.io/apache/activemq:6.3.0') }}" activemq_http_port_1: 8161 activemq_http_port_2: 61616 messagebus_http_port: 61613 @@ -63,7 +63,7 @@ mysqldb_name: "idrac_telemetrydb" idrac_telemetry_service_name: "idrac-telemetry-service" mysqldb_container_port1: 3306 mysqldb_container_port2: 33060 -mysql_image: "{{ telemetry_images['library/mysql'] | default('docker.io/library/mysql:9.3.0') }}" +mysql_image: "{{ telemetry_images['library/mysql'] | default('docker.io/library/mysql:9.7.2') }}" pod_wait_timeout: "10m" kafka_skip_verify: true @@ -147,7 +147,7 @@ victoria: container_name: "victoriametrics" service_name: "victoria-loadbalancer" container_port: 8443 - image: "{{ telemetry_images['victoriametrics/victoria-metrics'] | default('victoriametrics/victoria-metrics:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/victoria-metrics'] | default('victoriametrics/victoria-metrics:v1.149.0') }}" vmsingle: replicas: 2 @@ -177,7 +177,7 @@ victoria_cluster: vmstorage: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vmstorage.replicas | default(3) }}" replication_factor: 2 # Data redundancy: each metric written to 2 of 3 nodes - image: "{{ telemetry_images['victoriametrics/vmstorage'] | default('victoriametrics/vmstorage:v1.128.0-cluster') }}" + image: "{{ telemetry_images['victoriametrics/vmstorage'] | default('victoriametrics/vmstorage:v1.149.0-cluster') }}" dedup_min_scrape_interval: "1m" # Deduplication interval resources: requests: @@ -189,7 +189,7 @@ victoria_cluster: # VMInsert: Accepts data ingestion and routes to vmstorage vminsert: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vminsert.replicas | default(2) }}" - image: "{{ telemetry_images['victoriametrics/vminsert'] | default('victoriametrics/vminsert:v1.128.0-cluster') }}" + image: "{{ telemetry_images['victoriametrics/vminsert'] | default('victoriametrics/vminsert:v1.149.0-cluster') }}" # External access configuration external_access: true # Enable LoadBalancer service for external data ingestion resources: @@ -202,7 +202,7 @@ victoria_cluster: # VMSelect: Performs queries against vmstorage nodes vmselect: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vmselect.replicas | default(2) }}" - image: "{{ telemetry_images['victoriametrics/vmselect'] | default('victoriametrics/vmselect:v1.128.0-cluster') }}" + image: "{{ telemetry_images['victoriametrics/vmselect'] | default('victoriametrics/vmselect:v1.149.0-cluster') }}" max_query_duration: "5m" max_concurrent_requests: "8" cache_data_path: true # Enable query result caching @@ -216,7 +216,7 @@ victoria_cluster: vmagent: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vmagent.replicas | default(2) }}" - image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.149.0') }}" resources: requests: memory: "{{ telemetry_storage_config.victoria_cluster_storage.vmagent.resources.requests.memory | default('128Mi') }}" @@ -395,7 +395,7 @@ vmagent: rolebinding_name: "vmagent-sd-binding" app_name: "vmagent" container_name: "vmagent" - image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.149.0') }}" # Single-node URL remote_write_url: "https://victoria-loadbalancer.telemetry.svc.cluster.local:8443/api/v1/write" # Cluster URL (used when victoria_cluster.enabled: true) @@ -688,7 +688,7 @@ vector: service_name: "vmagent-vector" port: 8429 # prometheus_remote_write receiver metrics_port: 8429 # vmagent self-metrics - image: "{{ telemetry_images['victoriametrics/vmagent'] | default('docker.io/victoriametrics/vmagent:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/vmagent'] | default('docker.io/victoriametrics/vmagent:v1.149.0') }}" replicas: "{{ telemetry_storage_config.vector_storage.vmagent_vector.replicas | default(2) }}" pvc_size: "{{ telemetry_storage_config.vector_storage.vmagent_vector.pvc_size | default('5Gi') }}" # Disk WAL buffer remote_write_url: "https://vminsert-victoria-cluster.{{ telemetry_namespace }}.svc.cluster.local:8480/insert/0/prometheus/api/v1/write" From 153823d4808fbcfea765708ff29403108fdf9fa2 Mon Sep 17 00:00:00 2001 From: Katakam-Rakesh Date: Fri, 7 Aug 2026 10:10:46 +0530 Subject: [PATCH 07/22] Add kube-vip upgrade support for k8s addon upgrades - Detect kube-vip version from static pod manifest on each control plane node using SSH delegation since CRI-O stores images by digest without tags - Extract kube_vip_target_version from service_k8s.json - Add kube_vip status tracking in upgrade status structure - Upgrade kube-vip per-node with VIP leader upgraded last for availability - Abort addon upgrades on kube-vip failure since VIP is critical for kubectl - Add kube-vip pod validation in addon validation step - Add msg_addon_kube_vip_failed error message for failure reporting Signed-off-by: Katakam-Rakesh --- .../tasks/detect_addon_versions.yml | 104 +++++++ .../roles/upgrade_k8s/tasks/load_status.yml | 2 + .../upgrade_k8s/tasks/load_version_vars.yml | 7 + .../tasks/step_addon_validation.yml | 19 ++ .../tasks/step_kube_vip_upgrade.yml | 274 ++++++++++++++++++ .../upgrade_k8s/tasks/upgrade_addons.yml | 123 ++++++++ upgrade/roles/upgrade_k8s/vars/main.yml | 4 + 7 files changed, 533 insertions(+) create mode 100644 upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml diff --git a/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml b/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml index b981049e8b..2d527d4854 100644 --- a/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml +++ b/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml @@ -119,6 +119,104 @@ else 'pending' }} +# ── kube-vip version detection ──────────────────────────────────────── +# kube-vip runs as a static pod on EACH control plane node. The version +# is read from the manifest file (source of truth) rather than crictl, +# because CRI-O stores images by digest and crictl does not return tags. +# We check all CP nodes: if any node is not at target, status = pending. +# +# IMPORTANT: this role runs under `hosts: localhost, connection: local`. +# A delegate_to target only connects over SSH if that host has +# `ansible_connection: ssh` set in the inventory. Raw IPs (and file-based +# inventory hosts without that var) inherit connection: local and run on +# the controller instead — which is why the manifest was "not found". +# We therefore register each CP node in the in-memory inventory with an +# explicit SSH connection, then delegate to those hostnames. +- name: Build list of control plane nodes for kube-vip detection + ansible.builtin.set_fact: + _kube_vip_detect_cp_nodes: >- + {{ upgrade_status.nodes | dict2items + | selectattr('value.role', 'in', ['control_plane_first', 'control_plane']) + | list }} + +- name: Register control plane nodes for SSH delegation (kube-vip detection) + ansible.builtin.add_host: + name: "{{ item.key }}" + ansible_host: "{{ item.value.ip }}" + ansible_connection: ssh + ansible_user: root + ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + groups: _kube_vip_detect_targets + loop: "{{ _kube_vip_detect_cp_nodes }}" + loop_control: + label: "{{ item.key }} ({{ item.value.ip }})" + changed_when: false + +- name: Build list of control plane hostnames for kube-vip detection + ansible.builtin.set_fact: + _kube_vip_detect_cp_hosts: "{{ _kube_vip_detect_cp_nodes | map(attribute='key') | list }}" + +- name: Detect kube-vip version from manifest on each control plane + ansible.builtin.shell: + cmd: >- + set -o pipefail && + grep -oP 'image:\s+\S+:v\K\d+\.\d+\.\d+' + /etc/kubernetes/manifests/kube-vip.yaml 2>/dev/null || + echo "unknown" + executable: /bin/bash + delegate_to: "{{ item }}" + loop: "{{ _kube_vip_detect_cp_hosts }}" + register: kube_vip_version_per_node + changed_when: false + failed_when: false + +- name: Build per-node kube-vip version map + ansible.builtin.set_fact: + _kube_vip_node_versions: >- + {{ _kube_vip_node_versions | default({}) | combine({ + item.item: item.stdout | default('unknown') | trim + }) }} + loop: "{{ kube_vip_version_per_node.results }}" + loop_control: + label: "{{ item.item }}" + +- name: Display kube-vip version per control plane node + ansible.builtin.debug: + msg: "kube-vip versions: {{ _kube_vip_node_versions }}" + +# Use the first non-target version as the "current" version for overall +# reporting. If all nodes match the target, report the target. +- name: Set kube_vip_current_version fact + ansible.builtin.set_fact: + kube_vip_current_version: >- + {{ (_kube_vip_node_versions.values() | list + | reject('equalto', kube_vip_target_version) + | reject('equalto', 'unknown') | first | default('')) + or + (kube_vip_target_version + if (_kube_vip_node_versions.values() | list + | select('equalto', kube_vip_target_version) | list | length + == _kube_vip_node_versions | length) + else (_kube_vip_node_versions.values() | list + | reject('equalto', 'unknown') | first | default('unknown'))) }} + +- name: Set kube_vip_from_version fact (preserve existing if set) + ansible.builtin.set_fact: + kube_vip_from_version: >- + {{ upgrade_status.addon_upgrade.kube_vip.from_version + if (upgrade_status.addon_upgrade.kube_vip.from_version | default('unknown')) not in ['unknown', ''] + else kube_vip_current_version }} + +# completed only when ALL control plane nodes are at the target version +- name: Determine kube-vip status + ansible.builtin.set_fact: + kube_vip_status: >- + {{ 'completed' + if (_kube_vip_node_versions.values() | list + | select('equalto', kube_vip_target_version) | list | length + == _kube_vip_node_versions | length) + else 'pending' }} + - name: Load PowerScale target version from JSON ansible.builtin.set_fact: csi_powerscale_packages_json: "{{ lookup('file', input_project_dir + '/config/x86_64/rhel/' + cluster_os_version + '/csi_driver_powerscale.json') | from_json }}" # noqa yaml[line-length] @@ -189,6 +287,11 @@ vars: status_update: addon_upgrade: + kube_vip: + status: "{{ kube_vip_status }}" + current_version: "{{ kube_vip_current_version }}" + from_version: "{{ kube_vip_from_version }}" + target_version: "{{ kube_vip_target_version }}" calico: status: "{{ calico_status }}" current_version: "{{ calico_current_version }}" @@ -216,6 +319,7 @@ vars: addon_versions_banner: - "Detected addon versions:" + - " kube-vip: {{ kube_vip_current_version }} (target: {{ kube_vip_target_version }}) - {{ kube_vip_status }}" - " Calico: {{ calico_current_version }} (target: {{ calico_target_version }}) - {{ calico_status }}" - " MetalLB: {{ metallb_current_version }} (target: {{ metallb_target_version }}) - {{ metallb_status }}" - " Helm: {{ helm_current_version }} (target: {{ helm_target_version }}) - {{ helm_status }}" diff --git a/upgrade/roles/upgrade_k8s/tasks/load_status.yml b/upgrade/roles/upgrade_k8s/tasks/load_status.yml index a637eaa69d..048a52d3e2 100644 --- a/upgrade/roles/upgrade_k8s/tasks/load_status.yml +++ b/upgrade/roles/upgrade_k8s/tasks/load_status.yml @@ -192,6 +192,8 @@ error: addon_upgrade: status: pending + kube_vip: + status: pending calico: status: pending metallb: diff --git a/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml b/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml index 33b51ae6b6..5c966906e8 100644 --- a/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml +++ b/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml @@ -108,6 +108,13 @@ | regex_replace('^helm-v', '') | regex_replace('-.*$', '') }} +- name: Extract kube_vip_target_version from service_k8s.json + ansible.builtin.set_fact: + kube_vip_target_version: >- + {{ service_k8s_config.service_kube_control_plane_first.cluster + | selectattr('package', 'search', 'kube-vip') + | map(attribute='tag') | first | regex_replace('^v', '') }} + # ── Extract addon package names (for manifest file names) ───────────── - name: Extract addon package names from service_k8s.json ansible.builtin.set_fact: diff --git a/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml b/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml index 72acf94fa0..dfe19228b9 100644 --- a/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml +++ b/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml @@ -12,6 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. --- +- name: Verify kube-vip pods are Running on control planes + delegate_to: "{{ kube_vip }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail && + kubectl get pods -n kube-system --no-headers + -o custom-columns=':metadata.name,:status.phase' | + grep kube-vip + executable: /bin/bash + register: kube_vip_pods + changed_when: false + failed_when: false + +- name: Warn if kube-vip pods are not Running + ansible.builtin.debug: + msg: "WARNING: kube-vip issue: {{ kube_vip_pods.stdout | default('no pods found') }}" + when: kube_vip_pods.rc != 0 or kube_vip_pods.stdout | trim | length == 0 + - name: Verify calico-node pods are Running delegate_to: "{{ kube_vip }}" ansible.builtin.command: @@ -56,6 +74,7 @@ ansible.builtin.debug: msg: >- Addon validation complete. + kube-vip pods: {{ 'OK' if kube_vip_pods.rc == 0 else 'WARN' }} Calico pods: {{ 'OK' if calico_pods.rc == 0 else 'WARN' }} MetalLB pods: {{ 'OK' if metallb_pods.rc == 0 else 'WARN' }} LoadBalancer IPs: {{ lb_services.stdout | default('none') }} diff --git a/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml b/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml new file mode 100644 index 0000000000..4a232b70ee --- /dev/null +++ b/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml @@ -0,0 +1,274 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================ +# step_kube_vip_upgrade.yml - Upgrade kube-vip static pod on a control plane +# ============================================================================ +# kube-vip runs as a static pod managed by kubelet, not a DaemonSet. +# Upgrade = replace /etc/kubernetes/manifests/kube-vip.yaml on each CP node. +# +# Chicken-and-egg problem: +# When the old kube-vip pod dies, the VIP goes away. The new pod reads +# admin.conf which points to the VIP for leader election, so it cannot +# start. Fix: temporarily point admin.conf to the node's real IP, let +# kube-vip start and advertise the VIP, then restore admin.conf. +# +# Variables required (set by load_version_vars.yml / detect_addon_versions.yml): +# - kube_vip_target_version (e.g. "1.2.2") +# - kube_vip (VIP address, e.g. "10.40.5.176") +# - node_ip (real IP of the CP node being upgraded) +# - node_hostname (inventory hostname, e.g. "k8scp2") +# - backup_dir (path to store manifest backup) +# +# All tasks that touch the node run with delegate_to: "{{ node_hostname }}" +# to ensure SSH connection from the inventory is used (not connection: local). +# ============================================================================ + +- name: Check if kube-vip manifest exists + delegate_to: "{{ node_hostname }}" + ansible.builtin.stat: + path: /etc/kubernetes/manifests/kube-vip.yaml + register: _kube_vip_manifest_stat + +- name: Skip if manifest not found (not a control plane node) + ansible.builtin.debug: + msg: "No kube-vip manifest found on {{ node_ip }} — skipping" + when: not _kube_vip_manifest_stat.stat.exists + +- name: Upgrade kube-vip on {{ node_ip }} + when: _kube_vip_manifest_stat.stat.exists + block: + - name: Read current kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.slurp: + path: /etc/kubernetes/manifests/kube-vip.yaml + register: _kube_vip_current + + - name: Check if already at target version + ansible.builtin.set_fact: + _kube_vip_needs_upgrade: >- + {{ 'ghcr.io/kube-vip/kube-vip:v' + kube_vip_target_version + not in (_kube_vip_current.content | b64decode) }} + + - name: Skip if already at target version + ansible.builtin.debug: + msg: "kube-vip on {{ node_ip }} already at v{{ kube_vip_target_version }} — skipping" + when: not (_kube_vip_needs_upgrade | bool) + + - name: Perform kube-vip upgrade on {{ node_ip }} + when: _kube_vip_needs_upgrade | bool + block: + - name: Backup current kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + src: /etc/kubernetes/manifests/kube-vip.yaml + dest: "{{ backup_dir }}/kube-vip-{{ node_ip }}-pre-upgrade.yaml" + remote_src: true + mode: "0644" + + # ── Detect network facts (same approach as provision cloud-init) ── + - name: Detect VIP interface from node IP + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + ip -o addr show | + awk -v ip="{{ node_ip }}" '$4 ~ ip {print $2}' + register: _vip_interface + changed_when: false + + # ── Set facts matching provision template variables ────────────── + # admin_netmask_bits comes from network_data.admin_network.netmask_bits + # (same source as provision/roles/k8s_config create_k8s_config_nfs.yml) + - name: Set kube-vip template facts + ansible.builtin.set_fact: + kube_vip_interface: "{{ _vip_interface.stdout | trim }}" + admin_netmask_bits: "{{ hostvars['localhost']['admin_netmask_bits'] }}" + kube_vip_version: "{{ kube_vip_target_version }}" + + # ── Generate new manifest (same structure as provision template) ── + - name: Write upgraded kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + dest: /tmp/kube-vip-upgrade.yaml + mode: "0644" + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: {{ kube_vip_interface }} + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: {{ kube_vip }} + - name: vip_subnet + value: "{{ admin_netmask_bits }}" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v{{ kube_vip_version }} + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + + # ── Chicken-and-egg fix: temporarily use node IP ───────────────── + - name: Temporarily point admin.conf to node IP + delegate_to: "{{ node_hostname }}" + ansible.builtin.replace: + path: /etc/kubernetes/admin.conf + regexp: 'server: https://[^:]+:6443' + replace: "server: https://{{ node_ip }}:6443" + + # ── Replace manifest — kubelet auto-restarts the pod ───────────── + - name: Replace kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + src: /tmp/kube-vip-upgrade.yaml + dest: /etc/kubernetes/manifests/kube-vip.yaml + remote_src: true + mode: "0644" + + - name: Wait for new kube-vip pod to start + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + crictl ps --name kube-vip -o json | + python3 -c "import sys,json; + c=json.load(sys.stdin).get('containers',[]); + print(c[0]['state'] if c else 'missing')" + register: _kube_vip_status + changed_when: false + retries: 30 + delay: 5 + until: _kube_vip_status.stdout | trim == 'CONTAINER_RUNNING' + + - name: Verify VIP is reachable (API server responds via VIP) + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + curl -sk --connect-timeout 3 + https://{{ kube_vip }}:6443/healthz + register: _vip_check + changed_when: false + retries: 12 + delay: 5 + until: _vip_check.stdout | default('') == 'ok' + + # ── Restore admin.conf to VIP ──────────────────────────────────── + - name: Restore admin.conf to VIP address + delegate_to: "{{ node_hostname }}" + ansible.builtin.replace: + path: /etc/kubernetes/admin.conf + regexp: 'server: https://[^:]+:6443' + replace: "server: https://{{ kube_vip }}:6443" + + # Grep specifically for the version banner (not head -1) and retry + # until it reports the target version. This doubles as a stabilization + # gate: a pod that briefly starts then crash-loops logs a transient + # "level=fatal" line, which head -1 could mistake for the result. + - name: Verify kube-vip is running the target version + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + crictl ps --name kube-vip -q | head -1 | + xargs crictl logs 2>&1 | + grep -oE 'kube-vip.io version=v[0-9]+\.[0-9]+\.[0-9]+' | + head -1 + executable: /bin/bash + register: _kube_vip_log + changed_when: false + retries: 12 + delay: 5 + until: >- + ('version=v' + kube_vip_target_version) in (_kube_vip_log.stdout | default('')) + + # ── Verify API is reachable via VIP before moving to next node ─── + - name: Verify API server reachable via VIP + delegate_to: "{{ node_hostname }}" + ansible.builtin.command: + cmd: >- + kubectl --kubeconfig /etc/kubernetes/admin.conf get --raw /healthz + register: _api_health + changed_when: false + retries: 12 + delay: 5 + until: _api_health.rc == 0 + + - name: Display kube-vip upgrade result + ansible.builtin.debug: + msg: >- + kube-vip upgraded on {{ node_ip }}: + {{ _kube_vip_log.stdout | default('version not detected') | trim }} + + - name: Clean up temp manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.file: + path: /tmp/kube-vip-upgrade.yaml + state: absent diff --git a/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml b/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml index 4e85e45f73..9c6b14163d 100644 --- a/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml +++ b/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml @@ -23,6 +23,129 @@ status: in_progress started_at: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" +# ── kube-vip upgrade (ABORT on failure) ──────────────────────────── +# kube-vip MUST be upgraded BEFORE other addons because it provides +# the VIP that all kubectl commands are delegated through. +# Unlike Calico/MetalLB (cluster-wide kubectl apply), kube-vip runs +# as a static pod and must be upgraded per-node on each control plane. +- name: Starting kube-vip upgrade + ansible.builtin.debug: + msg: "Starting kube-vip upgrade from {{ kube_vip_from_version | default('unknown') }} to {{ kube_vip_target_version }}..." + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Mark kube-vip in_progress + ansible.builtin.include_tasks: update_addon_step.yml + vars: + addon_name: kube_vip + addon_status_update: + status: in_progress + current_version: "{{ kube_vip_current_version }}" + from_version: "{{ kube_vip_from_version | default('unknown') }}" + target_version: "{{ kube_vip_target_version }}" + timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Build list of control plane nodes for kube-vip upgrade + ansible.builtin.set_fact: + _cp_node_list: >- + {{ upgrade_status.nodes | dict2items + | selectattr('value.role', 'in', ['control_plane_first', 'control_plane']) + | list }} + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +# This role runs under `hosts: localhost, connection: local`. delegate_to +# only connects over SSH when the target host has `ansible_connection: ssh` +# set in the inventory; otherwise it inherits connection: local and runs on +# the controller. Register each CP node explicitly so step_kube_vip_upgrade.yml +# can delegate to hostnames over SSH. +- name: Register control plane nodes for SSH delegation (kube-vip upgrade) + ansible.builtin.add_host: + name: "{{ item.key }}" + ansible_host: "{{ item.value.ip }}" + ansible_connection: ssh + ansible_user: root + ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + groups: _kube_vip_upgrade_targets + loop: "{{ _cp_node_list }}" + loop_control: + label: "{{ item.key }} ({{ item.value.ip }})" + changed_when: false + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +# ── Detect VIP leader so we upgrade non-leader nodes first ──────── +# The VIP leader holds the virtual IP. Upgrading it last keeps the +# API server reachable through the VIP for as long as possible. +- name: Get kube-vip leader lease for upgrade ordering + ansible.builtin.command: + cmd: >- + kubectl get lease plndr-cp-lock -n kube-system + -o jsonpath='{.spec.holderIdentity}' + delegate_to: "{{ kube_vip }}" + register: _upgrade_vip_leader + changed_when: false + failed_when: false + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Set VIP leader IP for upgrade ordering + ansible.builtin.set_fact: + _vip_leader_ip: "{{ _upgrade_vip_leader.stdout | default('') | trim }}" + when: + - (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + - _upgrade_vip_leader.rc | default(1) == 0 + +- name: Reorder CP list — VIP leader last + ansible.builtin.set_fact: + _cp_node_list: >- + {{ (_cp_node_list | rejectattr('value.ip', 'equalto', _vip_leader_ip) | list) + + (_cp_node_list | selectattr('value.ip', 'equalto', _vip_leader_ip) | list) }} + when: + - (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + - _vip_leader_ip | default('') | length > 0 + +- name: Display kube-vip upgrade order + ansible.builtin.debug: + msg: >- + kube-vip upgrade order: {{ _cp_node_list | map(attribute='value.ip') | list | join(' -> ') }} + (VIP leader {{ _vip_leader_ip | default('unknown') }} upgraded last) + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Upgrade kube-vip + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + block: + - name: Execute kube-vip upgrade on each control plane + ansible.builtin.include_tasks: step_kube_vip_upgrade.yml + vars: + node_ip: "{{ _cp_item.value.ip }}" + node_hostname: "{{ _cp_item.key }}" + loop: "{{ _cp_node_list }}" + loop_control: + loop_var: _cp_item + label: "{{ _cp_item.key }} ({{ _cp_item.value.ip }})" + + - name: Mark kube-vip completed + ansible.builtin.include_tasks: update_addon_step.yml + vars: + addon_name: kube_vip + addon_status_update: + status: completed + current_version: "{{ kube_vip_target_version }}" + from_version: "{{ kube_vip_from_version | default('unknown') }}" + to_version: "{{ kube_vip_target_version }}" + timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + error: + rescue: + - name: Mark kube-vip failed + ansible.builtin.include_tasks: update_addon_step.yml + vars: + addon_name: kube_vip + addon_status_update: + status: failed + timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + + - name: ABORT — kube-vip upgrade failed + ansible.builtin.fail: + msg: "{{ msg_addon_kube_vip_failed }}" + # ── Calico upgrade (ABORT on failure) ────────────────────────────── - name: Starting Calico upgrade ansible.builtin.debug: diff --git a/upgrade/roles/upgrade_k8s/vars/main.yml b/upgrade/roles/upgrade_k8s/vars/main.yml index 2efe076c4f..379586542e 100644 --- a/upgrade/roles/upgrade_k8s/vars/main.yml +++ b/upgrade/roles/upgrade_k8s/vars/main.yml @@ -193,6 +193,10 @@ msg_backup_missing: >- msg_node_upgrade_failed: >- Node {{ current_node_name }} failed at step {{ current_step }}. Error: {{ step_result.stderr | default(step_result.msg | default('unknown')) }} +msg_addon_kube_vip_failed: >- + kube-vip upgrade failed. VIP may be unavailable. + Do NOT proceed to further addon upgrades. + Check /etc/kubernetes/manifests/kube-vip.yaml on each control plane node. msg_addon_calico_failed: >- Calico upgrade failed. Networking may be degraded. Do NOT proceed to worker upgrades. From fa68cab8ec070ec97b2ae604c5681f731e4f2d10 Mon Sep 17 00:00:00 2001 From: Katakam-Rakesh Date: Fri, 7 Aug 2026 10:11:56 +0530 Subject: [PATCH 08/22] Fix Calico pod validation namespace in post_validation - Changed Calico pod check from calico-system namespace to kube-system with label selector k8s-app=calico-node - Aligns with actual Calico deployment namespace in the cluster Signed-off-by: Katakam-Rakesh --- upgrade/roles/upgrade_k8s/tasks/post_validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade/roles/upgrade_k8s/tasks/post_validation.yml b/upgrade/roles/upgrade_k8s/tasks/post_validation.yml index 78016755bb..2fc617b1d4 100644 --- a/upgrade/roles/upgrade_k8s/tasks/post_validation.yml +++ b/upgrade/roles/upgrade_k8s/tasks/post_validation.yml @@ -105,7 +105,7 @@ ansible.builtin.shell: cmd: >- set -o pipefail && - kubectl get pods -n calico-system --no-headers + kubectl get pods -n kube-system -l k8s-app=calico-node --no-headers --field-selector status.phase!=Running,status.phase!=Succeeded 2>/dev/null | head -20 args: From 5d4cdd1f51831412834ab61020bbe4b86edb4bfd Mon Sep 17 00:00:00 2001 From: Katakam-Rakesh Date: Fri, 7 Aug 2026 10:12:25 +0530 Subject: [PATCH 09/22] Increase Pulp migration retries and enforce failure handling - Increase migration retries from 3 to 40 and delay from 10s to 15s to accommodate pg_upgrade (PG 13->16) and Django migrations which can take 5-10 minutes on first startup after upgrade - Remove failed_when: false so migration failures are properly caught - Update migration result message to show rc on failure - Add explanatory comment for startup timeout budget Signed-off-by: Katakam-Rakesh --- .../roles/upgrade_pulp/tasks/upgrade_pulp_container.yml | 7 +++---- upgrade/roles/upgrade_pulp/vars/main.yml | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml index f91084efe7..8450a700e7 100644 --- a/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml +++ b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml @@ -95,14 +95,13 @@ name: "{{ pulp_container_name }}" command: pulpcore-manager migrate --noinput register: pulp_migrate_result - retries: 3 - delay: 10 + retries: 40 + delay: 15 until: pulp_migrate_result.rc == 0 delegate_to: oim delegate_facts: true connection: ssh - failed_when: false - name: Display migration result ansible.builtin.debug: - msg: "Database migration: {{ 'completed' if pulp_migrate_result.rc | default(1) == 0 else 'skipped or not required' }}" + msg: "Database migration: {{ 'completed successfully' if pulp_migrate_result.rc == 0 else 'FAILED (rc=' ~ pulp_migrate_result.rc | string ~ ')' }}" diff --git a/upgrade/roles/upgrade_pulp/vars/main.yml b/upgrade/roles/upgrade_pulp/vars/main.yml index 66fc0629da..ed9fe59822 100644 --- a/upgrade/roles/upgrade_pulp/vars/main.yml +++ b/upgrade/roles/upgrade_pulp/vars/main.yml @@ -46,6 +46,8 @@ pull_image_retries: 5 pull_image_delay: 10 # Pulp startup and health check settings +# Note: First startup after upgrade runs pg_upgrade (PG 13→16) + Django migrations, +# which can take 5-10 minutes. The total wait budget must cover this one-time operation. pulp_startup_retries: 10 pulp_startup_delay: 10 pulp_init_wait: 30 From 0a0ee52eab9afc7ce5fa90bcb1a0c7754c4ddeb2 Mon Sep 17 00:00:00 2001 From: Katakam-Rakesh Date: Fri, 7 Aug 2026 11:41:31 +0530 Subject: [PATCH 10/22] Fix lint issues Signed-off-by: Katakam-Rakesh --- .../tasks/step_kube_vip_upgrade.yml | 278 ++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml diff --git a/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml b/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml new file mode 100644 index 0000000000..33fa0018a1 --- /dev/null +++ b/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml @@ -0,0 +1,278 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================ +# step_kube_vip_upgrade.yml - Upgrade kube-vip static pod on a control plane +# ============================================================================ +# kube-vip runs as a static pod managed by kubelet, not a DaemonSet. +# Upgrade = replace /etc/kubernetes/manifests/kube-vip.yaml on each CP node. +# +# Chicken-and-egg problem: +# When the old kube-vip pod dies, the VIP goes away. The new pod reads +# admin.conf which points to the VIP for leader election, so it cannot +# start. Fix: temporarily point admin.conf to the node's real IP, let +# kube-vip start and advertise the VIP, then restore admin.conf. +# +# Variables required (set by load_version_vars.yml / detect_addon_versions.yml): +# - kube_vip_target_version (e.g. "1.2.2") +# - kube_vip (VIP address, e.g. "10.40.5.176") +# - node_ip (real IP of the CP node being upgraded) +# - node_hostname (inventory hostname, e.g. "k8scp2") +# - backup_dir (path to store manifest backup) +# +# All tasks that touch the node run with delegate_to: "{{ node_hostname }}" +# to ensure SSH connection from the inventory is used (not connection: local). +# ============================================================================ + +- name: Check if kube-vip manifest exists + delegate_to: "{{ node_hostname }}" + ansible.builtin.stat: + path: /etc/kubernetes/manifests/kube-vip.yaml + register: _kube_vip_manifest_stat + +- name: Skip if manifest not found (not a control plane node) + ansible.builtin.debug: + msg: "No kube-vip manifest found on {{ node_ip }} — skipping" + when: not _kube_vip_manifest_stat.stat.exists + +- name: Upgrade kube-vip on {{ node_ip }} + when: _kube_vip_manifest_stat.stat.exists + block: + - name: Read current kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.slurp: + path: /etc/kubernetes/manifests/kube-vip.yaml + register: _kube_vip_current + + - name: Check if already at target version + ansible.builtin.set_fact: + _kube_vip_needs_upgrade: >- + {{ 'ghcr.io/kube-vip/kube-vip:v' + kube_vip_target_version + not in (_kube_vip_current.content | b64decode) }} + + - name: Skip if already at target version + ansible.builtin.debug: + msg: "kube-vip on {{ node_ip }} already at v{{ kube_vip_target_version }} — skipping" + when: not (_kube_vip_needs_upgrade | bool) + + - name: Perform kube-vip upgrade on {{ node_ip }} + when: _kube_vip_needs_upgrade | bool + block: + - name: Backup current kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + src: /etc/kubernetes/manifests/kube-vip.yaml + dest: "{{ backup_dir }}/kube-vip-{{ node_ip }}-pre-upgrade.yaml" + remote_src: true + mode: "0644" + + # ── Detect network facts (same approach as provision cloud-init) ── + - name: Detect VIP interface from node IP + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + ip -o addr show | + awk -v ip="{{ node_ip }}" '$4 ~ ip {print $2}' + executable: /bin/bash + register: _vip_interface + changed_when: false + + # ── Set facts matching provision template variables ────────────── + # admin_netmask_bits comes from network_data.admin_network.netmask_bits + # (same source as provision/roles/k8s_config create_k8s_config_nfs.yml) + - name: Set kube-vip template facts + ansible.builtin.set_fact: + kube_vip_interface: "{{ _vip_interface.stdout | trim }}" + admin_netmask_bits: "{{ hostvars['localhost']['admin_netmask_bits'] }}" + kube_vip_version: "{{ kube_vip_target_version }}" + + # ── Generate new manifest (same structure as provision template) ── + - name: Write upgraded kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + dest: /tmp/kube-vip-upgrade.yaml + mode: "0644" + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: {{ kube_vip_interface }} + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: {{ kube_vip }} + - name: vip_subnet + value: "{{ admin_netmask_bits }}" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v{{ kube_vip_version }} + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + + # ── Chicken-and-egg fix: temporarily use node IP ───────────────── + - name: Temporarily point admin.conf to node IP + delegate_to: "{{ node_hostname }}" + ansible.builtin.replace: + path: /etc/kubernetes/admin.conf + regexp: 'server: https://[^:]+:6443' + replace: "server: https://{{ node_ip }}:6443" + + # ── Replace manifest — kubelet auto-restarts the pod ───────────── + - name: Replace kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + src: /tmp/kube-vip-upgrade.yaml + dest: /etc/kubernetes/manifests/kube-vip.yaml + remote_src: true + mode: "0644" + + - name: Wait for new kube-vip pod to start + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + crictl ps --name kube-vip -o json | + python3 -c "import sys,json; + c=json.load(sys.stdin).get('containers',[]); + print(c[0]['state'] if c else 'missing')" + executable: /bin/bash + register: _kube_vip_status + changed_when: false + retries: 30 + delay: 5 + until: _kube_vip_status.stdout | trim == 'CONTAINER_RUNNING' + + - name: Verify VIP is reachable (API server responds via VIP) # noqa: command-instead-of-module command-instead-of-shell + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + curl -sk --connect-timeout 3 + https://{{ kube_vip }}:6443/healthz + register: _vip_check + changed_when: false + retries: 12 + delay: 5 + until: _vip_check.stdout | default('') == 'ok' + + # ── Restore admin.conf to VIP ──────────────────────────────────── + - name: Restore admin.conf to VIP address + delegate_to: "{{ node_hostname }}" + ansible.builtin.replace: + path: /etc/kubernetes/admin.conf + regexp: 'server: https://[^:]+:6443' + replace: "server: https://{{ kube_vip }}:6443" + + # Grep specifically for the version banner (not head -1) and retry + # until it reports the target version. This doubles as a stabilization + # gate: a pod that briefly starts then crash-loops logs a transient + # "level=fatal" line, which head -1 could mistake for the result. + - name: Verify kube-vip is running the target version + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + crictl ps --name kube-vip -q | head -1 | + xargs crictl logs 2>&1 | + grep -oE 'kube-vip.io version=v[0-9]+\.[0-9]+\.[0-9]+' | + head -1 + executable: /bin/bash + register: _kube_vip_log + changed_when: false + retries: 12 + delay: 5 + until: >- + ('version=v' + kube_vip_target_version) in (_kube_vip_log.stdout | default('')) + + # ── Verify API is reachable via VIP before moving to next node ─── + - name: Verify API server reachable via VIP + delegate_to: "{{ node_hostname }}" + ansible.builtin.command: + cmd: >- + kubectl --kubeconfig /etc/kubernetes/admin.conf get --raw /healthz + register: _api_health + changed_when: false + retries: 12 + delay: 5 + until: _api_health.rc == 0 + + - name: Display kube-vip upgrade result + ansible.builtin.debug: + msg: >- + kube-vip upgraded on {{ node_ip }}: + {{ _kube_vip_log.stdout | default('version not detected') | trim }} + + - name: Clean up temp manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.file: + path: /tmp/kube-vip-upgrade.yaml + state: absent From 368044ab05de3d564760eecd5069e57cea7b6601 Mon Sep 17 00:00:00 2001 From: Vrinda Marwah Date: Fri, 7 Aug 2026 14:05:57 +0530 Subject: [PATCH 11/22] Upgrading calico, metallb and kube-vip to latest versions (#4947) * bump calico,metallb,kube vip to latest versions Signed-off-by: Vrinda_Marwah * Updating catalog for service_k8s image version changes Signed-off-by: Vrinda_Marwah * Revert catalog_rhel_x86_64_with_slurm_only.json sinceit is not required for service_k8s catalog update Signed-off-by: Vrinda_Marwah * Reading kube vip and helm image version dynamically Signed-off-by: Vrinda_Marwah --------- Signed-off-by: Vrinda_Marwah Signed-off-by: Vrinda_Marwah --- examples/catalog/catalog_rhel.json | 33 ++++++++++--------- .../catalog_rhel_with_nfs_provisioner.json | 33 ++++++++++--------- examples/catalog/catalog_rhel_x86_64.json | 33 ++++++++++--------- .../x86_64/rhel/10.0/service_k8s_v1.35.1.json | 29 ++++++++-------- ...ce_kube_control_plane_first_x86_64.yaml.j2 | 8 +++-- ...-service_kube_control_plane_x86_64.yaml.j2 | 8 +++-- .../tasks/create_k8s_config_nfs.yml | 6 ++++ 7 files changed, 82 insertions(+), 68 deletions(-) diff --git a/examples/catalog/catalog_rhel.json b/examples/catalog/catalog_rhel.json index d040e7fe01..7244b3887a 100644 --- a/examples/catalog/catalog_rhel.json +++ b/examples/catalog/catalog_rhel.json @@ -126,6 +126,7 @@ "quay.io/jetstack/cert-manager-cainjector", "quay.io/jetstack/cert-manager-controller", "quay.io/jetstack/cert-manager-webhook", + "quay.io/metallb/controller", "quay.io/metallb/speaker", "quay.io/strimzi/kafka", "quay.io/strimzi/kafka-bridge", @@ -433,7 +434,7 @@ ] }, "calico": { - "Name": "calico-v3.31.4", + "Name": "calico-v3.32.1", "SupportedOS": [ { "Name": "RHEL", @@ -447,7 +448,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" + "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" } ] }, @@ -624,8 +625,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/kube-controllers": { "Name": "docker.io/calico/kube-controllers", @@ -639,8 +640,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/node": { "Name": "docker.io/calico/node", @@ -654,8 +655,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/curlimages/curl": { "Name": "docker.io/curlimages/curl", @@ -1021,8 +1022,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.8.9", - "Version": "v0.8.9" + "Tag": "v1.2.2", + "Version": "v1.2.2" }, "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector": { "Name": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector", @@ -1356,7 +1357,7 @@ ] }, "metallb-native": { - "Name": "metallb-native-v0.15.3", + "Name": "metallb-native-v0.16.1", "SupportedOS": [ { "Name": "RHEL", @@ -1370,7 +1371,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" + "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" } ] }, @@ -1832,8 +1833,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/metallb/speaker": { "Name": "quay.io/metallb/speaker", @@ -1847,8 +1848,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/strimzi/kafka": { "Name": "quay.io/strimzi/kafka", diff --git a/examples/catalog/catalog_rhel_with_nfs_provisioner.json b/examples/catalog/catalog_rhel_with_nfs_provisioner.json index be0cb11ada..da24a6d561 100644 --- a/examples/catalog/catalog_rhel_with_nfs_provisioner.json +++ b/examples/catalog/catalog_rhel_with_nfs_provisioner.json @@ -117,6 +117,7 @@ "quay.io/jetstack/cert-manager-cainjector", "quay.io/jetstack/cert-manager-controller", "quay.io/jetstack/cert-manager-webhook", + "quay.io/metallb/controller", "quay.io/metallb/speaker", "quay.io/strimzi/kafka", "quay.io/strimzi/kafka-bridge", @@ -402,7 +403,7 @@ ] }, "calico": { - "Name": "calico-v3.31.4", + "Name": "calico-v3.32.1", "SupportedOS": [ { "Name": "RHEL", @@ -416,7 +417,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" + "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" } ] }, @@ -593,8 +594,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/kube-controllers": { "Name": "docker.io/calico/kube-controllers", @@ -608,8 +609,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/node": { "Name": "docker.io/calico/node", @@ -623,8 +624,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/curlimages/curl": { "Name": "docker.io/curlimages/curl", @@ -990,8 +991,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.8.9", - "Version": "v0.8.9" + "Tag": "v1.2.2", + "Version": "v1.2.2" }, "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector": { "Name": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector", @@ -1325,7 +1326,7 @@ ] }, "metallb-native": { - "Name": "metallb-native-v0.15.3", + "Name": "metallb-native-v0.16.1", "SupportedOS": [ { "Name": "RHEL", @@ -1339,7 +1340,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" + "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" } ] }, @@ -1705,8 +1706,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/metallb/speaker": { "Name": "quay.io/metallb/speaker", @@ -1720,8 +1721,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/strimzi/kafka": { "Name": "quay.io/strimzi/kafka", diff --git a/examples/catalog/catalog_rhel_x86_64.json b/examples/catalog/catalog_rhel_x86_64.json index 72a354c8c3..2f1733e116 100644 --- a/examples/catalog/catalog_rhel_x86_64.json +++ b/examples/catalog/catalog_rhel_x86_64.json @@ -127,6 +127,7 @@ "quay.io/jetstack/cert-manager-cainjector", "quay.io/jetstack/cert-manager-controller", "quay.io/jetstack/cert-manager-webhook", + "quay.io/metallb/controller", "quay.io/metallb/speaker", "quay.io/strimzi/kafka", "quay.io/strimzi/kafka-bridge", @@ -429,7 +430,7 @@ ] }, "calico": { - "Name": "calico-v3.31.4", + "Name": "calico-v3.32.1", "SupportedOS": [ { "Name": "RHEL", @@ -443,7 +444,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" + "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" } ] }, @@ -610,8 +611,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/kube-controllers": { "Name": "docker.io/calico/kube-controllers", @@ -625,8 +626,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/node": { "Name": "docker.io/calico/node", @@ -640,8 +641,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/curlimages/curl": { "Name": "docker.io/curlimages/curl", @@ -997,8 +998,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.8.9", - "Version": "v0.8.9" + "Tag": "v1.2.2", + "Version": "v1.2.2" }, "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector": { "Name": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector", @@ -1297,7 +1298,7 @@ ] }, "metallb-native": { - "Name": "metallb-native-v0.15.3", + "Name": "metallb-native-v0.16.1", "SupportedOS": [ { "Name": "RHEL", @@ -1311,7 +1312,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" + "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" } ] }, @@ -1722,8 +1723,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/metallb/speaker": { "Name": "quay.io/metallb/speaker", @@ -1737,8 +1738,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/strimzi/kafka": { "Name": "quay.io/strimzi/kafka", diff --git a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json index 1c5bb469f7..981ae4fe32 100644 --- a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json +++ b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json @@ -62,7 +62,7 @@ }, "service_kube_control_plane": { "cluster": [ - { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v1.2.2", "type": "image" }, { "package": "docker.io/alpine/kubectl", "tag": "1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-apiserver", "tag": "v1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-controller-manager", "tag": "v1.35.1", "type": "image" }, @@ -71,10 +71,10 @@ { "package": "registry.k8s.io/coredns/coredns", "tag": "v1.13.1", "type": "image" }, { "package": "registry.k8s.io/pause", "tag": "3.10.1", "type": "image" }, { "package": "registry.k8s.io/etcd", "tag": "3.6.6-0", "type": "image" }, - { "package": "docker.io/calico/cni", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/kube-controllers", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/node", "tag": "v3.31.4", "type": "image" }, - { "package": "quay.io/metallb/speaker", "tag": "v0.15.3", "type": "image" }, + { "package": "docker.io/calico/cni", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/kube-controllers", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/node", "tag": "v3.32.1", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.16.1", "type": "image" }, { "package": "kubectl-1.35.1", "type": "rpm", "repo_name": "kubernetes-v1-35"}, { "package": "prettytable==3.14.0", "type": "pip_module" }, { "package": "python3-3.12.9", "type": "rpm", "repo_name": "baseos" }, @@ -86,7 +86,7 @@ }, "service_kube_control_plane_first": { "cluster": [ - { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v1.2.2", "type": "image" }, { "package": "registry.k8s.io/kube-apiserver", "tag": "v1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-controller-manager", "tag": "v1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-scheduler", "tag": "v1.35.1", "type": "image" }, @@ -95,12 +95,13 @@ { "package": "docker.io/alpine/kubectl", "tag": "1.35.1", "type": "image" }, { "package": "registry.k8s.io/pause", "tag": "3.10.1", "type": "image" }, { "package": "registry.k8s.io/etcd", "tag": "3.6.6-0", "type": "image" }, - { "package": "docker.io/calico/cni", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/kube-controllers", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/node", "tag": "v3.31.4", "type": "image" }, - { "package": "quay.io/metallb/speaker", "tag": "v0.15.3", "type": "image" }, - { "package": "calico-v3.31.4","type": "manifest", "url": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" }, - { "package": "metallb-native-v0.15.3", "type": "manifest", "url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" }, + { "package": "docker.io/calico/cni", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/kube-controllers", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/node", "tag": "v3.32.1", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.16.1", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.16.1", "type": "image" }, + { "package": "calico-v3.32.1","type": "manifest", "url": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" }, + { "package": "metallb-native-v0.16.1", "type": "manifest", "url": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" }, { "package": "helm-v3.20.1-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.20.1-linux-amd64.tar.gz" }, { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, { "package": "kubectl-1.35.1", "type": "rpm", "repo_name": "kubernetes-v1-35"}, @@ -115,8 +116,8 @@ "service_kube_node": { "cluster": [ { "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" }, - { "package": "quay.io/metallb/speaker", "tag": "v0.15.3", "type": "image" }, - { "package": "quay.io/metallb/controller", "tag": "v0.15.3", "type": "image" } + { "package": "quay.io/metallb/speaker", "tag": "v0.16.1", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.16.1", "type": "image" } ] } } diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 index f1cb099601..045052a05c 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 @@ -237,11 +237,13 @@ value: "3" - name: vip_retryperiod value: "1" - - name: vip_address + - name: address value: {{ kube_vip }} + - name: vip_subnet + value: "{{ admin_netmask_bits }}" - name: prometheus_server value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.8.9 + image: {{ kube_vip_image }} imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -306,7 +308,7 @@ content: | #!/bin/bash set -e - HELM_VERSION="v3.20.1" + HELM_VERSION="v{{ helm_version }}" ARCH="amd64" cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}-helm-$HELM_VERSION/helm /usr/local/bin/helm chmod +x /usr/local/bin/helm diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 index 4849f8bd88..22b3680373 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 @@ -207,11 +207,13 @@ value: "3" - name: vip_retryperiod value: "1" - - name: vip_address + - name: address value: {{ kube_vip }} + - name: vip_subnet + value: "{{ admin_netmask_bits }}" - name: prometheus_server value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.8.9 + image: {{ kube_vip_image }} imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -240,7 +242,7 @@ content: | #!/bin/bash set -e - HELM_VERSION="v3.20.1" + HELM_VERSION="v{{ helm_version }}" ARCH="amd64" cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}-helm-$HELM_VERSION/helm /usr/local/bin/helm chmod +x /usr/local/bin/helm diff --git a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml index f95df3668a..648fd69409 100644 --- a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml @@ -239,6 +239,12 @@ helm_package: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'tarball') | selectattr('package', 'search', 'helm') | map(attribute='package') | join }}" # noqa: yaml[line-length] nfs_subdir_external_provisioner_pkg: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'tarball') | selectattr('package', 'search', 'nfs-subdir-external-provisioner') | map(attribute='package') | join }}" # noqa: yaml[line-length] whereabouts_pkg: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'git') | selectattr('package', 'search', 'whereabouts') | map(attribute='package') | join }}" # noqa: yaml[line-length] + kube_vip_package: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'image') | selectattr('package', 'search', 'kube-vip') | map(attribute='package') | join }}" # noqa: yaml[line-length] + kube_vip_tag: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'image') | selectattr('package', 'search', 'kube-vip') | map(attribute='tag') | join }}" # noqa: yaml[line-length] + +- name: Compose kube-vip image reference from package and tag + ansible.builtin.set_fact: + kube_vip_image: "{{ kube_vip_package }}:{{ kube_vip_tag }}" - name: Detect CA trust anchors directory ansible.builtin.set_fact: From 9bedb68240551de150d2cd37811b97e945087479 Mon Sep 17 00:00:00 2001 From: Abhishek S A Date: Fri, 7 Aug 2026 17:01:14 +0530 Subject: [PATCH 12/22] Revert "Migrate to wolfi auth, buildstream and core containers (#4940)" (#4954) This reverts commit cb6dfeeefb586a272b0feae9ad18ec3721428470. --- .../library/module_utils/local_repo/config.py | 14 +- .../module_utils/local_repo/download_rpm.py | 277 ++++++------------ .../tasks/create_metadata.yml | 14 - .../tasks/process_rpm_repo.yml | 7 - .../roles/validation/tasks/prerequisites.yml | 7 - .../deploy_containers/auth/vars/main.yml | 2 +- .../build_stream/vars/main.yml | 2 +- .../deploy_containers/openchami/vars/main.yml | 2 +- .../pulp/tasks/create_pulp_config_https.yml | 78 +---- .../deploy_containers/pulp/vars/main.yml | 1 + .../tasks/validate_network_spec.yml | 20 +- .../tasks/create_k8s_config_nfs.yml | 23 +- provision/roles/k8s_config/vars/main.yml | 1 + 13 files changed, 99 insertions(+), 349 deletions(-) diff --git a/common/library/module_utils/local_repo/config.py b/common/library/module_utils/local_repo/config.py index 5ae64cff51..4bce5e64ca 100644 --- a/common/library/module_utils/local_repo/config.py +++ b/common/library/module_utils/local_repo/config.py @@ -108,7 +108,6 @@ "x86_64": ["dnf", "info", "--quiet"], "aarch64": ["dnf", "info", "--quiet", "--forcearch=aarch64"] } -PULP_RPM_PACKAGES_API = "/pulp/api/v3/content/rpm/packages/" # ---------------------------- # Cleanup File Types @@ -157,18 +156,7 @@ ISO_TIMEOUT_MIN = 45 # minutes TASK_POLL_INTERVAL = 10 # seconds FILE_URI = "/pulp/api/v3/content/file/files/" - -import os - -def _get_ca_cert_path(): - """Return CA cert path based on OS. Fedora/RHEL vs Wolfi/Debian.""" - rhel_path = "/etc/pki/ca-trust/source/anchors/pulp_webserver.crt" - wolfi_path = "/usr/local/share/ca-certificates/pulp_webserver.crt" - if os.path.exists("/etc/pki/ca-trust/source/anchors"): - return rhel_path - return wolfi_path - -PULP_SSL_CA_CERT = _get_ca_cert_path() +PULP_SSL_CA_CERT = "/etc/pki/ca-trust/source/anchors/pulp_webserver.crt" # ---------------------------- # Used by download_image.py # ---------------------------- diff --git a/common/library/module_utils/local_repo/download_rpm.py b/common/library/module_utils/local_repo/download_rpm.py index 1af2c4ea9d..e6fe9603a2 100644 --- a/common/library/module_utils/local_repo/download_rpm.py +++ b/common/library/module_utils/local_repo/download_rpm.py @@ -18,91 +18,17 @@ import subprocess import os import glob -import json import shutil from pathlib import Path from ansible.module_utils.local_repo.config import ( DNF_COMMANDS, - DNF_INFO_COMMANDS, - PULP_RPM_PACKAGES_API + DNF_INFO_COMMANDS ) from multiprocessing import Lock from ansible.module_utils.local_repo.parse_and_download import write_status_to_file, _prefix_repo_name_with_arch file_lock = Lock() - -def _is_dnf_available(): - """Check if dnf binary is available on the system.""" - return shutil.which('dnf') is not None - - -def _pulp_cmd(cmd_string, logger=None): - """Run a pulp CLI command and return parsed JSON output, or None on failure.""" - result = subprocess.run(cmd_string, shell=True, check=False, capture_output=True, text=True) - if result.returncode != 0: - if logger: - logger.debug(f"Pulp command failed: {cmd_string}, stderr: {result.stderr}") - return None - try: - return json.loads(result.stdout) - except (json.JSONDecodeError, ValueError): - return None - - -def _pulp_get_repo_version(repo_name, logger): - """Get latest_version_href for a Pulp RPM repository.""" - data = _pulp_cmd(f"pulp rpm repository show --name {repo_name}", logger) - if isinstance(data, dict): - return data.get("latest_version_href", "") - return "" - - -def _pulp_find_package(pkg_name, repo_name, logger): - """Find RPM package in a Pulp repository. Returns package info dict or None.""" - version_href = _pulp_get_repo_version(repo_name, logger) - if not version_href: - return None - api_url = f"{PULP_RPM_PACKAGES_API}?name={pkg_name}&repository_version={version_href}&limit=1" - data = _pulp_cmd(f"pulp show --href '{api_url}'", logger) - if isinstance(data, dict) and data.get("count", 0) > 0: - return data.get("results", [None])[0] - if isinstance(data, list) and len(data) > 0: - return data[0] - return None - - -def _pulp_validate_package(pkg_name, repo_name, logger): - """Check if package exists in Pulp repo (replaces dnf info).""" - return _pulp_find_package(pkg_name, repo_name, logger) is not None - - -def _pulp_download_rpm(pkg_name, repo_name, rpm_directory, logger): - """Download a single RPM from Pulp distribution (replaces dnf download).""" - pkg_info = _pulp_find_package(pkg_name, repo_name, logger) - if not pkg_info: - logger.error(f"Package '{pkg_name}' not found in Pulp repo '{repo_name}'") - return False - location_href = pkg_info.get("location_href", "") - if not location_href: - logger.error(f"No location_href for package '{pkg_name}'") - return False - dist_data = _pulp_cmd(f"pulp rpm distribution show --name {repo_name}", logger) - if not isinstance(dist_data, dict) or not dist_data.get("base_url"): - logger.error(f"Could not get distribution URL for '{repo_name}'") - return False - base_url = dist_data["base_url"].rstrip("/") - download_url = f"{base_url}/{location_href}" - logger.info(f"Downloading '{pkg_name}' from {download_url}") - dl_result = subprocess.run( - ["wget", "-c", "-q", "--no-check-certificate", "-P", rpm_directory, download_url], - check=False, capture_output=True, text=True - ) - if dl_result.returncode == 0 or _check_rpm_downloaded(rpm_directory, pkg_name): - return True - logger.error(f"wget failed for '{pkg_name}': {dl_result.stderr}") - return False - def _check_rpm_downloaded(rpm_directory, pkg_name): """ Check if an RPM file for the given package exists in the directory. @@ -174,107 +100,86 @@ def process_rpm(package, repo_store_path, status_file_path, cluster_os_type, arch_key = "x86_64" if arc.lower() in ("x86_64") else "aarch64" - downloaded = [] - failed = [] + # First try to download all at once + dnf_download_command = ( + DNF_COMMANDS[arch_key] + + [f"--destdir={rpm_directory}"] + + rpm_list + ) - if _is_dnf_available(): - # First try to download all at once - dnf_download_command = ( - DNF_COMMANDS[arch_key] - + [f"--destdir={rpm_directory}"] - + rpm_list - ) + result = subprocess.run( + dnf_download_command, + check=False, + capture_output=True, + text=True + ) + logger.info(f"Return code {result.returncode}") + logger.debug(f"STDOUT:\n{result.stdout}") + logger.debug(f"STDERR:\n{result.stderr}") - result = subprocess.run( - dnf_download_command, - check=False, - capture_output=True, - text=True - ) - logger.info(f"Return code {result.returncode}") - logger.debug(f"STDOUT:\n{result.stdout}") - logger.debug(f"STDERR:\n{result.stderr}") + stdout_lines = result.stdout.splitlines() + stderr_lines = result.stderr.splitlines() + + downloaded = [] + failed = [] - stdout_lines = result.stdout.splitlines() - stderr_lines = result.stderr.splitlines() + # Detect successes/failures from combined run + # Use filesystem check instead of parsing output (works with both DNF4 and DNF5) + for pkg in rpm_list: + # Get repo_name for this specific RPM from mapping + pkg_repo_name = repo_mapping.get(pkg, "") - # Detect successes/failures from combined run - # Use filesystem check instead of parsing output (works with both DNF4 and DNF5) - for pkg in rpm_list: + # Check if package was downloaded by looking for the RPM file + pkg_downloaded = _check_rpm_downloaded(rpm_directory, pkg) + + # Also check for "No match for argument" or "No package" errors in stderr + pkg_not_found = False + for line in stderr_lines: + if pkg in line and ("No match for argument" in line or + "No package" in line or + "not found" in line.lower()): + pkg_not_found = True + break + + if pkg_downloaded and not pkg_not_found: + downloaded.append(pkg) + write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) + logger.info(f"Package '{pkg}' downloaded successfully.") + else: + failed.append(pkg) + if pkg_not_found: + logger.warning(f"Package '{pkg}' not found in configured repositories") + + # Retry failed ones individually + if failed: + logger.warning(f"Retrying failed packages individually: {failed}") + for pkg in failed[:]: + cmd = DNF_COMMANDS[arch_key] + [f'--destdir={rpm_directory}', pkg] + retry_res = subprocess.run(cmd, check=False, capture_output=True, text=True) # Get repo_name for this specific RPM from mapping pkg_repo_name = repo_mapping.get(pkg, "") - # Check if package was downloaded by looking for the RPM file - pkg_downloaded = _check_rpm_downloaded(rpm_directory, pkg) - - # Also check for "No match for argument" or "No package" errors in stderr - pkg_not_found = False - for line in stderr_lines: - if pkg in line and ("No match for argument" in line or - "No package" in line or - "not found" in line.lower()): - pkg_not_found = True - break - - if pkg_downloaded and not pkg_not_found: + # Check for package not found errors + retry_stderr = retry_res.stderr.lower() + pkg_invalid = any(err in retry_stderr for err in [ + "no match for argument", + "no package", + "not found", + "unable to find a match" + ]) + + # Check if RPM file exists after retry (works with both DNF4 and DNF5) + if retry_res.returncode == 0 and _check_rpm_downloaded(rpm_directory, pkg): downloaded.append(pkg) + failed.remove(pkg) write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) - logger.info(f"Package '{pkg}' downloaded successfully.") + logger.info(f"Package '{pkg}' downloaded successfully on retry.") else: - failed.append(pkg) - if pkg_not_found: - logger.warning(f"Package '{pkg}' not found in configured repositories") - - # Retry failed ones individually - if failed: - logger.warning(f"Retrying failed packages individually: {failed}") - for pkg in failed[:]: - cmd = DNF_COMMANDS[arch_key] + [f'--destdir={rpm_directory}', pkg] - retry_res = subprocess.run(cmd, check=False, capture_output=True, text=True) - # Get repo_name for this specific RPM from mapping - pkg_repo_name = repo_mapping.get(pkg, "") - - # Check for package not found errors - retry_stderr = retry_res.stderr.lower() - pkg_invalid = any(err in retry_stderr for err in [ - "no match for argument", - "no package", - "not found", - "unable to find a match" - ]) - - # Check if RPM file exists after retry (works with both DNF4 and DNF5) - if retry_res.returncode == 0 and _check_rpm_downloaded(rpm_directory, pkg): - downloaded.append(pkg) - failed.remove(pkg) - write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) - logger.info(f"Package '{pkg}' downloaded successfully on retry.") - else: - write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, pkg_repo_name) - if pkg_invalid: - logger.error(f"Package '{pkg}' does not exist in configured repositories.") - else: - logger.error(f"Package '{pkg}' still failed after retry.") - - else: - # Pulp-based download (dnf not available, e.g. Wolfi container) - logger.info("dnf not available, using Pulp CLI for RPM download") - for pkg in rpm_list: - pkg_repo_name = repo_mapping.get(pkg, "") - if pkg_repo_name: - prefixed_repo_name = _prefix_repo_name_with_arch(pkg_repo_name, status_file_path, logger) - if _pulp_download_rpm(pkg, prefixed_repo_name, rpm_directory, logger): - downloaded.append(pkg) - write_status_to_file(status_file_path, pkg, "rpm", "Success", logger, file_lock, pkg_repo_name) - logger.info(f"Package '{pkg}' downloaded successfully via Pulp.") + write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, pkg_repo_name) + if pkg_invalid: + logger.error(f"Package '{pkg}' does not exist in configured repositories.") else: - failed.append(pkg) - write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, pkg_repo_name) - logger.error(f"Package '{pkg}' download failed via Pulp.") - else: - failed.append(pkg) - logger.warning(f"No repo_name mapping for package '{pkg}', cannot download via Pulp") - write_status_to_file(status_file_path, pkg, "rpm", "Failed", logger, file_lock, "") + logger.error(f"Package '{pkg}' still failed after retry.") # Determine final status if not failed: @@ -286,47 +191,35 @@ def process_rpm(package, repo_store_path, status_file_path, cluster_os_type, else: logger.info("RPM won't be downloaded when repo_config is partial or never") + logger.info("Validating package availability using dnf info...") arch_key = "x86_64" if arc.lower() in ("x86_64") else "aarch64" valid_packages = [] invalid_packages = [] - use_dnf = _is_dnf_available() - - if use_dnf: - logger.info("Validating package availability using dnf info...") - else: - logger.info("dnf not available, validating package availability using Pulp CLI...") for pkg in package["rpm_list"]: # Get repo_name for this specific RPM from mapping pkg_repo_name = repo_mapping.get(pkg, "") - if not pkg_repo_name: - # Skip validation if no specific repo is defined - logger.warning(f"No repo_name defined for package '{pkg}', skipping validation") - continue - - # Apply architecture prefixing if needed - prefixed_repo_name = _prefix_repo_name_with_arch(pkg_repo_name, status_file_path, logger) - - if use_dnf: - # Validate package using dnf info with specific repo + # Validate package using dnf info with specific repo only + if pkg_repo_name: + # Apply architecture prefixing if needed + prefixed_repo_name = _prefix_repo_name_with_arch(pkg_repo_name, status_file_path, logger) dnf_info_command = DNF_INFO_COMMANDS[arch_key] + [ f"--repo={prefixed_repo_name}", # Search specific repo from JSON pkg ] - result = subprocess.run( - dnf_info_command, - check=False, - capture_output=True, - text=True - ) - pkg_exists = result.returncode == 0 else: - # Validate package using Pulp CLI - pkg_exists = _pulp_validate_package(pkg, prefixed_repo_name, logger) - - if pkg_exists: + # Skip validation if no specific repo is defined + logger.warning(f"No repo_name defined for package '{pkg}', skipping validation") + continue + result = subprocess.run( + dnf_info_command, + check=False, + capture_output=True, + text=True + ) + if result.returncode == 0: # Package exists and is available valid_packages.append(pkg) write_status_to_file( diff --git a/local_repo/roles/parse_and_download/tasks/create_metadata.yml b/local_repo/roles/parse_and_download/tasks/create_metadata.yml index 54eaea6cc9..c994f225dc 100644 --- a/local_repo/roles/parse_and_download/tasks/create_metadata.yml +++ b/local_repo/roles/parse_and_download/tasks/create_metadata.yml @@ -15,13 +15,6 @@ - name: Clean DNF cache ansible.builtin.command: dnf clean all changed_when: false - when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] - -- name: Clean apk cache - ansible.builtin.command: apk cache clean - changed_when: false - failed_when: false - when: ansible_facts['pkg_mgr'] | default('') == 'apk' - name: Remove pulp.repo if exists ansible.builtin.file: @@ -32,13 +25,6 @@ ansible.builtin.dnf: update_cache: true changed_when: false - when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] - -- name: Regenerate apk package index - community.general.apk: - update_cache: true - changed_when: false - when: ansible_facts['pkg_mgr'] | default('') == 'apk' - name: Check for data folder existence ansible.builtin.stat: diff --git a/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml b/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml index 78de31f50e..166e6f8784 100644 --- a/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml +++ b/local_repo/roles/parse_and_download/tasks/process_rpm_repo.yml @@ -43,10 +43,3 @@ ansible.builtin.dnf: update_cache: true changed_when: false - when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] - -- name: Regenerate apk package index - community.general.apk: - update_cache: true - changed_when: false - when: ansible_facts['pkg_mgr'] | default('') == 'apk' diff --git a/local_repo/roles/validation/tasks/prerequisites.yml b/local_repo/roles/validation/tasks/prerequisites.yml index de6c5ffac8..a969bce84e 100644 --- a/local_repo/roles/validation/tasks/prerequisites.yml +++ b/local_repo/roles/validation/tasks/prerequisites.yml @@ -25,10 +25,3 @@ loop: - "{{ ['max_parallel_downloads', dnf_max_parallel] }}" - "{{ ['strict', 'False'] }}" - when: ansible_facts['pkg_mgr'] | default('') in ['dnf', 'dnf4', 'dnf5', 'yum'] - -- name: Refresh apk package index - community.general.apk: - update_cache: true - changed_when: false - when: ansible_facts['pkg_mgr'] | default('') == 'apk' diff --git a/prepare_oim/roles/deploy_containers/auth/vars/main.yml b/prepare_oim/roles/deploy_containers/auth/vars/main.yml index 55bf2e5ece..fcf2defe4f 100644 --- a/prepare_oim/roles/deploy_containers/auth/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/auth/vars/main.yml @@ -23,7 +23,7 @@ openldap_ports: - 636 wait_time: 10 auth_service_image_name: omnia_auth -auth_service_image_tag: "1.2" +auth_service_image_tag: "1.1" auth_service_registry: "docker.io/dellhpcomniaaisolution" auth_service_container_name: omnia_auth auth_service_image_pull_fail_msg: diff --git a/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml b/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml index baedf69322..896b876c6e 100644 --- a/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/build_stream/vars/main.yml @@ -33,7 +33,7 @@ admin_ip: "{{ hostvars['localhost']['admin_nic_ip'] | default('localhost') }}" # Build Stream Image (Docker Hub) build_stream_dockerhub_registry: "docker.io/dellhpcomniaaisolution" build_stream_image_name: "{{ build_stream_dockerhub_registry }}/omnia_build_stream" -build_stream_image_tag: "1.2" +build_stream_image_tag: "1.1" # Ports & Logs build_stream_port: "{{ hostvars['localhost']['build_stream_port'] }}" diff --git a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml index 8ab0d92a33..dfdf99a745 100644 --- a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml @@ -45,7 +45,7 @@ openchami_bss_tag: "v1.32.2" openchami_cloud_init_tag: "v1.3.0" openchami_coresmd_tag: "v0.4.3" # Third-party image tags for OpenCHAMI -minio_release_tag: "RELEASE.2026-06-18T00-00-00Z" +minio_release_tag: "RELEASE.2026-04-17T00-00-00Z" postgres_tag: "11.5-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" diff --git a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml index b1a7468652..a548bf6ed6 100644 --- a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml +++ b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml @@ -108,92 +108,16 @@ state: file loop: "{{ cert_items.values() }}" - - name: Detect CA trust anchors directory - ansible.builtin.set_fact: - ca_trust_anchors_dir: >- - {{ '/etc/pki/ca-trust/source/anchors' - if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] - else '/usr/local/share/ca-certificates' }} - when: ca_trust_anchors_dir is not defined - - - name: Set anchors path - ansible.builtin.set_fact: - ca_trust_path: "{{ ca_trust_anchors_dir }}/pulp_webserver.crt" - when: ca_trust_path is not defined - - - name: Ensure CA trust directory exists - ansible.builtin.file: - path: "{{ ca_trust_anchors_dir }}" - state: directory - mode: '0755' - - - name: Copy Pulp crt to container trust - ansible.builtin.copy: - src: "{{ pulp_cert_src }}" - dest: "{{ ca_trust_path }}" - mode: "{{ logs_dir_permission }}" - - - name: Change group ownership of SSL certificate and key - ansible.builtin.file: - path: "{{ item }}" - group: pulp - state: file - loop: "{{ cert_items.values() }}" - - # ── CA Trust: OS-aware cert installation ──────────────────────────────── - # Fedora/RHEL: use update-ca-trust (native command exists) - # Wolfi/Other: directly append cert to system CA bundle (no command exists) - - - name: Detect CA trust anchors directory - ansible.builtin.set_fact: - ca_trust_anchors_dir: >- - {{ '/etc/pki/ca-trust/source/anchors' - if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] - else '/usr/local/share/ca-certificates' }} - when: ca_trust_anchors_dir is not defined - - - name: Set anchors path - ansible.builtin.set_fact: - ca_trust_path: "{{ ca_trust_anchors_dir }}/pulp_webserver.crt" - when: ca_trust_path is not defined - - - name: Ensure CA trust directory exists - ansible.builtin.file: - path: "{{ ca_trust_anchors_dir }}" - state: directory - mode: '0755' - - name: Copy Pulp crt to container trust ansible.builtin.copy: src: "{{ pulp_cert_src }}" dest: "{{ ca_trust_path }}" mode: "{{ logs_dir_permission }}" - - name: Add Pulp Certificate to TrustStore (RHEL/Fedora) + - name: Add Pulp Certificate to TrustStore ansible.builtin.command: cmd: update-ca-trust extract changed_when: false - when: ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] - - - name: Add Pulp Certificate to TrustStore (Wolfi/Other) - ansible.builtin.shell: | - cat "{{ ca_trust_path }}" >> /etc/ssl/certs/ca-certificates.crt - changed_when: false - when: ansible_facts['os_family'] | default('') not in ['RedHat', 'Fedora'] - - # ── End CA Trust ──────────────────────────────────────────────────────── - - - name: Create a track file - ansible.builtin.file: - path: "{{ track_file_path }}" - state: touch - mode: "{{ logs_dir_permission }}" - - - name: Record current timestamp in track file - ansible.builtin.copy: - dest: "{{ track_file_path }}" - content: "Timestamp: {{ ansible_date_time.iso8601 }}" - mode: "{{ logs_dir_permission }}" - name: Create a track file ansible.builtin.file: diff --git a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml index bd2cf25d91..95b7afcdbe 100644 --- a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml @@ -138,6 +138,7 @@ generate_cert_cmd: > -addext {{ cert_san }} pulp_cert_src: "/opt/omnia/pulp/settings/certs/pulp_webserver.crt" +ca_trust_path: "/etc/pki/ca-trust/source/anchors/" # Usage: reload_pulp_nginx.yml nginx_reload_cmd: "nginx -s reload" diff --git a/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml b/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml index 3779f7767f..622f633d61 100644 --- a/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml +++ b/prepare_oim/roles/prepare_oim_validation/tasks/validate_network_spec.yml @@ -79,22 +79,14 @@ success_msg: "{{ admin_nic_ip_success_msg }}" when: fetch_oim_hostname.stdout in oim_hostname -# - name: Compute network address using ipcalc -# ansible.builtin.command: "/usr/bin/ipcalc -n {{ admin_nic_ip }}/{{ admin_netmask_bits }}" -# register: network_address_output -# changed_when: false - -# - name: Extract network address -# ansible.builtin.set_fact: -# admin_net_addr: "{{ network_address_output.stdout.split('=')[1] }}" - -- name: Compute network address - ansible.builtin.set_fact: - network_address: "{{ (admin_nic_ip + '/' + (admin_netmask_bits | string)) | ansible.utils.ipaddr('network') }}" +- name: Compute network address using ipcalc + ansible.builtin.command: "/usr/bin/ipcalc -n {{ admin_nic_ip }}/{{ admin_netmask_bits }}" + register: network_address_output + changed_when: false -- name: Set admin_net_addr from computed network address +- name: Extract network address ansible.builtin.set_fact: - admin_net_addr: "{{ network_address }}" + admin_net_addr: "{{ network_address_output.stdout.split('=')[1] }}" - name: Initialize network_interface_type ansible.builtin.set_fact: diff --git a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml index 648fd69409..bee7b47201 100644 --- a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml @@ -246,19 +246,6 @@ ansible.builtin.set_fact: kube_vip_image: "{{ kube_vip_package }}:{{ kube_vip_tag }}" -- name: Detect CA trust anchors directory - ansible.builtin.set_fact: - ca_trust_anchors_dir: >- - {{ '/etc/pki/ca-trust/source/anchors' - if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] - else '/usr/local/share/ca-certificates' }} - when: ca_trust_anchors_dir is not defined - -- name: Set anchors path - ansible.builtin.set_fact: - anchors_path: "{{ ca_trust_anchors_dir }}/pulp_webserver.crt" - when: anchors_path is not defined - - name: Copy pulp webserver certificate to target host ansible.builtin.copy: src: "{{ pulp_webserver_cert_path }}" @@ -266,16 +253,8 @@ mode: "{{ file_mode }}" become: true -- name: Detect CA trust update command - ansible.builtin.set_fact: - ca_trust_cmd: >- - {{ 'update-ca-trust' - if ansible_facts['os_family'] | default('') in ['RedHat', 'Fedora'] - else 'update-ca-certificates' }} - when: ca_trust_cmd is not defined - - name: Update CA trust on target host - ansible.builtin.command: "{{ ca_trust_cmd }}" + ansible.builtin.command: update-ca-trust register: update_ca changed_when: false diff --git a/provision/roles/k8s_config/vars/main.yml b/provision/roles/k8s_config/vars/main.yml index baa9583d29..c949564d70 100644 --- a/provision/roles/k8s_config/vars/main.yml +++ b/provision/roles/k8s_config/vars/main.yml @@ -27,6 +27,7 @@ whereabouts_git_url: "{{ offline_git_path }}/{{ whereabouts_pkg }}/{{ whereabout file_mode: "0644" ha_config_file: "{{ input_project_dir }}/high_availability_config.yml" pulp_webserver_cert_path: "/opt/omnia/pulp/settings/certs/pulp_webserver.crt" +anchors_path: "/etc/pki/ca-trust/source/anchors/pulp_webserver.crt" # Usage: create_node_dir.yml nodes_yaml: "{{ hostvars['localhost']['oim_shared_path'] }}/omnia/openchami/workdir/nodes/nodes.yaml" From 05ba05018dd96413afadb7c3a0f8d53eac026ca2 Mon Sep 17 00:00:00 2001 From: Abhishek S A Date: Fri, 7 Aug 2026 18:19:24 +0530 Subject: [PATCH 13/22] Fix DNF repo priority assignment for user repos in OS image builds (#4952) * repo priority fixes Signed-off-by: Abhishek S A * Update fetch_pulp_repos.yml Signed-off-by: Abhishek S A * update repo priority flow Signed-off-by: Abhishek S A * Update main.yml Signed-off-by: Abhishek S A --------- Signed-off-by: Abhishek S A --- .../fetch_packages/tasks/fetch_pulp_repos.yml | 38 ++++++++++++++++++- .../templates/images/rhel-base-config.yaml.j2 | 3 ++ .../images/rhel-compute-config.yaml.j2 | 3 ++ .../fetch_packages/tasks/fetch_pulp_repos.yml | 38 ++++++++++++++++++- .../templates/images/rhel-base-config.yaml.j2 | 3 ++ .../images/rhel-compute-config.yaml.j2 | 3 ++ .../deploy_containers/openchami/vars/main.yml | 2 +- 7 files changed, 85 insertions(+), 5 deletions(-) diff --git a/build_image_aarch64/roles/fetch_packages/tasks/fetch_pulp_repos.yml b/build_image_aarch64/roles/fetch_packages/tasks/fetch_pulp_repos.yml index b81fdfd494..9a47abe9de 100644 --- a/build_image_aarch64/roles/fetch_packages/tasks/fetch_pulp_repos.yml +++ b/build_image_aarch64/roles/fetch_packages/tasks/fetch_pulp_repos.yml @@ -28,10 +28,44 @@ | selectattr('name', 'match', '^aarch64') | list }} - - name: Build rhel_repos list from pulp_aarch_64_distributions + # -- Repo priority classification ---------------------------------- + # Load local_repo_config.yml to identify user vs RHEL vs omnia repos. + # Priority order: user (10) > RHEL (20) > omnia (99). + # This ensures DNF prefers user-supplied package versions over + # identically-named packages in omnia repos (e.g. slurm 25 from + # user repo beats slurm 26 from EPEL). + - name: Load local_repo_config for repo classification + ansible.builtin.include_vars: + file: "{{ input_project_dir }}/local_repo_config.yml" + name: _local_repo_cfg + failed_when: false + + - name: Build user repo name list + ansible.builtin.set_fact: + _user_repo_names: >- + {{ (_local_repo_cfg.user_repo_url_aarch64 | default([]) | map(attribute='name') | list) }} + _rhel_repo_names: >- + {{ (_local_repo_cfg.rhel_os_url_aarch64 | default([]) | map(attribute='name') | list) + + ['baseos', 'appstream', 'codeready-builder'] }} + _omnia_repo_names: >- + {{ (_local_repo_cfg.omnia_repo_url_rhel_aarch64 | default([]) | map(attribute='name') | list) }} + + - name: Build rhel_repos list with priority from pulp_aarch_64_distributions ansible.builtin.set_fact: rhel_aarch64_repos: >- - {{ pulp_aarch_64_distributions | map('combine', {'gpg': ''}) | list }} + {% set repos = [] -%} + {% for dist in pulp_aarch_64_distributions -%} + {% set repo_name = dist.name.split('_')[4:] | join('_') -%} + {% if repo_name in _user_repo_names -%} + {% set _priority = '10' -%} + {% elif repo_name in _rhel_repo_names -%} + {% set _priority = '20' -%} + {% else -%} + {% set _priority = '99' -%} + {% endif -%} + {% set _ = repos.append(dist | combine({'gpg': '', 'priority': _priority})) -%} + {% endfor -%} + {{ repos }} - name: Debug rhel_aarch64_repos ansible.builtin.debug: diff --git a/build_image_aarch64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 b/build_image_aarch64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 index 073f4336ec..dc404da3d0 100644 --- a/build_image_aarch64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 +++ b/build_image_aarch64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 @@ -13,6 +13,9 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' +{% if repo.priority is defined %} + priority: '{{ repo.priority }}' +{% endif %} {% endif %} {% if repo.gpg | length > 1 %} gpg: '{{ repo.gpg }}' diff --git a/build_image_aarch64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 b/build_image_aarch64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 index dfc7faa778..d239ad0dcd 100644 --- a/build_image_aarch64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 +++ b/build_image_aarch64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 @@ -18,6 +18,9 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' +{% if repo.priority is defined %} + priority: '{{ repo.priority }}' +{% endif %} {% endif %} {% if repo.gpg | length > 1 %} gpg: '{{ repo.gpg }}' diff --git a/build_image_x86_64/roles/fetch_packages/tasks/fetch_pulp_repos.yml b/build_image_x86_64/roles/fetch_packages/tasks/fetch_pulp_repos.yml index 24f9be6c52..d479def48a 100644 --- a/build_image_x86_64/roles/fetch_packages/tasks/fetch_pulp_repos.yml +++ b/build_image_x86_64/roles/fetch_packages/tasks/fetch_pulp_repos.yml @@ -28,10 +28,44 @@ | selectattr('name', 'match', '^x86_64') | list }} - - name: Build rhel_repos list from pulp_x86_64_distributions + # -- Repo priority classification ---------------------------------- + # Load local_repo_config.yml to identify user vs RHEL vs omnia repos. + # Priority order: user (10) > RHEL (20) > omnia (99). + # This ensures DNF prefers user-supplied package versions over + # identically-named packages in omnia repos (e.g. slurm 25 from + # user repo beats slurm 26 from EPEL). + - name: Load local_repo_config for repo classification + ansible.builtin.include_vars: + file: "{{ input_project_dir }}/local_repo_config.yml" + name: _local_repo_cfg + failed_when: false + + - name: Build user repo name list + ansible.builtin.set_fact: + _user_repo_names: >- + {{ (_local_repo_cfg.user_repo_url_x86_64 | default([]) | map(attribute='name') | list) }} + _rhel_repo_names: >- + {{ (_local_repo_cfg.rhel_os_url_x86_64 | default([]) | map(attribute='name') | list) + + ['baseos', 'appstream', 'codeready-builder'] }} + _omnia_repo_names: >- + {{ (_local_repo_cfg.omnia_repo_url_rhel_x86_64 | default([]) | map(attribute='name') | list) }} + + - name: Build rhel_repos list with priority from pulp_x86_64_distributions ansible.builtin.set_fact: rhel_x86_64_repos: >- - {{ pulp_x86_64_distributions | map('combine', {'gpg': ''}) | list }} + {% set repos = [] -%} + {% for dist in pulp_x86_64_distributions -%} + {% set repo_name = dist.name.split('_')[4:] | join('_') -%} + {% if repo_name in _user_repo_names -%} + {% set _priority = '10' -%} + {% elif repo_name in _rhel_repo_names -%} + {% set _priority = '20' -%} + {% else -%} + {% set _priority = '99' -%} + {% endif -%} + {% set _ = repos.append(dist | combine({'gpg': '', 'priority': _priority})) -%} + {% endfor -%} + {{ repos }} - name: Debug rhel_x86_64_repos ansible.builtin.debug: diff --git a/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 b/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 index 1b75508858..77de0a79c3 100644 --- a/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 +++ b/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 @@ -13,6 +13,9 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' +{% if repo.priority is defined and repo.priority | string | length > 0 %} + priority: '{{ repo.priority }}' +{% endif %} {% endif %} {% if repo.gpg | length > 1 %} gpg: '{{ repo.gpg }}' diff --git a/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 b/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 index a10790edf7..48fe492336 100644 --- a/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 +++ b/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 @@ -18,6 +18,9 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' +{% if repo.priority is defined and repo.priority | string | length > 0 %} + priority: '{{ repo.priority }}' +{% endif %} {% endif %} {% if repo.gpg | length > 1 %} gpg: '{{ repo.gpg }}' diff --git a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml index dfdf99a745..3300a3b13d 100644 --- a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml @@ -45,7 +45,7 @@ openchami_bss_tag: "v1.32.2" openchami_cloud_init_tag: "v1.3.0" openchami_coresmd_tag: "v0.4.3" # Third-party image tags for OpenCHAMI -minio_release_tag: "RELEASE.2026-04-17T00-00-00Z" +minio_release_tag: "RELEASE.2026-08-04T00-00-00Z" postgres_tag: "11.5-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" From 79ef23895dbfdceca6255e9c0de2d427efd589a8 Mon Sep 17 00:00:00 2001 From: Abhishek S A Date: Mon, 10 Aug 2026 10:39:53 +0530 Subject: [PATCH 14/22] Update minio and registry to latest available version (#4956) * repo priority fixes Signed-off-by: Abhishek S A * Update fetch_pulp_repos.yml Signed-off-by: Abhishek S A * update repo priority flow Signed-off-by: Abhishek S A * Update main.yml Signed-off-by: Abhishek S A * minio and registry tag updates Signed-off-by: Abhishek S A --------- Signed-off-by: Abhishek S A --- prepare_oim/roles/deploy_containers/openchami/vars/main.yml | 2 +- upgrade/roles/upgrade_openchami/vars/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml index 3300a3b13d..5205055b3e 100644 --- a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml @@ -49,7 +49,7 @@ minio_release_tag: "RELEASE.2026-08-04T00-00-00Z" postgres_tag: "11.5-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" -registry_tag: "3.1.0" +registry_tag: "3.1.1" curl_tag: "latest" acme_tag: "3.1.1" diff --git a/upgrade/roles/upgrade_openchami/vars/main.yml b/upgrade/roles/upgrade_openchami/vars/main.yml index a3b70aeb13..109eecca81 100644 --- a/upgrade/roles/upgrade_openchami/vars/main.yml +++ b/upgrade/roles/upgrade_openchami/vars/main.yml @@ -101,11 +101,11 @@ openchami_smd_tag: "v2.19.3" openchami_bss_tag: "v1.32.2" openchami_cloud_init_tag: "v1.3.0" openchami_coresmd_tag: "v0.4.3" -minio_release_tag: "RELEASE.2026-04-17T00-00-00Z" +minio_release_tag: "RELEASE.2026-08-04T00-00-00Z" postgres_tag: "11.5-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" -registry_tag: "3.1.0" +registry_tag: "3.1.1" curl_tag: "latest" acme_tag: "3.1.1" From 60f174e25d1808dbe215c0faf46c5c5558058e20 Mon Sep 17 00:00:00 2001 From: pullan1 Date: Mon, 10 Aug 2026 15:21:44 +0530 Subject: [PATCH 15/22] Fix for OS Command Injection Vulnerability in local_repo (#4960) Signed-off-by: pullan1 --- .../schema/local_repo_config.json | 30 ++++++++++++------- .../local_repo/parse_and_download.py | 6 ++-- common/library/modules/pulp_cleanup.py | 4 ++- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/common/library/module_utils/input_validation/schema/local_repo_config.json b/common/library/module_utils/input_validation/schema/local_repo_config.json index a89c648105..1d8f472837 100644 --- a/common/library/module_utils/input_validation/schema/local_repo_config.json +++ b/common/library/module_utils/input_validation/schema/local_repo_config.json @@ -83,7 +83,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -219,7 +220,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -355,7 +357,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -491,7 +494,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -644,7 +648,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "policy": { @@ -780,7 +785,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "policy": { @@ -898,7 +904,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "sslcacert": { @@ -1027,7 +1034,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "sslcacert": { @@ -1157,7 +1165,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -1215,7 +1224,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", diff --git a/common/library/module_utils/local_repo/parse_and_download.py b/common/library/module_utils/local_repo/parse_and_download.py index f9e605cbe8..1a661f8d7e 100644 --- a/common/library/module_utils/local_repo/parse_and_download.py +++ b/common/library/module_utils/local_repo/parse_and_download.py @@ -21,6 +21,7 @@ import os import subprocess +import shlex import json import re from multiprocessing import Lock @@ -57,12 +58,13 @@ def execute_command(cmd_string, logger, type_json=False): logger.info(f"Executing command: {safe_cmd_string}") # Run the command + cmd_list = shlex.split(cmd_string) cmd = subprocess.run( - cmd_string, + cmd_list, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True, + shell=False, ) status["returncode"] = cmd.returncode status["stdout"] = cmd.stdout.strip() if cmd.stdout else None diff --git a/common/library/modules/pulp_cleanup.py b/common/library/modules/pulp_cleanup.py index 1d076e7561..4c03a62fc0 100644 --- a/common/library/modules/pulp_cleanup.py +++ b/common/library/modules/pulp_cleanup.py @@ -29,6 +29,7 @@ import json import shutil import subprocess +import shlex import re import yaml from typing import Dict, List, Any, Tuple @@ -92,7 +93,8 @@ def format_pretty_table(results: List[Dict[str, Any]]) -> str: def run_cmd(cmd: str, logger) -> Dict[str, Any]: """Execute shell command and return result.""" try: - result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=300) + cmd_list = shlex.split(cmd) + result = subprocess.run(cmd_list, shell=False, capture_output=True, text=True, timeout=300) return {"rc": result.returncode, "stdout": result.stdout, "stderr": result.stderr} except (subprocess.SubprocessError, OSError) as e: logger.error(f"Command failed: {cmd} - {e}") From 3aae82d8dd3c0866b0be9427e8b663401f8b2b61 Mon Sep 17 00:00:00 2001 From: Sujit Jadhav Date: Mon, 10 Aug 2026 17:18:40 +0530 Subject: [PATCH 16/22] Fix OpenCHAMI service failures and improve Pulp deployment reliability on re-runs. (#4957) * fix(openchami): update image tags, RPM packaging, and coredhcp template - Update container image tags to latest versions: opaal v0.3.13, smd v2.20.5, bss v1.32.3, cloud-init v1.4.8, coresmd v0.6.3, postgres 17-alpine - Switch opaal, smd, bss, cloud-init images to Dell Docker Hub registry - Replace RPM download from GitHub with local tarball copy/extract using Git LFS for the tarball (openchami-0.1.7-1.noarch.tar.gz) - Fix coredhcp template to use key=value config format for coresmd v0.6.x+ regardless of additional_subnets presence - Add primary subnet_pool for bootloop plugin in v0.6.x format - Fix hostname rules to use type-specific patterns (nid/bmc) - Remove obsolete commented-out multi-subnet upgrade instructions - Add rescue block for ochami package install to fallback to Red Hat CDN AppStream repo if default (Pulp mirror) fails Signed-off-by: Sujit Jadhav * fix(pulp): ensure certs directory exists and fix password reset retry - Add 'Ensure certs directory exists' task before SSL certificate generation in create_pulp_config_https.yml to prevent failures when the directory is missing on re-runs - Fix password reset retry logic in deploy_pulp_container_https.yml: add register/until clause (required for Ansible retry to work) and increase retries from 8 to 15 to allow sufficient time for Pulp services to initialize after container restart Signed-off-by: Sujit Jadhav * fix(openchami): rebuild images with PR fixes and bump tags Rebuild container images incorporating fixes from upstream PRs: - opaal v0.3.14: SQLite driver fix (sqlite3 -> sqlite pure-Go driver) - cloud-init v1.4.9: replace deprecated middleware.RealIP with ClientIPFromRemoteAddr - bss v1.32.4: migrate chi v1 middleware import to chi/v5 - smd v2.20.6: security dependency updates (golang.org/x/crypto, golang.org/x/net) Update Ansible vars and RPM tarball to match. Signed-off-by: Sujit Jadhav * fix(upgrade): align image tags, registries, and RPM with prepare_oim Update upgrade_openchami vars/tasks to match prepare_oim: - Bump image tags: opaal v0.3.14, smd v2.20.6, bss v1.32.4, cloud-init v1.4.9, coresmd v0.6.3 - Switch opaal/smd/bss/cloud-init from ghcr.io/openchami to docker.io/dellhpcomniaaisolution (Dell Docker Hub images) - Update postgres 11.5-alpine -> 17-alpine, minio to latest release - Replace curl-based RPM download with local tarball copy+extract matching prepare_oim pattern - Add files/ directory with openchami-0.1.7-1.noarch.tar.gz Signed-off-by: Sujit Jadhav * fix: update minio_release_tag to RELEASE.2026-08-04T00-00-00Z Per reviewer feedback from abhishek-sa1. Signed-off-by: Sujit Jadhav * fix: remove duplicate RPM tarball and variablize pulp retries - Remove duplicate openchami tarball from upgrade role files/ - Reference prepare_oim files path via prepare_oim_files_path variable - Replace hardcoded retries: 15 with password_reset_retries variable in deploy_pulp_container_https.yml Signed-off-by: Sujit Jadhav --------- Signed-off-by: Sujit Jadhav Signed-off-by: Abhishek S A Co-authored-by: Abhishek S A --- .gitattributes | 1 + .../files/openchami-0.1.7-1.noarch.tar.gz | 3 ++ .../openchami/tasks/configs/ochami.yml | 18 ++++--- .../openchami/tasks/configs/packages.yml | 14 ++++-- .../templates/coredhcp/coredhcp.yaml.j2 | 50 ++++--------------- .../deploy_containers/openchami/vars/main.yml | 24 ++++----- .../pulp/tasks/create_pulp_config_https.yml | 6 +++ .../tasks/deploy_pulp_container_https.yml | 4 +- .../deploy_containers/pulp/vars/main.yml | 1 + .../tasks/upgrade_openchami_containers.yml | 21 ++++---- upgrade/roles/upgrade_openchami/vars/main.yml | 39 ++++++++------- 11 files changed, 86 insertions(+), 95 deletions(-) create mode 100644 prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz diff --git a/.gitattributes b/.gitattributes index 873288d963..a065796c70 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ *.yml linguist-detectable *.tar.gz filter=lfs diff=lfs merge=lfs -text +prepare_oim/roles/deploy_containers/openchami/files/*.tar.gz filter=lfs diff=lfs merge=lfs -text diff --git a/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz new file mode 100644 index 0000000000..91b2f3b9d5 --- /dev/null +++ b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4eaa990b4d51381198f702d61504aa86cc379e63eeb083c57e3cae011d168e9 +size 16024 diff --git a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml index de7b998c93..15c4b85a2f 100644 --- a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml +++ b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml @@ -13,15 +13,17 @@ # limitations under the License. --- -- name: Download the openchami RPM file - ansible.builtin.get_url: - url: "{{ openchami_rpm_url }}" - dest: "{{ openchami_work_dir }}/{{ openchami_rpm_name }}" +- name: Copy the openchami RPM tarball to OIM + ansible.builtin.copy: + src: "{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" mode: "{{ file_perm_rwx }}" - register: download_openchami - until: download_openchami is not failed - retries: "{{ max_retries }}" - delay: "{{ max_delay }}" + +- name: Extract the openchami RPM from tarball + ansible.builtin.unarchive: + src: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}" + remote_src: true - name: Install the openchami ansible.builtin.dnf: diff --git a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml index 2c75a6f853..e0bf639dcc 100644 --- a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml +++ b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml @@ -14,9 +14,17 @@ --- - name: Install ochami dependent packages - ansible.builtin.dnf: - name: "{{ ochami_package_dependencies }}" - state: present + block: + - name: Install ochami packages from default repos + ansible.builtin.dnf: + name: "{{ ochami_package_dependencies }}" + state: present + rescue: + - name: Fallback - install ochami packages from Red Hat CDN + ansible.builtin.dnf: + name: "{{ ochami_package_dependencies }}" + state: present + enablerepo: rhel-10-for-x86_64-appstream-rpms - name: Install pip packages ansible.builtin.pip: diff --git a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 index 1cf09db3f9..c546e4e901 100644 --- a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 +++ b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 @@ -9,8 +9,8 @@ server4: - dns: {{ coredhcp_dns_server }} - router: {{ coredhcp_router }} - netmask: {{ coredhcp_netmask }} -{% if multisubnet_native and has_additional_subnets %} - # Multi-subnet mode: key=value config format (coresmd v0.6.x+) +{% if multisubnet_native %} + # Key=value config format (coresmd v0.6.x+) - coresmd: | svc_base_uri=https://{{ cluster_name }}.{{ cluster_domain }}:8443 ipxe_base_uri=http://{{ cluster_boot_ip }}:8081 @@ -18,18 +18,20 @@ server4: cache_valid={{ coredhcp_cache_validity }} lease_time={{ coredhcp_lease_duration }} single_port={{ coredhcp_tftp_single_port_mode | lower }} -{% for s in additional_subnets %} +{% for s in additional_subnets | default([]) %} rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:Node,routers:{{ s.router }},cidr:{{ s.netmask_bits }} rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:NodeBMC,routers:{{ s.router }},cidr:{{ s.netmask_bits }} {% endfor %} - rule=type:Node - rule=type:NodeBMC - rule=hostname:unknown-{{'{'}}04d{{'}'}} + rule=type:Node,hostname:nid{{'{'}}04d{{'}'}} + rule=type:NodeBMC,hostname:bmc{{'{'}}04d{{'}'}} - bootloop: | lease_file=/tmp/coredhcp.db script_path={{ coredhcp_custom_ipxe }} lease_time={{ coredhcp_tmp_lease_duration }} -{% for s in additional_subnets %} +{% set primary_range = network_data.admin_network.dynamic_range | split('-') %} +{% set primary_subnet = (cluster_boot_ip + '/' + network_data.admin_network.netmask_bits | string) | ansible.utils.ipaddr('network') %} + subnet_pool={{ primary_subnet }}/{{ network_data.admin_network.netmask_bits }},{{ primary_range[0] }},{{ primary_range[1] }} +{% for s in additional_subnets | default([]) %} {% set range_parts = s.dynamic_range | split('-') %} subnet_pool={{ s.subnet }}/{{ s.netmask_bits }},{{ range_parts[0] }},{{ range_parts[1] }} {% endfor %} @@ -37,38 +39,4 @@ server4: # Single-subnet mode: positional argument format (coresmd v0.4.x) - coresmd: https://{{ cluster_name }}.{{ cluster_domain }}:8443 http://{{ cluster_boot_ip }}:8081 /root_ca/root_ca.crt {{ coredhcp_cache_validity }} {{ coredhcp_lease_duration }} {{ coredhcp_tftp_single_port_mode | lower }} - bootloop: /tmp/coredhcp.db {{ coredhcp_custom_ipxe }} {{ coredhcp_tmp_lease_duration }} {{ coredhcp_dhcp_pool }} -{% if has_additional_subnets %} - # ------------------------------------------------------------------- - # Multi-subnet configuration (requires coresmd v0.6.x+) - # To enable multi-subnet DHCP: - # 1. Pull the new coresmd image: podman pull ghcr.io/openchami/coresmd:v0.6.3 - # 2. Comment out the single-subnet coresmd and bootloop lines above - # 3. Uncomment the multi-subnet coresmd and bootloop blocks below - # 4. Replace the new coresmd image version in files: /etc/containers/systemd/coresmd-coredhcp.container /etc/containers/systemd/coresmd-coredns.container with the old version - # 5. Reload daemon: systemctl daemon-reload - # 6. Restart services: systemctl restart openchami.target - # ------------------------------------------------------------------- - # - coresmd: | - # svc_base_uri=https://{{ cluster_name }}.{{ cluster_domain }}:8443 - # ipxe_base_uri=http://{{ cluster_boot_ip }}:8081 - # ca_cert=/root_ca/root_ca.crt - # cache_valid={{ coredhcp_cache_validity }} - # lease_time={{ coredhcp_lease_duration }} - # single_port={{ coredhcp_tftp_single_port_mode | lower }} -{% for s in additional_subnets %} - # rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:Node,routers:{{ s.router }},cidr:{{ s.netmask_bits }} - # rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:NodeBMC,routers:{{ s.router }},cidr:{{ s.netmask_bits }} -{% endfor %} - # rule=type:Node - # rule=type:NodeBMC - # rule=hostname:unknown-{{'{'}}04d{{'}'}} - # - bootloop: | - # lease_file=/tmp/coredhcp.db - # script_path={{ coredhcp_custom_ipxe }} - # lease_time={{ coredhcp_tmp_lease_duration }} -{% for s in additional_subnets %} -{% set range_parts = s.dynamic_range | split('-') %} - # subnet_pool={{ s.subnet }}/{{ s.netmask_bits }},{{ range_parts[0] }},{{ range_parts[1] }} -{% endfor %} -{% endif %} {% endif %} diff --git a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml index 5205055b3e..3ff7a43463 100644 --- a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml @@ -39,14 +39,14 @@ pull_image_delay: 10 # OpenCHAMI image tags openchami_local_ca_tag: "v0.2.6" -openchami_opaal_tag: "v0.3.12" -openchami_smd_tag: "v2.19.3" -openchami_bss_tag: "v1.32.2" -openchami_cloud_init_tag: "v1.3.0" -openchami_coresmd_tag: "v0.4.3" +openchami_opaal_tag: "v0.3.14" +openchami_smd_tag: "v2.20.6" +openchami_bss_tag: "v1.32.4" +openchami_cloud_init_tag: "v1.4.9" +openchami_coresmd_tag: "v0.6.3" # Third-party image tags for OpenCHAMI minio_release_tag: "RELEASE.2026-08-04T00-00-00Z" -postgres_tag: "11.5-alpine" +postgres_tag: "17-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" registry_tag: "3.1.1" @@ -56,10 +56,10 @@ acme_tag: "3.1.1" # OpenCHAMI images list for podman pull on OIM openchami_images: - "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - - "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" - - "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" - - "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" - - "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + - "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" + - "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" + - "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" + - "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - "docker.io/pgsty/minio:{{ minio_release_tag }}" - "docker.io/library/postgres:{{ postgres_tag }}" @@ -92,8 +92,8 @@ ochami_cli_log_format: rfc3339 ochami_base_url: "https://{{ cluster_name }}.{{ cluster_domain }}:8443" # OpenCHAMI RPM packages -openchami_rpm_url: https://github.com/OpenCHAMI/release/releases/download/v0.1.5/openchami-0.1.5.rpm -openchami_rpm_name: openchami-0.1.5.rpm +openchami_rpm_tarball: openchami-0.1.7-1.noarch.tar.gz +openchami_rpm_name: openchami-0.1.7-1.noarch.rpm ochami_client_rpm_url: https://github.com/OpenCHAMI/ochami/releases/download/v0.7.1/ochami_0.7.1_linux_amd64.rpm ochami_client_rpm_name: ochami_0.7.1_linux_amd64.rpm diff --git a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml index a548bf6ed6..9409ae8f69 100644 --- a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml +++ b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml @@ -91,6 +91,12 @@ when: not hostvars['oim']['hostname_enabled'] no_log: true + - name: Ensure certs directory exists + ansible.builtin.file: + path: "{{ certs_dir }}" + state: directory + mode: "0755" + - name: Generating Pulp SSL Certificate ansible.builtin.command: cmd: "{{ generate_cert_cmd }}" diff --git a/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml b/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml index 2fe8968a1b..94928ef720 100644 --- a/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml +++ b/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml @@ -105,7 +105,9 @@ containers.podman.podman_container_exec: name: "{{ pulp_container_name }}" command: "{{ reset_password_cmd }}" - retries: "{{ retries_var }}" + register: pulp_password_reset + until: pulp_password_reset is not failed + retries: "{{ password_reset_retries }}" delay: "{{ delay_var }}" when: pulp_password | length > 1 no_log: true diff --git a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml index 95b7afcdbe..bd032493a3 100644 --- a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml @@ -47,6 +47,7 @@ pulp_deployed_fail_msg: Run utility/oim_cleanup.yml to cleanup, then re-run the playbook to deploy the {{ pulp_container_name }} container successfully." retries_var: 8 +password_reset_retries: 15 delay_var: 30 delay_var_sixty: 30 timeout_var: 60 diff --git a/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml b/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml index e01e7f2e57..b91c8a6fea 100644 --- a/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml +++ b/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml @@ -84,21 +84,20 @@ delegate_facts: true connection: ssh - - name: Download updated OpenCHAMI RPM - ansible.builtin.command: curl -L -o {{ openchami_work_dir }}/{{ openchami_rpm_name }} {{ openchami_rpm_url }} # noqa: command-instead-of-module - register: download_openchami - retries: "{{ max_retries }}" - delay: "{{ max_delay }}" - until: download_openchami.rc == 0 - changed_when: true + - name: Copy the openchami RPM tarball to OIM + ansible.builtin.copy: + src: "{{ prepare_oim_files_path }}/{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" + mode: "{{ dir_permissions_755 }}" delegate_to: oim delegate_facts: true connection: ssh - - name: Set permissions on downloaded OpenCHAMI RPM - ansible.builtin.file: - path: "{{ openchami_work_dir }}/{{ openchami_rpm_name }}" - mode: "{{ dir_permissions_755 }}" + - name: Extract the openchami RPM from tarball + ansible.builtin.unarchive: + src: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}" + remote_src: true delegate_to: oim delegate_facts: true connection: ssh diff --git a/upgrade/roles/upgrade_openchami/vars/main.yml b/upgrade/roles/upgrade_openchami/vars/main.yml index 109eecca81..f37e4f9f7a 100644 --- a/upgrade/roles/upgrade_openchami/vars/main.yml +++ b/upgrade/roles/upgrade_openchami/vars/main.yml @@ -112,10 +112,10 @@ acme_tag: "3.1.1" # OpenCHAMI target images for podman pull openchami_target_images: - "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - - "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" - - "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" - - "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" - - "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + - "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" + - "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" + - "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" + - "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - "docker.io/pgsty/minio:{{ minio_release_tag }}" - "docker.io/library/postgres:{{ postgres_tag }}" @@ -131,19 +131,19 @@ openchami_quadlet_updates: - file: "step-ca.container" image: "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - file: "opaal.container" - image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - file: "opaal-idp.container" - image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - file: "smd.container" - image: "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" + image: "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" - file: "smd-init.container" - image: "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" + image: "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" - file: "bss.container" - image: "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" + image: "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" - file: "bss-init.container" - image: "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" + image: "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" - file: "cloud-init-server.container" - image: "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + image: "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - file: "coresmd-coredhcp.container" image: "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - file: "coresmd-coredns.container" @@ -173,15 +173,15 @@ openchami_containers: - container: "step-ca" expected_image: "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - container: "opaal" - expected_image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - container: "opaal-idp" - expected_image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - container: "smd" - expected_image: "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" - container: "bss" - expected_image: "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" - container: "cloud-init-server" - expected_image: "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - container: "coresmd-coredhcp" expected_image: "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - container: "coresmd-coredns" @@ -195,8 +195,9 @@ openchami_containers: - container: "registry" expected_image: "docker.io/library/registry:{{ registry_tag }}" -# Path to prepare_oim templates (reuse for deploying new configs without duplication) +# Path to prepare_oim templates and files (reuse without duplication) prepare_oim_templates_path: "{{ role_path }}/../../../prepare_oim/roles/deploy_containers/openchami/templates" +prepare_oim_files_path: "{{ role_path }}/../../../prepare_oim/roles/deploy_containers/openchami/files" # OpenCHAMI configuration paths openchami_config_dir: "/etc/openchami/configs" @@ -228,8 +229,8 @@ network_spec_path: "{{ input_project_dir | default('/opt/omnia/input/project_def provision_config_path: "{{ input_project_dir | default('/opt/omnia/input/project_default') }}/provision_config.yml" # OpenCHAMI RPM packages (Omnia 2.2 target) -openchami_rpm_url: "https://github.com/OpenCHAMI/release/releases/download/v0.1.5/openchami-0.1.5.rpm" -openchami_rpm_name: "openchami-0.1.5.rpm" +openchami_rpm_tarball: openchami-0.1.7-1.noarch.tar.gz +openchami_rpm_name: openchami-0.1.7-1.noarch.rpm ochami_client_rpm_url: "https://github.com/OpenCHAMI/ochami/releases/download/v0.7.1/ochami_0.7.1_linux_amd64.rpm" ochami_client_rpm_name: "ochami_0.7.1_linux_amd64.rpm" From 84bc09170a2d9bc12067de2c72bb4303cf0b35a9 Mon Sep 17 00:00:00 2001 From: Sujit Jadhav Date: Mon, 10 Aug 2026 20:43:50 +0530 Subject: [PATCH 17/22] fix(openchami): rebuild RPM with correct image tags in quadlets (#4963) Release quadlet files had stale image tags (bss v1.32.3, smd v2.20.5, opaal v0.3.13, cloud-init v1.4.8) while Ansible vars and Dell Docker Hub had the newer versions (v1.32.4, v2.20.6, v0.3.14, v1.4.9). This caused BSS 503 errors because the quadlet referenced images that don't exist on the registry. Rebuild RPM tarball with corrected quadlet image tags. Signed-off-by: Sujit Jadhav --- .../openchami/files/openchami-0.1.7-1.noarch.tar.gz | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz index 91b2f3b9d5..3cce25255a 100644 --- a/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz +++ b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c4eaa990b4d51381198f702d61504aa86cc379e63eeb083c57e3cae011d168e9 -size 16024 +oid sha256:6be06242fc4c6d14c16fc44c2d259193eb3bf2ef7673dc75350a0e491f6b64e2 +size 15984 From eccc5ee50a3f68e1c44046cff9036d6f6082d1bb Mon Sep 17 00:00:00 2001 From: pullan1 Date: Mon, 10 Aug 2026 22:21:04 +0530 Subject: [PATCH 18/22] Added repo precedence order comments in local_repo_config (#4962) * Fix for OS Command Injection Vulnerability in local_repo Signed-off-by: pullan1 * Added repo precedence order comments in local_repo_config Signed-off-by: pullan1 --------- Signed-off-by: pullan1 --- input/local_repo_config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/input/local_repo_config.yml b/input/local_repo_config.yml index 9f14b476a6..63f7b6d22b 100644 --- a/input/local_repo_config.yml +++ b/input/local_repo_config.yml @@ -45,6 +45,12 @@ # To achieve proxy mode (on_demand): Set repo_config: partial # OR set policy: always + caching: true per repo #---------------------------------------------------------------------------------- +# REPO PRECEDENCE ORDER (package download priority): +# 1. user_repo_url_ (highest precedence) +# 2. rhel_os_url_ +# 3. omnia_repo_url_ (lowest precedence) +# If a package exists in multiple repos, the higher precedence repo is used. +#---------------------------------------------------------------------------------- # 1. user_registry #-------------------------- From db02ca6f4f1e13da895ddc48105edd9816de96d6 Mon Sep 17 00:00:00 2001 From: Sujit Jadhav Date: Tue, 11 Aug 2026 15:40:50 +0530 Subject: [PATCH 19/22] fix(openchami): rebuild RPM from correct source tree (openchami/release) (#4964) The openchami-0.1.7 RPM was previously built from release/ which ships an incomplete/newer architecture (tokensmith-based) that conflicts with the container images actually deployed (opaal, bss, hydra, cloud-init). Rebuild from openchami/release/ which has: - Complete bootstrap_openchami.sh with all required podman secrets (bss_postgres_password, hydra_postgres_password, hydra_system_secret, hydra_dsn, and full postgres_multiple_databases) - Correct openchami.env with BSS, OPAAL, and cloud-init env vars - openchami-certificate-update with opaal.container FQDN update - openchami_profile.sh with hydra-based gen_access_token (not tokensmith) - Correct openchami.target without tokensmith/boot-service/metadata-service - Only the containers matching the deployed images (opaal, bss, hydra, etc.) - cloud-init container rebuilt with go-viper/mapstructure/v2 v2.5.0 (fixes CVE-2025-11065 sensitive info leak) Signed-off-by: Sujit Jadhav --- .../openchami/files/openchami-0.1.7-1.noarch.tar.gz | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz index 3cce25255a..b211ae067f 100644 --- a/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz +++ b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6be06242fc4c6d14c16fc44c2d259193eb3bf2ef7673dc75350a0e491f6b64e2 -size 15984 +oid sha256:15f8e7e5c6623f6c966a64a7784c0ffd303e4c7bc87613abb81a8cc050005e07 +size 13818 From 96bfc012d94242f01dca23471acb5a643a0450c5 Mon Sep 17 00:00:00 2001 From: Sujit Jadhav Date: Tue, 11 Aug 2026 16:28:18 +0530 Subject: [PATCH 20/22] refactor(openchami): remove dead coresmd v0.4.x fallback from coredhcp template (#4968) Since coresmd v0.6.3 is the only version shipped, the version check and the v0.4.x positional argument fallback in coredhcp.yaml.j2 are dead code. Remove them to simplify the template. Both single-subnet and multi-subnet deployments continue to work through the same key=value config format - the additional_subnets for-loops simply produce zero iterations for single-subnet setups. Signed-off-by: Sujit Jadhav --- .../openchami/templates/coredhcp/coredhcp.yaml.j2 | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 index c546e4e901..fece22357b 100644 --- a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 +++ b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 @@ -1,6 +1,3 @@ -{% set coresmd_ver = openchami_coresmd_tag | regex_replace('^v', '') %} -{% set multisubnet_native = coresmd_ver is version('0.6.0', '>=') %} -{% set has_additional_subnets = additional_subnets | default([]) | length > 0 %} server4: listen: - "%{{ cluster_boot_interface }}" @@ -9,8 +6,6 @@ server4: - dns: {{ coredhcp_dns_server }} - router: {{ coredhcp_router }} - netmask: {{ coredhcp_netmask }} -{% if multisubnet_native %} - # Key=value config format (coresmd v0.6.x+) - coresmd: | svc_base_uri=https://{{ cluster_name }}.{{ cluster_domain }}:8443 ipxe_base_uri=http://{{ cluster_boot_ip }}:8081 @@ -35,8 +30,3 @@ server4: {% set range_parts = s.dynamic_range | split('-') %} subnet_pool={{ s.subnet }}/{{ s.netmask_bits }},{{ range_parts[0] }},{{ range_parts[1] }} {% endfor %} -{% else %} - # Single-subnet mode: positional argument format (coresmd v0.4.x) - - coresmd: https://{{ cluster_name }}.{{ cluster_domain }}:8443 http://{{ cluster_boot_ip }}:8081 /root_ca/root_ca.crt {{ coredhcp_cache_validity }} {{ coredhcp_lease_duration }} {{ coredhcp_tftp_single_port_mode | lower }} - - bootloop: /tmp/coredhcp.db {{ coredhcp_custom_ipxe }} {{ coredhcp_tmp_lease_duration }} {{ coredhcp_dhcp_pool }} -{% endif %} From 3873438c8f53b1c109b1ce2c39e5b632c8702e3f Mon Sep 17 00:00:00 2001 From: Kratika Patidar Date: Tue, 11 Aug 2026 16:50:27 +0530 Subject: [PATCH 21/22] mysql upgrade and rollback changes for version update. (#4961) * mysql upgrade and rollback fixes0 Signed-off-by: Kratika_Patidar * mysql upgrade and rollback changes for version update. Signed-off-by: Kratika_Patidar * Fix ansible-lint configuration Signed-off-by: Kratika_Patidar * lint fixes Signed-off-by: Kratika_Patidar * rollback backup handling fixes Signed-off-by: Kratika_Patidar * ansible-lint fixes Signed-off-by: Kratika_Patidar * Delete temp_upgrade_mysql Signed-off-by: Kratika Patidar * Update restore_mysql_data.yml Signed-off-by: Kratika Patidar --------- Signed-off-by: Kratika_Patidar Signed-off-by: Kratika Patidar --- input/telemetry_storage_config.yml | 8 +- provision/roles/telemetry/vars/main.yml | 8 +- rollback/roles/rollback_k8s/tasks/main.yml | 10 +- .../rollback_k8s/tasks/restore_mysql_data.yml | 537 ++++++++++++++++++ rollback/roles/rollback_k8s/vars/main.yml | 62 ++ .../templates/telemetry_storage_config.j2 | 8 +- .../tasks/backup_mysql_data.yml | 244 ++++++++ .../roles/upgrade_telemetry/tasks/main.yml | 8 + upgrade/roles/upgrade_telemetry/vars/main.yml | 29 + 9 files changed, 900 insertions(+), 14 deletions(-) create mode 100644 rollback/roles/rollback_k8s/tasks/restore_mysql_data.yml create mode 100644 upgrade/roles/upgrade_telemetry/tasks/backup_mysql_data.yml diff --git a/input/telemetry_storage_config.yml b/input/telemetry_storage_config.yml index c80dbdde65..32133b2c00 100644 --- a/input/telemetry_storage_config.yml +++ b/input/telemetry_storage_config.yml @@ -159,11 +159,11 @@ idrac_telemetry_storage: mysqldb: resources: requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "500m" + cpu: "250m" memory: "512Mi" + limits: + cpu: "1000m" + memory: "1Gi" activemq: resources: requests: diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml index 8ecf03ba6e..1e4eedbeec 100644 --- a/provision/roles/telemetry/vars/main.yml +++ b/provision/roles/telemetry/vars/main.yml @@ -74,11 +74,11 @@ kafka_skip_verify: true idrac_telemetry_resources: mysqldb: requests: - cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.cpu | default('100m') }}" - memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.memory | default('256Mi') }}" + cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.cpu | default('250m') }}" + memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.memory | default('512Mi') }}" limits: - cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.cpu | default('500m') }}" - memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.memory | default('512Mi') }}" + cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.cpu | default('1000m') }}" + memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.memory | default('1Gi') }}" activemq: requests: cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.activemq.resources.requests.cpu | default('100m') }}" diff --git a/rollback/roles/rollback_k8s/tasks/main.yml b/rollback/roles/rollback_k8s/tasks/main.yml index db660afd3a..b4db0e38c5 100644 --- a/rollback/roles/rollback_k8s/tasks/main.yml +++ b/rollback/roles/rollback_k8s/tasks/main.yml @@ -37,6 +37,8 @@ # 8b. Restart network pods (clear stale BIRD/speaker processes) # 8c. Restore Helm binary to rollback version # 8d. Clean up stale CSI VolumeAttachments (PowerScale/Isilon) +# 8e. Verify telemetry rollback +# 8f. Restore MySQL data if version downgrade detected # 9. Restore BSS boot params and cloud-init from backup # ══════════════════════════════════════════════════════════════════════ @@ -215,10 +217,14 @@ - name: "Stage 8d — Clean up stale CSI VolumeAttachments" ansible.builtin.include_tasks: cleanup_stale_volume_attachments.yml - # ── Stage 8d: Verify telemetry pods after etcd restore ─────── - - name: "Stage 8d — Verify telemetry rollback" + # ── Stage 8e: Verify telemetry pods after etcd restore ─────── + - name: "Stage 8e — Verify telemetry rollback" ansible.builtin.include_tasks: verify_telemetry_rollback.yml + # ── Stage 8f: Restore MySQL data if version downgrade detected ─ + - name: "Stage 8f — Restore MySQL data (if version mismatch)" + ansible.builtin.include_tasks: restore_mysql_data.yml + # ── Stage 9: Restore BSS boot params and cloud-init ────────── - name: "Stage 9 — Restore BSS boot params and cloud-init" ansible.builtin.include_tasks: restore_bss_cloud_init.yml diff --git a/rollback/roles/rollback_k8s/tasks/restore_mysql_data.yml b/rollback/roles/rollback_k8s/tasks/restore_mysql_data.yml new file mode 100644 index 0000000000..537ab74d77 --- /dev/null +++ b/rollback/roles/rollback_k8s/tasks/restore_mysql_data.yml @@ -0,0 +1,537 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================ +# Restore MySQL Data (Post-Rollback) — All Pods +# ============================================================================ +# After etcd restore, the idrac-telemetry StatefulSet spec is reverted to the +# pre-upgrade MySQL version (e.g., 9.3.0). However, each pod's PVC still +# contains data written by the upgraded version (e.g., 9.7.2). MySQL refuses +# to downgrade between minor versions: +# [ERROR] [MY-014061] Cannot downgrade from 90702 to 90300. +# +# The StatefulSet can have multiple replicas (one per parent service node), +# each with its own independent MySQL instance and PVC. This task handles +# ALL pods, not just pod-0. +# +# This task detects the version mismatch and fixes it: +# 1. Detects if mysqldb container is in CrashLoopBackOff +# 2. Confirms it is a downgrade error via container logs +# 3. Scales down the StatefulSet +# 4. Clears ALL PVC data (removes incompatible format files from every pod) +# 5. Scales up — MySQL initializes fresh at the old version on every pod +# 6. If per-pod mysqldump backups exist, restores schema+data to each pod +# ============================================================================ + +- name: Restore idrac-telemetry MySQL data (post-rollback) + when: + - kube_vip is defined + - kube_vip | length > 0 + block: + # ── Step 1: Check if idrac-telemetry StatefulSet exists ── + - name: Check if idrac-telemetry StatefulSet exists + ansible.builtin.command: + cmd: kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} --no-headers + delegate_to: "{{ kube_vip }}" + connection: ssh + register: idrac_sts_check + changed_when: false + failed_when: false + + - name: Skip MySQL restore if idrac-telemetry not deployed + ansible.builtin.debug: + msg: "{{ msg_mysql_sts_not_found }}" + when: idrac_sts_check.rc != 0 + + # ── Step 2: Detect MySQL CrashLoopBackOff on ANY pod ── + - name: Detect and fix MySQL version downgrade issue + when: idrac_sts_check.rc == 0 + block: + - name: Wait for pods to attempt startup (need multiple restart cycles for CrashLoopBackOff) + ansible.builtin.pause: + seconds: 90 + + - name: Check all idrac-telemetry pods status + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry \ + --no-headers 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: idrac_pod_raw_status + changed_when: false + failed_when: false + + - name: Check if any mysqldb container is crashing across all pods + ansible.builtin.shell: + cmd: | + set -o pipefail + COLUMNS="NAME:.metadata.name,WAITING:.status.containerStatuses[?(@.name==\"mysqldb\")].state.waiting.reason," + COLUMNS="${COLUMNS}LAST_EXIT:.status.containerStatuses[?(@.name==\"mysqldb\")].lastState.terminated.reason," + COLUMNS="${COLUMNS}RESTARTS:.status.containerStatuses[?(@.name==\"mysqldb\")].restartCount" + kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry \ + --no-headers \ + -o custom-columns="${COLUMNS}" \ + 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysqldb_waiting_reason + changed_when: false + failed_when: false + + - name: Check MySQL logs for downgrade error across all pods + ansible.builtin.shell: + cmd: | + set -o pipefail + for POD in $(kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry \ + --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do + kubectl logs "$POD" -n {{ telemetry_namespace }} -c mysqldb --previous --tail=50 2>/dev/null + kubectl logs "$POD" -n {{ telemetry_namespace }} -c mysqldb --tail=50 2>/dev/null + done + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_container_logs + changed_when: false + failed_when: false + + - name: Determine if MySQL downgrade issue exists + ansible.builtin.set_fact: + mysql_downgrade_detected: >- + {{ ('CrashLoopBackOff' in (mysqldb_waiting_reason.stdout | default('')) or + 'Error' in (mysqldb_waiting_reason.stdout | default('')) or + 'OOMKilled' in (mysqldb_waiting_reason.stdout | default(''))) and + ('Cannot downgrade' in (mysql_container_logs.stdout | default('')) or + 'Invalid MySQL server downgrade' in (mysql_container_logs.stdout | default(''))) }} + + - name: Display MySQL status + ansible.builtin.debug: + msg: + - "{{ msg_mysql_pod_status }}" + - "mysqldb waiting reasons: {{ mysqldb_waiting_reason.stdout | default('none') }}" + - "MySQL downgrade detected: {{ mysql_downgrade_detected }}" + + # ── Step 3: Fix MySQL downgrade — wipe ALL PVCs and reinitialize ── + - name: Fix MySQL version downgrade + when: + - idrac_sts_check.rc == 0 + - mysql_downgrade_detected | default(false) | bool + block: + - name: Display MySQL downgrade fix starting + ansible.builtin.debug: + msg: "{{ msg_mysql_downgrade_fix_start }}" + + # Capture original replica count before scaling down + - name: Get original replica count from StatefulSet + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.replicas}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: original_replica_count_raw + changed_when: false + + - name: Set original replica count + ansible.builtin.set_fact: + idrac_telemetry_original_replicas: "{{ original_replica_count_raw.stdout | default('1') | int }}" + + - name: Build list of all pod indices + ansible.builtin.set_fact: + idrac_telemetry_pod_indices: "{{ range(0, idrac_telemetry_original_replicas | int) | list }}" + + - name: Display replica count + ansible.builtin.debug: + msg: "{{ msg_mysql_replica_count }}" + + # Scale down + - name: Scale down idrac-telemetry StatefulSet + ansible.builtin.command: + cmd: kubectl scale statefulset idrac-telemetry -n {{ telemetry_namespace }} --replicas=0 + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + + - name: Wait for all idrac-telemetry pods to terminate + ansible.builtin.shell: + cmd: | + set -o pipefail + # Wait for all pods to be deleted + for i in {{ idrac_telemetry_pod_indices | join(' ') }}; do + kubectl wait --for=delete pod/idrac-telemetry-${i} -n {{ telemetry_namespace }} --timeout=120s 2>/dev/null || true + done + # Double-check no pods left + COUNT=$(kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry --no-headers 2>/dev/null | wc -l) + if [ "$COUNT" -gt 0 ]; then + kubectl delete pod -n {{ telemetry_namespace }} -l app=idrac-telemetry --force --grace-period=0 2>/dev/null || true + sleep 5 + fi + echo "pods_cleared" + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + register: pod_cleanup + + # Get the MySQL image from restored StatefulSet + - name: Get MySQL image from restored StatefulSet + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.template.spec.containers[?(@.name=="mysqldb")].image}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: restored_mysql_image + changed_when: false + + # Wipe ALL PVCs using temporary cleanup pods (one per PVC) + - name: Create PVC cleanup pod for each replica + ansible.builtin.shell: + cmd: | + kubectl run mysql-pvc-cleanup-{{ item }} --image={{ restored_mysql_image.stdout }} \ + --restart=Never -n {{ telemetry_namespace }} \ + --overrides='{ + "spec": { + "containers": [{ + "name": "mysql-pvc-cleanup-{{ item }}", + "image": "{{ restored_mysql_image.stdout }}", + "command": ["/bin/sh", "-c", "rm -rf /var/lib/mysql/* && echo PVC_DATA_CLEARED"], + "volumeMounts": [{ + "name": "mysqldb-pvc", + "mountPath": "/var/lib/mysql" + }] + }], + "volumes": [{ + "name": "mysqldb-pvc", + "persistentVolumeClaim": { + "claimName": "mysqldb-pvc-idrac-telemetry-{{ item }}" + } + }], + "restartPolicy": "Never" + } + }' + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Wait for all PVC cleanup pods to complete + ansible.builtin.shell: + cmd: > + kubectl wait --for=jsonpath='{.status.phase}'=Succeeded + pod/mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} + --timeout=120s 2>/dev/null || + kubectl wait --for=jsonpath='{.status.phase}'=Failed + pod/mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} + --timeout=120s 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: false + retries: 3 + delay: 10 + register: cleanup_wait + until: cleanup_wait.rc == 0 + failed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Verify PVC cleanup succeeded for each replica + ansible.builtin.command: + cmd: kubectl logs mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} + delegate_to: "{{ kube_vip }}" + connection: ssh + register: cleanup_logs_all + changed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Delete PVC if cleanup failed due to NFS lock files + ansible.builtin.command: + cmd: kubectl delete pvc mysqldb-pvc-idrac-telemetry-{{ item.item }} -n {{ telemetry_namespace }} --wait=true --timeout=60s + delegate_to: "{{ kube_vip }}" + connection: ssh + when: "'PVC_DATA_CLEARED' not in item.stdout" + loop: "{{ cleanup_logs_all.results }}" + loop_control: + label: "pod-{{ item.item }}" + changed_when: true + + - name: Delete all PVC cleanup pods + ansible.builtin.command: + cmd: kubectl delete pod mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} --wait=false + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + failed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + # Scale up — MySQL will initialize fresh at the old version on all pods + - name: Scale up idrac-telemetry StatefulSet + ansible.builtin.command: + cmd: kubectl scale statefulset idrac-telemetry -n {{ telemetry_namespace }} --replicas={{ idrac_telemetry_original_replicas | default(1) }} + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + + - name: Wait for all idrac-telemetry pods to be ready + ansible.builtin.shell: + cmd: | + kubectl wait --for=condition=Ready \ + pod/idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} \ + --timeout=300s + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: false + register: pod_ready_wait + retries: 3 + delay: 30 + until: pod_ready_wait.rc == 0 + loop: "{{ idrac_telemetry_pod_indices }}" + + # Get MySQL root password for restore + - name: Get MySQL root password from secret + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get secret mysqldb-credentials -n {{ telemetry_namespace }} \ + -o jsonpath='{.data.mysqldb_root_password}' | base64 -d + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_root_pw + changed_when: false + no_log: true + + # Wait for MySQL to accept connections on all pods + - name: Wait for MySQL to accept connections on each pod + ansible.builtin.shell: + cmd: > + kubectl exec idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} -c mysqldb -- + mysqladmin ping -u root -p'{{ mysql_root_pw.stdout }}' 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_ping + until: mysql_ping.rc == 0 + retries: 30 + delay: 10 + changed_when: false + no_log: true + loop: "{{ idrac_telemetry_pod_indices }}" + + # Search for per-pod mysqldump backups (new format: *_pod.sql.enc) + - name: Search for per-pod encrypted MySQL backups in backup directory + ansible.builtin.find: + paths: "{{ k8s_backup_dir }}" + patterns: "{{ mysql_backup_filename_prefix }}*.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_perpod_enc_backups_in_backup + failed_when: false + when: k8s_backup_dir is defined + + - name: Search for per-pod encrypted MySQL backups in archive directory + ansible.builtin.find: + paths: "{{ k8s_client_mount_path }}/upgrade/archive" + patterns: "{{ mysql_backup_filename_prefix }}*.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_perpod_enc_backups_in_archive + failed_when: false + when: k8s_client_mount_path is defined + + # Also search for legacy single-file backup (backward compatibility) + - name: Search for legacy encrypted MySQL backup in backup directory + ansible.builtin.find: + paths: "{{ k8s_backup_dir }}" + patterns: "idrac_telemetry_mysql_backup.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_legacy_enc_backup + failed_when: false + when: k8s_backup_dir is defined + + - name: Search for legacy encrypted MySQL backup in archive directory + ansible.builtin.find: + paths: "{{ k8s_client_mount_path }}/upgrade/archive" + patterns: "idrac_telemetry_mysql_backup.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_legacy_enc_archive + failed_when: false + when: k8s_client_mount_path is defined + + # Determine which backup format is available and set the backup directory + - name: Determine backup format and directory + ansible.builtin.set_fact: + mysql_perpod_backup_dir: >- + {% set perpod_backup = mysql_perpod_enc_backups_in_backup | default({'matched': 0}) -%} + {% set perpod_archive = mysql_perpod_enc_backups_in_archive | default({'matched': 0}) -%} + {% set legacy_backup = mysql_legacy_enc_backup | default({'matched': 0}) -%} + {% set legacy_archive = mysql_legacy_enc_archive | default({'matched': 0}) -%} + {% if (perpod_backup.matched | default(0)) > 0 -%} + {{ (perpod_backup.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- elif (perpod_archive.matched | default(0)) > 0 -%} + {{ (perpod_archive.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- elif (legacy_backup.matched | default(0)) > 0 -%} + {{ (legacy_backup.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- elif (legacy_archive.matched | default(0)) > 0 -%} + {{ (legacy_archive.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- else -%} + none + {%- endif %} + mysql_backup_is_perpod: >- + {{ ((mysql_perpod_enc_backups_in_backup | default({'matched': 0})).matched | default(0)) > 0 or + ((mysql_perpod_enc_backups_in_archive | default({'matched': 0})).matched | default(0)) > 0 }} + mysql_backup_is_legacy: >- + {{ ((mysql_perpod_enc_backups_in_backup | default({'matched': 0})).matched | default(0)) == 0 and + ((mysql_perpod_enc_backups_in_archive | default({'matched': 0})).matched | default(0)) == 0 and + (((mysql_legacy_enc_backup | default({'matched': 0})).matched | default(0)) > 0 or + ((mysql_legacy_enc_archive | default({'matched': 0})).matched | default(0)) > 0) }} + + - name: Display backup search results + ansible.builtin.debug: + msg: + - "{{ msg_mysql_backup_search_results }}" + - "Replica count: {{ idrac_telemetry_original_replicas }}" + + # ── Restore per-pod backups (new format) ── + - name: Restore per-pod MySQL backups + when: + - mysql_perpod_backup_dir | trim != 'none' + - mysql_backup_is_perpod | trim | bool + block: + - name: Display per-pod MySQL restore starting + ansible.builtin.debug: + msg: "{{ msg_mysql_perpod_restore_start }}" + + - name: Restore encrypted mysqldump to each pod + ansible.builtin.shell: + cmd: | + set -o pipefail + BACKUP_FILE="{{ mysql_perpod_backup_dir | trim }}/{{ mysql_backup_filename_prefix }}{{ item }}.sql.enc" + if [ ! -f "$BACKUP_FILE" ]; then + echo "NO_BACKUP_FOR_POD_{{ item }}" + exit 0 + fi + openssl enc -d -aes-256-cbc -pbkdf2 -iter 100000 \ + -in "$BACKUP_FILE" \ + -pass pass:'{{ mysql_root_pw.stdout }}' \ + | kubectl exec -i idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} -c mysqldb -- \ + mysql -u root -p'{{ mysql_root_pw.stdout }}' idrac_telemetrydb + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_perpod_restore_results + changed_when: true + failed_when: false + no_log: true + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Report per-pod restore results + ansible.builtin.debug: + msg: >- + {{ msg_mysql_perpod_restore_result }} + {{ '— no backup file found, MySQL reinitialized fresh' if 'NO_BACKUP_FOR_POD' in (item.stdout | default('')) else '' }} + loop: "{{ mysql_perpod_restore_results.results }}" + loop_control: + label: "pod-{{ item.item }}" + + # ── Restore legacy single-file backup (backward compatibility) ── + - name: Restore legacy MySQL backup (pod-0 only) + when: + - mysql_perpod_backup_dir | trim != 'none' + - mysql_backup_is_legacy | trim | bool + block: + - name: Display legacy MySQL restore starting + ansible.builtin.debug: + msg: "{{ msg_mysql_legacy_restore_start }}" + + - name: Restore legacy encrypted backup to pod-0 + ansible.builtin.shell: + cmd: | + set -o pipefail + openssl enc -d -aes-256-cbc -pbkdf2 -iter 100000 \ + -in {{ mysql_perpod_backup_dir | trim }}/idrac_telemetry_mysql_backup.sql.enc \ + -pass pass:'{{ mysql_root_pw.stdout }}' \ + | kubectl exec -i idrac-telemetry-0 -n {{ telemetry_namespace }} -c mysqldb -- \ + mysql -u root -p'{{ mysql_root_pw.stdout }}' idrac_telemetrydb + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_legacy_restore + changed_when: true + failed_when: false + no_log: true + + - name: Report legacy restore result + ansible.builtin.debug: + msg: "{{ msg_mysql_legacy_restore_result }}" + + # ── No backup found ── + - name: Log that no backup was found + ansible.builtin.debug: + msg: "{{ msg_mysql_no_backup_found }}" + when: mysql_perpod_backup_dir | trim == 'none' + + # Verify MySQL is healthy on all pods + - name: Verify MySQL tables after recovery on each pod + ansible.builtin.shell: + cmd: > + kubectl exec idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} -c mysqldb -- + mysql -u root -p'{{ mysql_root_pw.stdout }}' -e 'SHOW TABLES FROM idrac_telemetrydb;' + 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: tables_after_restore_all + changed_when: false + no_log: true + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Verify all containers are running on each pod + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get pod idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} --no-headers \ + -o custom-columns=READY:.status.containerStatuses[*].ready | tr ',' '\n' | grep -c 'true' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: ready_containers_all + changed_when: false + failed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Display MySQL recovery success + ansible.builtin.debug: + msg: + - "============================================================" + - "MySQL VERSION DOWNGRADE — RECOVERY COMPLETED" + - "============================================================" + - "MySQL image: {{ restored_mysql_image.stdout }}" + - "Replicas: {{ idrac_telemetry_original_replicas | default(1) }}" + - "Per-pod backup used: {{ mysql_backup_is_perpod | trim }}" + - "Legacy backup used: {{ mysql_backup_is_legacy | trim }}" + - "Pod recovery details:" + - "{% for idx in idrac_telemetry_pod_indices %}\ + pod-{{ idx }}: \ + ready_containers={{ (ready_containers_all.results[idx].stdout | default('0')) | trim }}/5, \ + tables={{ (tables_after_restore_all.results[idx].stdout_lines | default([])) | length }}\ + {{ ', ' if not loop.last else '' }}\ + {% endfor %}" + - "============================================================" + + # ── No downgrade detected — MySQL is healthy ── + - name: MySQL version compatible — no restore needed + ansible.builtin.debug: + msg: "{{ msg_mysql_no_restore_needed }}" + when: + - idrac_sts_check.rc == 0 + - not (mysql_downgrade_detected | default(false) | bool) diff --git a/rollback/roles/rollback_k8s/vars/main.yml b/rollback/roles/rollback_k8s/vars/main.yml index 0892fa9041..3eb34ec565 100644 --- a/rollback/roles/rollback_k8s/vars/main.yml +++ b/rollback/roles/rollback_k8s/vars/main.yml @@ -28,6 +28,14 @@ repo_file_path: "/etc/yum.repos.d/omnia-upgrade.repo" nfs_storage_name: "nfs_k8s" +# Telemetry namespace (used by restore_mysql_data.yml) +telemetry_namespace: telemetry + +# MySQL backup filename prefix (must match upgrade role's naming convention) +# Per-pod backups are named: .sql.enc +# e.g., idrac_telemetry_mysql_backup_pod0.sql.enc +mysql_backup_filename_prefix: "idrac_telemetry_mysql_backup_pod" + # nodes.yaml location (created by discovery) nodes_yaml_path: "/opt/omnia/openchami/workdir/nodes/nodes.yaml" @@ -152,3 +160,57 @@ msg_bss_backup_ci_missing: >- BSS restore failed: cloud-init/ directory not found in openchami backup workdir {{ _bss_backup_workdir_container }}/cloud-init. The backup may be incomplete or corrupted. + +# MySQL restore messages +msg_mysql_sts_not_found: >- + idrac-telemetry StatefulSet not found — skipping MySQL data restore. +msg_mysql_pod_status: | + idrac-telemetry pods status: + {{ idrac_pod_raw_status.stdout_lines | default(['unknown']) }} +msg_mysql_downgrade_fix_start: | + ============================================================ + MySQL VERSION DOWNGRADE DETECTED — STARTING PVC RECOVERY + ============================================================ + Current version: {{ restored_mysql_image.stdout }} + Expected version: {{ current_mysql_version.stdout }} + The rollback has downgraded the MySQL container image, but the + PVC contains data from a newer MySQL version. This is incompatible. + We will wipe the PVC data and restore from mysqldump backup. + ============================================================ +msg_mysql_replica_count: >- + idrac-telemetry has {{ idrac_telemetry_original_replicas }} replica(s) — will wipe and restore ALL PVCs +msg_mysql_backup_search_results: | + Backup directory: {{ mysql_perpod_backup_dir | trim }} + Per-pod backup format: {{ mysql_backup_is_perpod | trim }} + Legacy single-file format: {{ mysql_backup_is_legacy | trim }} +msg_mysql_perpod_restore_start: >- + Restoring per-pod MySQL data from: {{ mysql_perpod_backup_dir | trim }} +msg_mysql_perpod_restore_result: >- + Pod idrac-telemetry-{{ item.item }}: + {{ 'RESTORED' if item.rc == 0 else 'FAILED (rc=' ~ item.rc ~ ')' }} +msg_mysql_legacy_restore_start: | + Found legacy single-file backup (pre multi-pod support). + Restoring to pod-0 only. Other pods will reinitialize fresh. + Legacy backup: >- + {{ (mysql_legacy_enc_backup.files + mysql_legacy_enc_archive.files) + | sort(attribute='mtime', reverse=true) + | first + | default({'path': 'none'}) + | json_query('path') }} +msg_mysql_legacy_restore_result: | + Legacy backup restore to pod-0: {{ 'SUCCESS' if mysql_legacy_restore.rc == 0 else 'FAILED (rc=' ~ mysql_legacy_restore.rc ~ ')' }} + WARNING: Pods 1+ have no backup — MySQL reinitialized fresh with empty database. +msg_mysql_no_backup_found: | + No pre-upgrade mysqldump backup found. + ALL pods have been reinitialized fresh at {{ restored_mysql_image.stdout }}. + The database tables exist but will be empty. + You may need to manually repopulate data (e.g., iDRAC services list). +msg_mysql_recovery_success: | + ============================================================ + MySQL VERSION DOWNGRADE — RECOVERY COMPLETED + ============================================================ + All idrac-telemetry pods ({{ idrac_telemetry_original_replicas }}) are ready. + MySQL data has been restored from pre-upgrade backup. + ============================================================ +msg_mysql_no_restore_needed: >- + idrac-telemetry MySQL container is running normally — no version downgrade fix needed. diff --git a/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 b/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 index c80dbdde65..32133b2c00 100644 --- a/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 +++ b/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 @@ -159,11 +159,11 @@ idrac_telemetry_storage: mysqldb: resources: requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "500m" + cpu: "250m" memory: "512Mi" + limits: + cpu: "1000m" + memory: "1Gi" activemq: resources: requests: diff --git a/upgrade/roles/upgrade_telemetry/tasks/backup_mysql_data.yml b/upgrade/roles/upgrade_telemetry/tasks/backup_mysql_data.yml new file mode 100644 index 0000000000..17ba6fbed4 --- /dev/null +++ b/upgrade/roles/upgrade_telemetry/tasks/backup_mysql_data.yml @@ -0,0 +1,244 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================ +# Backup MySQL Data (Pre-Upgrade) +# ============================================================================ +# Performs a logical backup (mysqldump) of the idrac-telemetry MySQL database +# for ALL pods before the upgrade changes the MySQL version. Each pod has its +# own independent MySQL instance with its own PVC, so backups are taken +# per-pod to ensure no data loss. +# +# Why this is needed: +# MySQL only allows downgrades between patch versions (e.g., 9.3.1 -> 9.3.0). +# Minor/major version downgrades (e.g., 9.7.2 -> 9.3.0) are rejected because +# MySQL upgrades the internal data format on disk. When etcd rollback restores +# the old StatefulSet spec (with the old MySQL image), the new-format data on +# the PVC causes MySQL to crash with: +# [ERROR] [MY-014061] Cannot downgrade from 90702 to 90300. +# +# This task creates per-pod mysqldumps that can be restored into fresh MySQL +# instances at the old version, preserving all schema and data for every pod. +# +# NOTE: Failures in this task do NOT block the upgrade. The rollback playbook +# can handle the downgrade even without a backup (by wiping the PVC and +# letting MySQL reinitialize), but data will be lost in that case. +# ============================================================================ + +- name: Backup idrac-telemetry MySQL data (pre-upgrade) + when: + - kube_vip is defined + - kube_vip | length > 0 + - k8s_client_mount_path is defined + block: + # ── Check if idrac-telemetry StatefulSet exists ── + - name: Check if idrac-telemetry StatefulSet exists + ansible.builtin.command: + cmd: kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} --no-headers + delegate_to: "{{ kube_vip }}" + connection: ssh + register: idrac_sts_exists + changed_when: false + failed_when: false + + - name: Skip MySQL backup if idrac-telemetry not deployed + ansible.builtin.debug: + msg: "{{ mysql_backup_skip_msg }}" + when: idrac_sts_exists.rc != 0 + + # ── Perform MySQL backup for all pods ── + - name: Perform MySQL logical backup + when: idrac_sts_exists.rc == 0 + block: + - name: Create MySQL backup directory on NFS + ansible.builtin.file: + path: "{{ mysql_backup_dir }}" + state: directory + mode: '0755' + delegate_to: "{{ kube_vip }}" + connection: ssh + + # Get the MySQL root password from the K8s secret + - name: Get MySQL root password from secret + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get secret {{ mysqldb_secrets_name }} -n {{ telemetry_namespace }} \ + -o jsonpath='{.data.mysqldb_root_password}' | base64 -d + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_root_password + changed_when: false + no_log: true + + # Get the current MySQL image version (for metadata) + - name: Get current MySQL image version + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.template.spec.containers[?(@.name=="mysqldb")].image}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: current_mysql_image + changed_when: false + + # Discover all running idrac-telemetry pods + - name: Get current replica count from StatefulSet + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.replicas}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_backup_replica_count_raw + changed_when: false + + - name: Set replica count for backup + ansible.builtin.set_fact: + mysql_backup_replica_count: "{{ mysql_backup_replica_count_raw.stdout | default('1') | int }}" + + - name: Build list of pod names for backup + ansible.builtin.set_fact: + mysql_backup_pod_list: >- + {{ range(0, mysql_backup_replica_count | int) + | map('regex_replace', '^(.*)$', 'idrac-telemetry-\1') + | list }} + + - name: Display pods to backup + ansible.builtin.debug: + msg: "{{ mysql_backup_pods_discovered_msg }}" + + # Wait for MySQL to be ready on each pod before dumping + - name: Wait for MySQL container to be ready on each pod + ansible.builtin.shell: + cmd: | + kubectl exec {{ item }} -n {{ telemetry_namespace }} -c mysqldb -- \ + mysqladmin ping -u root -p'{{ mysql_root_password.stdout }}' 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_ping_results + until: mysql_ping_results.rc == 0 + retries: 12 + delay: 10 + changed_when: false + no_log: true + loop: "{{ mysql_backup_pod_list }}" + + # Perform mysqldump per pod (plaintext, temporary) + - name: Execute mysqldump for each pod + ansible.builtin.shell: + cmd: | + kubectl exec {{ item }} -n {{ telemetry_namespace }} -c mysqldb -- \ + mysqldump -u root -p'{{ mysql_root_password.stdout }}' \ + --single-transaction --routines --triggers --events \ + --set-gtid-purged=OFF \ + idrac_telemetrydb 2>/dev/null \ + > {{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ loop_index0 }}.sql + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysqldump_results + changed_when: true + no_log: true + loop: "{{ mysql_backup_pod_list }}" + loop_control: + index_var: loop_index0 + + # Verify and encrypt each dump file + - name: Verify plaintext mysqldump files were created + ansible.builtin.stat: + path: "{{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item }}.sql" + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysqldump_file_stats + loop: "{{ range(0, mysql_backup_replica_count | int) | list }}" + + - name: Warn about empty or missing mysqldump files + ansible.builtin.debug: + msg: "{{ mysql_backup_failed_msg }} Pod index: {{ item.item }}" + when: not (item.stat.exists | default(false)) or (item.stat.size | default(0)) < 100 + loop: "{{ mysqldump_file_stats.results }}" + loop_control: + label: "pod-{{ item.item }}" + + # Encrypt each dump file + - name: Encrypt mysqldump backup files + ansible.builtin.shell: + cmd: | + openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \ + -in {{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item.item }}.sql \ + -out {{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item.item }}.sql.enc \ + -pass pass:'{{ mysql_root_password.stdout }}' + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + no_log: true + when: (item.stat.exists | default(false)) and (item.stat.size | default(0)) >= 100 + loop: "{{ mysqldump_file_stats.results }}" + loop_control: + label: "pod-{{ item.item }}" + + - name: Remove plaintext mysqldump files + ansible.builtin.file: + path: "{{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item.item }}.sql" + state: absent + delegate_to: "{{ kube_vip }}" + connection: ssh + when: (item.stat.exists | default(false)) and (item.stat.size | default(0)) >= 100 + loop: "{{ mysqldump_file_stats.results }}" + loop_control: + label: "pod-{{ item.item }}" + + - name: Verify encrypted backup files exist + ansible.builtin.stat: + path: "{{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item }}.sql.enc" + delegate_to: "{{ kube_vip }}" + connection: ssh + register: encrypted_backup_file_stats + loop: "{{ range(0, mysql_backup_replica_count | int) | list }}" + + - name: Count successful backups + ansible.builtin.set_fact: + mysql_backup_success_count: >- + {{ encrypted_backup_file_stats.results + | selectattr('stat.exists', 'equalto', true) + | list | length }} + + # Save metadata about the backup + - name: Save MySQL backup metadata + ansible.builtin.copy: + content: | + # MySQL backup metadata (pre-upgrade) + backup_timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + mysql_image: "{{ current_mysql_image.stdout }}" + database: "idrac_telemetrydb" + replica_count: {{ mysql_backup_replica_count }} + encrypted: true + backup_filename_prefix: "{{ mysql_backup_filename_prefix }}" + pods_backed_up: {{ mysql_backup_pod_list }} + successful_backups: {{ mysql_backup_success_count }} + dest: "{{ mysql_backup_dir }}/{{ mysql_backup_metadata_filename }}" + mode: '0644' + delegate_to: "{{ kube_vip }}" + connection: ssh + when: mysql_backup_success_count | int > 0 + + - name: Display MySQL backup success + ansible.builtin.debug: + msg: "{{ mysql_backup_all_pods_success_msg }}" + when: mysql_backup_success_count | int > 0 + + rescue: + - name: Display MySQL backup warning (non-fatal) + ansible.builtin.debug: + msg: "{{ mysql_backup_rescue_msg }}" diff --git a/upgrade/roles/upgrade_telemetry/tasks/main.yml b/upgrade/roles/upgrade_telemetry/tasks/main.yml index 68c087306c..1dad6a148a 100644 --- a/upgrade/roles/upgrade_telemetry/tasks/main.yml +++ b/upgrade/roles/upgrade_telemetry/tasks/main.yml @@ -51,6 +51,14 @@ msg: "{{ victoria_upgrade_skipped }}" when: not (victoria_upgrade_needed | default(false) | bool) +# ── Phase 2b: Backup MySQL data before version change ── +- name: Phase 2b - Backup MySQL data (pre-upgrade) + ansible.builtin.include_tasks: backup_mysql_data.yml + when: + - k8s_client_mount_path is defined + - kube_vip is defined + - kube_vip | length > 0 + # ── Phase 3: Execute telemetry.sh to redeploy telemetry stack ── - name: Phase 3 - Execute telemetry.sh to redeploy telemetry stack ansible.builtin.include_tasks: execute_telemetry_sh.yml diff --git a/upgrade/roles/upgrade_telemetry/vars/main.yml b/upgrade/roles/upgrade_telemetry/vars/main.yml index b726a4be10..4e0c238781 100644 --- a/upgrade/roles/upgrade_telemetry/vars/main.yml +++ b/upgrade/roles/upgrade_telemetry/vars/main.yml @@ -186,3 +186,32 @@ idrac_patch_msg: >- MySQL will have enough time to flush on NFS during pod restart. idrac_skip_patch_msg: "idrac-telemetry StatefulSet not found (first deploy). Skipping patch." idrac_replica_restore_msg: "idrac-telemetry scaled back to {{ idrac_replica_count.stdout }} replicas" + +# ============================================================================ +# MYSQL BACKUP VARIABLES (Pre-Upgrade) +# ============================================================================ +mysqldb_secrets_name: "mysqldb-credentials" +mysql_backup_dir: "{{ telemetry_backup_dir }}/mysql" +# Per-pod backup naming: .sql / .sql.enc +# e.g., idrac_telemetry_mysql_backup_pod0.sql.enc, idrac_telemetry_mysql_backup_pod1.sql.enc +mysql_backup_filename_prefix: "idrac_telemetry_mysql_backup_pod" +mysql_backup_metadata_filename: "mysql_backup_metadata.yml" +mysql_backup_skip_msg: "idrac-telemetry StatefulSet not found — skipping MySQL data backup." +mysql_backup_pods_discovered_msg: >- + Discovered {{ mysql_backup_replica_count }} idrac-telemetry pod(s) for backup: + {{ mysql_backup_pod_list | join(', ') }} +mysql_backup_all_pods_success_msg: >- + MySQL data backup completed successfully for {{ mysql_backup_success_count }}/{{ mysql_backup_replica_count }} pod(s). + Backup directory: {{ mysql_backup_dir }} + These per-pod backups will be used by the rollback playbook to restore MySQL data + if a version downgrade makes the PVC data incompatible. +mysql_backup_failed_msg: >- + WARNING: MySQL data backup FAILED — mysqldump file is empty or missing. + The upgrade will proceed but rollback may not be able to restore MySQL data + if the MySQL version changes. Check pod logs for MySQL connectivity issues. +mysql_backup_rescue_msg: + - "WARNING: MySQL data backup failed — this is non-fatal." + - "The upgrade will proceed. If rollback is needed later, MySQL" + - "will be reinitialized fresh (empty database with auto-created" + - "schema from MYSQL_DATABASE env var in the StatefulSet)." + - "Error: {{ ansible_failed_result.msg | default('unknown') }}" From 086ee1411ea7fc925a90f19e5f02697ee656bffd Mon Sep 17 00:00:00 2001 From: "balajikumaran.cs" Date: Wed, 12 Aug 2026 16:10:16 +0530 Subject: [PATCH 22/22] Remove subnet validation for aarch64_inventory_host_ip in build_stream_config (#4977) The subnet membership check for aarch64_inventory_host_ip only validated against the primary admin subnet from network_spec.yml, ignoring additional_subnets. In multi-subnet environments, this incorrectly rejected valid IPs on additional subnets (e.g., 192.168.3.48 on 192.168.3.0/24 was rejected because it wasn't in the primary 192.168.0.0/24 subnet). Removed the subnet check entirely. IPv4 format validation and SSH reachability checks are retained, which provide sufficient validation. Signed-off-by: balajikumaran-c-s --- .../common_utils/en_us_validation_msg.py | 5 ----- .../validation_flows/build_stream_validation.py | 13 ------------- 2 files changed, 18 deletions(-) diff --git a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py index e64d68762e..559a1619c6 100644 --- a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py +++ b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py @@ -702,11 +702,6 @@ def tls_ext_fail_msg(valid_extensions): "build_stream_config.yml file is empty or has syntax errors. " "It must contain valid YAML with 'enable_build_stream' field." ) -AARCH64_INVENTORY_HOST_IP_INVALID_SUBNET_MSG = ( - "Field 'aarch64_inventory_host_ip' must be in the same subnet as OIM admin IP. " - "Check network_spec.yml for admin network configuration." -) - AARCH64_INVENTORY_HOST_IP_REQUIRED_MSG = ( "Field 'aarch64_inventory_host_ip' is required when PXE mapping file " "contains aarch64 functional groups. Provide the admin IP of the " diff --git a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py index 350073dd37..efc02949db 100644 --- a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py @@ -273,19 +273,6 @@ def validate_build_stream_config(input_file_path, data, "Invalid IPv4 address format")) return errors - # Check if it's in the same subnet as admin IP - try: - admin_network = ipaddress.IPv4Network(f"{admin_ip}/{netmask_bits}", strict=False) - - if aarch64_ip not in admin_network: - errors.append(create_error_msg( - build_stream_yml, - "aarch64_inventory_host_ip", - msg.AARCH64_INVENTORY_HOST_IP_INVALID_SUBNET_MSG - )) - except ValueError as e: - logger.error("Failed to validate subnet for aarch64_inventory_host_ip: %s", str(e)) - # Check aarch64 host IP reachability using socket (safer than subprocess) try: # Try to connect to SSH port which is usually open on inventory hosts