diff --git a/.gitattributes b/.gitattributes index 873288d963..a065796c70 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ *.yml linguist-detectable *.tar.gz filter=lfs diff=lfs merge=lfs -text +prepare_oim/roles/deploy_containers/openchami/files/*.tar.gz filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index aea0698715..58fe64e4d5 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -6,11 +6,7 @@ on: - main - staging - release_1.7.1 - - pub/build_stream - - pub/q2_dev - - pub/telemetry - - pub/q2_upgrade - - pub/q2_ansible + - pub/omnia_2.2.0.0_fix jobs: build: diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 3aaded93be..f7351452e5 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -6,11 +6,7 @@ on: - main - staging - release_1.7.1 - - pub/build_stream - - pub/q2_dev - - pub/telemetry - - pub/q2_upgrade - - pub/q2_ansible + - pub/omnia_2.2.0.0_fix jobs: build: diff --git a/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 b/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 index 9068f0ca6e..77de0a79c3 100644 --- a/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 +++ b/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 @@ -13,7 +13,7 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' -{% if repo.priority is defined %} +{% if repo.priority is defined and repo.priority | string | length > 0 %} priority: '{{ repo.priority }}' {% endif %} {% endif %} diff --git a/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 b/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 index a078a0a134..48fe492336 100644 --- a/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 +++ b/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 @@ -18,7 +18,7 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' -{% if repo.priority is defined %} +{% if repo.priority is defined and repo.priority | string | length > 0 %} priority: '{{ repo.priority }}' {% endif %} {% endif %} diff --git a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py index e64d68762e..559a1619c6 100644 --- a/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py +++ b/common/library/module_utils/input_validation/common_utils/en_us_validation_msg.py @@ -702,11 +702,6 @@ def tls_ext_fail_msg(valid_extensions): "build_stream_config.yml file is empty or has syntax errors. " "It must contain valid YAML with 'enable_build_stream' field." ) -AARCH64_INVENTORY_HOST_IP_INVALID_SUBNET_MSG = ( - "Field 'aarch64_inventory_host_ip' must be in the same subnet as OIM admin IP. " - "Check network_spec.yml for admin network configuration." -) - AARCH64_INVENTORY_HOST_IP_REQUIRED_MSG = ( "Field 'aarch64_inventory_host_ip' is required when PXE mapping file " "contains aarch64 functional groups. Provide the admin IP of the " diff --git a/common/library/module_utils/input_validation/schema/local_repo_config.json b/common/library/module_utils/input_validation/schema/local_repo_config.json index a89c648105..1d8f472837 100644 --- a/common/library/module_utils/input_validation/schema/local_repo_config.json +++ b/common/library/module_utils/input_validation/schema/local_repo_config.json @@ -83,7 +83,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -219,7 +220,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -355,7 +357,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -491,7 +494,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -644,7 +648,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "policy": { @@ -780,7 +785,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "policy": { @@ -898,7 +904,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "sslcacert": { @@ -1027,7 +1034,8 @@ }, "name": { "type": "string", - "pattern": "^(?!\\s*$).+", + "pattern": "^[A-Za-z0-9._-]+$", + "maxLength": 64, "minLength": 1 }, "sslcacert": { @@ -1157,7 +1165,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", @@ -1215,7 +1224,8 @@ "name": { "type": "string", "minLength": 1, - "pattern": "^(?!\\s*$).+" + "maxLength": 64, + "pattern": "^[A-Za-z0-9._-]+$" }, "policy": { "type": "string", diff --git a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py index 350073dd37..efc02949db 100644 --- a/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/build_stream_validation.py @@ -273,19 +273,6 @@ def validate_build_stream_config(input_file_path, data, "Invalid IPv4 address format")) return errors - # Check if it's in the same subnet as admin IP - try: - admin_network = ipaddress.IPv4Network(f"{admin_ip}/{netmask_bits}", strict=False) - - if aarch64_ip not in admin_network: - errors.append(create_error_msg( - build_stream_yml, - "aarch64_inventory_host_ip", - msg.AARCH64_INVENTORY_HOST_IP_INVALID_SUBNET_MSG - )) - except ValueError as e: - logger.error("Failed to validate subnet for aarch64_inventory_host_ip: %s", str(e)) - # Check aarch64 host IP reachability using socket (safer than subprocess) try: # Try to connect to SSH port which is usually open on inventory hosts diff --git a/common/library/module_utils/local_repo/parse_and_download.py b/common/library/module_utils/local_repo/parse_and_download.py index f9e605cbe8..1a661f8d7e 100644 --- a/common/library/module_utils/local_repo/parse_and_download.py +++ b/common/library/module_utils/local_repo/parse_and_download.py @@ -21,6 +21,7 @@ import os import subprocess +import shlex import json import re from multiprocessing import Lock @@ -57,12 +58,13 @@ def execute_command(cmd_string, logger, type_json=False): logger.info(f"Executing command: {safe_cmd_string}") # Run the command + cmd_list = shlex.split(cmd_string) cmd = subprocess.run( - cmd_string, + cmd_list, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True, + shell=False, ) status["returncode"] = cmd.returncode status["stdout"] = cmd.stdout.strip() if cmd.stdout else None diff --git a/common/library/modules/pulp_cleanup.py b/common/library/modules/pulp_cleanup.py index 1d076e7561..4c03a62fc0 100644 --- a/common/library/modules/pulp_cleanup.py +++ b/common/library/modules/pulp_cleanup.py @@ -29,6 +29,7 @@ import json import shutil import subprocess +import shlex import re import yaml from typing import Dict, List, Any, Tuple @@ -92,7 +93,8 @@ def format_pretty_table(results: List[Dict[str, Any]]) -> str: def run_cmd(cmd: str, logger) -> Dict[str, Any]: """Execute shell command and return result.""" try: - result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=300) + cmd_list = shlex.split(cmd) + result = subprocess.run(cmd_list, shell=False, capture_output=True, text=True, timeout=300) return {"rc": result.returncode, "stdout": result.stdout, "stderr": result.stderr} except (subprocess.SubprocessError, OSError) as e: logger.error(f"Command failed: {cmd} - {e}") diff --git a/common/library/modules/pulp_pgsql_backup_restore.py b/common/library/modules/pulp_pgsql_backup_restore.py index b35aeccdc0..a5c51158f9 100644 --- a/common/library/modules/pulp_pgsql_backup_restore.py +++ b/common/library/modules/pulp_pgsql_backup_restore.py @@ -20,7 +20,7 @@ Handles the PostgreSQL version incompatibility between Pulp versions: - Pulp 3.80 uses PostgreSQL 12/13 - - Pulp 3.113 uses PostgreSQL 16 + - Pulp 3.114.2 uses PostgreSQL 16 Backup (action=backup): - Validates source PostgreSQL data exists diff --git a/common/vars/image_vars.yml b/common/vars/image_vars.yml index 97c07d9a75..a7ad163749 100644 --- a/common/vars/image_vars.yml +++ b/common/vars/image_vars.yml @@ -16,10 +16,10 @@ # Usage: get_container_image_list.yml container_tag: "latest" squid_tag: "6.6-24.04_beta" -pulp_tag: "3.113" -mysql_tag: "9.3.0" +pulp_tag: "3.114.2" +mysql_tag: "9.7.2" prometheus_tag: "v3.4.1" -activemq_tag: "5.19.7" +activemq_tag: "6.3.0" grafana_image_tag: "12.0.1" loki_image_tag: "3.5.1" promtail_image_tag: "3.5.1" diff --git a/examples/catalog/catalog_rhel.json b/examples/catalog/catalog_rhel.json index d5f5c5dd70..7244b3887a 100644 --- a/examples/catalog/catalog_rhel.json +++ b/examples/catalog/catalog_rhel.json @@ -126,6 +126,7 @@ "quay.io/jetstack/cert-manager-cainjector", "quay.io/jetstack/cert-manager-controller", "quay.io/jetstack/cert-manager-webhook", + "quay.io/metallb/controller", "quay.io/metallb/speaker", "quay.io/strimzi/kafka", "quay.io/strimzi/kafka-bridge", @@ -433,7 +434,7 @@ ] }, "calico": { - "Name": "calico-v3.31.4", + "Name": "calico-v3.32.1", "SupportedOS": [ { "Name": "RHEL", @@ -447,7 +448,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" + "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" } ] }, @@ -609,8 +610,8 @@ "x86_64" ], "Type": "image", - "Tag": "5.19.7", - "Version": "5.19.7" + "Tag": "6.3.0", + "Version": "6.3.0" }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", @@ -624,8 +625,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/kube-controllers": { "Name": "docker.io/calico/kube-controllers", @@ -639,8 +640,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/node": { "Name": "docker.io/calico/node", @@ -654,8 +655,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/curlimages/curl": { "Name": "docker.io/curlimages/curl", @@ -759,8 +760,8 @@ "x86_64" ], "Type": "image", - "Tag": "9.3.0", - "Version": "9.3.0" + "Tag": "9.7.2", + "Version": "9.7.2" }, "docker.io/library/python": { "Name": "docker.io/library/python", @@ -864,8 +865,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vlagent": { "Name": "docker.io/victoriametrics/vlagent", @@ -894,8 +895,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vminsert": { "Name": "docker.io/victoriametrics/vminsert", @@ -909,8 +910,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmselect": { "Name": "docker.io/victoriametrics/vmselect", @@ -924,8 +925,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmstorage": { "Name": "docker.io/victoriametrics/vmstorage", @@ -939,8 +940,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "firewalld": { "Name": "firewalld", @@ -1021,8 +1022,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.8.9", - "Version": "v0.8.9" + "Tag": "v1.2.2", + "Version": "v1.2.2" }, "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector": { "Name": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector", @@ -1356,7 +1357,7 @@ ] }, "metallb-native": { - "Name": "metallb-native-v0.15.3", + "Name": "metallb-native-v0.16.1", "SupportedOS": [ { "Name": "RHEL", @@ -1370,7 +1371,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" + "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" } ] }, @@ -1832,8 +1833,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/metallb/speaker": { "Name": "quay.io/metallb/speaker", @@ -1847,8 +1848,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/strimzi/kafka": { "Name": "quay.io/strimzi/kafka", diff --git a/examples/catalog/catalog_rhel_with_nfs_provisioner.json b/examples/catalog/catalog_rhel_with_nfs_provisioner.json index 665438cf9a..da24a6d561 100644 --- a/examples/catalog/catalog_rhel_with_nfs_provisioner.json +++ b/examples/catalog/catalog_rhel_with_nfs_provisioner.json @@ -117,6 +117,7 @@ "quay.io/jetstack/cert-manager-cainjector", "quay.io/jetstack/cert-manager-controller", "quay.io/jetstack/cert-manager-webhook", + "quay.io/metallb/controller", "quay.io/metallb/speaker", "quay.io/strimzi/kafka", "quay.io/strimzi/kafka-bridge", @@ -402,7 +403,7 @@ ] }, "calico": { - "Name": "calico-v3.31.4", + "Name": "calico-v3.32.1", "SupportedOS": [ { "Name": "RHEL", @@ -416,7 +417,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" + "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" } ] }, @@ -578,8 +579,8 @@ "x86_64" ], "Type": "image", - "Tag": "5.19.7", - "Version": "5.19.7" + "Tag": "6.3.0", + "Version": "6.3.0" }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", @@ -593,8 +594,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/kube-controllers": { "Name": "docker.io/calico/kube-controllers", @@ -608,8 +609,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/node": { "Name": "docker.io/calico/node", @@ -623,8 +624,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/curlimages/curl": { "Name": "docker.io/curlimages/curl", @@ -728,8 +729,8 @@ "x86_64" ], "Type": "image", - "Tag": "9.3.0", - "Version": "9.3.0" + "Tag": "9.7.2", + "Version": "9.7.2" }, "docker.io/library/python": { "Name": "docker.io/library/python", @@ -833,8 +834,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vlagent": { "Name": "docker.io/victoriametrics/vlagent", @@ -863,8 +864,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vminsert": { "Name": "docker.io/victoriametrics/vminsert", @@ -878,8 +879,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmselect": { "Name": "docker.io/victoriametrics/vmselect", @@ -893,8 +894,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmstorage": { "Name": "docker.io/victoriametrics/vmstorage", @@ -908,8 +909,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "firewalld": { "Name": "firewalld", @@ -990,8 +991,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.8.9", - "Version": "v0.8.9" + "Tag": "v1.2.2", + "Version": "v1.2.2" }, "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector": { "Name": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector", @@ -1325,7 +1326,7 @@ ] }, "metallb-native": { - "Name": "metallb-native-v0.15.3", + "Name": "metallb-native-v0.16.1", "SupportedOS": [ { "Name": "RHEL", @@ -1339,7 +1340,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" + "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" } ] }, @@ -1705,8 +1706,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/metallb/speaker": { "Name": "quay.io/metallb/speaker", @@ -1720,8 +1721,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/strimzi/kafka": { "Name": "quay.io/strimzi/kafka", diff --git a/examples/catalog/catalog_rhel_x86_64.json b/examples/catalog/catalog_rhel_x86_64.json index 9f34bfc706..2f1733e116 100644 --- a/examples/catalog/catalog_rhel_x86_64.json +++ b/examples/catalog/catalog_rhel_x86_64.json @@ -127,6 +127,7 @@ "quay.io/jetstack/cert-manager-cainjector", "quay.io/jetstack/cert-manager-controller", "quay.io/jetstack/cert-manager-webhook", + "quay.io/metallb/controller", "quay.io/metallb/speaker", "quay.io/strimzi/kafka", "quay.io/strimzi/kafka-bridge", @@ -429,7 +430,7 @@ ] }, "calico": { - "Name": "calico-v3.31.4", + "Name": "calico-v3.32.1", "SupportedOS": [ { "Name": "RHEL", @@ -443,7 +444,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" + "Uri": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" } ] }, @@ -595,8 +596,8 @@ "x86_64" ], "Type": "image", - "Tag": "5.19.7", - "Version": "5.19.7" + "Tag": "6.3.0", + "Version": "6.3.0" }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", @@ -610,8 +611,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/kube-controllers": { "Name": "docker.io/calico/kube-controllers", @@ -625,8 +626,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/calico/node": { "Name": "docker.io/calico/node", @@ -640,8 +641,8 @@ "x86_64" ], "Type": "image", - "Tag": "v3.31.4", - "Version": "v3.31.4" + "Tag": "v3.32.1", + "Version": "v3.32.1" }, "docker.io/curlimages/curl": { "Name": "docker.io/curlimages/curl", @@ -745,8 +746,8 @@ "x86_64" ], "Type": "image", - "Tag": "9.3.0", - "Version": "9.3.0" + "Tag": "9.7.2", + "Version": "9.7.2" }, "docker.io/library/python": { "Name": "docker.io/library/python", @@ -850,8 +851,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vlagent": { "Name": "docker.io/victoriametrics/vlagent", @@ -880,8 +881,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0", - "Version": "v1.128.0" + "Tag": "v1.149.0", + "Version": "v1.149.0" }, "docker.io/victoriametrics/vminsert": { "Name": "docker.io/victoriametrics/vminsert", @@ -895,8 +896,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmselect": { "Name": "docker.io/victoriametrics/vmselect", @@ -910,8 +911,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "docker.io/victoriametrics/vmstorage": { "Name": "docker.io/victoriametrics/vmstorage", @@ -925,8 +926,8 @@ "x86_64" ], "Type": "image", - "Tag": "v1.128.0-cluster", - "Version": "v1.128.0-cluster" + "Tag": "v1.149.0-cluster", + "Version": "v1.149.0-cluster" }, "firewalld": { "Name": "firewalld", @@ -997,8 +998,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.8.9", - "Version": "v0.8.9" + "Tag": "v1.2.2", + "Version": "v1.2.2" }, "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector": { "Name": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector", @@ -1297,7 +1298,7 @@ ] }, "metallb-native": { - "Name": "metallb-native-v0.15.3", + "Name": "metallb-native-v0.16.1", "SupportedOS": [ { "Name": "RHEL", @@ -1311,7 +1312,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" + "Uri": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" } ] }, @@ -1722,8 +1723,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/metallb/speaker": { "Name": "quay.io/metallb/speaker", @@ -1737,8 +1738,8 @@ "x86_64" ], "Type": "image", - "Tag": "v0.15.3", - "Version": "v0.15.3" + "Tag": "v0.16.1", + "Version": "v0.16.1" }, "quay.io/strimzi/kafka": { "Name": "quay.io/strimzi/kafka", diff --git a/gitlab/roles/hosted_gitlab/vars/main.yml b/gitlab/roles/hosted_gitlab/vars/main.yml index 693efa38f2..a5797d9e15 100644 --- a/gitlab/roles/hosted_gitlab/vars/main.yml +++ b/gitlab/roles/hosted_gitlab/vars/main.yml @@ -78,10 +78,10 @@ gitlab_initial_root_password_path: "/etc/gitlab/initial_root_password" gitlab_root_token_file_path: "/root/.gitlab_root_token" # Runner container -gitlab_runner_image: "docker.io/gitlab/gitlab-runner:v18.8.0" +gitlab_runner_image: "docker.io/gitlab/gitlab-runner:v19.2.0" gitlab_runner_default_image: "docker.io/library/alpine:3.23.3" gitlab_runner_helper_image_registry: "registry.gitlab.com/gitlab-org/gitlab-runner/gitlab-runner-helper" -gitlab_runner_helper_image_version: "v18.8.0" +gitlab_runner_helper_image_version: "v19.2.0" gitlab_runner_container_name: "gitlab-runner" gitlab_restart_policy: "always" gitlab_runner_description: "Omnia Hosted Runner" diff --git a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json index b120abd367..981ae4fe32 100644 --- a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json +++ b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json @@ -13,17 +13,17 @@ { "package": "kubelet-1.35.1", "type": "rpm", "repo_name": "kubernetes-v1-35"}, { "package": "container-selinux", "type": "rpm", "repo_name": "appstream"}, { "package": "cri-o-1.35.1", "type": "rpm", "repo_name": "cri-o-v1-35"}, - { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.128.0" }, - { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.128.0" }, - { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.128.0-cluster" }, - { "package": "docker.io/victoriametrics/vminsert", "type": "image", "tag": "v1.128.0-cluster" }, - { "package": "docker.io/victoriametrics/vmselect", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.149.0" }, + { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.149.0" }, + { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.149.0-cluster" }, + { "package": "docker.io/victoriametrics/vminsert", "type": "image", "tag": "v1.149.0-cluster" }, + { "package": "docker.io/victoriametrics/vmselect", "type": "image", "tag": "v1.149.0-cluster" }, { "package": "docker.io/victoriametrics/victoria-logs", "type": "image", "tag": "v1.50.0" }, { "package": "docker.io/victoriametrics/vlagent", "type": "image", "tag": "v1.50.0" }, { "package": "docker.io/alpine/kubectl", "tag": "1.35.1", "type": "image" }, { "package": "docker.io/curlimages/curl", "type": "image", "tag": "8.17.0" }, - { "package": "docker.io/apache/activemq", "type": "image", "tag": "5.19.7" }, - { "package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0" }, + { "package": "docker.io/apache/activemq", "type": "image", "tag": "6.3.0" }, + { "package": "docker.io/library/mysql", "type": "image", "tag": "9.7.2" }, { "package": "docker.io/library/python", "type": "image", "tag": "3.12-slim" }, { "package": "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "type": "image", "tag": "1.3" }, { "package": "docker.io/dellhpcomniaaisolution/kafkapump", "type": "image", "tag": "1.3" }, @@ -62,7 +62,7 @@ }, "service_kube_control_plane": { "cluster": [ - { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v1.2.2", "type": "image" }, { "package": "docker.io/alpine/kubectl", "tag": "1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-apiserver", "tag": "v1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-controller-manager", "tag": "v1.35.1", "type": "image" }, @@ -71,10 +71,10 @@ { "package": "registry.k8s.io/coredns/coredns", "tag": "v1.13.1", "type": "image" }, { "package": "registry.k8s.io/pause", "tag": "3.10.1", "type": "image" }, { "package": "registry.k8s.io/etcd", "tag": "3.6.6-0", "type": "image" }, - { "package": "docker.io/calico/cni", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/kube-controllers", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/node", "tag": "v3.31.4", "type": "image" }, - { "package": "quay.io/metallb/speaker", "tag": "v0.15.3", "type": "image" }, + { "package": "docker.io/calico/cni", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/kube-controllers", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/node", "tag": "v3.32.1", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.16.1", "type": "image" }, { "package": "kubectl-1.35.1", "type": "rpm", "repo_name": "kubernetes-v1-35"}, { "package": "prettytable==3.14.0", "type": "pip_module" }, { "package": "python3-3.12.9", "type": "rpm", "repo_name": "baseos" }, @@ -86,7 +86,7 @@ }, "service_kube_control_plane_first": { "cluster": [ - { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v1.2.2", "type": "image" }, { "package": "registry.k8s.io/kube-apiserver", "tag": "v1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-controller-manager", "tag": "v1.35.1", "type": "image" }, { "package": "registry.k8s.io/kube-scheduler", "tag": "v1.35.1", "type": "image" }, @@ -95,12 +95,13 @@ { "package": "docker.io/alpine/kubectl", "tag": "1.35.1", "type": "image" }, { "package": "registry.k8s.io/pause", "tag": "3.10.1", "type": "image" }, { "package": "registry.k8s.io/etcd", "tag": "3.6.6-0", "type": "image" }, - { "package": "docker.io/calico/cni", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/kube-controllers", "tag": "v3.31.4", "type": "image" }, - { "package": "docker.io/calico/node", "tag": "v3.31.4", "type": "image" }, - { "package": "quay.io/metallb/speaker", "tag": "v0.15.3", "type": "image" }, - { "package": "calico-v3.31.4","type": "manifest", "url": "https://raw.githubusercontent.com/projectcalico/calico/v3.31.4/manifests/calico.yaml" }, - { "package": "metallb-native-v0.15.3", "type": "manifest", "url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml" }, + { "package": "docker.io/calico/cni", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/kube-controllers", "tag": "v3.32.1", "type": "image" }, + { "package": "docker.io/calico/node", "tag": "v3.32.1", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.16.1", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.16.1", "type": "image" }, + { "package": "calico-v3.32.1","type": "manifest", "url": "https://raw.githubusercontent.com/projectcalico/calico/v3.32.1/manifests/calico.yaml" }, + { "package": "metallb-native-v0.16.1", "type": "manifest", "url": "https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-native.yaml" }, { "package": "helm-v3.20.1-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.20.1-linux-amd64.tar.gz" }, { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, { "package": "kubectl-1.35.1", "type": "rpm", "repo_name": "kubernetes-v1-35"}, @@ -115,8 +116,8 @@ "service_kube_node": { "cluster": [ { "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" }, - { "package": "quay.io/metallb/speaker", "tag": "v0.15.3", "type": "image" }, - { "package": "quay.io/metallb/controller", "tag": "v0.15.3", "type": "image" } + { "package": "quay.io/metallb/speaker", "tag": "v0.16.1", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.16.1", "type": "image" } ] } } diff --git a/input/local_repo_config.yml b/input/local_repo_config.yml index 9f14b476a6..63f7b6d22b 100644 --- a/input/local_repo_config.yml +++ b/input/local_repo_config.yml @@ -45,6 +45,12 @@ # To achieve proxy mode (on_demand): Set repo_config: partial # OR set policy: always + caching: true per repo #---------------------------------------------------------------------------------- +# REPO PRECEDENCE ORDER (package download priority): +# 1. user_repo_url_ (highest precedence) +# 2. rhel_os_url_ +# 3. omnia_repo_url_ (lowest precedence) +# If a package exists in multiple repos, the higher precedence repo is used. +#---------------------------------------------------------------------------------- # 1. user_registry #-------------------------- diff --git a/input/telemetry_storage_config.yml b/input/telemetry_storage_config.yml index c80dbdde65..32133b2c00 100644 --- a/input/telemetry_storage_config.yml +++ b/input/telemetry_storage_config.yml @@ -159,11 +159,11 @@ idrac_telemetry_storage: mysqldb: resources: requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "500m" + cpu: "250m" memory: "512Mi" + limits: + cpu: "1000m" + memory: "1Gi" activemq: resources: requests: diff --git a/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz new file mode 100644 index 0000000000..b211ae067f --- /dev/null +++ b/prepare_oim/roles/deploy_containers/openchami/files/openchami-0.1.7-1.noarch.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15f8e7e5c6623f6c966a64a7784c0ffd303e4c7bc87613abb81a8cc050005e07 +size 13818 diff --git a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml index de7b998c93..15c4b85a2f 100644 --- a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml +++ b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/ochami.yml @@ -13,15 +13,17 @@ # limitations under the License. --- -- name: Download the openchami RPM file - ansible.builtin.get_url: - url: "{{ openchami_rpm_url }}" - dest: "{{ openchami_work_dir }}/{{ openchami_rpm_name }}" +- name: Copy the openchami RPM tarball to OIM + ansible.builtin.copy: + src: "{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" mode: "{{ file_perm_rwx }}" - register: download_openchami - until: download_openchami is not failed - retries: "{{ max_retries }}" - delay: "{{ max_delay }}" + +- name: Extract the openchami RPM from tarball + ansible.builtin.unarchive: + src: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}" + remote_src: true - name: Install the openchami ansible.builtin.dnf: diff --git a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml index 2c75a6f853..e0bf639dcc 100644 --- a/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml +++ b/prepare_oim/roles/deploy_containers/openchami/tasks/configs/packages.yml @@ -14,9 +14,17 @@ --- - name: Install ochami dependent packages - ansible.builtin.dnf: - name: "{{ ochami_package_dependencies }}" - state: present + block: + - name: Install ochami packages from default repos + ansible.builtin.dnf: + name: "{{ ochami_package_dependencies }}" + state: present + rescue: + - name: Fallback - install ochami packages from Red Hat CDN + ansible.builtin.dnf: + name: "{{ ochami_package_dependencies }}" + state: present + enablerepo: rhel-10-for-x86_64-appstream-rpms - name: Install pip packages ansible.builtin.pip: diff --git a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 index 1cf09db3f9..fece22357b 100644 --- a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 +++ b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 @@ -1,6 +1,3 @@ -{% set coresmd_ver = openchami_coresmd_tag | regex_replace('^v', '') %} -{% set multisubnet_native = coresmd_ver is version('0.6.0', '>=') %} -{% set has_additional_subnets = additional_subnets | default([]) | length > 0 %} server4: listen: - "%{{ cluster_boot_interface }}" @@ -9,8 +6,6 @@ server4: - dns: {{ coredhcp_dns_server }} - router: {{ coredhcp_router }} - netmask: {{ coredhcp_netmask }} -{% if multisubnet_native and has_additional_subnets %} - # Multi-subnet mode: key=value config format (coresmd v0.6.x+) - coresmd: | svc_base_uri=https://{{ cluster_name }}.{{ cluster_domain }}:8443 ipxe_base_uri=http://{{ cluster_boot_ip }}:8081 @@ -18,57 +13,20 @@ server4: cache_valid={{ coredhcp_cache_validity }} lease_time={{ coredhcp_lease_duration }} single_port={{ coredhcp_tftp_single_port_mode | lower }} -{% for s in additional_subnets %} +{% for s in additional_subnets | default([]) %} rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:Node,routers:{{ s.router }},cidr:{{ s.netmask_bits }} rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:NodeBMC,routers:{{ s.router }},cidr:{{ s.netmask_bits }} {% endfor %} - rule=type:Node - rule=type:NodeBMC - rule=hostname:unknown-{{'{'}}04d{{'}'}} + rule=type:Node,hostname:nid{{'{'}}04d{{'}'}} + rule=type:NodeBMC,hostname:bmc{{'{'}}04d{{'}'}} - bootloop: | lease_file=/tmp/coredhcp.db script_path={{ coredhcp_custom_ipxe }} lease_time={{ coredhcp_tmp_lease_duration }} -{% for s in additional_subnets %} +{% set primary_range = network_data.admin_network.dynamic_range | split('-') %} +{% set primary_subnet = (cluster_boot_ip + '/' + network_data.admin_network.netmask_bits | string) | ansible.utils.ipaddr('network') %} + subnet_pool={{ primary_subnet }}/{{ network_data.admin_network.netmask_bits }},{{ primary_range[0] }},{{ primary_range[1] }} +{% for s in additional_subnets | default([]) %} {% set range_parts = s.dynamic_range | split('-') %} subnet_pool={{ s.subnet }}/{{ s.netmask_bits }},{{ range_parts[0] }},{{ range_parts[1] }} {% endfor %} -{% else %} - # Single-subnet mode: positional argument format (coresmd v0.4.x) - - coresmd: https://{{ cluster_name }}.{{ cluster_domain }}:8443 http://{{ cluster_boot_ip }}:8081 /root_ca/root_ca.crt {{ coredhcp_cache_validity }} {{ coredhcp_lease_duration }} {{ coredhcp_tftp_single_port_mode | lower }} - - bootloop: /tmp/coredhcp.db {{ coredhcp_custom_ipxe }} {{ coredhcp_tmp_lease_duration }} {{ coredhcp_dhcp_pool }} -{% if has_additional_subnets %} - # ------------------------------------------------------------------- - # Multi-subnet configuration (requires coresmd v0.6.x+) - # To enable multi-subnet DHCP: - # 1. Pull the new coresmd image: podman pull ghcr.io/openchami/coresmd:v0.6.3 - # 2. Comment out the single-subnet coresmd and bootloop lines above - # 3. Uncomment the multi-subnet coresmd and bootloop blocks below - # 4. Replace the new coresmd image version in files: /etc/containers/systemd/coresmd-coredhcp.container /etc/containers/systemd/coresmd-coredns.container with the old version - # 5. Reload daemon: systemctl daemon-reload - # 6. Restart services: systemctl restart openchami.target - # ------------------------------------------------------------------- - # - coresmd: | - # svc_base_uri=https://{{ cluster_name }}.{{ cluster_domain }}:8443 - # ipxe_base_uri=http://{{ cluster_boot_ip }}:8081 - # ca_cert=/root_ca/root_ca.crt - # cache_valid={{ coredhcp_cache_validity }} - # lease_time={{ coredhcp_lease_duration }} - # single_port={{ coredhcp_tftp_single_port_mode | lower }} -{% for s in additional_subnets %} - # rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:Node,routers:{{ s.router }},cidr:{{ s.netmask_bits }} - # rule=subnet:{{ s.subnet }}/{{ s.netmask_bits }},type:NodeBMC,routers:{{ s.router }},cidr:{{ s.netmask_bits }} -{% endfor %} - # rule=type:Node - # rule=type:NodeBMC - # rule=hostname:unknown-{{'{'}}04d{{'}'}} - # - bootloop: | - # lease_file=/tmp/coredhcp.db - # script_path={{ coredhcp_custom_ipxe }} - # lease_time={{ coredhcp_tmp_lease_duration }} -{% for s in additional_subnets %} -{% set range_parts = s.dynamic_range | split('-') %} - # subnet_pool={{ s.subnet }}/{{ s.netmask_bits }},{{ range_parts[0] }},{{ range_parts[1] }} -{% endfor %} -{% endif %} -{% endif %} diff --git a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml index dfdf99a745..3ff7a43463 100644 --- a/prepare_oim/roles/deploy_containers/openchami/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/openchami/vars/main.yml @@ -39,27 +39,27 @@ pull_image_delay: 10 # OpenCHAMI image tags openchami_local_ca_tag: "v0.2.6" -openchami_opaal_tag: "v0.3.12" -openchami_smd_tag: "v2.19.3" -openchami_bss_tag: "v1.32.2" -openchami_cloud_init_tag: "v1.3.0" -openchami_coresmd_tag: "v0.4.3" +openchami_opaal_tag: "v0.3.14" +openchami_smd_tag: "v2.20.6" +openchami_bss_tag: "v1.32.4" +openchami_cloud_init_tag: "v1.4.9" +openchami_coresmd_tag: "v0.6.3" # Third-party image tags for OpenCHAMI -minio_release_tag: "RELEASE.2026-04-17T00-00-00Z" -postgres_tag: "11.5-alpine" +minio_release_tag: "RELEASE.2026-08-04T00-00-00Z" +postgres_tag: "17-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" -registry_tag: "3.1.0" +registry_tag: "3.1.1" curl_tag: "latest" acme_tag: "3.1.1" # OpenCHAMI images list for podman pull on OIM openchami_images: - "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - - "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" - - "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" - - "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" - - "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + - "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" + - "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" + - "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" + - "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - "docker.io/pgsty/minio:{{ minio_release_tag }}" - "docker.io/library/postgres:{{ postgres_tag }}" @@ -92,8 +92,8 @@ ochami_cli_log_format: rfc3339 ochami_base_url: "https://{{ cluster_name }}.{{ cluster_domain }}:8443" # OpenCHAMI RPM packages -openchami_rpm_url: https://github.com/OpenCHAMI/release/releases/download/v0.1.5/openchami-0.1.5.rpm -openchami_rpm_name: openchami-0.1.5.rpm +openchami_rpm_tarball: openchami-0.1.7-1.noarch.tar.gz +openchami_rpm_name: openchami-0.1.7-1.noarch.rpm ochami_client_rpm_url: https://github.com/OpenCHAMI/ochami/releases/download/v0.7.1/ochami_0.7.1_linux_amd64.rpm ochami_client_rpm_name: ochami_0.7.1_linux_amd64.rpm diff --git a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml index a548bf6ed6..9409ae8f69 100644 --- a/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml +++ b/prepare_oim/roles/deploy_containers/pulp/tasks/create_pulp_config_https.yml @@ -91,6 +91,12 @@ when: not hostvars['oim']['hostname_enabled'] no_log: true + - name: Ensure certs directory exists + ansible.builtin.file: + path: "{{ certs_dir }}" + state: directory + mode: "0755" + - name: Generating Pulp SSL Certificate ansible.builtin.command: cmd: "{{ generate_cert_cmd }}" diff --git a/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml b/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml index 2fe8968a1b..94928ef720 100644 --- a/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml +++ b/prepare_oim/roles/deploy_containers/pulp/tasks/deploy_pulp_container_https.yml @@ -105,7 +105,9 @@ containers.podman.podman_container_exec: name: "{{ pulp_container_name }}" command: "{{ reset_password_cmd }}" - retries: "{{ retries_var }}" + register: pulp_password_reset + until: pulp_password_reset is not failed + retries: "{{ password_reset_retries }}" delay: "{{ delay_var }}" when: pulp_password | length > 1 no_log: true diff --git a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml index f5935b79fb..bd032493a3 100644 --- a/prepare_oim/roles/deploy_containers/pulp/vars/main.yml +++ b/prepare_oim/roles/deploy_containers/pulp/vars/main.yml @@ -25,7 +25,7 @@ device_name: "/dev/fuse:/dev/fuse:rwm" pulp_container_name: "pulp" pulp_protocol_https: true # Tag is fixed for the Pulp container image as of 10-06-2025 -pulp_image: "docker.io/pulp/pulp:3.113" +pulp_image: "docker.io/pulp/pulp:3.114.2" # Usage: deployment_prereq.yml - pull image retries pull_image_retries: 5 @@ -47,6 +47,7 @@ pulp_deployed_fail_msg: Run utility/oim_cleanup.yml to cleanup, then re-run the playbook to deploy the {{ pulp_container_name }} container successfully." retries_var: 8 +password_reset_retries: 15 delay_var: 30 delay_var_sixty: 30 timeout_var: 60 diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 index f1cb099601..045052a05c 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 @@ -237,11 +237,13 @@ value: "3" - name: vip_retryperiod value: "1" - - name: vip_address + - name: address value: {{ kube_vip }} + - name: vip_subnet + value: "{{ admin_netmask_bits }}" - name: prometheus_server value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.8.9 + image: {{ kube_vip_image }} imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -306,7 +308,7 @@ content: | #!/bin/bash set -e - HELM_VERSION="v3.20.1" + HELM_VERSION="v{{ helm_version }}" ARCH="amd64" cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}-helm-$HELM_VERSION/helm /usr/local/bin/helm chmod +x /usr/local/bin/helm diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 index 4849f8bd88..22b3680373 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2 @@ -207,11 +207,13 @@ value: "3" - name: vip_retryperiod value: "1" - - name: vip_address + - name: address value: {{ kube_vip }} + - name: vip_subnet + value: "{{ admin_netmask_bits }}" - name: prometheus_server value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.8.9 + image: {{ kube_vip_image }} imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -240,7 +242,7 @@ content: | #!/bin/bash set -e - HELM_VERSION="v3.20.1" + HELM_VERSION="v{{ helm_version }}" ARCH="amd64" cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}-helm-$HELM_VERSION/helm /usr/local/bin/helm chmod +x /usr/local/bin/helm diff --git a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml index 5ca3d4101b..bee7b47201 100644 --- a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml @@ -239,6 +239,12 @@ helm_package: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'tarball') | selectattr('package', 'search', 'helm') | map(attribute='package') | join }}" # noqa: yaml[line-length] nfs_subdir_external_provisioner_pkg: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'tarball') | selectattr('package', 'search', 'nfs-subdir-external-provisioner') | map(attribute='package') | join }}" # noqa: yaml[line-length] whereabouts_pkg: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'git') | selectattr('package', 'search', 'whereabouts') | map(attribute='package') | join }}" # noqa: yaml[line-length] + kube_vip_package: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'image') | selectattr('package', 'search', 'kube-vip') | map(attribute='package') | join }}" # noqa: yaml[line-length] + kube_vip_tag: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] | selectattr('type', 'equalto', 'image') | selectattr('package', 'search', 'kube-vip') | map(attribute='tag') | join }}" # noqa: yaml[line-length] + +- name: Compose kube-vip image reference from package and tag + ansible.builtin.set_fact: + kube_vip_image: "{{ kube_vip_package }}:{{ kube_vip_tag }}" - name: Copy pulp webserver certificate to target host ansible.builtin.copy: diff --git a/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 index 5538a366e6..fa1420c944 100644 --- a/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 @@ -96,6 +96,4 @@ data: - - diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml index 1be2a599e7..1e4eedbeec 100644 --- a/provision/roles/telemetry/vars/main.yml +++ b/provision/roles/telemetry/vars/main.yml @@ -53,7 +53,7 @@ idrac_telemetry_receiver_image: "{{ telemetry_images['dellhpcomniaaisolution/idr kafkapump_image: "{{ telemetry_images['dellhpcomniaaisolution/kafkapump'] | default('docker.io/dellhpcomniaaisolution/kafkapump:1.2') }}" victoriapump_image: "{{ telemetry_images['dellhpcomniaaisolution/victoriapump'] | default('docker.io/dellhpcomniaaisolution/victoriapump:1.2') }}" -activemq_image: "{{ telemetry_images['apache/activemq'] | default('docker.io/apache/activemq:5.19.7') }}" +activemq_image: "{{ telemetry_images['apache/activemq'] | default('docker.io/apache/activemq:6.3.0') }}" activemq_http_port_1: 8161 activemq_http_port_2: 61616 messagebus_http_port: 61613 @@ -63,7 +63,7 @@ mysqldb_name: "idrac_telemetrydb" idrac_telemetry_service_name: "idrac-telemetry-service" mysqldb_container_port1: 3306 mysqldb_container_port2: 33060 -mysql_image: "{{ telemetry_images['library/mysql'] | default('docker.io/library/mysql:9.3.0') }}" +mysql_image: "{{ telemetry_images['library/mysql'] | default('docker.io/library/mysql:9.7.2') }}" pod_wait_timeout: "10m" kafka_skip_verify: true @@ -74,11 +74,11 @@ kafka_skip_verify: true idrac_telemetry_resources: mysqldb: requests: - cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.cpu | default('100m') }}" - memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.memory | default('256Mi') }}" + cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.cpu | default('250m') }}" + memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.requests.memory | default('512Mi') }}" limits: - cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.cpu | default('500m') }}" - memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.memory | default('512Mi') }}" + cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.cpu | default('1000m') }}" + memory: "{{ telemetry_storage_config.idrac_telemetry_storage.mysqldb.resources.limits.memory | default('1Gi') }}" activemq: requests: cpu: "{{ telemetry_storage_config.idrac_telemetry_storage.activemq.resources.requests.cpu | default('100m') }}" @@ -147,7 +147,7 @@ victoria: container_name: "victoriametrics" service_name: "victoria-loadbalancer" container_port: 8443 - image: "{{ telemetry_images['victoriametrics/victoria-metrics'] | default('victoriametrics/victoria-metrics:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/victoria-metrics'] | default('victoriametrics/victoria-metrics:v1.149.0') }}" vmsingle: replicas: 2 @@ -177,7 +177,7 @@ victoria_cluster: vmstorage: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vmstorage.replicas | default(3) }}" replication_factor: 2 # Data redundancy: each metric written to 2 of 3 nodes - image: "{{ telemetry_images['victoriametrics/vmstorage'] | default('victoriametrics/vmstorage:v1.128.0-cluster') }}" + image: "{{ telemetry_images['victoriametrics/vmstorage'] | default('victoriametrics/vmstorage:v1.149.0-cluster') }}" dedup_min_scrape_interval: "1m" # Deduplication interval resources: requests: @@ -189,7 +189,7 @@ victoria_cluster: # VMInsert: Accepts data ingestion and routes to vmstorage vminsert: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vminsert.replicas | default(2) }}" - image: "{{ telemetry_images['victoriametrics/vminsert'] | default('victoriametrics/vminsert:v1.128.0-cluster') }}" + image: "{{ telemetry_images['victoriametrics/vminsert'] | default('victoriametrics/vminsert:v1.149.0-cluster') }}" # External access configuration external_access: true # Enable LoadBalancer service for external data ingestion resources: @@ -202,7 +202,7 @@ victoria_cluster: # VMSelect: Performs queries against vmstorage nodes vmselect: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vmselect.replicas | default(2) }}" - image: "{{ telemetry_images['victoriametrics/vmselect'] | default('victoriametrics/vmselect:v1.128.0-cluster') }}" + image: "{{ telemetry_images['victoriametrics/vmselect'] | default('victoriametrics/vmselect:v1.149.0-cluster') }}" max_query_duration: "5m" max_concurrent_requests: "8" cache_data_path: true # Enable query result caching @@ -216,7 +216,7 @@ victoria_cluster: vmagent: replicas: "{{ telemetry_storage_config.victoria_cluster_storage.vmagent.replicas | default(2) }}" - image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.149.0') }}" resources: requests: memory: "{{ telemetry_storage_config.victoria_cluster_storage.vmagent.resources.requests.memory | default('128Mi') }}" @@ -395,7 +395,7 @@ vmagent: rolebinding_name: "vmagent-sd-binding" app_name: "vmagent" container_name: "vmagent" - image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/vmagent'] | default('victoriametrics/vmagent:v1.149.0') }}" # Single-node URL remote_write_url: "https://victoria-loadbalancer.telemetry.svc.cluster.local:8443/api/v1/write" # Cluster URL (used when victoria_cluster.enabled: true) @@ -688,7 +688,7 @@ vector: service_name: "vmagent-vector" port: 8429 # prometheus_remote_write receiver metrics_port: 8429 # vmagent self-metrics - image: "{{ telemetry_images['victoriametrics/vmagent'] | default('docker.io/victoriametrics/vmagent:v1.128.0') }}" + image: "{{ telemetry_images['victoriametrics/vmagent'] | default('docker.io/victoriametrics/vmagent:v1.149.0') }}" replicas: "{{ telemetry_storage_config.vector_storage.vmagent_vector.replicas | default(2) }}" pvc_size: "{{ telemetry_storage_config.vector_storage.vmagent_vector.pvc_size | default('5Gi') }}" # Disk WAL buffer remote_write_url: "https://vminsert-victoria-cluster.{{ telemetry_namespace }}.svc.cluster.local:8480/insert/0/prometheus/api/v1/write" diff --git a/rollback/roles/rollback_k8s/tasks/main.yml b/rollback/roles/rollback_k8s/tasks/main.yml index db660afd3a..b4db0e38c5 100644 --- a/rollback/roles/rollback_k8s/tasks/main.yml +++ b/rollback/roles/rollback_k8s/tasks/main.yml @@ -37,6 +37,8 @@ # 8b. Restart network pods (clear stale BIRD/speaker processes) # 8c. Restore Helm binary to rollback version # 8d. Clean up stale CSI VolumeAttachments (PowerScale/Isilon) +# 8e. Verify telemetry rollback +# 8f. Restore MySQL data if version downgrade detected # 9. Restore BSS boot params and cloud-init from backup # ══════════════════════════════════════════════════════════════════════ @@ -215,10 +217,14 @@ - name: "Stage 8d — Clean up stale CSI VolumeAttachments" ansible.builtin.include_tasks: cleanup_stale_volume_attachments.yml - # ── Stage 8d: Verify telemetry pods after etcd restore ─────── - - name: "Stage 8d — Verify telemetry rollback" + # ── Stage 8e: Verify telemetry pods after etcd restore ─────── + - name: "Stage 8e — Verify telemetry rollback" ansible.builtin.include_tasks: verify_telemetry_rollback.yml + # ── Stage 8f: Restore MySQL data if version downgrade detected ─ + - name: "Stage 8f — Restore MySQL data (if version mismatch)" + ansible.builtin.include_tasks: restore_mysql_data.yml + # ── Stage 9: Restore BSS boot params and cloud-init ────────── - name: "Stage 9 — Restore BSS boot params and cloud-init" ansible.builtin.include_tasks: restore_bss_cloud_init.yml diff --git a/rollback/roles/rollback_k8s/tasks/restore_mysql_data.yml b/rollback/roles/rollback_k8s/tasks/restore_mysql_data.yml new file mode 100644 index 0000000000..537ab74d77 --- /dev/null +++ b/rollback/roles/rollback_k8s/tasks/restore_mysql_data.yml @@ -0,0 +1,537 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================ +# Restore MySQL Data (Post-Rollback) — All Pods +# ============================================================================ +# After etcd restore, the idrac-telemetry StatefulSet spec is reverted to the +# pre-upgrade MySQL version (e.g., 9.3.0). However, each pod's PVC still +# contains data written by the upgraded version (e.g., 9.7.2). MySQL refuses +# to downgrade between minor versions: +# [ERROR] [MY-014061] Cannot downgrade from 90702 to 90300. +# +# The StatefulSet can have multiple replicas (one per parent service node), +# each with its own independent MySQL instance and PVC. This task handles +# ALL pods, not just pod-0. +# +# This task detects the version mismatch and fixes it: +# 1. Detects if mysqldb container is in CrashLoopBackOff +# 2. Confirms it is a downgrade error via container logs +# 3. Scales down the StatefulSet +# 4. Clears ALL PVC data (removes incompatible format files from every pod) +# 5. Scales up — MySQL initializes fresh at the old version on every pod +# 6. If per-pod mysqldump backups exist, restores schema+data to each pod +# ============================================================================ + +- name: Restore idrac-telemetry MySQL data (post-rollback) + when: + - kube_vip is defined + - kube_vip | length > 0 + block: + # ── Step 1: Check if idrac-telemetry StatefulSet exists ── + - name: Check if idrac-telemetry StatefulSet exists + ansible.builtin.command: + cmd: kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} --no-headers + delegate_to: "{{ kube_vip }}" + connection: ssh + register: idrac_sts_check + changed_when: false + failed_when: false + + - name: Skip MySQL restore if idrac-telemetry not deployed + ansible.builtin.debug: + msg: "{{ msg_mysql_sts_not_found }}" + when: idrac_sts_check.rc != 0 + + # ── Step 2: Detect MySQL CrashLoopBackOff on ANY pod ── + - name: Detect and fix MySQL version downgrade issue + when: idrac_sts_check.rc == 0 + block: + - name: Wait for pods to attempt startup (need multiple restart cycles for CrashLoopBackOff) + ansible.builtin.pause: + seconds: 90 + + - name: Check all idrac-telemetry pods status + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry \ + --no-headers 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: idrac_pod_raw_status + changed_when: false + failed_when: false + + - name: Check if any mysqldb container is crashing across all pods + ansible.builtin.shell: + cmd: | + set -o pipefail + COLUMNS="NAME:.metadata.name,WAITING:.status.containerStatuses[?(@.name==\"mysqldb\")].state.waiting.reason," + COLUMNS="${COLUMNS}LAST_EXIT:.status.containerStatuses[?(@.name==\"mysqldb\")].lastState.terminated.reason," + COLUMNS="${COLUMNS}RESTARTS:.status.containerStatuses[?(@.name==\"mysqldb\")].restartCount" + kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry \ + --no-headers \ + -o custom-columns="${COLUMNS}" \ + 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysqldb_waiting_reason + changed_when: false + failed_when: false + + - name: Check MySQL logs for downgrade error across all pods + ansible.builtin.shell: + cmd: | + set -o pipefail + for POD in $(kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry \ + --no-headers -o custom-columns=NAME:.metadata.name 2>/dev/null); do + kubectl logs "$POD" -n {{ telemetry_namespace }} -c mysqldb --previous --tail=50 2>/dev/null + kubectl logs "$POD" -n {{ telemetry_namespace }} -c mysqldb --tail=50 2>/dev/null + done + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_container_logs + changed_when: false + failed_when: false + + - name: Determine if MySQL downgrade issue exists + ansible.builtin.set_fact: + mysql_downgrade_detected: >- + {{ ('CrashLoopBackOff' in (mysqldb_waiting_reason.stdout | default('')) or + 'Error' in (mysqldb_waiting_reason.stdout | default('')) or + 'OOMKilled' in (mysqldb_waiting_reason.stdout | default(''))) and + ('Cannot downgrade' in (mysql_container_logs.stdout | default('')) or + 'Invalid MySQL server downgrade' in (mysql_container_logs.stdout | default(''))) }} + + - name: Display MySQL status + ansible.builtin.debug: + msg: + - "{{ msg_mysql_pod_status }}" + - "mysqldb waiting reasons: {{ mysqldb_waiting_reason.stdout | default('none') }}" + - "MySQL downgrade detected: {{ mysql_downgrade_detected }}" + + # ── Step 3: Fix MySQL downgrade — wipe ALL PVCs and reinitialize ── + - name: Fix MySQL version downgrade + when: + - idrac_sts_check.rc == 0 + - mysql_downgrade_detected | default(false) | bool + block: + - name: Display MySQL downgrade fix starting + ansible.builtin.debug: + msg: "{{ msg_mysql_downgrade_fix_start }}" + + # Capture original replica count before scaling down + - name: Get original replica count from StatefulSet + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.replicas}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: original_replica_count_raw + changed_when: false + + - name: Set original replica count + ansible.builtin.set_fact: + idrac_telemetry_original_replicas: "{{ original_replica_count_raw.stdout | default('1') | int }}" + + - name: Build list of all pod indices + ansible.builtin.set_fact: + idrac_telemetry_pod_indices: "{{ range(0, idrac_telemetry_original_replicas | int) | list }}" + + - name: Display replica count + ansible.builtin.debug: + msg: "{{ msg_mysql_replica_count }}" + + # Scale down + - name: Scale down idrac-telemetry StatefulSet + ansible.builtin.command: + cmd: kubectl scale statefulset idrac-telemetry -n {{ telemetry_namespace }} --replicas=0 + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + + - name: Wait for all idrac-telemetry pods to terminate + ansible.builtin.shell: + cmd: | + set -o pipefail + # Wait for all pods to be deleted + for i in {{ idrac_telemetry_pod_indices | join(' ') }}; do + kubectl wait --for=delete pod/idrac-telemetry-${i} -n {{ telemetry_namespace }} --timeout=120s 2>/dev/null || true + done + # Double-check no pods left + COUNT=$(kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry --no-headers 2>/dev/null | wc -l) + if [ "$COUNT" -gt 0 ]; then + kubectl delete pod -n {{ telemetry_namespace }} -l app=idrac-telemetry --force --grace-period=0 2>/dev/null || true + sleep 5 + fi + echo "pods_cleared" + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + register: pod_cleanup + + # Get the MySQL image from restored StatefulSet + - name: Get MySQL image from restored StatefulSet + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.template.spec.containers[?(@.name=="mysqldb")].image}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: restored_mysql_image + changed_when: false + + # Wipe ALL PVCs using temporary cleanup pods (one per PVC) + - name: Create PVC cleanup pod for each replica + ansible.builtin.shell: + cmd: | + kubectl run mysql-pvc-cleanup-{{ item }} --image={{ restored_mysql_image.stdout }} \ + --restart=Never -n {{ telemetry_namespace }} \ + --overrides='{ + "spec": { + "containers": [{ + "name": "mysql-pvc-cleanup-{{ item }}", + "image": "{{ restored_mysql_image.stdout }}", + "command": ["/bin/sh", "-c", "rm -rf /var/lib/mysql/* && echo PVC_DATA_CLEARED"], + "volumeMounts": [{ + "name": "mysqldb-pvc", + "mountPath": "/var/lib/mysql" + }] + }], + "volumes": [{ + "name": "mysqldb-pvc", + "persistentVolumeClaim": { + "claimName": "mysqldb-pvc-idrac-telemetry-{{ item }}" + } + }], + "restartPolicy": "Never" + } + }' + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Wait for all PVC cleanup pods to complete + ansible.builtin.shell: + cmd: > + kubectl wait --for=jsonpath='{.status.phase}'=Succeeded + pod/mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} + --timeout=120s 2>/dev/null || + kubectl wait --for=jsonpath='{.status.phase}'=Failed + pod/mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} + --timeout=120s 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: false + retries: 3 + delay: 10 + register: cleanup_wait + until: cleanup_wait.rc == 0 + failed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Verify PVC cleanup succeeded for each replica + ansible.builtin.command: + cmd: kubectl logs mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} + delegate_to: "{{ kube_vip }}" + connection: ssh + register: cleanup_logs_all + changed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Delete PVC if cleanup failed due to NFS lock files + ansible.builtin.command: + cmd: kubectl delete pvc mysqldb-pvc-idrac-telemetry-{{ item.item }} -n {{ telemetry_namespace }} --wait=true --timeout=60s + delegate_to: "{{ kube_vip }}" + connection: ssh + when: "'PVC_DATA_CLEARED' not in item.stdout" + loop: "{{ cleanup_logs_all.results }}" + loop_control: + label: "pod-{{ item.item }}" + changed_when: true + + - name: Delete all PVC cleanup pods + ansible.builtin.command: + cmd: kubectl delete pod mysql-pvc-cleanup-{{ item }} -n {{ telemetry_namespace }} --wait=false + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + failed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + # Scale up — MySQL will initialize fresh at the old version on all pods + - name: Scale up idrac-telemetry StatefulSet + ansible.builtin.command: + cmd: kubectl scale statefulset idrac-telemetry -n {{ telemetry_namespace }} --replicas={{ idrac_telemetry_original_replicas | default(1) }} + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + + - name: Wait for all idrac-telemetry pods to be ready + ansible.builtin.shell: + cmd: | + kubectl wait --for=condition=Ready \ + pod/idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} \ + --timeout=300s + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: false + register: pod_ready_wait + retries: 3 + delay: 30 + until: pod_ready_wait.rc == 0 + loop: "{{ idrac_telemetry_pod_indices }}" + + # Get MySQL root password for restore + - name: Get MySQL root password from secret + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get secret mysqldb-credentials -n {{ telemetry_namespace }} \ + -o jsonpath='{.data.mysqldb_root_password}' | base64 -d + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_root_pw + changed_when: false + no_log: true + + # Wait for MySQL to accept connections on all pods + - name: Wait for MySQL to accept connections on each pod + ansible.builtin.shell: + cmd: > + kubectl exec idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} -c mysqldb -- + mysqladmin ping -u root -p'{{ mysql_root_pw.stdout }}' 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_ping + until: mysql_ping.rc == 0 + retries: 30 + delay: 10 + changed_when: false + no_log: true + loop: "{{ idrac_telemetry_pod_indices }}" + + # Search for per-pod mysqldump backups (new format: *_pod.sql.enc) + - name: Search for per-pod encrypted MySQL backups in backup directory + ansible.builtin.find: + paths: "{{ k8s_backup_dir }}" + patterns: "{{ mysql_backup_filename_prefix }}*.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_perpod_enc_backups_in_backup + failed_when: false + when: k8s_backup_dir is defined + + - name: Search for per-pod encrypted MySQL backups in archive directory + ansible.builtin.find: + paths: "{{ k8s_client_mount_path }}/upgrade/archive" + patterns: "{{ mysql_backup_filename_prefix }}*.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_perpod_enc_backups_in_archive + failed_when: false + when: k8s_client_mount_path is defined + + # Also search for legacy single-file backup (backward compatibility) + - name: Search for legacy encrypted MySQL backup in backup directory + ansible.builtin.find: + paths: "{{ k8s_backup_dir }}" + patterns: "idrac_telemetry_mysql_backup.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_legacy_enc_backup + failed_when: false + when: k8s_backup_dir is defined + + - name: Search for legacy encrypted MySQL backup in archive directory + ansible.builtin.find: + paths: "{{ k8s_client_mount_path }}/upgrade/archive" + patterns: "idrac_telemetry_mysql_backup.sql.enc" + recurse: true + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_legacy_enc_archive + failed_when: false + when: k8s_client_mount_path is defined + + # Determine which backup format is available and set the backup directory + - name: Determine backup format and directory + ansible.builtin.set_fact: + mysql_perpod_backup_dir: >- + {% set perpod_backup = mysql_perpod_enc_backups_in_backup | default({'matched': 0}) -%} + {% set perpod_archive = mysql_perpod_enc_backups_in_archive | default({'matched': 0}) -%} + {% set legacy_backup = mysql_legacy_enc_backup | default({'matched': 0}) -%} + {% set legacy_archive = mysql_legacy_enc_archive | default({'matched': 0}) -%} + {% if (perpod_backup.matched | default(0)) > 0 -%} + {{ (perpod_backup.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- elif (perpod_archive.matched | default(0)) > 0 -%} + {{ (perpod_archive.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- elif (legacy_backup.matched | default(0)) > 0 -%} + {{ (legacy_backup.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- elif (legacy_archive.matched | default(0)) > 0 -%} + {{ (legacy_archive.files | sort(attribute='mtime', reverse=true) | first).path | dirname }} + {%- else -%} + none + {%- endif %} + mysql_backup_is_perpod: >- + {{ ((mysql_perpod_enc_backups_in_backup | default({'matched': 0})).matched | default(0)) > 0 or + ((mysql_perpod_enc_backups_in_archive | default({'matched': 0})).matched | default(0)) > 0 }} + mysql_backup_is_legacy: >- + {{ ((mysql_perpod_enc_backups_in_backup | default({'matched': 0})).matched | default(0)) == 0 and + ((mysql_perpod_enc_backups_in_archive | default({'matched': 0})).matched | default(0)) == 0 and + (((mysql_legacy_enc_backup | default({'matched': 0})).matched | default(0)) > 0 or + ((mysql_legacy_enc_archive | default({'matched': 0})).matched | default(0)) > 0) }} + + - name: Display backup search results + ansible.builtin.debug: + msg: + - "{{ msg_mysql_backup_search_results }}" + - "Replica count: {{ idrac_telemetry_original_replicas }}" + + # ── Restore per-pod backups (new format) ── + - name: Restore per-pod MySQL backups + when: + - mysql_perpod_backup_dir | trim != 'none' + - mysql_backup_is_perpod | trim | bool + block: + - name: Display per-pod MySQL restore starting + ansible.builtin.debug: + msg: "{{ msg_mysql_perpod_restore_start }}" + + - name: Restore encrypted mysqldump to each pod + ansible.builtin.shell: + cmd: | + set -o pipefail + BACKUP_FILE="{{ mysql_perpod_backup_dir | trim }}/{{ mysql_backup_filename_prefix }}{{ item }}.sql.enc" + if [ ! -f "$BACKUP_FILE" ]; then + echo "NO_BACKUP_FOR_POD_{{ item }}" + exit 0 + fi + openssl enc -d -aes-256-cbc -pbkdf2 -iter 100000 \ + -in "$BACKUP_FILE" \ + -pass pass:'{{ mysql_root_pw.stdout }}' \ + | kubectl exec -i idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} -c mysqldb -- \ + mysql -u root -p'{{ mysql_root_pw.stdout }}' idrac_telemetrydb + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_perpod_restore_results + changed_when: true + failed_when: false + no_log: true + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Report per-pod restore results + ansible.builtin.debug: + msg: >- + {{ msg_mysql_perpod_restore_result }} + {{ '— no backup file found, MySQL reinitialized fresh' if 'NO_BACKUP_FOR_POD' in (item.stdout | default('')) else '' }} + loop: "{{ mysql_perpod_restore_results.results }}" + loop_control: + label: "pod-{{ item.item }}" + + # ── Restore legacy single-file backup (backward compatibility) ── + - name: Restore legacy MySQL backup (pod-0 only) + when: + - mysql_perpod_backup_dir | trim != 'none' + - mysql_backup_is_legacy | trim | bool + block: + - name: Display legacy MySQL restore starting + ansible.builtin.debug: + msg: "{{ msg_mysql_legacy_restore_start }}" + + - name: Restore legacy encrypted backup to pod-0 + ansible.builtin.shell: + cmd: | + set -o pipefail + openssl enc -d -aes-256-cbc -pbkdf2 -iter 100000 \ + -in {{ mysql_perpod_backup_dir | trim }}/idrac_telemetry_mysql_backup.sql.enc \ + -pass pass:'{{ mysql_root_pw.stdout }}' \ + | kubectl exec -i idrac-telemetry-0 -n {{ telemetry_namespace }} -c mysqldb -- \ + mysql -u root -p'{{ mysql_root_pw.stdout }}' idrac_telemetrydb + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_legacy_restore + changed_when: true + failed_when: false + no_log: true + + - name: Report legacy restore result + ansible.builtin.debug: + msg: "{{ msg_mysql_legacy_restore_result }}" + + # ── No backup found ── + - name: Log that no backup was found + ansible.builtin.debug: + msg: "{{ msg_mysql_no_backup_found }}" + when: mysql_perpod_backup_dir | trim == 'none' + + # Verify MySQL is healthy on all pods + - name: Verify MySQL tables after recovery on each pod + ansible.builtin.shell: + cmd: > + kubectl exec idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} -c mysqldb -- + mysql -u root -p'{{ mysql_root_pw.stdout }}' -e 'SHOW TABLES FROM idrac_telemetrydb;' + 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: tables_after_restore_all + changed_when: false + no_log: true + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Verify all containers are running on each pod + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get pod idrac-telemetry-{{ item }} -n {{ telemetry_namespace }} --no-headers \ + -o custom-columns=READY:.status.containerStatuses[*].ready | tr ',' '\n' | grep -c 'true' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: ready_containers_all + changed_when: false + failed_when: false + loop: "{{ idrac_telemetry_pod_indices }}" + + - name: Display MySQL recovery success + ansible.builtin.debug: + msg: + - "============================================================" + - "MySQL VERSION DOWNGRADE — RECOVERY COMPLETED" + - "============================================================" + - "MySQL image: {{ restored_mysql_image.stdout }}" + - "Replicas: {{ idrac_telemetry_original_replicas | default(1) }}" + - "Per-pod backup used: {{ mysql_backup_is_perpod | trim }}" + - "Legacy backup used: {{ mysql_backup_is_legacy | trim }}" + - "Pod recovery details:" + - "{% for idx in idrac_telemetry_pod_indices %}\ + pod-{{ idx }}: \ + ready_containers={{ (ready_containers_all.results[idx].stdout | default('0')) | trim }}/5, \ + tables={{ (tables_after_restore_all.results[idx].stdout_lines | default([])) | length }}\ + {{ ', ' if not loop.last else '' }}\ + {% endfor %}" + - "============================================================" + + # ── No downgrade detected — MySQL is healthy ── + - name: MySQL version compatible — no restore needed + ansible.builtin.debug: + msg: "{{ msg_mysql_no_restore_needed }}" + when: + - idrac_sts_check.rc == 0 + - not (mysql_downgrade_detected | default(false) | bool) diff --git a/rollback/roles/rollback_k8s/vars/main.yml b/rollback/roles/rollback_k8s/vars/main.yml index 0892fa9041..3eb34ec565 100644 --- a/rollback/roles/rollback_k8s/vars/main.yml +++ b/rollback/roles/rollback_k8s/vars/main.yml @@ -28,6 +28,14 @@ repo_file_path: "/etc/yum.repos.d/omnia-upgrade.repo" nfs_storage_name: "nfs_k8s" +# Telemetry namespace (used by restore_mysql_data.yml) +telemetry_namespace: telemetry + +# MySQL backup filename prefix (must match upgrade role's naming convention) +# Per-pod backups are named: .sql.enc +# e.g., idrac_telemetry_mysql_backup_pod0.sql.enc +mysql_backup_filename_prefix: "idrac_telemetry_mysql_backup_pod" + # nodes.yaml location (created by discovery) nodes_yaml_path: "/opt/omnia/openchami/workdir/nodes/nodes.yaml" @@ -152,3 +160,57 @@ msg_bss_backup_ci_missing: >- BSS restore failed: cloud-init/ directory not found in openchami backup workdir {{ _bss_backup_workdir_container }}/cloud-init. The backup may be incomplete or corrupted. + +# MySQL restore messages +msg_mysql_sts_not_found: >- + idrac-telemetry StatefulSet not found — skipping MySQL data restore. +msg_mysql_pod_status: | + idrac-telemetry pods status: + {{ idrac_pod_raw_status.stdout_lines | default(['unknown']) }} +msg_mysql_downgrade_fix_start: | + ============================================================ + MySQL VERSION DOWNGRADE DETECTED — STARTING PVC RECOVERY + ============================================================ + Current version: {{ restored_mysql_image.stdout }} + Expected version: {{ current_mysql_version.stdout }} + The rollback has downgraded the MySQL container image, but the + PVC contains data from a newer MySQL version. This is incompatible. + We will wipe the PVC data and restore from mysqldump backup. + ============================================================ +msg_mysql_replica_count: >- + idrac-telemetry has {{ idrac_telemetry_original_replicas }} replica(s) — will wipe and restore ALL PVCs +msg_mysql_backup_search_results: | + Backup directory: {{ mysql_perpod_backup_dir | trim }} + Per-pod backup format: {{ mysql_backup_is_perpod | trim }} + Legacy single-file format: {{ mysql_backup_is_legacy | trim }} +msg_mysql_perpod_restore_start: >- + Restoring per-pod MySQL data from: {{ mysql_perpod_backup_dir | trim }} +msg_mysql_perpod_restore_result: >- + Pod idrac-telemetry-{{ item.item }}: + {{ 'RESTORED' if item.rc == 0 else 'FAILED (rc=' ~ item.rc ~ ')' }} +msg_mysql_legacy_restore_start: | + Found legacy single-file backup (pre multi-pod support). + Restoring to pod-0 only. Other pods will reinitialize fresh. + Legacy backup: >- + {{ (mysql_legacy_enc_backup.files + mysql_legacy_enc_archive.files) + | sort(attribute='mtime', reverse=true) + | first + | default({'path': 'none'}) + | json_query('path') }} +msg_mysql_legacy_restore_result: | + Legacy backup restore to pod-0: {{ 'SUCCESS' if mysql_legacy_restore.rc == 0 else 'FAILED (rc=' ~ mysql_legacy_restore.rc ~ ')' }} + WARNING: Pods 1+ have no backup — MySQL reinitialized fresh with empty database. +msg_mysql_no_backup_found: | + No pre-upgrade mysqldump backup found. + ALL pods have been reinitialized fresh at {{ restored_mysql_image.stdout }}. + The database tables exist but will be empty. + You may need to manually repopulate data (e.g., iDRAC services list). +msg_mysql_recovery_success: | + ============================================================ + MySQL VERSION DOWNGRADE — RECOVERY COMPLETED + ============================================================ + All idrac-telemetry pods ({{ idrac_telemetry_original_replicas }}) are ready. + MySQL data has been restored from pre-upgrade backup. + ============================================================ +msg_mysql_no_restore_needed: >- + idrac-telemetry MySQL container is running normally — no version downgrade fix needed. diff --git a/rollback/roles/rollback_pulp/tasks/main.yml b/rollback/roles/rollback_pulp/tasks/main.yml index ba34c235d2..748c603c34 100644 --- a/rollback/roles/rollback_pulp/tasks/main.yml +++ b/rollback/roles/rollback_pulp/tasks/main.yml @@ -16,7 +16,7 @@ # ============================================================================ # rollback_pulp — Main Orchestration # ============================================================================ -# Reverse the Pulp upgrade (3.113 → 3.80) by: +# Reverse the Pulp upgrade (3.114.2 → 3.80) by: # 1. Resolving admin NIC IP for health checks # 2. Pre-rollback checks (verify Pulp is deployed, backup exists) # 3. Stopping the current Pulp container @@ -29,7 +29,7 @@ # # IMPORTANT: PostgreSQL data must be restored because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions # # No idempotency check — rollback always executes regardless of current diff --git a/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml b/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml index cb74c969aa..f613674d3c 100644 --- a/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml +++ b/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml @@ -14,7 +14,7 @@ --- # ============================================================================ -# Pulp Container Rollback (3.113 → 3.80) +# Pulp Container Rollback (3.114.2 → 3.80) # # Steps: # 1. Pull the v2.1 Pulp image (3.80) diff --git a/rollback/roles/rollback_pulp/vars/main.yml b/rollback/roles/rollback_pulp/vars/main.yml index 21ff282c46..87926ad40a 100644 --- a/rollback/roles/rollback_pulp/vars/main.yml +++ b/rollback/roles/rollback_pulp/vars/main.yml @@ -16,7 +16,7 @@ # ============================================================================ # rollback_pulp — Variables # ============================================================================ -# Rollback target: Pulp 3.80 (from Pulp 3.113) +# Rollback target: Pulp 3.80 (from Pulp 3.114.2) # Reverses the upgrade performed by upgrade_pulp role. # ============================================================================ @@ -74,7 +74,7 @@ pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql" # PostgreSQL backup paths # PostgreSQL data must be restored during rollback because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp" pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql" diff --git a/upgrade/playbooks/upgrade_oim.yml b/upgrade/playbooks/upgrade_oim.yml index 65ff749392..2d46cbcc71 100644 --- a/upgrade/playbooks/upgrade_oim.yml +++ b/upgrade/playbooks/upgrade_oim.yml @@ -21,7 +21,7 @@ # # Flow: # 1. Pre-flight: read manifest, check idempotency -# 2. Phase 1: Pulp container upgrade (3.80 → 3.113) +# 2. Phase 1: Pulp container upgrade (3.80 → 3.114.2) # 3. Phase 2: OpenCHAMI container upgrade (pg_dump, deployment-recipes, # image pull, ordered restart, DB migration, validation) # 4. Mark OIM as completed in manifest diff --git a/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 b/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 index c80dbdde65..32133b2c00 100644 --- a/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 +++ b/upgrade/roles/import_input_parameters/templates/telemetry_storage_config.j2 @@ -159,11 +159,11 @@ idrac_telemetry_storage: mysqldb: resources: requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "500m" + cpu: "250m" memory: "512Mi" + limits: + cpu: "1000m" + memory: "1Gi" activemq: resources: requests: diff --git a/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml b/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml index b981049e8b..2d527d4854 100644 --- a/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml +++ b/upgrade/roles/upgrade_k8s/tasks/detect_addon_versions.yml @@ -119,6 +119,104 @@ else 'pending' }} +# ── kube-vip version detection ──────────────────────────────────────── +# kube-vip runs as a static pod on EACH control plane node. The version +# is read from the manifest file (source of truth) rather than crictl, +# because CRI-O stores images by digest and crictl does not return tags. +# We check all CP nodes: if any node is not at target, status = pending. +# +# IMPORTANT: this role runs under `hosts: localhost, connection: local`. +# A delegate_to target only connects over SSH if that host has +# `ansible_connection: ssh` set in the inventory. Raw IPs (and file-based +# inventory hosts without that var) inherit connection: local and run on +# the controller instead — which is why the manifest was "not found". +# We therefore register each CP node in the in-memory inventory with an +# explicit SSH connection, then delegate to those hostnames. +- name: Build list of control plane nodes for kube-vip detection + ansible.builtin.set_fact: + _kube_vip_detect_cp_nodes: >- + {{ upgrade_status.nodes | dict2items + | selectattr('value.role', 'in', ['control_plane_first', 'control_plane']) + | list }} + +- name: Register control plane nodes for SSH delegation (kube-vip detection) + ansible.builtin.add_host: + name: "{{ item.key }}" + ansible_host: "{{ item.value.ip }}" + ansible_connection: ssh + ansible_user: root + ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + groups: _kube_vip_detect_targets + loop: "{{ _kube_vip_detect_cp_nodes }}" + loop_control: + label: "{{ item.key }} ({{ item.value.ip }})" + changed_when: false + +- name: Build list of control plane hostnames for kube-vip detection + ansible.builtin.set_fact: + _kube_vip_detect_cp_hosts: "{{ _kube_vip_detect_cp_nodes | map(attribute='key') | list }}" + +- name: Detect kube-vip version from manifest on each control plane + ansible.builtin.shell: + cmd: >- + set -o pipefail && + grep -oP 'image:\s+\S+:v\K\d+\.\d+\.\d+' + /etc/kubernetes/manifests/kube-vip.yaml 2>/dev/null || + echo "unknown" + executable: /bin/bash + delegate_to: "{{ item }}" + loop: "{{ _kube_vip_detect_cp_hosts }}" + register: kube_vip_version_per_node + changed_when: false + failed_when: false + +- name: Build per-node kube-vip version map + ansible.builtin.set_fact: + _kube_vip_node_versions: >- + {{ _kube_vip_node_versions | default({}) | combine({ + item.item: item.stdout | default('unknown') | trim + }) }} + loop: "{{ kube_vip_version_per_node.results }}" + loop_control: + label: "{{ item.item }}" + +- name: Display kube-vip version per control plane node + ansible.builtin.debug: + msg: "kube-vip versions: {{ _kube_vip_node_versions }}" + +# Use the first non-target version as the "current" version for overall +# reporting. If all nodes match the target, report the target. +- name: Set kube_vip_current_version fact + ansible.builtin.set_fact: + kube_vip_current_version: >- + {{ (_kube_vip_node_versions.values() | list + | reject('equalto', kube_vip_target_version) + | reject('equalto', 'unknown') | first | default('')) + or + (kube_vip_target_version + if (_kube_vip_node_versions.values() | list + | select('equalto', kube_vip_target_version) | list | length + == _kube_vip_node_versions | length) + else (_kube_vip_node_versions.values() | list + | reject('equalto', 'unknown') | first | default('unknown'))) }} + +- name: Set kube_vip_from_version fact (preserve existing if set) + ansible.builtin.set_fact: + kube_vip_from_version: >- + {{ upgrade_status.addon_upgrade.kube_vip.from_version + if (upgrade_status.addon_upgrade.kube_vip.from_version | default('unknown')) not in ['unknown', ''] + else kube_vip_current_version }} + +# completed only when ALL control plane nodes are at the target version +- name: Determine kube-vip status + ansible.builtin.set_fact: + kube_vip_status: >- + {{ 'completed' + if (_kube_vip_node_versions.values() | list + | select('equalto', kube_vip_target_version) | list | length + == _kube_vip_node_versions | length) + else 'pending' }} + - name: Load PowerScale target version from JSON ansible.builtin.set_fact: csi_powerscale_packages_json: "{{ lookup('file', input_project_dir + '/config/x86_64/rhel/' + cluster_os_version + '/csi_driver_powerscale.json') | from_json }}" # noqa yaml[line-length] @@ -189,6 +287,11 @@ vars: status_update: addon_upgrade: + kube_vip: + status: "{{ kube_vip_status }}" + current_version: "{{ kube_vip_current_version }}" + from_version: "{{ kube_vip_from_version }}" + target_version: "{{ kube_vip_target_version }}" calico: status: "{{ calico_status }}" current_version: "{{ calico_current_version }}" @@ -216,6 +319,7 @@ vars: addon_versions_banner: - "Detected addon versions:" + - " kube-vip: {{ kube_vip_current_version }} (target: {{ kube_vip_target_version }}) - {{ kube_vip_status }}" - " Calico: {{ calico_current_version }} (target: {{ calico_target_version }}) - {{ calico_status }}" - " MetalLB: {{ metallb_current_version }} (target: {{ metallb_target_version }}) - {{ metallb_status }}" - " Helm: {{ helm_current_version }} (target: {{ helm_target_version }}) - {{ helm_status }}" diff --git a/upgrade/roles/upgrade_k8s/tasks/load_status.yml b/upgrade/roles/upgrade_k8s/tasks/load_status.yml index a637eaa69d..048a52d3e2 100644 --- a/upgrade/roles/upgrade_k8s/tasks/load_status.yml +++ b/upgrade/roles/upgrade_k8s/tasks/load_status.yml @@ -192,6 +192,8 @@ error: addon_upgrade: status: pending + kube_vip: + status: pending calico: status: pending metallb: diff --git a/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml b/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml index 33b51ae6b6..5c966906e8 100644 --- a/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml +++ b/upgrade/roles/upgrade_k8s/tasks/load_version_vars.yml @@ -108,6 +108,13 @@ | regex_replace('^helm-v', '') | regex_replace('-.*$', '') }} +- name: Extract kube_vip_target_version from service_k8s.json + ansible.builtin.set_fact: + kube_vip_target_version: >- + {{ service_k8s_config.service_kube_control_plane_first.cluster + | selectattr('package', 'search', 'kube-vip') + | map(attribute='tag') | first | regex_replace('^v', '') }} + # ── Extract addon package names (for manifest file names) ───────────── - name: Extract addon package names from service_k8s.json ansible.builtin.set_fact: diff --git a/upgrade/roles/upgrade_k8s/tasks/post_validation.yml b/upgrade/roles/upgrade_k8s/tasks/post_validation.yml index 78016755bb..2fc617b1d4 100644 --- a/upgrade/roles/upgrade_k8s/tasks/post_validation.yml +++ b/upgrade/roles/upgrade_k8s/tasks/post_validation.yml @@ -105,7 +105,7 @@ ansible.builtin.shell: cmd: >- set -o pipefail && - kubectl get pods -n calico-system --no-headers + kubectl get pods -n kube-system -l k8s-app=calico-node --no-headers --field-selector status.phase!=Running,status.phase!=Succeeded 2>/dev/null | head -20 args: diff --git a/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml b/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml index 72acf94fa0..dfe19228b9 100644 --- a/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml +++ b/upgrade/roles/upgrade_k8s/tasks/step_addon_validation.yml @@ -12,6 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. --- +- name: Verify kube-vip pods are Running on control planes + delegate_to: "{{ kube_vip }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail && + kubectl get pods -n kube-system --no-headers + -o custom-columns=':metadata.name,:status.phase' | + grep kube-vip + executable: /bin/bash + register: kube_vip_pods + changed_when: false + failed_when: false + +- name: Warn if kube-vip pods are not Running + ansible.builtin.debug: + msg: "WARNING: kube-vip issue: {{ kube_vip_pods.stdout | default('no pods found') }}" + when: kube_vip_pods.rc != 0 or kube_vip_pods.stdout | trim | length == 0 + - name: Verify calico-node pods are Running delegate_to: "{{ kube_vip }}" ansible.builtin.command: @@ -56,6 +74,7 @@ ansible.builtin.debug: msg: >- Addon validation complete. + kube-vip pods: {{ 'OK' if kube_vip_pods.rc == 0 else 'WARN' }} Calico pods: {{ 'OK' if calico_pods.rc == 0 else 'WARN' }} MetalLB pods: {{ 'OK' if metallb_pods.rc == 0 else 'WARN' }} LoadBalancer IPs: {{ lb_services.stdout | default('none') }} diff --git a/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml b/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml new file mode 100644 index 0000000000..33fa0018a1 --- /dev/null +++ b/upgrade/roles/upgrade_k8s/tasks/step_kube_vip_upgrade.yml @@ -0,0 +1,278 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================ +# step_kube_vip_upgrade.yml - Upgrade kube-vip static pod on a control plane +# ============================================================================ +# kube-vip runs as a static pod managed by kubelet, not a DaemonSet. +# Upgrade = replace /etc/kubernetes/manifests/kube-vip.yaml on each CP node. +# +# Chicken-and-egg problem: +# When the old kube-vip pod dies, the VIP goes away. The new pod reads +# admin.conf which points to the VIP for leader election, so it cannot +# start. Fix: temporarily point admin.conf to the node's real IP, let +# kube-vip start and advertise the VIP, then restore admin.conf. +# +# Variables required (set by load_version_vars.yml / detect_addon_versions.yml): +# - kube_vip_target_version (e.g. "1.2.2") +# - kube_vip (VIP address, e.g. "10.40.5.176") +# - node_ip (real IP of the CP node being upgraded) +# - node_hostname (inventory hostname, e.g. "k8scp2") +# - backup_dir (path to store manifest backup) +# +# All tasks that touch the node run with delegate_to: "{{ node_hostname }}" +# to ensure SSH connection from the inventory is used (not connection: local). +# ============================================================================ + +- name: Check if kube-vip manifest exists + delegate_to: "{{ node_hostname }}" + ansible.builtin.stat: + path: /etc/kubernetes/manifests/kube-vip.yaml + register: _kube_vip_manifest_stat + +- name: Skip if manifest not found (not a control plane node) + ansible.builtin.debug: + msg: "No kube-vip manifest found on {{ node_ip }} — skipping" + when: not _kube_vip_manifest_stat.stat.exists + +- name: Upgrade kube-vip on {{ node_ip }} + when: _kube_vip_manifest_stat.stat.exists + block: + - name: Read current kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.slurp: + path: /etc/kubernetes/manifests/kube-vip.yaml + register: _kube_vip_current + + - name: Check if already at target version + ansible.builtin.set_fact: + _kube_vip_needs_upgrade: >- + {{ 'ghcr.io/kube-vip/kube-vip:v' + kube_vip_target_version + not in (_kube_vip_current.content | b64decode) }} + + - name: Skip if already at target version + ansible.builtin.debug: + msg: "kube-vip on {{ node_ip }} already at v{{ kube_vip_target_version }} — skipping" + when: not (_kube_vip_needs_upgrade | bool) + + - name: Perform kube-vip upgrade on {{ node_ip }} + when: _kube_vip_needs_upgrade | bool + block: + - name: Backup current kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + src: /etc/kubernetes/manifests/kube-vip.yaml + dest: "{{ backup_dir }}/kube-vip-{{ node_ip }}-pre-upgrade.yaml" + remote_src: true + mode: "0644" + + # ── Detect network facts (same approach as provision cloud-init) ── + - name: Detect VIP interface from node IP + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + ip -o addr show | + awk -v ip="{{ node_ip }}" '$4 ~ ip {print $2}' + executable: /bin/bash + register: _vip_interface + changed_when: false + + # ── Set facts matching provision template variables ────────────── + # admin_netmask_bits comes from network_data.admin_network.netmask_bits + # (same source as provision/roles/k8s_config create_k8s_config_nfs.yml) + - name: Set kube-vip template facts + ansible.builtin.set_fact: + kube_vip_interface: "{{ _vip_interface.stdout | trim }}" + admin_netmask_bits: "{{ hostvars['localhost']['admin_netmask_bits'] }}" + kube_vip_version: "{{ kube_vip_target_version }}" + + # ── Generate new manifest (same structure as provision template) ── + - name: Write upgraded kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + dest: /tmp/kube-vip-upgrade.yaml + mode: "0644" + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: {{ kube_vip_interface }} + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: {{ kube_vip }} + - name: vip_subnet + value: "{{ admin_netmask_bits }}" + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v{{ kube_vip_version }} + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig + status: {} + + # ── Chicken-and-egg fix: temporarily use node IP ───────────────── + - name: Temporarily point admin.conf to node IP + delegate_to: "{{ node_hostname }}" + ansible.builtin.replace: + path: /etc/kubernetes/admin.conf + regexp: 'server: https://[^:]+:6443' + replace: "server: https://{{ node_ip }}:6443" + + # ── Replace manifest — kubelet auto-restarts the pod ───────────── + - name: Replace kube-vip manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.copy: + src: /tmp/kube-vip-upgrade.yaml + dest: /etc/kubernetes/manifests/kube-vip.yaml + remote_src: true + mode: "0644" + + - name: Wait for new kube-vip pod to start + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + crictl ps --name kube-vip -o json | + python3 -c "import sys,json; + c=json.load(sys.stdin).get('containers',[]); + print(c[0]['state'] if c else 'missing')" + executable: /bin/bash + register: _kube_vip_status + changed_when: false + retries: 30 + delay: 5 + until: _kube_vip_status.stdout | trim == 'CONTAINER_RUNNING' + + - name: Verify VIP is reachable (API server responds via VIP) # noqa: command-instead-of-module command-instead-of-shell + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + curl -sk --connect-timeout 3 + https://{{ kube_vip }}:6443/healthz + register: _vip_check + changed_when: false + retries: 12 + delay: 5 + until: _vip_check.stdout | default('') == 'ok' + + # ── Restore admin.conf to VIP ──────────────────────────────────── + - name: Restore admin.conf to VIP address + delegate_to: "{{ node_hostname }}" + ansible.builtin.replace: + path: /etc/kubernetes/admin.conf + regexp: 'server: https://[^:]+:6443' + replace: "server: https://{{ kube_vip }}:6443" + + # Grep specifically for the version banner (not head -1) and retry + # until it reports the target version. This doubles as a stabilization + # gate: a pod that briefly starts then crash-loops logs a transient + # "level=fatal" line, which head -1 could mistake for the result. + - name: Verify kube-vip is running the target version + delegate_to: "{{ node_hostname }}" + ansible.builtin.shell: + cmd: >- + set -o pipefail; + crictl ps --name kube-vip -q | head -1 | + xargs crictl logs 2>&1 | + grep -oE 'kube-vip.io version=v[0-9]+\.[0-9]+\.[0-9]+' | + head -1 + executable: /bin/bash + register: _kube_vip_log + changed_when: false + retries: 12 + delay: 5 + until: >- + ('version=v' + kube_vip_target_version) in (_kube_vip_log.stdout | default('')) + + # ── Verify API is reachable via VIP before moving to next node ─── + - name: Verify API server reachable via VIP + delegate_to: "{{ node_hostname }}" + ansible.builtin.command: + cmd: >- + kubectl --kubeconfig /etc/kubernetes/admin.conf get --raw /healthz + register: _api_health + changed_when: false + retries: 12 + delay: 5 + until: _api_health.rc == 0 + + - name: Display kube-vip upgrade result + ansible.builtin.debug: + msg: >- + kube-vip upgraded on {{ node_ip }}: + {{ _kube_vip_log.stdout | default('version not detected') | trim }} + + - name: Clean up temp manifest + delegate_to: "{{ node_hostname }}" + ansible.builtin.file: + path: /tmp/kube-vip-upgrade.yaml + state: absent diff --git a/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml b/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml index 4e85e45f73..9c6b14163d 100644 --- a/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml +++ b/upgrade/roles/upgrade_k8s/tasks/upgrade_addons.yml @@ -23,6 +23,129 @@ status: in_progress started_at: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" +# ── kube-vip upgrade (ABORT on failure) ──────────────────────────── +# kube-vip MUST be upgraded BEFORE other addons because it provides +# the VIP that all kubectl commands are delegated through. +# Unlike Calico/MetalLB (cluster-wide kubectl apply), kube-vip runs +# as a static pod and must be upgraded per-node on each control plane. +- name: Starting kube-vip upgrade + ansible.builtin.debug: + msg: "Starting kube-vip upgrade from {{ kube_vip_from_version | default('unknown') }} to {{ kube_vip_target_version }}..." + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Mark kube-vip in_progress + ansible.builtin.include_tasks: update_addon_step.yml + vars: + addon_name: kube_vip + addon_status_update: + status: in_progress + current_version: "{{ kube_vip_current_version }}" + from_version: "{{ kube_vip_from_version | default('unknown') }}" + target_version: "{{ kube_vip_target_version }}" + timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Build list of control plane nodes for kube-vip upgrade + ansible.builtin.set_fact: + _cp_node_list: >- + {{ upgrade_status.nodes | dict2items + | selectattr('value.role', 'in', ['control_plane_first', 'control_plane']) + | list }} + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +# This role runs under `hosts: localhost, connection: local`. delegate_to +# only connects over SSH when the target host has `ansible_connection: ssh` +# set in the inventory; otherwise it inherits connection: local and runs on +# the controller. Register each CP node explicitly so step_kube_vip_upgrade.yml +# can delegate to hostnames over SSH. +- name: Register control plane nodes for SSH delegation (kube-vip upgrade) + ansible.builtin.add_host: + name: "{{ item.key }}" + ansible_host: "{{ item.value.ip }}" + ansible_connection: ssh + ansible_user: root + ansible_ssh_common_args: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" + groups: _kube_vip_upgrade_targets + loop: "{{ _cp_node_list }}" + loop_control: + label: "{{ item.key }} ({{ item.value.ip }})" + changed_when: false + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +# ── Detect VIP leader so we upgrade non-leader nodes first ──────── +# The VIP leader holds the virtual IP. Upgrading it last keeps the +# API server reachable through the VIP for as long as possible. +- name: Get kube-vip leader lease for upgrade ordering + ansible.builtin.command: + cmd: >- + kubectl get lease plndr-cp-lock -n kube-system + -o jsonpath='{.spec.holderIdentity}' + delegate_to: "{{ kube_vip }}" + register: _upgrade_vip_leader + changed_when: false + failed_when: false + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Set VIP leader IP for upgrade ordering + ansible.builtin.set_fact: + _vip_leader_ip: "{{ _upgrade_vip_leader.stdout | default('') | trim }}" + when: + - (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + - _upgrade_vip_leader.rc | default(1) == 0 + +- name: Reorder CP list — VIP leader last + ansible.builtin.set_fact: + _cp_node_list: >- + {{ (_cp_node_list | rejectattr('value.ip', 'equalto', _vip_leader_ip) | list) + + (_cp_node_list | selectattr('value.ip', 'equalto', _vip_leader_ip) | list) }} + when: + - (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + - _vip_leader_ip | default('') | length > 0 + +- name: Display kube-vip upgrade order + ansible.builtin.debug: + msg: >- + kube-vip upgrade order: {{ _cp_node_list | map(attribute='value.ip') | list | join(' -> ') }} + (VIP leader {{ _vip_leader_ip | default('unknown') }} upgraded last) + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + +- name: Upgrade kube-vip + when: (upgrade_status.addon_upgrade.kube_vip.status | default('pending')) != 'completed' + block: + - name: Execute kube-vip upgrade on each control plane + ansible.builtin.include_tasks: step_kube_vip_upgrade.yml + vars: + node_ip: "{{ _cp_item.value.ip }}" + node_hostname: "{{ _cp_item.key }}" + loop: "{{ _cp_node_list }}" + loop_control: + loop_var: _cp_item + label: "{{ _cp_item.key }} ({{ _cp_item.value.ip }})" + + - name: Mark kube-vip completed + ansible.builtin.include_tasks: update_addon_step.yml + vars: + addon_name: kube_vip + addon_status_update: + status: completed + current_version: "{{ kube_vip_target_version }}" + from_version: "{{ kube_vip_from_version | default('unknown') }}" + to_version: "{{ kube_vip_target_version }}" + timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + error: + rescue: + - name: Mark kube-vip failed + ansible.builtin.include_tasks: update_addon_step.yml + vars: + addon_name: kube_vip + addon_status_update: + status: failed + timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + + - name: ABORT — kube-vip upgrade failed + ansible.builtin.fail: + msg: "{{ msg_addon_kube_vip_failed }}" + # ── Calico upgrade (ABORT on failure) ────────────────────────────── - name: Starting Calico upgrade ansible.builtin.debug: diff --git a/upgrade/roles/upgrade_k8s/vars/main.yml b/upgrade/roles/upgrade_k8s/vars/main.yml index 2efe076c4f..379586542e 100644 --- a/upgrade/roles/upgrade_k8s/vars/main.yml +++ b/upgrade/roles/upgrade_k8s/vars/main.yml @@ -193,6 +193,10 @@ msg_backup_missing: >- msg_node_upgrade_failed: >- Node {{ current_node_name }} failed at step {{ current_step }}. Error: {{ step_result.stderr | default(step_result.msg | default('unknown')) }} +msg_addon_kube_vip_failed: >- + kube-vip upgrade failed. VIP may be unavailable. + Do NOT proceed to further addon upgrades. + Check /etc/kubernetes/manifests/kube-vip.yaml on each control plane node. msg_addon_calico_failed: >- Calico upgrade failed. Networking may be degraded. Do NOT proceed to worker upgrades. diff --git a/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml b/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml index e01e7f2e57..b91c8a6fea 100644 --- a/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml +++ b/upgrade/roles/upgrade_openchami/tasks/upgrade_openchami_containers.yml @@ -84,21 +84,20 @@ delegate_facts: true connection: ssh - - name: Download updated OpenCHAMI RPM - ansible.builtin.command: curl -L -o {{ openchami_work_dir }}/{{ openchami_rpm_name }} {{ openchami_rpm_url }} # noqa: command-instead-of-module - register: download_openchami - retries: "{{ max_retries }}" - delay: "{{ max_delay }}" - until: download_openchami.rc == 0 - changed_when: true + - name: Copy the openchami RPM tarball to OIM + ansible.builtin.copy: + src: "{{ prepare_oim_files_path }}/{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" + mode: "{{ dir_permissions_755 }}" delegate_to: oim delegate_facts: true connection: ssh - - name: Set permissions on downloaded OpenCHAMI RPM - ansible.builtin.file: - path: "{{ openchami_work_dir }}/{{ openchami_rpm_name }}" - mode: "{{ dir_permissions_755 }}" + - name: Extract the openchami RPM from tarball + ansible.builtin.unarchive: + src: "{{ openchami_work_dir }}/{{ openchami_rpm_tarball }}" + dest: "{{ openchami_work_dir }}" + remote_src: true delegate_to: oim delegate_facts: true connection: ssh diff --git a/upgrade/roles/upgrade_openchami/vars/main.yml b/upgrade/roles/upgrade_openchami/vars/main.yml index a3b70aeb13..f37e4f9f7a 100644 --- a/upgrade/roles/upgrade_openchami/vars/main.yml +++ b/upgrade/roles/upgrade_openchami/vars/main.yml @@ -101,21 +101,21 @@ openchami_smd_tag: "v2.19.3" openchami_bss_tag: "v1.32.2" openchami_cloud_init_tag: "v1.3.0" openchami_coresmd_tag: "v0.4.3" -minio_release_tag: "RELEASE.2026-04-17T00-00-00Z" +minio_release_tag: "RELEASE.2026-08-04T00-00-00Z" postgres_tag: "11.5-alpine" hydra_tag: "v2.3" haproxy_tag: "latest" -registry_tag: "3.1.0" +registry_tag: "3.1.1" curl_tag: "latest" acme_tag: "3.1.1" # OpenCHAMI target images for podman pull openchami_target_images: - "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - - "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" - - "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" - - "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" - - "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + - "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" + - "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" + - "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" + - "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - "docker.io/pgsty/minio:{{ minio_release_tag }}" - "docker.io/library/postgres:{{ postgres_tag }}" @@ -131,19 +131,19 @@ openchami_quadlet_updates: - file: "step-ca.container" image: "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - file: "opaal.container" - image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - file: "opaal-idp.container" - image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - file: "smd.container" - image: "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" + image: "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" - file: "smd-init.container" - image: "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" + image: "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" - file: "bss.container" - image: "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" + image: "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" - file: "bss-init.container" - image: "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" + image: "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" - file: "cloud-init-server.container" - image: "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + image: "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - file: "coresmd-coredhcp.container" image: "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - file: "coresmd-coredns.container" @@ -173,15 +173,15 @@ openchami_containers: - container: "step-ca" expected_image: "ghcr.io/openchami/local-ca:{{ openchami_local_ca_tag }}" - container: "opaal" - expected_image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - container: "opaal-idp" - expected_image: "ghcr.io/openchami/opaal:{{ openchami_opaal_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/opaal:{{ openchami_opaal_tag }}" - container: "smd" - expected_image: "ghcr.io/openchami/smd:{{ openchami_smd_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/smd:{{ openchami_smd_tag }}" - container: "bss" - expected_image: "ghcr.io/openchami/bss:{{ openchami_bss_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/bss:{{ openchami_bss_tag }}" - container: "cloud-init-server" - expected_image: "ghcr.io/openchami/cloud-init:{{ openchami_cloud_init_tag }}" + expected_image: "docker.io/dellhpcomniaaisolution/cloud-init:{{ openchami_cloud_init_tag }}" - container: "coresmd-coredhcp" expected_image: "ghcr.io/openchami/coresmd:{{ openchami_coresmd_tag }}" - container: "coresmd-coredns" @@ -195,8 +195,9 @@ openchami_containers: - container: "registry" expected_image: "docker.io/library/registry:{{ registry_tag }}" -# Path to prepare_oim templates (reuse for deploying new configs without duplication) +# Path to prepare_oim templates and files (reuse without duplication) prepare_oim_templates_path: "{{ role_path }}/../../../prepare_oim/roles/deploy_containers/openchami/templates" +prepare_oim_files_path: "{{ role_path }}/../../../prepare_oim/roles/deploy_containers/openchami/files" # OpenCHAMI configuration paths openchami_config_dir: "/etc/openchami/configs" @@ -228,8 +229,8 @@ network_spec_path: "{{ input_project_dir | default('/opt/omnia/input/project_def provision_config_path: "{{ input_project_dir | default('/opt/omnia/input/project_default') }}/provision_config.yml" # OpenCHAMI RPM packages (Omnia 2.2 target) -openchami_rpm_url: "https://github.com/OpenCHAMI/release/releases/download/v0.1.5/openchami-0.1.5.rpm" -openchami_rpm_name: "openchami-0.1.5.rpm" +openchami_rpm_tarball: openchami-0.1.7-1.noarch.tar.gz +openchami_rpm_name: openchami-0.1.7-1.noarch.rpm ochami_client_rpm_url: "https://github.com/OpenCHAMI/ochami/releases/download/v0.7.1/ochami_0.7.1_linux_amd64.rpm" ochami_client_rpm_name: "ochami_0.7.1_linux_amd64.rpm" diff --git a/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml b/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml index 2b19607817..3408ce26d2 100644 --- a/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml +++ b/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml @@ -18,7 +18,7 @@ # # PostgreSQL data must be backed up before upgrade because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions # # Without this backup, rollback is not possible. diff --git a/upgrade/roles/upgrade_pulp/tasks/main.yml b/upgrade/roles/upgrade_pulp/tasks/main.yml index 4ec5803ca2..771211b5a8 100644 --- a/upgrade/roles/upgrade_pulp/tasks/main.yml +++ b/upgrade/roles/upgrade_pulp/tasks/main.yml @@ -16,13 +16,13 @@ # =========================================================================== # Upgrade Pulp Role - Main Entry Point # -# This role upgrades the Pulp container from 3.80 to 3.113. +# This role upgrades the Pulp container from 3.80 to 3.114.2. # Pulp stores all data in persistent volumes mounted from shared storage, # so data is preserved during the upgrade. # # IMPORTANT: PostgreSQL data must be backed up before upgrade because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions # Without the backup, rollback is not possible. # @@ -30,7 +30,7 @@ # 1. Resolve admin NIC IP for health check endpoints # 2. Pre-upgrade health check (verify Pulp is deployed and accessible) # 3. Backup PostgreSQL data (required for rollback) -# 4. Pull the new Pulp image (3.113) +# 4. Pull the new Pulp image (3.114.2) # 5. Stop the Pulp container # 6. Update the quadlet file with the new image tag # 7. Reload systemd daemon and restart container diff --git a/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml index 937558189f..8450a700e7 100644 --- a/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml +++ b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml @@ -14,7 +14,7 @@ --- # =========================================================================== -# Upgrade Pulp Container (3.80 → 3.113) +# Upgrade Pulp Container (3.80 → 3.114.2) # # Steps: # 1. Pull new Pulp image @@ -95,14 +95,13 @@ name: "{{ pulp_container_name }}" command: pulpcore-manager migrate --noinput register: pulp_migrate_result - retries: 3 - delay: 10 + retries: 40 + delay: 15 until: pulp_migrate_result.rc == 0 delegate_to: oim delegate_facts: true connection: ssh - failed_when: false - name: Display migration result ansible.builtin.debug: - msg: "Database migration: {{ 'completed' if pulp_migrate_result.rc | default(1) == 0 else 'skipped or not required' }}" + msg: "Database migration: {{ 'completed successfully' if pulp_migrate_result.rc == 0 else 'FAILED (rc=' ~ pulp_migrate_result.rc | string ~ ')' }}" diff --git a/upgrade/roles/upgrade_pulp/vars/main.yml b/upgrade/roles/upgrade_pulp/vars/main.yml index 722272dde5..ed9fe59822 100644 --- a/upgrade/roles/upgrade_pulp/vars/main.yml +++ b/upgrade/roles/upgrade_pulp/vars/main.yml @@ -14,7 +14,7 @@ --- # =========================================================================== -# Pulp Container Upgrade Configuration (3.80 → 3.113) +# Pulp Container Upgrade Configuration (3.80 → 3.114.2) # =========================================================================== # File permissions @@ -23,7 +23,7 @@ file_permissions_644: "0644" # Pulp container settings pulp_container_name: "pulp" -pulp_target_tag: "3.113" +pulp_target_tag: "3.114.2" pulp_target_image: "docker.io/pulp/pulp:{{ pulp_target_tag }}" # Pulp quadlet file path @@ -46,6 +46,8 @@ pull_image_retries: 5 pull_image_delay: 10 # Pulp startup and health check settings +# Note: First startup after upgrade runs pg_upgrade (PG 13→16) + Django migrations, +# which can take 5-10 minutes. The total wait budget must cover this one-time operation. pulp_startup_retries: 10 pulp_startup_delay: 10 pulp_init_wait: 30 @@ -60,7 +62,7 @@ pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql" # Backup settings # PostgreSQL data must be backed up before upgrade because: # - Pulp 3.80 uses PostgreSQL 12/13 -# - Pulp 3.113 uses PostgreSQL 16 +# - Pulp 3.114.2 uses PostgreSQL 16 # - PostgreSQL cannot downgrade data files between major versions pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp" pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql" diff --git a/upgrade/roles/upgrade_telemetry/tasks/backup_mysql_data.yml b/upgrade/roles/upgrade_telemetry/tasks/backup_mysql_data.yml new file mode 100644 index 0000000000..17ba6fbed4 --- /dev/null +++ b/upgrade/roles/upgrade_telemetry/tasks/backup_mysql_data.yml @@ -0,0 +1,244 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +# ============================================================================ +# Backup MySQL Data (Pre-Upgrade) +# ============================================================================ +# Performs a logical backup (mysqldump) of the idrac-telemetry MySQL database +# for ALL pods before the upgrade changes the MySQL version. Each pod has its +# own independent MySQL instance with its own PVC, so backups are taken +# per-pod to ensure no data loss. +# +# Why this is needed: +# MySQL only allows downgrades between patch versions (e.g., 9.3.1 -> 9.3.0). +# Minor/major version downgrades (e.g., 9.7.2 -> 9.3.0) are rejected because +# MySQL upgrades the internal data format on disk. When etcd rollback restores +# the old StatefulSet spec (with the old MySQL image), the new-format data on +# the PVC causes MySQL to crash with: +# [ERROR] [MY-014061] Cannot downgrade from 90702 to 90300. +# +# This task creates per-pod mysqldumps that can be restored into fresh MySQL +# instances at the old version, preserving all schema and data for every pod. +# +# NOTE: Failures in this task do NOT block the upgrade. The rollback playbook +# can handle the downgrade even without a backup (by wiping the PVC and +# letting MySQL reinitialize), but data will be lost in that case. +# ============================================================================ + +- name: Backup idrac-telemetry MySQL data (pre-upgrade) + when: + - kube_vip is defined + - kube_vip | length > 0 + - k8s_client_mount_path is defined + block: + # ── Check if idrac-telemetry StatefulSet exists ── + - name: Check if idrac-telemetry StatefulSet exists + ansible.builtin.command: + cmd: kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} --no-headers + delegate_to: "{{ kube_vip }}" + connection: ssh + register: idrac_sts_exists + changed_when: false + failed_when: false + + - name: Skip MySQL backup if idrac-telemetry not deployed + ansible.builtin.debug: + msg: "{{ mysql_backup_skip_msg }}" + when: idrac_sts_exists.rc != 0 + + # ── Perform MySQL backup for all pods ── + - name: Perform MySQL logical backup + when: idrac_sts_exists.rc == 0 + block: + - name: Create MySQL backup directory on NFS + ansible.builtin.file: + path: "{{ mysql_backup_dir }}" + state: directory + mode: '0755' + delegate_to: "{{ kube_vip }}" + connection: ssh + + # Get the MySQL root password from the K8s secret + - name: Get MySQL root password from secret + ansible.builtin.shell: + cmd: | + set -o pipefail + kubectl get secret {{ mysqldb_secrets_name }} -n {{ telemetry_namespace }} \ + -o jsonpath='{.data.mysqldb_root_password}' | base64 -d + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_root_password + changed_when: false + no_log: true + + # Get the current MySQL image version (for metadata) + - name: Get current MySQL image version + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.template.spec.containers[?(@.name=="mysqldb")].image}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: current_mysql_image + changed_when: false + + # Discover all running idrac-telemetry pods + - name: Get current replica count from StatefulSet + ansible.builtin.shell: + cmd: > + kubectl get statefulset idrac-telemetry -n {{ telemetry_namespace }} + -o jsonpath='{.spec.replicas}' + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_backup_replica_count_raw + changed_when: false + + - name: Set replica count for backup + ansible.builtin.set_fact: + mysql_backup_replica_count: "{{ mysql_backup_replica_count_raw.stdout | default('1') | int }}" + + - name: Build list of pod names for backup + ansible.builtin.set_fact: + mysql_backup_pod_list: >- + {{ range(0, mysql_backup_replica_count | int) + | map('regex_replace', '^(.*)$', 'idrac-telemetry-\1') + | list }} + + - name: Display pods to backup + ansible.builtin.debug: + msg: "{{ mysql_backup_pods_discovered_msg }}" + + # Wait for MySQL to be ready on each pod before dumping + - name: Wait for MySQL container to be ready on each pod + ansible.builtin.shell: + cmd: | + kubectl exec {{ item }} -n {{ telemetry_namespace }} -c mysqldb -- \ + mysqladmin ping -u root -p'{{ mysql_root_password.stdout }}' 2>/dev/null + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysql_ping_results + until: mysql_ping_results.rc == 0 + retries: 12 + delay: 10 + changed_when: false + no_log: true + loop: "{{ mysql_backup_pod_list }}" + + # Perform mysqldump per pod (plaintext, temporary) + - name: Execute mysqldump for each pod + ansible.builtin.shell: + cmd: | + kubectl exec {{ item }} -n {{ telemetry_namespace }} -c mysqldb -- \ + mysqldump -u root -p'{{ mysql_root_password.stdout }}' \ + --single-transaction --routines --triggers --events \ + --set-gtid-purged=OFF \ + idrac_telemetrydb 2>/dev/null \ + > {{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ loop_index0 }}.sql + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysqldump_results + changed_when: true + no_log: true + loop: "{{ mysql_backup_pod_list }}" + loop_control: + index_var: loop_index0 + + # Verify and encrypt each dump file + - name: Verify plaintext mysqldump files were created + ansible.builtin.stat: + path: "{{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item }}.sql" + delegate_to: "{{ kube_vip }}" + connection: ssh + register: mysqldump_file_stats + loop: "{{ range(0, mysql_backup_replica_count | int) | list }}" + + - name: Warn about empty or missing mysqldump files + ansible.builtin.debug: + msg: "{{ mysql_backup_failed_msg }} Pod index: {{ item.item }}" + when: not (item.stat.exists | default(false)) or (item.stat.size | default(0)) < 100 + loop: "{{ mysqldump_file_stats.results }}" + loop_control: + label: "pod-{{ item.item }}" + + # Encrypt each dump file + - name: Encrypt mysqldump backup files + ansible.builtin.shell: + cmd: | + openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \ + -in {{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item.item }}.sql \ + -out {{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item.item }}.sql.enc \ + -pass pass:'{{ mysql_root_password.stdout }}' + delegate_to: "{{ kube_vip }}" + connection: ssh + changed_when: true + no_log: true + when: (item.stat.exists | default(false)) and (item.stat.size | default(0)) >= 100 + loop: "{{ mysqldump_file_stats.results }}" + loop_control: + label: "pod-{{ item.item }}" + + - name: Remove plaintext mysqldump files + ansible.builtin.file: + path: "{{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item.item }}.sql" + state: absent + delegate_to: "{{ kube_vip }}" + connection: ssh + when: (item.stat.exists | default(false)) and (item.stat.size | default(0)) >= 100 + loop: "{{ mysqldump_file_stats.results }}" + loop_control: + label: "pod-{{ item.item }}" + + - name: Verify encrypted backup files exist + ansible.builtin.stat: + path: "{{ mysql_backup_dir }}/{{ mysql_backup_filename_prefix }}{{ item }}.sql.enc" + delegate_to: "{{ kube_vip }}" + connection: ssh + register: encrypted_backup_file_stats + loop: "{{ range(0, mysql_backup_replica_count | int) | list }}" + + - name: Count successful backups + ansible.builtin.set_fact: + mysql_backup_success_count: >- + {{ encrypted_backup_file_stats.results + | selectattr('stat.exists', 'equalto', true) + | list | length }} + + # Save metadata about the backup + - name: Save MySQL backup metadata + ansible.builtin.copy: + content: | + # MySQL backup metadata (pre-upgrade) + backup_timestamp: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + mysql_image: "{{ current_mysql_image.stdout }}" + database: "idrac_telemetrydb" + replica_count: {{ mysql_backup_replica_count }} + encrypted: true + backup_filename_prefix: "{{ mysql_backup_filename_prefix }}" + pods_backed_up: {{ mysql_backup_pod_list }} + successful_backups: {{ mysql_backup_success_count }} + dest: "{{ mysql_backup_dir }}/{{ mysql_backup_metadata_filename }}" + mode: '0644' + delegate_to: "{{ kube_vip }}" + connection: ssh + when: mysql_backup_success_count | int > 0 + + - name: Display MySQL backup success + ansible.builtin.debug: + msg: "{{ mysql_backup_all_pods_success_msg }}" + when: mysql_backup_success_count | int > 0 + + rescue: + - name: Display MySQL backup warning (non-fatal) + ansible.builtin.debug: + msg: "{{ mysql_backup_rescue_msg }}" diff --git a/upgrade/roles/upgrade_telemetry/tasks/main.yml b/upgrade/roles/upgrade_telemetry/tasks/main.yml index 68c087306c..1dad6a148a 100644 --- a/upgrade/roles/upgrade_telemetry/tasks/main.yml +++ b/upgrade/roles/upgrade_telemetry/tasks/main.yml @@ -51,6 +51,14 @@ msg: "{{ victoria_upgrade_skipped }}" when: not (victoria_upgrade_needed | default(false) | bool) +# ── Phase 2b: Backup MySQL data before version change ── +- name: Phase 2b - Backup MySQL data (pre-upgrade) + ansible.builtin.include_tasks: backup_mysql_data.yml + when: + - k8s_client_mount_path is defined + - kube_vip is defined + - kube_vip | length > 0 + # ── Phase 3: Execute telemetry.sh to redeploy telemetry stack ── - name: Phase 3 - Execute telemetry.sh to redeploy telemetry stack ansible.builtin.include_tasks: execute_telemetry_sh.yml diff --git a/upgrade/roles/upgrade_telemetry/vars/main.yml b/upgrade/roles/upgrade_telemetry/vars/main.yml index b726a4be10..4e0c238781 100644 --- a/upgrade/roles/upgrade_telemetry/vars/main.yml +++ b/upgrade/roles/upgrade_telemetry/vars/main.yml @@ -186,3 +186,32 @@ idrac_patch_msg: >- MySQL will have enough time to flush on NFS during pod restart. idrac_skip_patch_msg: "idrac-telemetry StatefulSet not found (first deploy). Skipping patch." idrac_replica_restore_msg: "idrac-telemetry scaled back to {{ idrac_replica_count.stdout }} replicas" + +# ============================================================================ +# MYSQL BACKUP VARIABLES (Pre-Upgrade) +# ============================================================================ +mysqldb_secrets_name: "mysqldb-credentials" +mysql_backup_dir: "{{ telemetry_backup_dir }}/mysql" +# Per-pod backup naming: .sql / .sql.enc +# e.g., idrac_telemetry_mysql_backup_pod0.sql.enc, idrac_telemetry_mysql_backup_pod1.sql.enc +mysql_backup_filename_prefix: "idrac_telemetry_mysql_backup_pod" +mysql_backup_metadata_filename: "mysql_backup_metadata.yml" +mysql_backup_skip_msg: "idrac-telemetry StatefulSet not found — skipping MySQL data backup." +mysql_backup_pods_discovered_msg: >- + Discovered {{ mysql_backup_replica_count }} idrac-telemetry pod(s) for backup: + {{ mysql_backup_pod_list | join(', ') }} +mysql_backup_all_pods_success_msg: >- + MySQL data backup completed successfully for {{ mysql_backup_success_count }}/{{ mysql_backup_replica_count }} pod(s). + Backup directory: {{ mysql_backup_dir }} + These per-pod backups will be used by the rollback playbook to restore MySQL data + if a version downgrade makes the PVC data incompatible. +mysql_backup_failed_msg: >- + WARNING: MySQL data backup FAILED — mysqldump file is empty or missing. + The upgrade will proceed but rollback may not be able to restore MySQL data + if the MySQL version changes. Check pod logs for MySQL connectivity issues. +mysql_backup_rescue_msg: + - "WARNING: MySQL data backup failed — this is non-fatal." + - "The upgrade will proceed. If rollback is needed later, MySQL" + - "will be reinitialized fresh (empty database with auto-created" + - "schema from MYSQL_DATABASE env var in the StatefulSet)." + - "Error: {{ ansible_failed_result.msg | default('unknown') }}" diff --git a/upgrade/upgrade.yml b/upgrade/upgrade.yml index e2c27ad10b..1d90679129 100644 --- a/upgrade/upgrade.yml +++ b/upgrade/upgrade.yml @@ -453,7 +453,7 @@ ── Omnia Upgrade Execution Plan (in order) ────────────────── 1. oim → Upgrade OpenCHAMI control-plane containers - and Pulp container (3.80 → 3.113) on the OIM + and Pulp container (3.80 → 3.114.2) on the OIM 2. build_stream → SKIPPED (not enabled in build_stream_config.yml) 3. local_repo → Synchronize Omnia 2.2 packages into the local Pulp repository for cluster nodes