From 6fddc00af892d051cb7ee75a1192dca5db8b16b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 23 Oct 2026 16:09:37 +0200 Subject: [PATCH 01/16] fix(ci): unblock disconnected OCP nightlies The Helm and Operator disconnected nightlies have not passed for over five weeks. Three independent gaps kept them red: - Backend egress to the mirror registry. The chart and the Operator both default-deny backend egress except DNS, HTTPS, Redis, and PostgreSQL, while the CI mirror serves plain HTTP on port 5000. install-dynamic-plugins timed out pulling the catalog index from the bastion. - Operator catalog. A filtered oc-mirror run against the published Red Hat catalog rejects it since rhdh-operator#3822 removed the default `fast` channel, and that catalog carries no 2.x bundles anyway. CI runs now mirror the RHDH CI catalog (quay.io/rhdh/iib:-v-x86_64); its registry.redhat.io/rhdh relatedImages are redirected to quay.io/rhdh with CONTAINERS_REGISTRIES_CONF for the prepare-restricted-environment.sh call. - Smoke plugin set. On the 2.x New Frontend System hub the sign-in page, sidebar, SCM integrations, and guest auth provider are dynamic plugins, so the homepage-only set could not render a login. Mirror the minimal digest pinned set and configure the NFS extensions and signInPage in the app configs. The 2.x hub image and the new plugin set were verified on a live OCP cluster: guest sign-in, sidebar, and the "Welcome back!" homepage heading all render. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 10 +- .../jobs/ocp-disconnected-operator.sh | 65 +++++++-- .ci/pipelines/lib/disconnected.sh | 7 +- .ci/pipelines/lib/disconnected/local.sh | 2 +- .../lib/disconnected/local/plugins.sh | 12 +- .ci/pipelines/lib/disconnected/namespace.sh | 69 ++++++++++ .ci/pipelines/lib/disconnected/plugins.sh | 124 ++++++++++++------ .../app-config-rhdh-disconnected-smoke.yaml | 13 ++ .../dynamic-plugins-homepage.yaml | 15 --- .../disconnected/dynamic-plugins-smoke.yaml | 18 +++ .../rhdh-start-disconnected-smoke.yaml | 6 +- .../values_disconnected-smoke.yaml | 37 ++++-- 12 files changed, 292 insertions(+), 86 deletions(-) delete mode 100644 .ci/pipelines/resources/disconnected/dynamic-plugins-homepage.yaml create mode 100644 .ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index d4ef2a0872..2d132016b7 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -143,13 +143,15 @@ handle_ocp_disconnected_helm() { log::section "Plugin Mirroring" disconnected::mirror_plugins || return 1 # Pin CATALOG_INDEX_IMAGE to the digest actually pushed (hub profile default - # digest is often absent from the mirror). Homepage package is the digest-pinned - # OCI exported as HOMEPAGE_PLUGIN_PACKAGE (ref:// cannot resolve with includes: []). + # digest is often absent from the mirror). mirror_plugins exported the + # digest-pinned OCI package for each smoke plugin (ref:// cannot resolve with + # includes: []). disconnected::resolve_catalog_index_image || return 1 log::section "Namespace and Secrets" namespace::configure "${NAME_SPACE}" + disconnected::allow_mirror_registry_egress "${NAME_SPACE}" || return 1 if [[ "${LOCAL_DISCONNECTED:-}" == "1" ]]; then disconnected::ensure_local_image_pull_access "${NAME_SPACE}" || return 1 fi @@ -205,7 +207,9 @@ handle_ocp_disconnected_helm() { # to the rendered Deployment, avoiding the Helm "array clobber" pitfall. local post_renderer="${DIR}/resources/disconnected/helm-post-renderer.sh" local helm_values="${DISCONNECTED_TMPDIR}/values_disconnected-smoke.yaml" - common::require_vars HOMEPAGE_PLUGIN_PACKAGE || return 1 + common::require_vars HOMEPAGE_PLUGIN_PACKAGE APP_AUTH_PLUGIN_PACKAGE \ + APP_DEFAULTS_PLUGIN_PACKAGE APP_INTEGRATIONS_PLUGIN_PACKAGE \ + GUEST_PROVIDER_PLUGIN_PACKAGE || return 1 envsubst < "${DIR}/value_files/values_disconnected-smoke.yaml" > "${helm_values}" || { log::error "Failed to render disconnected Helm values" return 1 diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 8146802b9b..b4da39e46e 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -51,8 +51,8 @@ handle_ocp_disconnected_operator() { # default skopeo/umoci/podman-build path. Nested Podman in this CI pod cannot # initialize storage (newuidmap / VFS chown both fail under hostUsers: false). # CATALOG_INDEX_IMAGE is the plugin catalog index — do not pass it as - # --index-image (OLM operator catalog). Used only for mirror-plugins.sh below, - # then cleared so deploy does not inject :next defaults onto the GA hub. + # --index-image (OLM operator catalog). It is consumed by mirror-plugins.sh and + # injected into the Backstage CR by deploy_rhdh_operator. # # OLM version: leave default (auto). On OCP 4.21+ this selects OLM v1. local filter_versions="${RELEASE_VERSION}" @@ -63,7 +63,6 @@ handle_ocp_disconnected_operator() { # Mirroring every historical RHDH operator version (* / next) is unnecessary # for smoke and flakes on quay CDN EOF (CI) or overwhelms the integrated # registry (local). Pin to the chart major.minor (e.g. 2.0-80-CI → 2.0). - # Release branches already pass a concrete version (e.g. 1.9) and skip this. if [[ "${filter_versions}" == "*" ]]; then local chart_mm="" if [[ -n "${CHART_VERSION:-}" && "${CHART_VERSION}" =~ ^([0-9]+\.[0-9]+) ]]; then @@ -84,7 +83,47 @@ handle_ocp_disconnected_operator() { log::error "CONTAINER_PLATFORM_VERSION is unset or invalid ('${ocp_version}'); expected OpenShift major.minor" return 1 fi - local index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}" + # GA/RC verification mirrors the published Red Hat Operator catalog, the only + # one carrying released bundles. A filtered oc-mirror run rejects it until the + # rhdh-operator defaultChannel fix lands (rhdh-operator#3822). CI runs + # (IMAGE_REGISTRY=quay.io) cannot use it at all: it has no 2.x bundles, so they + # mirror the RHDH CI catalog built from the same commit as the hub image. + local index_image="" + local registries_conf="" + if [[ "${IMAGE_REGISTRY}" == "registry.redhat.io" ]]; then + index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}" + else + # rhdh-operator publishes one catalog per release stream and OCP minor, + # e.g. quay.io/rhdh/iib:next-v4.21-x86_64 (amd64 only; the jobs run on + # amd64 clusters). + index_image="quay.io/rhdh/iib:${RELEASE_VERSION}-v${ocp_version}-x86_64" + skopeo inspect --raw "docker://${index_image}" > /dev/null || { + log::error "RHDH CI operator catalog ${index_image} not found or unreadable" + return 1 + } + + # The CI bundle's RELATED_IMAGE refs point at registry.redhat.io/rhdh/* + # (plus the stage/access aliases), whose digests exist on quay.io/rhdh only. + # Rewrite those pulls for the oc-mirror run with containers/image + # registries.conf (CONTAINERS_REGISTRIES_CONF), the same mechanism the + # medik8s-disconnected-catalogsource CI step uses. Images in other + # repositories (e.g. registry.redhat.io/rhel10/postgresql-18) keep their + # original source and are pulled with the CI credentials. + registries_conf="${DISCONNECTED_TMPDIR}/oc-mirror-registries.conf" + cat > "${registries_conf}" << 'EOF' +[[registry]] +prefix = "registry.redhat.io/rhdh" +location = "quay.io/rhdh" + +[[registry]] +prefix = "registry.stage.redhat.io/rhdh" +location = "quay.io/rhdh" + +[[registry]] +prefix = "registry.access.redhat.com/rhdh" +location = "quay.io/rhdh" +EOF + fi log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}" # Local connected OCP: OCP_INTERNAL sentinel (same as omit-on-OCP). CI bastion @@ -123,10 +162,18 @@ handle_ocp_disconnected_operator() { log::info "LOCAL_DISCONNECTED=1: skipping external bastion CA/pull-secret pre-prepare helpers" fi + local -a prepare_cmd=(bash "${prepare_script_path}" "${prepare_args[@]}") + if [[ -n "${registries_conf}" ]]; then + # Scoped to this call: CATALOG_INDEX_IMAGE and the plugin pulls keep their + # own registry configuration. + prepare_cmd=(env CONTAINERS_REGISTRIES_CONF="${registries_conf}" "${prepare_cmd[@]}") + log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" + fi + log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}" if ! disconnected::retry_on_local_registry 5 \ disconnected::with_unset_registry_auth_file \ - bash "${prepare_script_path}" "${prepare_args[@]}"; then + "${prepare_cmd[@]}"; then log::error "prepare-restricted-environment.sh failed — aborting" return 1 fi @@ -193,13 +240,15 @@ handle_ocp_disconnected_operator() { disconnected::mirror_plugins || return 1 # Pin CATALOG_INDEX_IMAGE to the digest actually pushed (hub profile default - # digest is often absent from the mirror). Homepage package is the digest-pinned - # OCI exported as HOMEPAGE_PLUGIN_PACKAGE (ref:// cannot resolve with includes: []). + # digest is often absent from the mirror). mirror_plugins exported the + # digest-pinned OCI package for each smoke plugin (ref:// cannot resolve with + # includes: []). disconnected::resolve_catalog_index_image || return 1 log::section "Namespace and Secrets" namespace::configure "${NAME_SPACE}" + disconnected::allow_mirror_registry_egress "${NAME_SPACE}" || return 1 if [[ "${LOCAL_DISCONNECTED:-}" == "1" ]]; then disconnected::ensure_local_image_pull_access "${NAME_SPACE}" || return 1 fi @@ -208,7 +257,7 @@ handle_ocp_disconnected_operator() { # from the mirror (registries.conf alone is not enough — TLS verify fails). disconnected::create_mirror_registry_ca_configmap "${NAME_SPACE}" || return 1 disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" || return 1 - disconnected::create_homepage_plugins_configmap "${NAME_SPACE}" || return 1 + disconnected::create_smoke_plugins_configmap "${NAME_SPACE}" || return 1 # Operator mounts one volume per extraFiles ConfigMap name. policy.json cannot # share rhdh-plugin-mirror-conf or reconcile fails with duplicate volume keys. diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index f8b5b45f71..4025037608 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -63,10 +63,12 @@ export -f disconnected::_hook_rewrite_mirror_host # plugins.sh export -f disconnected::mirror_plugins export -f disconnected::resolve_catalog_index_image -export -f disconnected::create_homepage_plugins_configmap +export -f disconnected::create_smoke_plugins_configmap export -f disconnected::apply_plugin_mirror_configmap export -f disconnected::write_digest_plugin_list -export -f disconnected::set_homepage_plugin_package_from_list +export -f disconnected::write_smoke_plugin_list +export -f disconnected::set_smoke_plugin_packages_from_list +export -f disconnected::_smoke_plugin_package_var export -f disconnected::copy_catalog_index_to_mirror export -f disconnected::_catalog_index_source_ref export -f disconnected::_catalog_index_mirror_dest @@ -75,6 +77,7 @@ export -f disconnected::_hook_catalog_index_exports # namespace.sh export -f disconnected::create_mirror_registry_ca_configmap export -f disconnected::create_plugin_registry_auth_secret +export -f disconnected::allow_mirror_registry_egress export -f disconnected::ensure_mirror_registry_ca export -f disconnected::ensure_olm_mirror_pull_secret # operator.sh diff --git a/.ci/pipelines/lib/disconnected/local.sh b/.ci/pipelines/lib/disconnected/local.sh index 6d99bee71f..40d0440613 100644 --- a/.ci/pipelines/lib/disconnected/local.sh +++ b/.ci/pipelines/lib/disconnected/local.sh @@ -7,7 +7,7 @@ # registry.sh integrated-registry bring-up, retry-on-503, MIRROR_* bootstrap # access.sh mirror push-target projects, workload + OLM pull access # mirror-host.sh IDMS/ITMS push-route -> in-cluster registry service rewrite -# plugins.sh homepage-only mirror_plugins override + imagestream tags +# plugins.sh smoke-plugin mirror_plugins override + imagestream tags [[ -n "${_DISCONNECTED_LOCAL_SOURCED:-}" ]] && return 0 readonly _DISCONNECTED_LOCAL_SOURCED=1 diff --git a/.ci/pipelines/lib/disconnected/local/plugins.sh b/.ci/pipelines/lib/disconnected/local/plugins.sh index 2cb87fbfc7..0d89e2a4eb 100644 --- a/.ci/pipelines/lib/disconnected/local/plugins.sh +++ b/.ci/pipelines/lib/disconnected/local/plugins.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# LOCAL_DISCONNECTED plugin mirroring: homepage-only mirror_plugins override +# LOCAL_DISCONNECTED plugin mirroring: smoke-plugin mirror_plugins override # (retry loop, skopeo catalog copy, imagestream tagging). Sourced only when # LOCAL_DISCONNECTED=1. [[ -n "${_DISCONNECTED_LOCAL_PLUGINS_SOURCED:-}" ]] && return 0 readonly _DISCONNECTED_LOCAL_PLUGINS_SOURCED=1 -# Full override: homepage-only plugin list, retry loop, skopeo catalog copy, +# Full override: smoke plugin list, retry loop, skopeo catalog copy, # summary handling, and imagestream tagging. disconnected::mirror_plugins() { local mirror_script="${DISCONNECTED_TMPDIR}/mirror-plugins.sh" @@ -19,10 +19,8 @@ disconnected::mirror_plugins() { local plugin_index plugin_index="oci://$(disconnected::_catalog_index_source_ref)" - local list_file="${DISCONNECTED_TMPDIR}/homepage-plugins.txt" - disconnected::write_digest_plugin_list "${plugin_index}" "${list_file}" \ - "${DISCONNECTED_HOMEPAGE_PLUGIN_NAME}" || return 1 - disconnected::set_homepage_plugin_package_from_list "${list_file}" || return 1 + local list_file="${DISCONNECTED_TMPDIR}/smoke-plugins.txt" + disconnected::write_smoke_plugin_list "${list_file}" || return 1 # --plugin-list only (not --plugin-index): index mode re-adds fragile tags. # Catalog index is mirrored after plugins so the summary still records it. @@ -40,7 +38,7 @@ disconnected::mirror_plugins() { log::warn "mirror-plugins.sh attempt ${attempt}/${max_attempts} failed (registry 503 during Recreate is typical)" done if [[ "${mirrored}" != "1" ]]; then - log::error "mirror-plugins.sh (homepage plugin list) failed after ${max_attempts} attempts -- aborting" + log::error "mirror-plugins.sh (smoke plugin list) failed after ${max_attempts} attempts -- aborting" return 1 fi diff --git a/.ci/pipelines/lib/disconnected/namespace.sh b/.ci/pipelines/lib/disconnected/namespace.sh index 1f6158de14..23da8cbe7f 100644 --- a/.ci/pipelines/lib/disconnected/namespace.sh +++ b/.ci/pipelines/lib/disconnected/namespace.sh @@ -116,3 +116,72 @@ disconnected::ensure_olm_mirror_pull_secret() { } log::success "Merged mirror registry credentials into openshift-config/pull-secret" } + +# Allow backend egress to the mirror registry. The chart and the Operator both +# default-deny backend egress except DNS, HTTPS, Redis, and PostgreSQL, while CI +# mirrors serve plain HTTP on a non-443 port: the install-dynamic-plugins init +# container (skopeo catalog/plugin pulls) times out without this rule. Kubelet +# pulls are unaffected. NetworkPolicies are additive and separate selectors cover +# the Helm chart and the Operator pod labels (same split as +# resources/network_policy/netpol-ci-allow-backend-egress.yaml). +# Mirrors served on 443 (e.g. the LOCAL_DISCONNECTED route) need no extra rule. +# Args: +# $1 - namespace +disconnected::allow_mirror_registry_egress() { + local namespace=$1 + local registry_host="${MIRROR_REGISTRY_URL%%/*}" + local registry_port="" + + if [[ "${registry_host}" =~ :([0-9]+)$ ]]; then + registry_port="${BASH_REMATCH[1]}" + fi + + if [[ -z "${registry_port}" || "${registry_port}" == "443" ]]; then + log::info "Mirror registry ${registry_host} is served on HTTPS; skipping dedicated egress rule" + return 0 + fi + + local policy_file="${DISCONNECTED_TMPDIR}/netpol-mirror-registry-egress.yaml" + cat > "${policy_file}" << EOF +# Allow backend egress to the CI mirror registry (${registry_host}). +# install-dynamic-plugins pulls the catalog index and plugins over plain HTTP. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rhdh-mirror-registry-egress +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: developer-hub + app.kubernetes.io/component: backstage + policyTypes: + - Egress + egress: + - ports: + - port: ${registry_port} + protocol: TCP +--- +# Operator Backend pods derive rhdh.redhat.com/app from the Backstage CR name. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: rhdh-operator-mirror-registry-egress +spec: + podSelector: + matchLabels: + rhdh.redhat.com/app: backstage-${RELEASE_NAME} + policyTypes: + - Egress + egress: + - ports: + - port: ${registry_port} + protocol: TCP +EOF + + oc apply -f "${policy_file}" -n "${namespace}" || { + log::error "Failed to apply mirror registry egress NetworkPolicy in ${namespace}" + return 1 + } + cp "${policy_file}" "${ARTIFACT_DIR}/disconnected-netpol-mirror-registry-egress.yaml" 2> /dev/null || true + log::success "NetworkPolicy allows backend egress to ${registry_host} in ${namespace}" +} diff --git a/.ci/pipelines/lib/disconnected/plugins.sh b/.ci/pipelines/lib/disconnected/plugins.sh index 59a265e241..0aaa8ca7cc 100644 --- a/.ci/pipelines/lib/disconnected/plugins.sh +++ b/.ci/pipelines/lib/disconnected/plugins.sh @@ -1,13 +1,21 @@ #!/usr/bin/env bash -# Plugin mirroring, catalog index resolution, and homepage ConfigMap. +# Plugin mirroring, catalog index resolution, and smoke plugin ConfigMap. [[ -n "${_DISCONNECTED_PLUGINS_SOURCED:-}" ]] && return 0 readonly _DISCONNECTED_PLUGINS_SOURCED=1 -# Smoke only enables this plugin (includes: []). Full-catalog mirroring is RHIDP-13967. -readonly DISCONNECTED_HOMEPAGE_PLUGIN_NAME="red-hat-developer-hub-backstage-plugin-homepage" -export DISCONNECTED_HOMEPAGE_PLUGIN_NAME +# Smoke only installs this plugin set (includes: []). Full-catalog mirroring is +# RHIDP-13967. On the 2.x New Frontend System hub the sign-in page (app-auth), +# sidebar (app-defaults), SCM integrations (app-integrations) and guest auth +# provider are dynamic plugins, so the homepage alone cannot render a login. +readonly -a DISCONNECTED_SMOKE_PLUGIN_NAMES=( + "red-hat-developer-hub-backstage-plugin-app-auth" + "red-hat-developer-hub-backstage-plugin-app-defaults" + "red-hat-developer-hub-backstage-plugin-app-integrations" + "red-hat-developer-hub-backstage-plugin-homepage" + "backstage-plugin-auth-backend-module-guest-provider" +) # --------------------------------------------------------------------------- # Hooks — overridable by local.sh (or other consumers) to customise CI-only @@ -69,11 +77,11 @@ disconnected::_catalog_index_mirror_dest() { # Args: # $1 - plugin_index (oci://...) # $2 - output list file path -# $3 - optional name substring; keep only matching refs (e.g. homepage) +# $3 - optional space-separated package names; keep only matching refs disconnected::write_digest_plugin_list() { local plugin_index=$1 local list_file=$2 - local name_filter=${3:-} + local name_filters=${3:-} local index_ref="${plugin_index#oci://}" local extract_dir extract_dir=$(mktemp -d) @@ -122,11 +130,12 @@ disconnected::write_digest_plugin_list() { mv "${tmp}" "${list_file}" fi - if [[ -n "${name_filter}" && -s "${list_file}" ]]; then - local filtered + if [[ -n "${name_filters}" && -s "${list_file}" ]]; then + local filtered pattern filtered=$(mktemp) # Path-segment match so "...-homepage" does not also keep "...-homepage-backend". - grep -E "/${name_filter}(@|:)" "${list_file}" > "${filtered}" || true + pattern="${name_filters// /|}" + grep -E "/(${pattern})(@|:)" "${list_file}" > "${filtered}" || true mv "${filtered}" "${list_file}" fi @@ -135,15 +144,15 @@ disconnected::write_digest_plugin_list() { rm -rf "${extract_dir}" if [[ "${count}" -lt 1 ]]; then - if [[ -n "${name_filter}" ]]; then - log::error "No digest-pinned ${name_filter} plugin found in catalog index" + if [[ -n "${name_filters}" ]]; then + log::error "No digest-pinned plugin matched: ${name_filters}" else log::error "No digest-pinned plugins found in catalog index" fi return 1 fi - if [[ -n "${name_filter}" ]]; then - log::info "Plugin list filtered to ${name_filter}: ${count} digest-pinned ref(s)" + if [[ -n "${name_filters}" ]]; then + log::info "Plugin list filtered to ${count} digest-pinned ref(s) for: ${name_filters}" else log::info "${count} digest-pinned plugins (tag-only refs skipped)" fi @@ -173,32 +182,69 @@ disconnected::copy_catalog_index_to_mirror() { fi } -# Export HOMEPAGE_PLUGIN_PACKAGE from a one-line digest plugin list. +# Package name -> env var rendered by the envsubst templates. +# Args: +# $1 - plugin package name (last OCI path segment) +disconnected::_smoke_plugin_package_var() { + case "$1" in + red-hat-developer-hub-backstage-plugin-app-auth) echo "APP_AUTH_PLUGIN_PACKAGE" ;; + red-hat-developer-hub-backstage-plugin-app-defaults) echo "APP_DEFAULTS_PLUGIN_PACKAGE" ;; + red-hat-developer-hub-backstage-plugin-app-integrations) echo "APP_INTEGRATIONS_PLUGIN_PACKAGE" ;; + red-hat-developer-hub-backstage-plugin-homepage) echo "HOMEPAGE_PLUGIN_PACKAGE" ;; + backstage-plugin-auth-backend-module-guest-provider) echo "GUEST_PROVIDER_PLUGIN_PACKAGE" ;; + *) return 1 ;; + esac +} + +# Export one _PLUGIN_PACKAGE per smoke plugin from the digest plugin list. # Adds !name if the list line has no OCI subpath (install-dynamic-plugins needs it). # Args: # $1 - plugin list file from write_digest_plugin_list -disconnected::set_homepage_plugin_package_from_list() { +disconnected::set_smoke_plugin_packages_from_list() { local list_file=$1 - local line="" - line=$(head -1 "${list_file}" 2> /dev/null || true) - line="${line#"${line%%[![:space:]]*}"}" - line="${line%"${line##*[![:space:]]}"}" - if [[ -z "${line}" ]]; then - log::error "Homepage plugin list is empty: ${list_file}" - return 1 - fi - if [[ "${line}" != *'!'* ]]; then - line="${line}!${DISCONNECTED_HOMEPAGE_PLUGIN_NAME}" - fi - export HOMEPAGE_PLUGIN_PACKAGE="${line}" - log::success "HOMEPAGE_PLUGIN_PACKAGE=${HOMEPAGE_PLUGIN_PACKAGE}" + local name="" var="" line="" exported=0 + + for name in "${DISCONNECTED_SMOKE_PLUGIN_NAMES[@]}"; do + line=$(grep -E "/${name}(@|:)" "${list_file}" 2> /dev/null | head -1 || true) + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + if [[ -z "${line}" ]]; then + log::error "No digest-pinned ${name} plugin in ${list_file}" + return 1 + fi + if [[ "${line}" != *'!'* ]]; then + line="${line}!${name}" + fi + var=$(disconnected::_smoke_plugin_package_var "${name}") || { + log::error "No package variable mapped for plugin ${name}" + return 1 + } + export "${var}=${line}" + log::success "${var}=${line}" + exported=$((exported + 1)) + done + + log::info "Exported ${exported} smoke plugin package(s)" +} + +# Build the digest-pinned smoke plugin list and export its package variables. +# Args: +# $1 - output list file path +disconnected::write_smoke_plugin_list() { + local list_file=$1 + local plugin_index + plugin_index="oci://$(disconnected::_catalog_index_source_ref)" + + disconnected::write_digest_plugin_list "${plugin_index}" "${list_file}" \ + "${DISCONNECTED_SMOKE_PLUGIN_NAMES[*]}" || return 1 + disconnected::set_smoke_plugin_packages_from_list "${list_file}" || return 1 } # --------------------------------------------------------------------------- # mirror_plugins — fetch and run mirror-plugins.sh against the mirror registry # --------------------------------------------------------------------------- -# Smoke: homepage plugin only (--plugin-list) plus the catalog index +# Smoke: the minimal NFS plugin set (--plugin-list) plus the catalog index # (CATALOG_INDEX_IMAGE). --plugin-index would enumerate every catalog plugin, # including unpublished :tag refs. disconnected::mirror_plugins() { @@ -212,10 +258,8 @@ disconnected::mirror_plugins() { local plugin_index plugin_index="oci://$(disconnected::_catalog_index_source_ref)" - local list_file="${DISCONNECTED_TMPDIR}/homepage-plugins.txt" - disconnected::write_digest_plugin_list "${plugin_index}" "${list_file}" \ - "${DISCONNECTED_HOMEPAGE_PLUGIN_NAME}" || return 1 - disconnected::set_homepage_plugin_package_from_list "${list_file}" || return 1 + local list_file="${DISCONNECTED_TMPDIR}/smoke-plugins.txt" + disconnected::write_smoke_plugin_list "${list_file}" || return 1 disconnected::wait_mirror_registry_route 300 || return 1 bash "${mirror_script}" \ @@ -313,18 +357,20 @@ disconnected::resolve_catalog_index_image() { } # --------------------------------------------------------------------------- -# Homepage plugin ConfigMap (Operator) +# Smoke plugin ConfigMap (Operator) # --------------------------------------------------------------------------- -# Create a ConfigMap from the homepage-only dynamic-plugins.yaml -# (digest-pinned OCI package; includes: [] skips catalog defaults). +# Create a ConfigMap from the smoke dynamic-plugins.yaml +# (digest-pinned OCI packages; includes: [] skips catalog defaults). # Args: # $1 - namespace -disconnected::create_homepage_plugins_configmap() { - common::require_vars HOMEPAGE_PLUGIN_PACKAGE || return 1 +disconnected::create_smoke_plugins_configmap() { + common::require_vars HOMEPAGE_PLUGIN_PACKAGE APP_AUTH_PLUGIN_PACKAGE \ + APP_DEFAULTS_PLUGIN_PACKAGE APP_INTEGRATIONS_PLUGIN_PACKAGE \ + GUEST_PROVIDER_PLUGIN_PACKAGE || return 1 local namespace=$1 local cm_name="dynamic-plugins-disconnected-smoke" - local template="${DIR}/resources/disconnected/dynamic-plugins-homepage.yaml" + local template="${DIR}/resources/disconnected/dynamic-plugins-smoke.yaml" local tmp_yaml="${DISCONNECTED_TMPDIR}/dynamic-plugins-disconnected-smoke.yaml" envsubst < "${template}" > "${tmp_yaml}" || { diff --git a/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml b/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml index 3a6f4e7a38..047861bd9a 100644 --- a/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml +++ b/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml @@ -1,11 +1,24 @@ # Minimal app-config for disconnected operator smoke (guest login only). app: title: Red Hat Developer Hub + # New Frontend System: the homepage plugin owns page:homepage and its layout; + # disable the community home page so only one Home destination is registered. + # Same extension list as value_files/values_disconnected-smoke.yaml. + extensions: + - page:home: false + - page:homepage: + config: + path: / + title: Home + - translation:app/homepage-translations auth: environment: development providers: guest: dangerouslyAllowOutsideDevelopment: true +# app-auth does not inject the Guest provider (auth.providers is not +# frontend-visible), so the provider list must be explicit. +signInPage: guest # Hub defaults enable permission/RBAC; without a policy module the backend # stays not-ready (readiness 503) even though / can return 200. permission: diff --git a/.ci/pipelines/resources/disconnected/dynamic-plugins-homepage.yaml b/.ci/pipelines/resources/disconnected/dynamic-plugins-homepage.yaml deleted file mode 100644 index 9cf0f851e1..0000000000 --- a/.ci/pipelines/resources/disconnected/dynamic-plugins-homepage.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# Disconnected smoke: homepage-only dynamic plugins (Operator ConfigMap). -# Rendered via envsubst (requires HOMEPAGE_PLUGIN_PACKAGE). -# includes: [] skips dynamic-plugins.default.yaml (unpublished :tag refs). -# package is the digest-pinned OCI from the catalog (ref:// needs includes). -includes: [] -plugins: - - package: ${HOMEPAGE_PLUGIN_PACKAGE} - enabled: true - pluginConfig: - dynamicPlugins: - frontend: - red-hat-developer-hub.backstage-plugin-homepage: - dynamicRoutes: - - path: / - importName: DynamicHomePage diff --git a/.ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml b/.ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml new file mode 100644 index 0000000000..566e44dcb0 --- /dev/null +++ b/.ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml @@ -0,0 +1,18 @@ +# Disconnected smoke: minimal New Frontend System plugin set (Operator ConfigMap). +# Rendered via envsubst (requires the *_PLUGIN_PACKAGE variables). +# includes: [] skips dynamic-plugins.default.yaml (unpublished :tag refs). +# packages are the digest-pinned OCI refs from the catalog (ref:// needs includes). +# The 2.x hub provides the sign-in page, sidebar, SCM integrations and guest auth +# as dynamic plugins, so they must be installed alongside the homepage. +includes: [] +plugins: + - package: ${APP_AUTH_PLUGIN_PACKAGE} + enabled: true + - package: ${APP_DEFAULTS_PLUGIN_PACKAGE} + enabled: true + - package: ${APP_INTEGRATIONS_PLUGIN_PACKAGE} + enabled: true + - package: ${GUEST_PROVIDER_PLUGIN_PACKAGE} + enabled: true + - package: ${HOMEPAGE_PLUGIN_PACKAGE} + enabled: true diff --git a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml index dbfb6d09f4..794346ce20 100644 --- a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml +++ b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml @@ -4,8 +4,10 @@ # Guest login + homepage only. flavours: [] disables default flavours # (lightspeed) to avoid extra sidecar images. # -# Hub image: CSV RELATED_IMAGE_backstage (GA rhdh-hub-rhel9). prepare + -# oc-mirror IDMS rewrites pulls for kubelet. +# Hub image: CSV RELATED_IMAGE_backstage. CI runs install the RHDH CI catalog +# (quay.io/rhdh/iib), whose bundle points at rhdh-hub-rhel10; GA runs use the +# released rhdh-hub-rhel9. Either way the CSV ref is mirrored by prepare + +# oc-mirror and the generated IDMS rewrites the kubelet pull. # # CATALOG_INDEX_IMAGE is injected by deploy_rhdh_operator from the mirrored # catalog digest so install-dynamic-plugins does not pull the hub profile diff --git a/.ci/pipelines/value_files/values_disconnected-smoke.yaml b/.ci/pipelines/value_files/values_disconnected-smoke.yaml index c5129b9dff..a6e2ff2174 100644 --- a/.ci/pipelines/value_files/values_disconnected-smoke.yaml +++ b/.ci/pipelines/value_files/values_disconnected-smoke.yaml @@ -7,20 +7,25 @@ # Do not include dynamic-plugins.default.yaml from the catalog index: next # catalogs can list :tag refs that do not exist on quay (manifest unknown), # which CrashLoops install-dynamic-plugins after registries.conf rewrites -# those pulls to the mirror. Homepage package is HOMEPAGE_PLUGIN_PACKAGE -# (digest-pinned OCI from the catalog). ref:// needs includes and cannot be used. +# those pulls to the mirror. Every package below is a digest-pinned OCI from +# the catalog. ref:// needs includes and cannot be used. +# +# The 2.x New Frontend System hub serves the sign-in page (app-auth), sidebar +# (app-defaults), SCM integrations (app-integrations) and guest auth provider +# as dynamic plugins, so they are installed next to the homepage. dynamicPlugins: includes: [] plugins: + - package: ${APP_AUTH_PLUGIN_PACKAGE} + enabled: true + - package: ${APP_DEFAULTS_PLUGIN_PACKAGE} + enabled: true + - package: ${APP_INTEGRATIONS_PLUGIN_PACKAGE} + enabled: true + - package: ${GUEST_PROVIDER_PLUGIN_PACKAGE} + enabled: true - package: ${HOMEPAGE_PLUGIN_PACKAGE} enabled: true - pluginConfig: - dynamicPlugins: - frontend: - red-hat-developer-hub.backstage-plugin-homepage: - dynamicRoutes: - - path: / - importName: DynamicHomePage intelligentAssistant: enabled: false orchestrator: @@ -38,3 +43,17 @@ appConfig: providers: guest: dangerouslyAllowOutsideDevelopment: true + # app-auth does not inject the Guest provider (auth.providers is not + # frontend-visible), so the provider list must be explicit. + signInPage: guest + app: + # New Frontend System: the homepage plugin owns page:homepage and its + # layout; disable the community home page so only one Home destination is + # registered. Mirrors resources/config_map/dynamic-plugins-config.yaml. + extensions: + - page:home: false + - page:homepage: + config: + path: / + title: Home + - translation:app/homepage-translations From 07a367a528b82d2150853e9c927ea387086e16c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 23 Oct 2026 16:09:37 +0200 Subject: [PATCH 02/16] fix(ci): address disconnected review feedback - Pick the operator catalog from the chart build type instead of the image registry alone: a CI chart (x.y-N-CI) mirrors the RHDH CI catalog, a released chart (x.y.z) keeps the published catalog. A pinned RC/GA verification on main (TAG_NAME and CHART_VERSION 1.9.4) was handed a `next` catalog for a 1.9 channel filter. - Export the smoke plugin list as a space-separated scalar. bash cannot export arrays, so the `export -f` helpers saw an empty list in a child shell and would have mirrored every digest-pinned plugin. Fail when no package variable was exported. - Document why the mirror egress rule is scoped by port and not by destination: the bastion address is assigned per run and reachable by DNS name only, so an ipBlock would break whenever the bastion is replaced. Assisted-by: OpenCode --- .../jobs/ocp-disconnected-operator.sh | 22 ++++++++++++------- .ci/pipelines/lib/disconnected/namespace.sh | 6 +++++ .ci/pipelines/lib/disconnected/plugins.sh | 19 ++++++++++++++-- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index b4da39e46e..df0bc2efae 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -83,16 +83,20 @@ handle_ocp_disconnected_operator() { log::error "CONTAINER_PLATFORM_VERSION is unset or invalid ('${ocp_version}'); expected OpenShift major.minor" return 1 fi - # GA/RC verification mirrors the published Red Hat Operator catalog, the only - # one carrying released bundles. A filtered oc-mirror run rejects it until the - # rhdh-operator defaultChannel fix lands (rhdh-operator#3822). CI runs - # (IMAGE_REGISTRY=quay.io) cannot use it at all: it has no 2.x bundles, so they - # mirror the RHDH CI catalog built from the same commit as the hub image. + # A CI chart tag (x.y-N-CI) means the run deploys a stream build, so mirror the + # RHDH CI catalog built from the same commit as the hub and plugin catalogs. + # Released charts (x.y.z) pair with the published Red Hat catalog, the only one + # carrying released bundles, even when the image itself is pulled from quay. + # That catalog still needs the rhdh-operator defaultChannel fix + # (rhdh-operator#3822) for a filtered oc-mirror run to accept it. + local use_ci_catalog="false" + if [[ "${IMAGE_REGISTRY}" != "registry.redhat.io" && "${CHART_VERSION:-}" == *-CI ]]; then + use_ci_catalog="true" + fi + local index_image="" local registries_conf="" - if [[ "${IMAGE_REGISTRY}" == "registry.redhat.io" ]]; then - index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}" - else + if [[ "${use_ci_catalog}" == "true" ]]; then # rhdh-operator publishes one catalog per release stream and OCP minor, # e.g. quay.io/rhdh/iib:next-v4.21-x86_64 (amd64 only; the jobs run on # amd64 clusters). @@ -123,6 +127,8 @@ location = "quay.io/rhdh" prefix = "registry.access.redhat.com/rhdh" location = "quay.io/rhdh" EOF + else + index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}" fi log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}" diff --git a/.ci/pipelines/lib/disconnected/namespace.sh b/.ci/pipelines/lib/disconnected/namespace.sh index 23da8cbe7f..3a9449e02e 100644 --- a/.ci/pipelines/lib/disconnected/namespace.sh +++ b/.ci/pipelines/lib/disconnected/namespace.sh @@ -124,6 +124,12 @@ disconnected::ensure_olm_mirror_pull_secret() { # pulls are unaffected. NetworkPolicies are additive and separate selectors cover # the Helm chart and the Operator pod labels (same split as # resources/network_policy/netpol-ci-allow-backend-egress.yaml). +# +# The rule is destination-agnostic on purpose. The bastion mirror's address is +# assigned per run and only reachable by DNS name, so an ipBlock would break as +# soon as the bastion is replaced. The namespace is ephemeral, and the connected +# CI lanes use a broader allow-all rule, so scoping by port is the meaningful +# boundary here. # Mirrors served on 443 (e.g. the LOCAL_DISCONNECTED route) need no extra rule. # Args: # $1 - namespace diff --git a/.ci/pipelines/lib/disconnected/plugins.sh b/.ci/pipelines/lib/disconnected/plugins.sh index 0aaa8ca7cc..a49f04055f 100644 --- a/.ci/pipelines/lib/disconnected/plugins.sh +++ b/.ci/pipelines/lib/disconnected/plugins.sh @@ -16,6 +16,12 @@ readonly -a DISCONNECTED_SMOKE_PLUGIN_NAMES=( "red-hat-developer-hub-backstage-plugin-homepage" "backstage-plugin-auth-backend-module-guest-provider" ) +# Space-separated copy for the `export -f` helpers: bash cannot export arrays, so +# a child shell (e.g. `timeout bash -c`) would otherwise see an empty list and +# mirror every digest-pinned plugin in the catalog. +DISCONNECTED_SMOKE_PLUGIN_NAMES_STR="${DISCONNECTED_SMOKE_PLUGIN_NAMES[*]}" +readonly DISCONNECTED_SMOKE_PLUGIN_NAMES_STR +export DISCONNECTED_SMOKE_PLUGIN_NAMES_STR # --------------------------------------------------------------------------- # Hooks — overridable by local.sh (or other consumers) to customise CI-only @@ -204,7 +210,12 @@ disconnected::set_smoke_plugin_packages_from_list() { local list_file=$1 local name="" var="" line="" exported=0 - for name in "${DISCONNECTED_SMOKE_PLUGIN_NAMES[@]}"; do + if [[ -z "${DISCONNECTED_SMOKE_PLUGIN_NAMES_STR:-}" ]]; then + log::error "DISCONNECTED_SMOKE_PLUGIN_NAMES_STR is empty; cannot resolve smoke plugin packages" + return 1 + fi + + for name in ${DISCONNECTED_SMOKE_PLUGIN_NAMES_STR}; do line=$(grep -E "/${name}(@|:)" "${list_file}" 2> /dev/null | head -1 || true) line="${line#"${line%%[![:space:]]*}"}" line="${line%"${line##*[![:space:]]}"}" @@ -224,6 +235,10 @@ disconnected::set_smoke_plugin_packages_from_list() { exported=$((exported + 1)) done + if [[ "${exported}" -eq 0 ]]; then + log::error "No smoke plugin package variables exported from ${list_file}" + return 1 + fi log::info "Exported ${exported} smoke plugin package(s)" } @@ -236,7 +251,7 @@ disconnected::write_smoke_plugin_list() { plugin_index="oci://$(disconnected::_catalog_index_source_ref)" disconnected::write_digest_plugin_list "${plugin_index}" "${list_file}" \ - "${DISCONNECTED_SMOKE_PLUGIN_NAMES[*]}" || return 1 + "${DISCONNECTED_SMOKE_PLUGIN_NAMES_STR}" || return 1 disconnected::set_smoke_plugin_packages_from_list "${list_file}" || return 1 } From 99f617d322616e7f9b1bfd864ab1b10b00fd575b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Fri, 23 Oct 2026 18:12:40 +0200 Subject: [PATCH 03/16] fix(ci): mount the Helm registry auth secret under the chart name The chart mounts -dynamic-plugins-registry-auth, and fullname is - with nameOverride pinned to developer-hub. The job created rhdh-dynamic-plugins-registry-auth instead, so the optional volume was never mounted, install-dynamic-plugins ran without auth.json, and skopeo failed the mirror pull with "authentication required". Pin nameOverride in the values and create the secret under the chart's name. Verified on a live OCP 4.21 cluster (LOCAL_DISCONNECTED): the init container now pulls the catalog index and all five smoke plugins through the mirror. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 8 +++++++- .ci/pipelines/value_files/values_disconnected-smoke.yaml | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 2d132016b7..e5a811efd2 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -162,7 +162,13 @@ handle_ocp_disconnected_helm() { # the chart mounts ${RELEASE_NAME}-dynamic-plugins-registry-auth at # /opt/app-root/src/.config/containers. disconnected::create_mirror_registry_ca_configmap "${NAME_SPACE}" || return 1 - disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" || return 1 + # The chart mounts -dynamic-plugins-registry-auth (optional: true) at + # /opt/app-root/src/.config/containers, and fullname is + # - with nameOverride pinned to developer-hub in the + # values file. A differently named secret leaves the init container without + # auth.json and skopeo then fails the mirror pull with "authentication required". + disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" \ + "${RELEASE_NAME}-developer-hub-dynamic-plugins-registry-auth" || return 1 log::section "Helm Deployment" diff --git a/.ci/pipelines/value_files/values_disconnected-smoke.yaml b/.ci/pipelines/value_files/values_disconnected-smoke.yaml index a6e2ff2174..dc3de1ffd6 100644 --- a/.ci/pipelines/value_files/values_disconnected-smoke.yaml +++ b/.ci/pipelines/value_files/values_disconnected-smoke.yaml @@ -13,6 +13,9 @@ # The 2.x New Frontend System hub serves the sign-in page (app-auth), sidebar # (app-defaults), SCM integrations (app-integrations) and guest auth provider # as dynamic plugins, so they are installed next to the homepage. +# Pins the chart fullname (-developer-hub). The disconnected job creates +# the registry-auth secret and the backend egress NetworkPolicy for that name. +nameOverride: developer-hub dynamicPlugins: includes: [] plugins: From 0be78ba893c8c5b53a4dab203d5be7e196f2a841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Sat, 24 Oct 2026 12:01:37 +0200 Subject: [PATCH 04/16] fix(ci): stop MCP drains mid-deploy and fail fast on OLM v1 RBAC errors Helm: after the IDMS/ITMS apply, `oc wait machineconfigpool --for=condition=Updated=True` returned while the MCO had not rendered the new config yet. The nodes were drained a few seconds later, while the hub and PostgreSQL pods were starting, so they were evicted mid-startup. The hub recovery path then ran `oc rollout restart`, starting a second backend next to the first against a freshly initialised database; the two raced on the knex migration lock and the losing backend stayed up with readiness 503 forever. - snapshot each pool's rendered config before the IDMS/ITMS and additionalTrustedCA changes, and wait for the MCO to render before waiting for convergence (poll Updated + updatedMachineCount; warn and continue when nothing is rolling) - start the hub restart grace period only once the backend container has started, and replace the rollout restart with scale to 0, wait for the pods to be gone, scale to 1, so two backends never overlap Operator: OLM v1 installs the bundle with the rhdh-operator-installer ServiceAccount, which cannot grant RBAC it does not hold itself. The installer ClusterRole rendered by prepare-restricted-environment.sh lags behind config/rbac/role.yaml (pods, apiservers/ingresses, devhubplugincatalogs), so the operator ClusterRole creation is refused and the operator ends up running with no permissions. The half-applied release still creates the CRD, so the CRD-only wait passed and the job failed later with a misleading message. - require ClusterExtension Installed=True as well as the CRD, log the install condition at info level, and dump OLM v1 status when the Backstage CR produces no deployment - TEMPORARY: grant the installer the operator's own ClusterRole rules from the same release branch before prepare runs, so the first install attempt succeeds. Remove once redhat-developer/rhdh-operator ships the fix. Verified the RBAC workaround on a live OCP cluster: the same ClusterRole creation is refused before the grant and succeeds after it. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 7 +- .../jobs/ocp-disconnected-operator.sh | 23 +++- .ci/pipelines/lib/disconnected.sh | 3 + .ci/pipelines/lib/disconnected/helm.sh | 49 +++++++-- .ci/pipelines/lib/disconnected/mirror.sh | 83 +++++++++++++- .ci/pipelines/lib/disconnected/operator.sh | 101 +++++++++++++++--- 6 files changed, 233 insertions(+), 33 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index e5a811efd2..3a76b1b425 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -121,6 +121,11 @@ handle_ocp_disconnected_helm() { disconnected::patch_idms "${OC_MIRROR_IDMS_FILE}" + # Snapshot before the apply so wait_mcp_updated can tell whether the MCO has + # already rendered this change or the pool is still on the previous config. + local mcp_snapshot + mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) + oc apply -f "${OC_MIRROR_IDMS_FILE}" || { log::error "Failed to apply IDMS — aborting" return 1 @@ -138,7 +143,7 @@ handle_ocp_disconnected_helm() { # IDMS/ITMS changes trigger a MachineConfig update which rolls worker nodes # (drain → apply config → reboot). Wait for all MachineConfigPools to finish # before deploying workloads, otherwise pods get evicted mid-startup. - disconnected::wait_mcp_updated + disconnected::wait_mcp_updated "${mcp_snapshot}" log::section "Plugin Mirroring" disconnected::mirror_plugins || return 1 diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index df0bc2efae..33f791a705 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -160,10 +160,14 @@ EOF # external registries. Catalogd must trust the mirror CA and authenticate # before ClusterCatalog can reach Serving=True. Skip on LOCAL_DISCONNECTED — # OCP_INTERNAL path configures internal registry trust itself. + local mcp_snapshot if [[ "${LOCAL_DISCONNECTED:-}" != "1" ]]; then + # Snapshot before the additionalTrustedCA patch: it also rolls the nodes. + mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) + disconnected::ensure_olm_mirror_pull_secret || return 1 disconnected::ensure_mirror_registry_ca || return 1 - disconnected::wait_mcp_updated + disconnected::wait_mcp_updated "${mcp_snapshot}" else log::info "LOCAL_DISCONNECTED=1: skipping external bastion CA/pull-secret pre-prepare helpers" fi @@ -176,6 +180,15 @@ EOF log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" fi + # prepare applies the IDMS; snapshot first so the wait below can tell whether + # the MCO has already rendered it. + mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) + + # TEMPORARY: the installer ClusterRole rendered by + # prepare-restricted-environment.sh lags behind the bundle's required + # permissions on OLM v1. Remove once the upstream fix is released. + disconnected::grant_operator_installer_rbac || return 1 + log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}" if ! disconnected::retry_on_local_registry 5 \ disconnected::with_unset_registry_auth_file \ @@ -215,7 +228,7 @@ EOF # prepare-restricted-environment.sh applies IDMS/CatalogSource which triggers # a MachineConfig update and node rolling. Wait for completion before deploying # workloads, same as the Helm path. - disconnected::wait_mcp_updated + disconnected::wait_mcp_updated "${mcp_snapshot}" if [[ "${LOCAL_DISCONNECTED:-}" == "1" ]]; then # Catalogd needs the registry + image-puller grants above before Serving=True. @@ -295,7 +308,11 @@ EOF cp "${cr_temp}" "${ARTIFACT_DIR}/disconnected-backstage-cr.yaml" 2> /dev/null || true - deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" + deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" || { + log::error "Backstage CR did not produce a deployment — dumping OLM v1 status" + disconnected::dump_olm_v1_status "rhdh-operator" + return 1 + } log::success "Backstage CR deployed in ${NAME_SPACE}" log::section "Smoke Test" diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index 4025037608..3c38da9b52 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -55,6 +55,8 @@ export -f disconnected::cluster_mirror_host export -f disconnected::build_imageset_config export -f disconnected::run_oc_mirror export -f disconnected::patch_idms +export -f disconnected::snapshot_mcp_rendered_configs +export -f disconnected::mcp_all_converged export -f disconnected::wait_mcp_updated export -f disconnected::_hook_should_add_hub_image export -f disconnected::_hook_adjust_oc_mirror_args @@ -83,6 +85,7 @@ export -f disconnected::ensure_olm_mirror_pull_secret # operator.sh export -f disconnected::dump_olm_v1_status export -f disconnected::wait_operator_crd_olm_v1 +export -f disconnected::grant_operator_installer_rbac # helm.sh export -f disconnected::ensure_helm_hub_after_postgres # mirror.sh / plugins.sh CI-safe defaults (always defined; local.sh overrides) diff --git a/.ci/pipelines/lib/disconnected/helm.sh b/.ci/pipelines/lib/disconnected/helm.sh index ae9e2c192a..6a6fa59cf1 100644 --- a/.ci/pipelines/lib/disconnected/helm.sh +++ b/.ci/pipelines/lib/disconnected/helm.sh @@ -10,16 +10,20 @@ readonly _DISCONNECTED_HELM_SOURCED=1 # it. Wait for Postgres Ready, then bounce the hub only if it is not Available. # No-op when the hub becomes Ready after Postgres (typical local timing). # -# The bounce uses a rolling `oc rollout restart deployment` (never `oc delete -# pod`): a pod-delete restart was observed to coincide with the PostgreSQL -# StatefulSet pod being rescheduled, which on AWS triggers a slow (~5 min) -# EBS CSI volume detach/re-attach (FailedAttachVolume) and cascades into a -# smoke-test timeout. A rollout restart only touches the hub Deployment. +# The bounce scales the Deployment to zero and back up instead of using +# `oc rollout restart`: a rolling restart keeps the old pod serving while the +# new one starts, and two backends booting against a freshly initialised +# database race on the knex migration lock ("Migration table is already +# locked"), after which the losing backend never becomes Ready and the rollout +# times out. Scaling to zero guarantees a single backend at a time. +# +# The grace period starts only once the backend container has started, so a pod +# still extracting plugins is not mistaken for a stuck backend. # Args: # $1 - namespace # $2 - Helm release name (default: rhdh) # $3 - Postgres Ready timeout seconds (default: 600) -# $4 - Hub Available grace seconds before restart (default: 180) +# $4 - Hub Available grace seconds after the backend starts (default: 180) # $5 - Hub rollout timeout seconds after restart (default: 420) disconnected::ensure_helm_hub_after_postgres() { local namespace=$1 @@ -35,6 +39,8 @@ disconnected::ensure_helm_hub_after_postgres() { local pg_pod="${release_name}-postgresql-0" local hub_deploy="${release_name}-developer-hub" + # Same label the disconnected NetworkPolicies select the backend with. + local hub_selector="app.kubernetes.io/name=developer-hub" local pg_ready=0 local start=$SECONDS @@ -58,6 +64,24 @@ disconnected::ensure_helm_hub_after_postgres() { return 1 fi + log::info "Waiting for the hub backend container to start (timeout ${hub_grace}s)..." + local backend_started=0 + start=$SECONDS + while ((SECONDS - start < hub_grace)); do + local backend_state + backend_state=$(oc get pods -n "${namespace}" -l "${hub_selector}" \ + -o jsonpath='{range .items[*].status.containerStatuses[?(@.name=="backstage-backend")]}{.started}{"\n"}{end}' \ + 2> /dev/null || true) + if grep -q true <<< "${backend_state}"; then + backend_started=1 + break + fi + sleep 5 + done + if [[ "${backend_started}" -ne 1 ]]; then + log::warn "Hub backend did not start within ${hub_grace}s; checking deployment availability anyway" + fi + if oc wait --for=condition=Available "deployment/${hub_deploy}" \ -n "${namespace}" --timeout="${hub_grace}s"; then log::success "Hub ${hub_deploy} already Available — skipping restart" @@ -66,8 +90,17 @@ disconnected::ensure_helm_hub_after_postgres() { log::warn "Hub ${hub_deploy} not Available after Postgres Ready — restarting to recover from DB connect race" oc get pods -n "${namespace}" || true - if ! oc rollout restart "deployment/${hub_deploy}" -n "${namespace}"; then - log::error "Failed to restart deployment/${hub_deploy} in ${namespace}" + if ! oc scale "deployment/${hub_deploy}" --replicas=0 -n "${namespace}"; then + log::error "Failed to scale deployment/${hub_deploy} down in ${namespace}" + return 1 + fi + # Wait for the old pod to be gone before starting the replacement, otherwise + # both backends initialise at once. + if ! oc wait --for=delete pod -l "${hub_selector}" -n "${namespace}" --timeout=180s; then + log::warn "Old hub pods still present after 180s — scaling back up anyway" + fi + if ! oc scale "deployment/${hub_deploy}" --replicas=1 -n "${namespace}"; then + log::error "Failed to scale deployment/${hub_deploy} back up in ${namespace}" return 1 fi if ! oc rollout status "deployment/${hub_deploy}" -n "${namespace}" --timeout="${rollout_timeout}s"; then diff --git a/.ci/pipelines/lib/disconnected/mirror.sh b/.ci/pipelines/lib/disconnected/mirror.sh index b572c615b5..16cf75fe9a 100644 --- a/.ci/pipelines/lib/disconnected/mirror.sh +++ b/.ci/pipelines/lib/disconnected/mirror.sh @@ -256,13 +256,86 @@ disconnected::patch_idms() { cp "${idms_file}" "${ARTIFACT_DIR}/disconnected-idms-patched.yaml" 2> /dev/null || true } +# Record the rendered MachineConfig every pool currently targets. +# Call this before applying an IDMS/ITMS/additionalTrustedCA change and pass the +# output to disconnected::wait_mcp_updated. +disconnected::snapshot_mcp_rendered_configs() { + oc get machineconfigpool \ + -o jsonpath='{range .items[*]}{.metadata.name}={.spec.configuration.name}{"\n"}{end}' \ + 2> /dev/null || true +} + +# True when every MachineConfigPool runs its rendered config and no node is +# still updating or degraded. +disconnected::mcp_all_converged() { + local snapshot + snapshot=$(oc get machineconfigpool \ + -o jsonpath='{range .items[*]}{.spec.configuration.name}{"|"}{.status.configuration.name}{"|"}{.status.updatedMachineCount}{"|"}{.status.machineCount}{"|"}{.status.degradedMachineCount}{"\n"}{end}' \ + 2> /dev/null) || return 1 + [[ -n "${snapshot}" ]] || return 1 + + local spec_name status_name updated total degraded + while IFS='|' read -r spec_name status_name updated total degraded; do + if [[ "${spec_name}" != "${status_name}" || + "${updated}" != "${total}" || + "${degraded:-0}" != "0" ]]; then + return 1 + fi + done <<< "${snapshot}" + return 0 +} + # Wait for MachineConfigPool updates after IDMS/CatalogSource changes. +# +# `oc wait --for=condition=Updated=True` alone is not enough right after an +# apply: the MCO needs a few seconds to render the new config and bump the pool +# generation, and until then the condition is still True from the previous +# rollout, so the wait returns immediately. The nodes are drained later, while +# workloads are starting (observed: workers drained mid-deploy, hub and +# PostgreSQL pods evicted). With a snapshot from +# disconnected::snapshot_mcp_rendered_configs (taken before the change) this +# waits for the MCO to pick the change up first. +# # Warns and continues on timeout (same behavior as both handlers historically). +# Args: +# $1 - snapshot from disconnected::snapshot_mcp_rendered_configs (optional) +# $2 - seconds to wait for the MCO to render the change (default: 180) +# $3 - seconds to wait for the rollout to finish (default: 1200) disconnected::wait_mcp_updated() { - log::info "Waiting for MachineConfigPool updates to complete (up to 20m)..." - if oc wait machineconfigpool --all --for=condition=Updated=True --timeout=20m; then - log::success "All MachineConfigPools are Updated" - else - log::warn "MachineConfigPool wait timed out -- proceeding anyway" + local before_snapshot=${1:-} + local render_timeout=${2:-180} + local rollout_timeout=${3:-1200} + + if [[ -n "${before_snapshot}" ]]; then + log::info "Waiting for MachineConfigPool to pick up the new rendered config (up to ${render_timeout}s)..." + local rendered=0 + local render_start=$SECONDS + while ((SECONDS - render_start < render_timeout)); do + if [[ "$(disconnected::snapshot_mcp_rendered_configs)" != "${before_snapshot}" ]]; then + rendered=1 + break + fi + sleep 5 + done + if ((rendered != 1)); then + # Re-applying identical content (e.g. an unchanged IDMS) does not roll + # the nodes, so there is nothing to wait for. + log::warn "No MachineConfigPool rendered a new config within ${render_timeout}s -- nothing to roll, continuing" + return 0 + fi + log::success "MachineConfigPool picked up the new rendered config" fi + + log::info "Waiting for MachineConfigPool updates to complete (up to 20m)..." + local rollout_start=$SECONDS + while ((SECONDS - rollout_start < rollout_timeout)); do + if disconnected::mcp_all_converged; then + log::success "All MachineConfigPools are Updated" + return 0 + fi + sleep 10 + done + + log::warn "MachineConfigPool wait timed out -- proceeding anyway" + oc get machineconfigpool 2>&1 | tee -a "${ARTIFACT_DIR}/disconnected-mcp-status.txt" || true } diff --git a/.ci/pipelines/lib/disconnected/operator.sh b/.ci/pipelines/lib/disconnected/operator.sh index eb056234c3..60c520deb7 100644 --- a/.ci/pipelines/lib/disconnected/operator.sh +++ b/.ci/pipelines/lib/disconnected/operator.sh @@ -21,6 +21,13 @@ disconnected::dump_olm_v1_status() { } # Wait for OLM v1 ClusterExtension to report Installed, then for the CRD. +# +# The CRD alone is not proof of a successful install: with OLM v1 the bundle is +# applied as a Helm chart, so a release that fails halfway (e.g. the operator +# ClusterRole is refused) still leaves the CRD, the Deployment and a +# ClusterRoleBinding behind. The operator then runs without permissions and the +# job fails much later with a misleading "deployment not created". Require +# Installed=True as well. # Args: # $1 - extension name (default: rhdh-operator) # $2 - crd name (default: backstages.rhdh.redhat.com) @@ -32,33 +39,95 @@ disconnected::wait_operator_crd_olm_v1() { local interval=15 local elapsed=0 - log::info "Waiting for ClusterExtension/${extension_name} and CRD ${crd_name} (timeout: ${timeout}s)..." + log::info "Waiting for ClusterExtension/${extension_name} Installed=True and CRD ${crd_name} (timeout: ${timeout}s)..." while ((elapsed < timeout)); do - if oc get crd "${crd_name}" > /dev/null 2>&1; then - log::success "CRD '${crd_name}' is available" - return 0 - fi - local installed installed=$(oc get clusterextension "${extension_name}" \ -o jsonpath='{range .status.conditions[?(@.type=="Installed")]}{.status}{end}' 2> /dev/null || true) - if [[ "${installed}" == "True" ]]; then - log::info "ClusterExtension/${extension_name} reports Installed=True; waiting for CRD..." - else - local progressing reason - progressing=$(oc get clusterextension "${extension_name}" \ - -o jsonpath='{range .status.conditions[?(@.type=="Progressing")]}{.status}{end}' 2> /dev/null || true) - reason=$(oc get clusterextension "${extension_name}" \ - -o jsonpath='{range .status.conditions[?(@.type=="Installed")]}{.reason}{" "}{.message}{end}' 2> /dev/null || true) - log::debug "ClusterExtension Installed=${installed:-unknown} Progressing=${progressing:-unknown} ${reason}" + + if [[ "${installed}" == "True" ]] && oc get crd "${crd_name}" > /dev/null 2>&1; then + log::success "ClusterExtension/${extension_name} Installed=True; CRD '${crd_name}' is available" + return 0 fi + # Info level on purpose: install errors (RBAC escalation, missing images) + # are the useful signal in the build log when this wait times out. + local progressing reason + progressing=$(oc get clusterextension "${extension_name}" \ + -o jsonpath='{range .status.conditions[?(@.type=="Progressing")]}{.status}{end}' 2> /dev/null || true) + reason=$(oc get clusterextension "${extension_name}" \ + -o jsonpath='{range .status.conditions[?(@.type=="Installed")]}{.reason}{": "}{.message}{end}' 2> /dev/null || true) + log::info "ClusterExtension/${extension_name} Installed=${installed:-unknown} Progressing=${progressing:-unknown} ${reason}" + sleep "${interval}" elapsed=$((elapsed + interval)) done - log::error "Timeout waiting for CRD '${crd_name}' after ${timeout}s" + log::error "Timeout waiting for ClusterExtension/${extension_name} Installed=True and CRD '${crd_name}' after ${timeout}s" disconnected::dump_olm_v1_status "${extension_name}" return 1 } + +# TEMPORARY WORKAROUND — remove once the upstream fix is released. +# +# OLM v1 installs the bundle with a dedicated installer ServiceAccount, and the +# API server refuses to grant RBAC the installer does not hold itself +# ("attempting to grant RBAC permissions not currently held"). The ClusterRole +# rhdh-operator-installer-role rendered by prepare-restricted-environment.sh +# lags behind config/rbac/role.yaml (pods, apiservers/ingresses, +# devhubplugincatalogs were added to the operator without updating the +# installer role), so the operator ClusterRole cannot be created and the +# operator runs with no permissions. +# +# Grant the installer the operator's own rules, fetched from the same release +# branch the bundle comes from, so this cannot drift. Applied before prepare so +# the first install attempt already has the permissions. +# Args: +# $1 - operator namespace (default: rhdh-operator) +# $2 - installer ServiceAccount name (default: rhdh-operator-installer) +disconnected::grant_operator_installer_rbac() { + local operator_ns=${1:-rhdh-operator} + local installer_sa=${2:-rhdh-operator-installer} + local role_name="${installer_sa}-bundle-permissions" + local binding_name="${role_name}-binding" + local role_url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/refs/heads/${RELEASE_BRANCH_NAME:-main}/config/rbac/role.yaml" + local role_file="${DISCONNECTED_TMPDIR}/operator-installer-bundle-role.yaml" + + if [[ -z "${RELEASE_BRANCH_NAME:-}" ]]; then + log::error "grant_operator_installer_rbac requires RELEASE_BRANCH_NAME" + return 1 + fi + + log::info "Granting ${installer_sa} the operator ClusterRole rules (OLM v1 installer RBAC workaround)..." + if ! curl -fL -o "${role_file}" "${role_url}"; then + log::error "Failed to download ${role_url}" + return 1 + fi + + if ! yq eval ".metadata.name = \"${role_name}\"" "${role_file}" | oc apply -f -; then + log::error "Failed to apply ClusterRole ${role_name}" + return 1 + fi + + if ! oc apply -f - << EOF; then +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ${binding_name} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ${role_name} +subjects: +- kind: ServiceAccount + name: ${installer_sa} + namespace: ${operator_ns} +EOF + log::error "Failed to apply ClusterRoleBinding ${binding_name}" + return 1 + fi + + cp "${role_file}" "${ARTIFACT_DIR}/disconnected-operator-installer-role-source.yaml" 2> /dev/null || true + log::success "Installer SA ${installer_sa} may now grant the operator's ClusterRole" +} From b3a8b08c09aecc540f787f1f616ecaf2e46aeb96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Sat, 24 Oct 2026 12:38:09 +0200 Subject: [PATCH 05/16] chore(ci): reference the installer RBAC fix and ticket Point the temporary OLM v1 installer workaround at redhat-developer/rhdh-operator#3923 (backport #3924) and RHDHBUGS-4056, so the patch is easy to find and remove once the fix is released. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-operator.sh | 3 ++- .ci/pipelines/lib/disconnected/operator.sh | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 33f791a705..663559eabd 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -186,7 +186,8 @@ EOF # TEMPORARY: the installer ClusterRole rendered by # prepare-restricted-environment.sh lags behind the bundle's required - # permissions on OLM v1. Remove once the upstream fix is released. + # permissions on OLM v1. Remove once redhat-developer/rhdh-operator#3923 is + # released (RHDHBUGS-4056). disconnected::grant_operator_installer_rbac || return 1 log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}" diff --git a/.ci/pipelines/lib/disconnected/operator.sh b/.ci/pipelines/lib/disconnected/operator.sh index 60c520deb7..4dc92f2fdf 100644 --- a/.ci/pipelines/lib/disconnected/operator.sh +++ b/.ci/pipelines/lib/disconnected/operator.sh @@ -83,6 +83,8 @@ disconnected::wait_operator_crd_olm_v1() { # Grant the installer the operator's own rules, fetched from the same release # branch the bundle comes from, so this cannot drift. Applied before prepare so # the first install attempt already has the permissions. +# Upstream fix: redhat-developer/rhdh-operator#3923 (+ #3924 for release-2.1), +# tracked in RHDHBUGS-4056. # Args: # $1 - operator namespace (default: rhdh-operator) # $2 - installer ServiceAccount name (default: rhdh-operator-installer) From 78be6087495780096276ca74ac851bff754d16b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 26 Oct 2026 08:57:16 +0100 Subject: [PATCH 06/16] chore(ci): drop the temporary OLM v1 installer RBAC workaround redhat-developer/rhdh-operator#3923 is merged, so the prepare-restricted-environment.sh fetched from rhdh-operator main renders an installer ClusterRole that covers the bundle permissions (pods, apiservers/ingresses, devhubplugincatalogs). The disconnected operator job downloads that script at runtime, so the workaround is no longer needed on main. release-2.1 keeps the workaround until #3924 merges, because that job fetches the script from rhdh-operator release-2.1. Assisted-by: OpenCode --- .../jobs/ocp-disconnected-operator.sh | 6 -- .ci/pipelines/lib/disconnected.sh | 1 - .ci/pipelines/lib/disconnected/operator.sh | 65 ------------------- 3 files changed, 72 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 663559eabd..0d548628a7 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -184,12 +184,6 @@ EOF # the MCO has already rendered it. mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) - # TEMPORARY: the installer ClusterRole rendered by - # prepare-restricted-environment.sh lags behind the bundle's required - # permissions on OLM v1. Remove once redhat-developer/rhdh-operator#3923 is - # released (RHDHBUGS-4056). - disconnected::grant_operator_installer_rbac || return 1 - log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}" if ! disconnected::retry_on_local_registry 5 \ disconnected::with_unset_registry_auth_file \ diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index 3c38da9b52..aca7b0a9d2 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -85,7 +85,6 @@ export -f disconnected::ensure_olm_mirror_pull_secret # operator.sh export -f disconnected::dump_olm_v1_status export -f disconnected::wait_operator_crd_olm_v1 -export -f disconnected::grant_operator_installer_rbac # helm.sh export -f disconnected::ensure_helm_hub_after_postgres # mirror.sh / plugins.sh CI-safe defaults (always defined; local.sh overrides) diff --git a/.ci/pipelines/lib/disconnected/operator.sh b/.ci/pipelines/lib/disconnected/operator.sh index 4dc92f2fdf..b45eff0e3c 100644 --- a/.ci/pipelines/lib/disconnected/operator.sh +++ b/.ci/pipelines/lib/disconnected/operator.sh @@ -68,68 +68,3 @@ disconnected::wait_operator_crd_olm_v1() { disconnected::dump_olm_v1_status "${extension_name}" return 1 } - -# TEMPORARY WORKAROUND — remove once the upstream fix is released. -# -# OLM v1 installs the bundle with a dedicated installer ServiceAccount, and the -# API server refuses to grant RBAC the installer does not hold itself -# ("attempting to grant RBAC permissions not currently held"). The ClusterRole -# rhdh-operator-installer-role rendered by prepare-restricted-environment.sh -# lags behind config/rbac/role.yaml (pods, apiservers/ingresses, -# devhubplugincatalogs were added to the operator without updating the -# installer role), so the operator ClusterRole cannot be created and the -# operator runs with no permissions. -# -# Grant the installer the operator's own rules, fetched from the same release -# branch the bundle comes from, so this cannot drift. Applied before prepare so -# the first install attempt already has the permissions. -# Upstream fix: redhat-developer/rhdh-operator#3923 (+ #3924 for release-2.1), -# tracked in RHDHBUGS-4056. -# Args: -# $1 - operator namespace (default: rhdh-operator) -# $2 - installer ServiceAccount name (default: rhdh-operator-installer) -disconnected::grant_operator_installer_rbac() { - local operator_ns=${1:-rhdh-operator} - local installer_sa=${2:-rhdh-operator-installer} - local role_name="${installer_sa}-bundle-permissions" - local binding_name="${role_name}-binding" - local role_url="https://raw.githubusercontent.com/redhat-developer/rhdh-operator/refs/heads/${RELEASE_BRANCH_NAME:-main}/config/rbac/role.yaml" - local role_file="${DISCONNECTED_TMPDIR}/operator-installer-bundle-role.yaml" - - if [[ -z "${RELEASE_BRANCH_NAME:-}" ]]; then - log::error "grant_operator_installer_rbac requires RELEASE_BRANCH_NAME" - return 1 - fi - - log::info "Granting ${installer_sa} the operator ClusterRole rules (OLM v1 installer RBAC workaround)..." - if ! curl -fL -o "${role_file}" "${role_url}"; then - log::error "Failed to download ${role_url}" - return 1 - fi - - if ! yq eval ".metadata.name = \"${role_name}\"" "${role_file}" | oc apply -f -; then - log::error "Failed to apply ClusterRole ${role_name}" - return 1 - fi - - if ! oc apply -f - << EOF; then -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ${binding_name} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ${role_name} -subjects: -- kind: ServiceAccount - name: ${installer_sa} - namespace: ${operator_ns} -EOF - log::error "Failed to apply ClusterRoleBinding ${binding_name}" - return 1 - fi - - cp "${role_file}" "${ARTIFACT_DIR}/disconnected-operator-installer-role-source.yaml" 2> /dev/null || true - log::success "Installer SA ${installer_sa} may now grant the operator's ClusterRole" -} From 2df5494dcc89cfd7c21b8227d1aeb3d598eec7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 26 Oct 2026 14:36:10 +0100 Subject: [PATCH 07/16] fix(ci): derive the disconnected operator filter from the release stream The operator job took the bundle filter from CHART_VERSION while the catalog tag came from RELEASE_VERSION, so a pinned run on main (for example an RC tag) asked oc-mirror for bundles the `next` catalog does not carry. Both now come from the branch: RELEASE_VERSION for the catalog tag (next on main, 2.1 on release-2.1) and common::get_release_stream for the filter, which needs a major.minor and resolves main to the package.json stream (2.2). Move that helper out of lib/helm.sh while at it: the operator and upgrade jobs use it too, so it lives in lib/common.sh as common::get_release_stream. Assisted-by: OpenCode --- .../jobs/ocp-disconnected-operator.sh | 35 ++++---------- .ci/pipelines/jobs/upgrade.sh | 2 +- .ci/pipelines/lib/README.md | 3 +- .ci/pipelines/lib/common.sh | 45 ++++++++++++++++++ .ci/pipelines/lib/helm.sh | 47 +------------------ 5 files changed, 59 insertions(+), 73 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 0d548628a7..120e55a36d 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -55,26 +55,14 @@ handle_ocp_disconnected_operator() { # injected into the Backstage CR by deploy_rhdh_operator. # # OLM version: leave default (auto). On OCP 4.21+ this selects OLM v1. - local filter_versions="${RELEASE_VERSION}" - if [[ "${filter_versions}" == "next" || "${filter_versions}" == "*" ]]; then - filter_versions="*" - fi - - # Mirroring every historical RHDH operator version (* / next) is unnecessary - # for smoke and flakes on quay CDN EOF (CI) or overwhelms the integrated - # registry (local). Pin to the chart major.minor (e.g. 2.0-80-CI → 2.0). - if [[ "${filter_versions}" == "*" ]]; then - local chart_mm="" - if [[ -n "${CHART_VERSION:-}" && "${CHART_VERSION}" =~ ^([0-9]+\.[0-9]+) ]]; then - chart_mm="${BASH_REMATCH[1]}" - fi - if [[ -z "${chart_mm}" ]]; then - log::error "Cannot derive --filter-versions from CHART_VERSION='${CHART_VERSION:-unset}'; expected major.minor prefix (e.g. 2.0-80-CI)" - return 1 - fi - filter_versions="${chart_mm}" - log::info "filter-versions=${filter_versions} (from CHART_VERSION=${CHART_VERSION})" - fi + # + # The operator stream follows the checkout branch, like install_rhdh_operator: + # main mirrors the 2.2 bundles, release-2.1 the 2.1 ones. A pinned image or + # chart does not change it, and an unfiltered catalog would pull every + # historical version. + local filter_versions + filter_versions=$(common::get_release_stream) || return 1 + log::info "filter-versions=${filter_versions} (from RELEASE_BRANCH_NAME=${RELEASE_BRANCH_NAME})" # CONTAINER_PLATFORM_VERSION is set by e2e-tests/container-init.sh from the # live cluster (oc version → major.minor). @@ -87,8 +75,6 @@ handle_ocp_disconnected_operator() { # RHDH CI catalog built from the same commit as the hub and plugin catalogs. # Released charts (x.y.z) pair with the published Red Hat catalog, the only one # carrying released bundles, even when the image itself is pulled from quay. - # That catalog still needs the rhdh-operator defaultChannel fix - # (rhdh-operator#3822) for a filtered oc-mirror run to accept it. local use_ci_catalog="false" if [[ "${IMAGE_REGISTRY}" != "registry.redhat.io" && "${CHART_VERSION:-}" == *-CI ]]; then use_ci_catalog="true" @@ -97,9 +83,8 @@ handle_ocp_disconnected_operator() { local index_image="" local registries_conf="" if [[ "${use_ci_catalog}" == "true" ]]; then - # rhdh-operator publishes one catalog per release stream and OCP minor, - # e.g. quay.io/rhdh/iib:next-v4.21-x86_64 (amd64 only; the jobs run on - # amd64 clusters). + # rhdh-operator publishes one catalog per stream and OCP minor, e.g. + # quay.io/rhdh/iib:next-v4.21-x86_64 (amd64 only; the jobs run on amd64). index_image="quay.io/rhdh/iib:${RELEASE_VERSION}-v${ocp_version}-x86_64" skopeo inspect --raw "docker://${index_image}" > /dev/null || { log::error "RHDH CI operator catalog ${index_image} not found or unreadable" diff --git a/.ci/pipelines/jobs/upgrade.sh b/.ci/pipelines/jobs/upgrade.sh index 5b39f85150..997a309de5 100644 --- a/.ci/pipelines/jobs/upgrade.sh +++ b/.ci/pipelines/jobs/upgrade.sh @@ -98,7 +98,7 @@ handle_ocp_helm_upgrade() { # Resolve the previous release's chart and values for the baseline deployment. local current_release_version previous_release_version - current_release_version=$(helm::get_chart_stream) + current_release_version=$(common::get_release_stream) if [[ -z "$current_release_version" ]]; then log::error "Failed to determine current release version. Exiting." save_overall_result 1 diff --git a/.ci/pipelines/lib/README.md b/.ci/pipelines/lib/README.md index 14652fe092..9f3d429cf3 100644 --- a/.ci/pipelines/lib/README.md +++ b/.ci/pipelines/lib/README.md @@ -14,7 +14,8 @@ Functions: `log::info`, `log::warn`, `log::error`, `log::success`, `log::debug`, Common utilities. -Functions: `common::oc_login`, `common::sed_inplace`, `common::get_previous_release_version` +Functions: `common::oc_login`, `common::sed_inplace`, `common::get_release_stream`, +`common::get_previous_release_version` ### `k8s-wait.sh` diff --git a/.ci/pipelines/lib/common.sh b/.ci/pipelines/lib/common.sh index 324627e7c4..cf64074c1e 100644 --- a/.ci/pipelines/lib/common.sh +++ b/.ci/pipelines/lib/common.sh @@ -72,6 +72,51 @@ common::sed_inplace() { return $? } +# Get the release stream this run belongs to: a major.minor pair like "1.10", +# not a bare major. 'main' takes it from package.json; 'release-x.y' from the +# branch name. +# Args: +# $1 - (optional) version_override: Specific stream to use (e.g., "1.8" for upgrade base) +# Returns: +# Prints the stream (e.g., "1.10") +common::get_release_stream() { + local version_override=${1:-} + + if [[ -n "$version_override" ]]; then + echo "$version_override" + return 0 + fi + + if [[ -z "${RELEASE_BRANCH_NAME:-}" ]]; then + log::error "RELEASE_BRANCH_NAME is not set" + return 1 + fi + + if [[ "$RELEASE_BRANCH_NAME" == "main" ]]; then + # main carries no version in its name. The published tags cannot answer + # this either - they hold every stream anyone pushed, so "highest on quay" + # follows whichever stream is ahead, not this checkout's. + local package_json="${DIR}/../../package.json" + if [[ ! -f "$package_json" ]]; then + log::error "Cannot determine the release stream: ${package_json} not found" + return 1 + fi + + local release_stream + release_stream=$(jq -r '.version // empty' "$package_json" | grep -oE '^[0-9]+\.[0-9]+') + if [[ -z "$release_stream" ]]; then + log::error "Cannot determine the release stream: no usable .version in ${package_json}" + return 1 + fi + echo "$release_stream" + elif echo "$RELEASE_BRANCH_NAME" | grep -qE '^release-[0-9]+\.[0-9]+$'; then + echo "$RELEASE_BRANCH_NAME" | grep -oE '[0-9]+\.[0-9]+' + else + log::error "Invalid RELEASE_BRANCH_NAME: $RELEASE_BRANCH_NAME (expected 'main' or 'release-x.y')" + return 1 + fi +} + # Find the highest existing release branch below the current version. # Usage: prev=$(common::get_previous_release_version "1.6") # Returns: "1.5" # prev=$(common::get_previous_release_version "2.0") # Returns: "1.10" diff --git a/.ci/pipelines/lib/helm.sh b/.ci/pipelines/lib/helm.sh index e64fa5a393..91891f333a 100644 --- a/.ci/pipelines/lib/helm.sh +++ b/.ci/pipelines/lib/helm.sh @@ -171,51 +171,6 @@ helm::resolve_chart_version() { helm::get_chart_version } -# Get the release stream this run belongs to: a major.minor pair like "1.10", -# not a bare major. 'main' takes it from package.json; 'release-x.y' from the -# branch name. -# Args: -# $1 - (optional) version_override: Specific stream to use (e.g., "1.8" for upgrade base) -# Returns: -# Prints the stream (e.g., "1.10") -helm::get_chart_stream() { - local version_override=${1:-} - - if [[ -n "$version_override" ]]; then - echo "$version_override" - return 0 - fi - - if [[ -z "${RELEASE_BRANCH_NAME:-}" ]]; then - log::error "RELEASE_BRANCH_NAME is not set" - return 1 - fi - - if [[ "$RELEASE_BRANCH_NAME" == "main" ]]; then - # main carries no version in its name. The published tags cannot answer - # this either - they hold every stream anyone pushed, so "highest on quay" - # follows whichever stream is ahead, not this checkout's. - local package_json="${DIR}/../../package.json" - if [[ ! -f "$package_json" ]]; then - log::error "Cannot determine the chart stream: ${package_json} not found" - return 1 - fi - - local chart_stream - chart_stream=$(jq -r '.version // empty' "$package_json" | grep -oE '^[0-9]+\.[0-9]+') - if [[ -z "$chart_stream" ]]; then - log::error "Cannot determine the chart stream: no usable .version in ${package_json}" - return 1 - fi - echo "$chart_stream" - elif echo "$RELEASE_BRANCH_NAME" | grep -qE '^release-[0-9]+\.[0-9]+$'; then - echo "$RELEASE_BRANCH_NAME" | grep -oE '[0-9]+\.[0-9]+' - else - log::error "Invalid RELEASE_BRANCH_NAME: $RELEASE_BRANCH_NAME (expected 'main' or 'release-x.y')" - return 1 - fi -} - # Get the latest chart version based on RELEASE_BRANCH_NAME or an optional version override. # Args: # $1 - (optional) version_override: Specific version to use (e.g., "1.8" for upgrade base) @@ -223,7 +178,7 @@ helm::get_chart_stream() { # Prints the chart version (e.g., "1.4-123-CI") helm::get_chart_version() { local chart_stream - chart_stream=$(helm::get_chart_stream "${1:-}") + chart_stream=$(common::get_release_stream "${1:-}") if [[ -z "$chart_stream" ]]; then return 1 fi From f8976953237afd050b2e8fcdd6cabd6a4b02e1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 26 Oct 2026 14:54:24 +0100 Subject: [PATCH 08/16] chore(ci): trim disconnected code comments Drop the comments that only describe the PR history or restate the code (merged rhdh-operator PRs, the NFS plugin rationale repeated in three files, snapshot double-comments, a duplicated secret-mount paragraph). Keep the notes that explain non-obvious behaviour: the guest sign-in page, nameOverride coupling, CI vs released catalogs, the registries.conf redirect and the additionalTrustedCA node roll. Also log the configured MachineConfigPool timeout instead of a hardcoded 20m, and correct the mirror egress rationale: MIRROR_REGISTRY_URL is the bastion's public EC2 DNS name, which resolves differently from inside the cluster, so there is no stable ipBlock to use. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 18 +++++++----------- .../jobs/ocp-disconnected-operator.sh | 13 ++++--------- .ci/pipelines/lib/disconnected/helm.sh | 15 +++++---------- .ci/pipelines/lib/disconnected/mirror.sh | 16 ++++++---------- .ci/pipelines/lib/disconnected/namespace.sh | 10 +++++----- .ci/pipelines/lib/disconnected/operator.sh | 9 ++------- .ci/pipelines/lib/disconnected/plugins.sh | 6 ++---- .../disconnected/dynamic-plugins-smoke.yaml | 3 +-- .../value_files/values_disconnected-smoke.yaml | 6 ++---- 9 files changed, 34 insertions(+), 62 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 3a76b1b425..e1abba3b4a 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -121,8 +121,6 @@ handle_ocp_disconnected_helm() { disconnected::patch_idms "${OC_MIRROR_IDMS_FILE}" - # Snapshot before the apply so wait_mcp_updated can tell whether the MCO has - # already rendered this change or the pool is still on the previous config. local mcp_snapshot mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) @@ -162,16 +160,14 @@ handle_ocp_disconnected_helm() { fi disconnected::apply_plugin_mirror_configmap "${NAME_SPACE}" || return 1 - # Mirror CA + registry auth for install-dynamic-plugins (skopeo). - # Post-renderer mounts CA at /etc/containers/certs.d//ca.crt; - # the chart mounts ${RELEASE_NAME}-dynamic-plugins-registry-auth at - # /opt/app-root/src/.config/containers. + # Mirror CA + registry auth for install-dynamic-plugins (skopeo). The + # post-renderer mounts the CA at /etc/containers/certs.d//ca.crt; the + # chart mounts -dynamic-plugins-registry-auth (optional: true) at + # /opt/app-root/src/.config/containers, where fullname is - + # and nameOverride is pinned to developer-hub in the values file. A differently + # named secret leaves the init container without auth.json and the mirror pull + # fails with "authentication required". disconnected::create_mirror_registry_ca_configmap "${NAME_SPACE}" || return 1 - # The chart mounts -dynamic-plugins-registry-auth (optional: true) at - # /opt/app-root/src/.config/containers, and fullname is - # - with nameOverride pinned to developer-hub in the - # values file. A differently named secret leaves the init container without - # auth.json and skopeo then fails the mirror pull with "authentication required". disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" \ "${RELEASE_NAME}-developer-hub-dynamic-plugins-registry-auth" || return 1 diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 120e55a36d..1febef562d 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -38,8 +38,7 @@ handle_ocp_disconnected_operator() { log::section "Operator Mirroring and Installation" # Fetch prepare-restricted-environment.sh from RELEASE_BRANCH_NAME (default - # main). rhdh-operator#3259 (native oc-mirror cc-* catalogs for OLM v1) is - # merged on main. + # main). local prepare_script_path="${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh" disconnected::fetch_operator_repo_script "prepare-restricted-environment.sh" \ "${prepare_script_path}" || { @@ -93,11 +92,9 @@ handle_ocp_disconnected_operator() { # The CI bundle's RELATED_IMAGE refs point at registry.redhat.io/rhdh/* # (plus the stage/access aliases), whose digests exist on quay.io/rhdh only. - # Rewrite those pulls for the oc-mirror run with containers/image - # registries.conf (CONTAINERS_REGISTRIES_CONF), the same mechanism the - # medik8s-disconnected-catalogsource CI step uses. Images in other - # repositories (e.g. registry.redhat.io/rhel10/postgresql-18) keep their - # original source and are pulled with the CI credentials. + # Rewrite those pulls for the oc-mirror run via CONTAINERS_REGISTRIES_CONF. + # Images in other repositories (e.g. registry.redhat.io/rhel10/postgresql-18) + # keep their original source and are pulled with the CI credentials. registries_conf="${DISCONNECTED_TMPDIR}/oc-mirror-registries.conf" cat > "${registries_conf}" << 'EOF' [[registry]] @@ -165,8 +162,6 @@ EOF log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" fi - # prepare applies the IDMS; snapshot first so the wait below can tell whether - # the MCO has already rendered it. mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}" diff --git a/.ci/pipelines/lib/disconnected/helm.sh b/.ci/pipelines/lib/disconnected/helm.sh index 6a6fa59cf1..fba752f367 100644 --- a/.ci/pipelines/lib/disconnected/helm.sh +++ b/.ci/pipelines/lib/disconnected/helm.sh @@ -10,15 +10,11 @@ readonly _DISCONNECTED_HELM_SOURCED=1 # it. Wait for Postgres Ready, then bounce the hub only if it is not Available. # No-op when the hub becomes Ready after Postgres (typical local timing). # -# The bounce scales the Deployment to zero and back up instead of using -# `oc rollout restart`: a rolling restart keeps the old pod serving while the -# new one starts, and two backends booting against a freshly initialised -# database race on the knex migration lock ("Migration table is already -# locked"), after which the losing backend never becomes Ready and the rollout -# times out. Scaling to zero guarantees a single backend at a time. -# -# The grace period starts only once the backend container has started, so a pod -# still extracting plugins is not mistaken for a stuck backend. +# The bounce scales the Deployment to 0 and back to 1 instead of using +# `oc rollout restart`: a rolling restart runs two backends against the freshly +# initialised database, which race on the knex migration lock and leave the +# loser never Ready. The grace period starts once the backend container starts, +# so a pod still extracting plugins is not mistaken for a stuck backend. # Args: # $1 - namespace # $2 - Helm release name (default: rhdh) @@ -39,7 +35,6 @@ disconnected::ensure_helm_hub_after_postgres() { local pg_pod="${release_name}-postgresql-0" local hub_deploy="${release_name}-developer-hub" - # Same label the disconnected NetworkPolicies select the backend with. local hub_selector="app.kubernetes.io/name=developer-hub" local pg_ready=0 local start=$SECONDS diff --git a/.ci/pipelines/lib/disconnected/mirror.sh b/.ci/pipelines/lib/disconnected/mirror.sh index 16cf75fe9a..6282a7cd1a 100644 --- a/.ci/pipelines/lib/disconnected/mirror.sh +++ b/.ci/pipelines/lib/disconnected/mirror.sh @@ -287,16 +287,12 @@ disconnected::mcp_all_converged() { # Wait for MachineConfigPool updates after IDMS/CatalogSource changes. # -# `oc wait --for=condition=Updated=True` alone is not enough right after an -# apply: the MCO needs a few seconds to render the new config and bump the pool -# generation, and until then the condition is still True from the previous -# rollout, so the wait returns immediately. The nodes are drained later, while -# workloads are starting (observed: workers drained mid-deploy, hub and -# PostgreSQL pods evicted). With a snapshot from -# disconnected::snapshot_mcp_rendered_configs (taken before the change) this -# waits for the MCO to pick the change up first. +# `oc wait --for=condition=Updated=True` alone is not enough: right after an +# apply the MCO has not rendered the new config yet, so the condition is still +# True from the previous rollout and the wait returns immediately. With the +# snapshot taken before the change, wait until the MCO picks it up first. # -# Warns and continues on timeout (same behavior as both handlers historically). +# Warns and continues on timeout. # Args: # $1 - snapshot from disconnected::snapshot_mcp_rendered_configs (optional) # $2 - seconds to wait for the MCO to render the change (default: 180) @@ -326,7 +322,7 @@ disconnected::wait_mcp_updated() { log::success "MachineConfigPool picked up the new rendered config" fi - log::info "Waiting for MachineConfigPool updates to complete (up to 20m)..." + log::info "Waiting for MachineConfigPool updates to complete (up to ${rollout_timeout}s)..." local rollout_start=$SECONDS while ((SECONDS - rollout_start < rollout_timeout)); do if disconnected::mcp_all_converged; then diff --git a/.ci/pipelines/lib/disconnected/namespace.sh b/.ci/pipelines/lib/disconnected/namespace.sh index 3a9449e02e..0c70c81ea7 100644 --- a/.ci/pipelines/lib/disconnected/namespace.sh +++ b/.ci/pipelines/lib/disconnected/namespace.sh @@ -125,11 +125,11 @@ disconnected::ensure_olm_mirror_pull_secret() { # the Helm chart and the Operator pod labels (same split as # resources/network_policy/netpol-ci-allow-backend-egress.yaml). # -# The rule is destination-agnostic on purpose. The bastion mirror's address is -# assigned per run and only reachable by DNS name, so an ipBlock would break as -# soon as the bastion is replaced. The namespace is ephemeral, and the connected -# CI lanes use a broader allow-all rule, so scoping by port is the meaningful -# boundary here. +# The rule is scoped by port only. MIRROR_REGISTRY_URL is the bastion's public +# EC2 DNS name, which resolves to a different address from the CI pod and from +# inside the cluster, so there is no stable ipBlock to use. The namespace is +# ephemeral and the connected CI lanes allow all egress, so the port is the +# meaningful boundary here. # Mirrors served on 443 (e.g. the LOCAL_DISCONNECTED route) need no extra rule. # Args: # $1 - namespace diff --git a/.ci/pipelines/lib/disconnected/operator.sh b/.ci/pipelines/lib/disconnected/operator.sh index b45eff0e3c..75f1121684 100644 --- a/.ci/pipelines/lib/disconnected/operator.sh +++ b/.ci/pipelines/lib/disconnected/operator.sh @@ -22,11 +22,8 @@ disconnected::dump_olm_v1_status() { # Wait for OLM v1 ClusterExtension to report Installed, then for the CRD. # -# The CRD alone is not proof of a successful install: with OLM v1 the bundle is -# applied as a Helm chart, so a release that fails halfway (e.g. the operator -# ClusterRole is refused) still leaves the CRD, the Deployment and a -# ClusterRoleBinding behind. The operator then runs without permissions and the -# job fails much later with a misleading "deployment not created". Require +# A half-failed OLM v1 install still leaves the CRD behind (the bundle is applied +# as a Helm chart), so the CRD alone is not proof of success. Require # Installed=True as well. # Args: # $1 - extension name (default: rhdh-operator) @@ -51,8 +48,6 @@ disconnected::wait_operator_crd_olm_v1() { return 0 fi - # Info level on purpose: install errors (RBAC escalation, missing images) - # are the useful signal in the build log when this wait times out. local progressing reason progressing=$(oc get clusterextension "${extension_name}" \ -o jsonpath='{range .status.conditions[?(@.type=="Progressing")]}{.status}{end}' 2> /dev/null || true) diff --git a/.ci/pipelines/lib/disconnected/plugins.sh b/.ci/pipelines/lib/disconnected/plugins.sh index a49f04055f..265b3bffc4 100644 --- a/.ci/pipelines/lib/disconnected/plugins.sh +++ b/.ci/pipelines/lib/disconnected/plugins.sh @@ -6,7 +6,7 @@ readonly _DISCONNECTED_PLUGINS_SOURCED=1 # Smoke only installs this plugin set (includes: []). Full-catalog mirroring is -# RHIDP-13967. On the 2.x New Frontend System hub the sign-in page (app-auth), +# RHIDP-13967. On the 2.1+ New Frontend System hub the sign-in page (app-auth), # sidebar (app-defaults), SCM integrations (app-integrations) and guest auth # provider are dynamic plugins, so the homepage alone cannot render a login. readonly -a DISCONNECTED_SMOKE_PLUGIN_NAMES=( @@ -16,9 +16,7 @@ readonly -a DISCONNECTED_SMOKE_PLUGIN_NAMES=( "red-hat-developer-hub-backstage-plugin-homepage" "backstage-plugin-auth-backend-module-guest-provider" ) -# Space-separated copy for the `export -f` helpers: bash cannot export arrays, so -# a child shell (e.g. `timeout bash -c`) would otherwise see an empty list and -# mirror every digest-pinned plugin in the catalog. +# Bash cannot export arrays, so the exported helpers read this string. DISCONNECTED_SMOKE_PLUGIN_NAMES_STR="${DISCONNECTED_SMOKE_PLUGIN_NAMES[*]}" readonly DISCONNECTED_SMOKE_PLUGIN_NAMES_STR export DISCONNECTED_SMOKE_PLUGIN_NAMES_STR diff --git a/.ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml b/.ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml index 566e44dcb0..3a0a08ed60 100644 --- a/.ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml +++ b/.ci/pipelines/resources/disconnected/dynamic-plugins-smoke.yaml @@ -2,8 +2,7 @@ # Rendered via envsubst (requires the *_PLUGIN_PACKAGE variables). # includes: [] skips dynamic-plugins.default.yaml (unpublished :tag refs). # packages are the digest-pinned OCI refs from the catalog (ref:// needs includes). -# The 2.x hub provides the sign-in page, sidebar, SCM integrations and guest auth -# as dynamic plugins, so they must be installed alongside the homepage. +# The plugin set is explained in .ci/pipelines/lib/disconnected/plugins.sh. includes: [] plugins: - package: ${APP_AUTH_PLUGIN_PACKAGE} diff --git a/.ci/pipelines/value_files/values_disconnected-smoke.yaml b/.ci/pipelines/value_files/values_disconnected-smoke.yaml index dc3de1ffd6..8f6d45a692 100644 --- a/.ci/pipelines/value_files/values_disconnected-smoke.yaml +++ b/.ci/pipelines/value_files/values_disconnected-smoke.yaml @@ -8,11 +8,9 @@ # catalogs can list :tag refs that do not exist on quay (manifest unknown), # which CrashLoops install-dynamic-plugins after registries.conf rewrites # those pulls to the mirror. Every package below is a digest-pinned OCI from -# the catalog. ref:// needs includes and cannot be used. +# the catalog. ref:// needs includes and cannot be used. The plugin set is +# explained in .ci/pipelines/lib/disconnected/plugins.sh. # -# The 2.x New Frontend System hub serves the sign-in page (app-auth), sidebar -# (app-defaults), SCM integrations (app-integrations) and guest auth provider -# as dynamic plugins, so they are installed next to the homepage. # Pins the chart fullname (-developer-hub). The disconnected job creates # the registry-auth secret and the backend egress NetworkPolicy for that name. nameOverride: developer-hub From 24db15ce3d5c41b02ab84a4f8af546bdfdfb1a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 26 Oct 2026 16:01:35 +0100 Subject: [PATCH 09/16] fix(ci): take the CI operator catalog tag from the release stream The operator job picked the catalog tag from RELEASE_VERSION and the --filter-versions value from common::get_release_stream. On main those are "next" and the package.json stream (2.2), which agree only while rhdh and rhdh-operator bump together; when one side lags, prepare keeps a fast-X.Y channel the catalog does not carry, or oc-mirror rejects the catalog because its default channel is filtered out. Use filter_versions for the tag as well. Both catalog tags resolve to the same digest today (iib:next-v4.21 == iib:2.2-v4.21), so nightlies are unchanged, and a stream without a published catalog now fails at the skopeo check with a readable error. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-operator.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 1febef562d..4e1be95518 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -55,10 +55,9 @@ handle_ocp_disconnected_operator() { # # OLM version: leave default (auto). On OCP 4.21+ this selects OLM v1. # - # The operator stream follows the checkout branch, like install_rhdh_operator: - # main mirrors the 2.2 bundles, release-2.1 the 2.1 ones. A pinned image or - # chart does not change it, and an unfiltered catalog would pull every - # historical version. + # The operator stream follows the checkout branch: main mirrors the 2.2 + # bundles, release-2.1 the 2.1 ones. A pinned image or chart does not change + # it, and an unfiltered catalog would pull every historical version. local filter_versions filter_versions=$(common::get_release_stream) || return 1 log::info "filter-versions=${filter_versions} (from RELEASE_BRANCH_NAME=${RELEASE_BRANCH_NAME})" @@ -83,8 +82,11 @@ handle_ocp_disconnected_operator() { local registries_conf="" if [[ "${use_ci_catalog}" == "true" ]]; then # rhdh-operator publishes one catalog per stream and OCP minor, e.g. - # quay.io/rhdh/iib:next-v4.21-x86_64 (amd64 only; the jobs run on amd64). - index_image="quay.io/rhdh/iib:${RELEASE_VERSION}-v${ocp_version}-x86_64" + # quay.io/rhdh/iib:2.2-v4.21-x86_64 (amd64 only; the jobs run on amd64). + # The tag follows filter_versions, so the tag and --filter-versions cannot + # disagree, and a stream without a published catalog fails here instead of + # in oc-mirror. + index_image="quay.io/rhdh/iib:${filter_versions}-v${ocp_version}-x86_64" skopeo inspect --raw "docker://${index_image}" > /dev/null || { log::error "RHDH CI operator catalog ${index_image} not found or unreadable" return 1 From a9dc37379b042991a1ba2118e23ca54ad4ec437a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Mon, 26 Oct 2026 16:02:17 +0100 Subject: [PATCH 10/16] chore(ci): tidy disconnected helpers - list jq in common.sh dependencies: the moved get_release_stream uses it - rename mcp_all_converged to the underscore prefix the other private helpers use - drop the unreachable exported -eq 0 branch in the smoke plugin export - document the hub grace period as it behaves: it covers the backend start wait, then the Available wait Assisted-by: OpenCode --- .ci/pipelines/lib/common.sh | 2 +- .ci/pipelines/lib/disconnected.sh | 2 +- .ci/pipelines/lib/disconnected/helm.sh | 7 ++++--- .ci/pipelines/lib/disconnected/mirror.sh | 10 ++++------ .ci/pipelines/lib/disconnected/plugins.sh | 4 ---- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/.ci/pipelines/lib/common.sh b/.ci/pipelines/lib/common.sh index cf64074c1e..723c20fa46 100644 --- a/.ci/pipelines/lib/common.sh +++ b/.ci/pipelines/lib/common.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Common utility functions for pipeline scripts -# Dependencies: oc, kubectl, lib/log.sh +# Dependencies: oc, kubectl, jq, lib/log.sh if [[ -n "${COMMON_LIB_SOURCED:-}" ]]; then return 0 diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index aca7b0a9d2..2b51ec8dbb 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -56,7 +56,7 @@ export -f disconnected::build_imageset_config export -f disconnected::run_oc_mirror export -f disconnected::patch_idms export -f disconnected::snapshot_mcp_rendered_configs -export -f disconnected::mcp_all_converged +export -f disconnected::_mcp_all_converged export -f disconnected::wait_mcp_updated export -f disconnected::_hook_should_add_hub_image export -f disconnected::_hook_adjust_oc_mirror_args diff --git a/.ci/pipelines/lib/disconnected/helm.sh b/.ci/pipelines/lib/disconnected/helm.sh index fba752f367..45b668179d 100644 --- a/.ci/pipelines/lib/disconnected/helm.sh +++ b/.ci/pipelines/lib/disconnected/helm.sh @@ -13,13 +13,14 @@ readonly _DISCONNECTED_HELM_SOURCED=1 # The bounce scales the Deployment to 0 and back to 1 instead of using # `oc rollout restart`: a rolling restart runs two backends against the freshly # initialised database, which race on the knex migration lock and leave the -# loser never Ready. The grace period starts once the backend container starts, -# so a pod still extracting plugins is not mistaken for a stuck backend. +# loser never Ready. The wait gives a pod that is still extracting plugins up to +# the grace period to start its backend before the hub is considered stuck. # Args: # $1 - namespace # $2 - Helm release name (default: rhdh) # $3 - Postgres Ready timeout seconds (default: 600) -# $4 - Hub Available grace seconds after the backend starts (default: 180) +# $4 - Hub grace seconds: waits this long for the backend container to start, +# then this long again for the Deployment to become Available (default: 180) # $5 - Hub rollout timeout seconds after restart (default: 420) disconnected::ensure_helm_hub_after_postgres() { local namespace=$1 diff --git a/.ci/pipelines/lib/disconnected/mirror.sh b/.ci/pipelines/lib/disconnected/mirror.sh index 6282a7cd1a..7bbc4b055f 100644 --- a/.ci/pipelines/lib/disconnected/mirror.sh +++ b/.ci/pipelines/lib/disconnected/mirror.sh @@ -267,7 +267,7 @@ disconnected::snapshot_mcp_rendered_configs() { # True when every MachineConfigPool runs its rendered config and no node is # still updating or degraded. -disconnected::mcp_all_converged() { +disconnected::_mcp_all_converged() { local snapshot snapshot=$(oc get machineconfigpool \ -o jsonpath='{range .items[*]}{.spec.configuration.name}{"|"}{.status.configuration.name}{"|"}{.status.updatedMachineCount}{"|"}{.status.machineCount}{"|"}{.status.degradedMachineCount}{"\n"}{end}' \ @@ -295,12 +295,10 @@ disconnected::mcp_all_converged() { # Warns and continues on timeout. # Args: # $1 - snapshot from disconnected::snapshot_mcp_rendered_configs (optional) -# $2 - seconds to wait for the MCO to render the change (default: 180) -# $3 - seconds to wait for the rollout to finish (default: 1200) disconnected::wait_mcp_updated() { local before_snapshot=${1:-} - local render_timeout=${2:-180} - local rollout_timeout=${3:-1200} + local render_timeout=180 + local rollout_timeout=1200 if [[ -n "${before_snapshot}" ]]; then log::info "Waiting for MachineConfigPool to pick up the new rendered config (up to ${render_timeout}s)..." @@ -325,7 +323,7 @@ disconnected::wait_mcp_updated() { log::info "Waiting for MachineConfigPool updates to complete (up to ${rollout_timeout}s)..." local rollout_start=$SECONDS while ((SECONDS - rollout_start < rollout_timeout)); do - if disconnected::mcp_all_converged; then + if disconnected::_mcp_all_converged; then log::success "All MachineConfigPools are Updated" return 0 fi diff --git a/.ci/pipelines/lib/disconnected/plugins.sh b/.ci/pipelines/lib/disconnected/plugins.sh index 265b3bffc4..d9c2839816 100644 --- a/.ci/pipelines/lib/disconnected/plugins.sh +++ b/.ci/pipelines/lib/disconnected/plugins.sh @@ -233,10 +233,6 @@ disconnected::set_smoke_plugin_packages_from_list() { exported=$((exported + 1)) done - if [[ "${exported}" -eq 0 ]]; then - log::error "No smoke plugin package variables exported from ${list_file}" - return 1 - fi log::info "Exported ${exported} smoke plugin package(s)" } From 5efa5073881dc232af78d085438003d443611652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 27 Oct 2026 09:30:14 +0100 Subject: [PATCH 11/16] refactor(ci): always mirror the branch's CI operator catalog The operator job fell back to the published Red Hat catalog whenever the image registry was not registry.redhat.io or the chart was not a CI tag. That path cannot work: prepare-restricted-environment.sh writes only fast-X.Y channels into the ImageSetConfiguration, so oc-mirror rejects the published catalog because its default fast channel is filtered out, and the catalog carries no 2.x bundles at all. Every disconnected run is branch-aligned, and the hub image comes from the bundle's RELATED_IMAGE_backstage, so TAG_NAME, IMAGE_* and CHART_VERSION never reached the deployment. Use the CI catalog (quay.io/rhdh/iib:-v-x86_64) and its registries.conf redirect unconditionally, and log that only the newest bundle of the branch's stream is tested. Assisted-by: OpenCode --- .../jobs/ocp-disconnected-operator.sh | 70 ++++++++----------- .../rhdh-start-disconnected-smoke.yaml | 7 +- 2 files changed, 31 insertions(+), 46 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 4e1be95518..65f1225741 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -56,8 +56,10 @@ handle_ocp_disconnected_operator() { # OLM version: leave default (auto). On OCP 4.21+ this selects OLM v1. # # The operator stream follows the checkout branch: main mirrors the 2.2 - # bundles, release-2.1 the 2.1 ones. A pinned image or chart does not change - # it, and an unfiltered catalog would pull every historical version. + # bundles, release-2.1 the 2.1 ones. TAG_NAME, IMAGE_* and CHART_VERSION do + # not change it: the hub image comes from the bundle, so only the newest + # bundle of the branch's stream can be tested. An unfiltered catalog would + # pull every historical version. local filter_versions filter_versions=$(common::get_release_stream) || return 1 log::info "filter-versions=${filter_versions} (from RELEASE_BRANCH_NAME=${RELEASE_BRANCH_NAME})" @@ -69,36 +71,26 @@ handle_ocp_disconnected_operator() { log::error "CONTAINER_PLATFORM_VERSION is unset or invalid ('${ocp_version}'); expected OpenShift major.minor" return 1 fi - # A CI chart tag (x.y-N-CI) means the run deploys a stream build, so mirror the - # RHDH CI catalog built from the same commit as the hub and plugin catalogs. - # Released charts (x.y.z) pair with the published Red Hat catalog, the only one - # carrying released bundles, even when the image itself is pulled from quay. - local use_ci_catalog="false" - if [[ "${IMAGE_REGISTRY}" != "registry.redhat.io" && "${CHART_VERSION:-}" == *-CI ]]; then - use_ci_catalog="true" - fi - - local index_image="" - local registries_conf="" - if [[ "${use_ci_catalog}" == "true" ]]; then - # rhdh-operator publishes one catalog per stream and OCP minor, e.g. - # quay.io/rhdh/iib:2.2-v4.21-x86_64 (amd64 only; the jobs run on amd64). - # The tag follows filter_versions, so the tag and --filter-versions cannot - # disagree, and a stream without a published catalog fails here instead of - # in oc-mirror. - index_image="quay.io/rhdh/iib:${filter_versions}-v${ocp_version}-x86_64" - skopeo inspect --raw "docker://${index_image}" > /dev/null || { - log::error "RHDH CI operator catalog ${index_image} not found or unreadable" - return 1 - } - - # The CI bundle's RELATED_IMAGE refs point at registry.redhat.io/rhdh/* - # (plus the stage/access aliases), whose digests exist on quay.io/rhdh only. - # Rewrite those pulls for the oc-mirror run via CONTAINERS_REGISTRIES_CONF. - # Images in other repositories (e.g. registry.redhat.io/rhel10/postgresql-18) - # keep their original source and are pulled with the CI credentials. - registries_conf="${DISCONNECTED_TMPDIR}/oc-mirror-registries.conf" - cat > "${registries_conf}" << 'EOF' + # rhdh-operator publishes one catalog per stream and OCP minor, e.g. + # quay.io/rhdh/iib:2.2-v4.21-x86_64 (amd64 only; the jobs run on amd64), built + # from the same commit as the hub and plugin catalogs. The tag follows + # filter_versions, so the tag and --filter-versions cannot disagree, and a + # stream without a published catalog fails here instead of in oc-mirror. + local index_image="quay.io/rhdh/iib:${filter_versions}-v${ocp_version}-x86_64" + skopeo inspect --raw "docker://${index_image}" > /dev/null || { + log::error "RHDH CI operator catalog ${index_image} not found or unreadable" + return 1 + } + log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}" + log::info "Operator and hub come from the newest ${filter_versions} bundle in ${index_image}; TAG_NAME/IMAGE_* do not apply" + + # The CI bundle's RELATED_IMAGE refs point at registry.redhat.io/rhdh/* + # (plus the stage/access aliases), whose digests exist on quay.io/rhdh only. + # Rewrite those pulls for the oc-mirror run via CONTAINERS_REGISTRIES_CONF. + # Images in other repositories (e.g. registry.redhat.io/rhel10/postgresql-18) + # keep their original source and are pulled with the CI credentials. + local registries_conf="${DISCONNECTED_TMPDIR}/oc-mirror-registries.conf" + cat > "${registries_conf}" << 'EOF' [[registry]] prefix = "registry.redhat.io/rhdh" location = "quay.io/rhdh" @@ -111,10 +103,6 @@ location = "quay.io/rhdh" prefix = "registry.access.redhat.com/rhdh" location = "quay.io/rhdh" EOF - else - index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}" - fi - log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}" # Local connected OCP: OCP_INTERNAL sentinel (same as omit-on-OCP). CI bastion # keeps the external MIRROR_REGISTRY_URL. Route URL stays in MIRROR_REGISTRY_URL @@ -157,12 +145,10 @@ EOF fi local -a prepare_cmd=(bash "${prepare_script_path}" "${prepare_args[@]}") - if [[ -n "${registries_conf}" ]]; then - # Scoped to this call: CATALOG_INDEX_IMAGE and the plugin pulls keep their - # own registry configuration. - prepare_cmd=(env CONTAINERS_REGISTRIES_CONF="${registries_conf}" "${prepare_cmd[@]}") - log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" - fi + # Scoped to this call: CATALOG_INDEX_IMAGE and the plugin pulls keep their own + # registry configuration. + prepare_cmd=(env CONTAINERS_REGISTRIES_CONF="${registries_conf}" "${prepare_cmd[@]}") + log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) diff --git a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml index 794346ce20..ce83841da6 100644 --- a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml +++ b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml @@ -4,10 +4,9 @@ # Guest login + homepage only. flavours: [] disables default flavours # (lightspeed) to avoid extra sidecar images. # -# Hub image: CSV RELATED_IMAGE_backstage. CI runs install the RHDH CI catalog -# (quay.io/rhdh/iib), whose bundle points at rhdh-hub-rhel10; GA runs use the -# released rhdh-hub-rhel9. Either way the CSV ref is mirrored by prepare + -# oc-mirror and the generated IDMS rewrites the kubelet pull. +# Hub image: CSV RELATED_IMAGE_backstage from the newest CI bundle of the +# branch's operator stream. prepare + oc-mirror mirror that ref and the generated +# IDMS rewrites the kubelet pull. # # CATALOG_INDEX_IMAGE is injected by deploy_rhdh_operator from the mirrored # catalog digest so install-dynamic-plugins does not pull the hub profile From 41e5fedccb44471f4cfe9722a5c85d1989b47043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 27 Oct 2026 09:31:00 +0100 Subject: [PATCH 12/16] fix(ci): fail disconnected deploys on errors instead of waiting Putting deploy_rhdh_operator on the left of || turns errexit off for its whole body, so a failed envsubst, yq or oc apply fell through to the 5-minute deployment poll and then reported a missing deployment, which pointed at OLM. Return from deploy_rhdh_operator as soon as rendering or applying the CR fails, and use a neutral message in the operator job (the OLM v1 dump stays as diagnostics). The Helm hub bounce scaled the Deployment back up even when the old pod was still present after 180s, recreating the two-backend race it is meant to prevent. Treat that timeout as a recovery failure. Assisted-by: OpenCode --- .ci/pipelines/install-methods/operator.sh | 15 ++++++++++++--- .ci/pipelines/jobs/ocp-disconnected-operator.sh | 2 +- .ci/pipelines/lib/disconnected/helm.sh | 7 +++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.ci/pipelines/install-methods/operator.sh b/.ci/pipelines/install-methods/operator.sh index 844fa26f78..52f1d06bb9 100755 --- a/.ci/pipelines/install-methods/operator.sh +++ b/.ci/pipelines/install-methods/operator.sh @@ -212,14 +212,23 @@ deploy_rhdh_operator() { # Verify Backstage CRD is available k8s_wait::crd "backstages.rhdh.redhat.com" 60 5 || return 1 - rendered_yaml=$(envsubst < "$backstage_crd_path") + rendered_yaml=$(envsubst < "$backstage_crd_path") || { + log::error "Failed to render Backstage CR from $backstage_crd_path" + return 1 + } if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then # Dynamically inject CATALOG_INDEX_IMAGE environment variable if specified - rendered_yaml=$(echo "$rendered_yaml" | yq eval '.spec.application.extraEnvs.envs += [{"name": "CATALOG_INDEX_IMAGE", "value": "'"$CATALOG_INDEX_IMAGE"'", "containers": ["install-dynamic-plugins"]}]' -) + rendered_yaml=$(echo "$rendered_yaml" | yq eval '.spec.application.extraEnvs.envs += [{"name": "CATALOG_INDEX_IMAGE", "value": "'"$CATALOG_INDEX_IMAGE"'", "containers": ["install-dynamic-plugins"]}]' -) || { + log::error "Failed to inject CATALOG_INDEX_IMAGE into the Backstage CR" + return 1 + } fi log::info "Applying Backstage CR from: $backstage_crd_path" log::debug "$rendered_yaml" - echo "$rendered_yaml" | oc apply -f - -n "$namespace" + echo "$rendered_yaml" | oc apply -f - -n "$namespace" || { + log::error "Failed to apply Backstage CR from $backstage_crd_path in $namespace" + return 1 + } # Wait for the operator to create the Backstage deployment (5 minutes max) if ! common::poll_until \ diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 65f1225741..886485785e 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -272,7 +272,7 @@ EOF cp "${cr_temp}" "${ARTIFACT_DIR}/disconnected-backstage-cr.yaml" 2> /dev/null || true deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" || { - log::error "Backstage CR did not produce a deployment — dumping OLM v1 status" + log::error "deploy_rhdh_operator failed in ${NAME_SPACE} — dumping OLM v1 status" disconnected::dump_olm_v1_status "rhdh-operator" return 1 } diff --git a/.ci/pipelines/lib/disconnected/helm.sh b/.ci/pipelines/lib/disconnected/helm.sh index 45b668179d..76be44fb32 100644 --- a/.ci/pipelines/lib/disconnected/helm.sh +++ b/.ci/pipelines/lib/disconnected/helm.sh @@ -91,9 +91,12 @@ disconnected::ensure_helm_hub_after_postgres() { return 1 fi # Wait for the old pod to be gone before starting the replacement, otherwise - # both backends initialise at once. + # both backends initialise at once. kubectl treats --for=delete as satisfied + # when nothing matches, so this only fails while a pod is still there. if ! oc wait --for=delete pod -l "${hub_selector}" -n "${namespace}" --timeout=180s; then - log::warn "Old hub pods still present after 180s — scaling back up anyway" + log::error "Old hub pods still present after 180s — not scaling up next to them" + oc get pods -l "${hub_selector}" -n "${namespace}" -o wide || true + return 1 fi if ! oc scale "deployment/${hub_deploy}" --replicas=1 -n "${namespace}"; then log::error "Failed to scale deployment/${hub_deploy} back up in ${namespace}" From cad623f9951310b4851c0dbc412e68987d721853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 27 Oct 2026 09:32:53 +0100 Subject: [PATCH 13/16] fix(ci): retry MachineConfigPool reads and drop the MCP abbreviation disconnected::snapshot_mcp_rendered_configs hid oc failures by returning an empty success: a failed pre-change read skipped render detection, and a failed read during detection looked like a changed config. Retry the read three times and fail on an error or an empty answer, skip failed reads in the render loop, and require the pre-change snapshot. Both jobs now stop when it cannot be taken. Rename the MCP identifiers to machineconfigpool, since MCP reads as Model Context Protocol in this repository. Document why the IDMS and ITMS applies share one wait: a pool's spec moves to the newest rendered config, so a config rendered during the first rollout is still awaited. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 6 +- .../jobs/ocp-disconnected-operator.sh | 10 +-- .ci/pipelines/lib/disconnected.sh | 6 +- .ci/pipelines/lib/disconnected/mirror.sh | 84 +++++++++++-------- .ci/pipelines/lib/disconnected/namespace.sh | 3 +- 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index e1abba3b4a..4756e43c4f 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -121,8 +121,8 @@ handle_ocp_disconnected_helm() { disconnected::patch_idms "${OC_MIRROR_IDMS_FILE}" - local mcp_snapshot - mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) + local machineconfigpool_snapshot + machineconfigpool_snapshot=$(disconnected::snapshot_machineconfigpool_rendered_configs) || return 1 oc apply -f "${OC_MIRROR_IDMS_FILE}" || { log::error "Failed to apply IDMS — aborting" @@ -141,7 +141,7 @@ handle_ocp_disconnected_helm() { # IDMS/ITMS changes trigger a MachineConfig update which rolls worker nodes # (drain → apply config → reboot). Wait for all MachineConfigPools to finish # before deploying workloads, otherwise pods get evicted mid-startup. - disconnected::wait_mcp_updated "${mcp_snapshot}" + disconnected::wait_machineconfigpool_updated "${machineconfigpool_snapshot}" || return 1 log::section "Plugin Mirroring" disconnected::mirror_plugins || return 1 diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index 886485785e..e0e632f56f 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -132,14 +132,14 @@ EOF # external registries. Catalogd must trust the mirror CA and authenticate # before ClusterCatalog can reach Serving=True. Skip on LOCAL_DISCONNECTED — # OCP_INTERNAL path configures internal registry trust itself. - local mcp_snapshot + local machineconfigpool_snapshot if [[ "${LOCAL_DISCONNECTED:-}" != "1" ]]; then # Snapshot before the additionalTrustedCA patch: it also rolls the nodes. - mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) + machineconfigpool_snapshot=$(disconnected::snapshot_machineconfigpool_rendered_configs) || return 1 disconnected::ensure_olm_mirror_pull_secret || return 1 disconnected::ensure_mirror_registry_ca || return 1 - disconnected::wait_mcp_updated "${mcp_snapshot}" + disconnected::wait_machineconfigpool_updated "${machineconfigpool_snapshot}" || return 1 else log::info "LOCAL_DISCONNECTED=1: skipping external bastion CA/pull-secret pre-prepare helpers" fi @@ -150,7 +150,7 @@ EOF prepare_cmd=(env CONTAINERS_REGISTRIES_CONF="${registries_conf}" "${prepare_cmd[@]}") log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" - mcp_snapshot=$(disconnected::snapshot_mcp_rendered_configs) + machineconfigpool_snapshot=$(disconnected::snapshot_machineconfigpool_rendered_configs) || return 1 log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}" if ! disconnected::retry_on_local_registry 5 \ @@ -191,7 +191,7 @@ EOF # prepare-restricted-environment.sh applies IDMS/CatalogSource which triggers # a MachineConfig update and node rolling. Wait for completion before deploying # workloads, same as the Helm path. - disconnected::wait_mcp_updated "${mcp_snapshot}" + disconnected::wait_machineconfigpool_updated "${machineconfigpool_snapshot}" || return 1 if [[ "${LOCAL_DISCONNECTED:-}" == "1" ]]; then # Catalogd needs the registry + image-puller grants above before Serving=True. diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index 2b51ec8dbb..ed750fac6f 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -55,9 +55,9 @@ export -f disconnected::cluster_mirror_host export -f disconnected::build_imageset_config export -f disconnected::run_oc_mirror export -f disconnected::patch_idms -export -f disconnected::snapshot_mcp_rendered_configs -export -f disconnected::_mcp_all_converged -export -f disconnected::wait_mcp_updated +export -f disconnected::snapshot_machineconfigpool_rendered_configs +export -f disconnected::_machineconfigpool_all_converged +export -f disconnected::wait_machineconfigpool_updated export -f disconnected::_hook_should_add_hub_image export -f disconnected::_hook_adjust_oc_mirror_args export -f disconnected::_hook_run_oc_mirror diff --git a/.ci/pipelines/lib/disconnected/mirror.sh b/.ci/pipelines/lib/disconnected/mirror.sh index 7bbc4b055f..7d435a4ddb 100644 --- a/.ci/pipelines/lib/disconnected/mirror.sh +++ b/.ci/pipelines/lib/disconnected/mirror.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# oc-mirror image mirroring, IDMS/ITMS patching, and MCP wait helpers. +# oc-mirror image mirroring, IDMS/ITMS patching, and MachineConfigPool wait helpers. [[ -n "${_DISCONNECTED_MIRROR_SOURCED:-}" ]] && return 0 readonly _DISCONNECTED_MIRROR_SOURCED=1 @@ -256,18 +256,30 @@ disconnected::patch_idms() { cp "${idms_file}" "${ARTIFACT_DIR}/disconnected-idms-patched.yaml" 2> /dev/null || true } -# Record the rendered MachineConfig every pool currently targets. -# Call this before applying an IDMS/ITMS/additionalTrustedCA change and pass the -# output to disconnected::wait_mcp_updated. -disconnected::snapshot_mcp_rendered_configs() { - oc get machineconfigpool \ - -o jsonpath='{range .items[*]}{.metadata.name}={.spec.configuration.name}{"\n"}{end}' \ - 2> /dev/null || true +# Record the rendered MachineConfig every pool currently targets. Fails when the +# read fails or returns nothing, so a caller cannot mistake a broken API call for +# an unchanged cluster. +disconnected::snapshot_machineconfigpool_rendered_configs() { + local attempt snapshot="" + for attempt in 1 2 3; do + if snapshot=$(oc get machineconfigpool \ + -o jsonpath='{range .items[*]}{.metadata.name}={.spec.configuration.name}{"\n"}{end}' \ + 2> /dev/null) && [[ -n "${snapshot}" ]]; then + printf '%s\n' "${snapshot}" + return 0 + fi + log::debug "MachineConfigPool snapshot failed (attempt ${attempt}/3)" + if ((attempt < 3)); then + sleep 5 + fi + done + log::error "Could not read the MachineConfigPool rendered configs" + return 1 } # True when every MachineConfigPool runs its rendered config and no node is # still updating or degraded. -disconnected::_mcp_all_converged() { +disconnected::_machineconfigpool_all_converged() { local snapshot snapshot=$(oc get machineconfigpool \ -o jsonpath='{range .items[*]}{.spec.configuration.name}{"|"}{.status.configuration.name}{"|"}{.status.updatedMachineCount}{"|"}{.status.machineCount}{"|"}{.status.degradedMachineCount}{"\n"}{end}' \ @@ -290,40 +302,46 @@ disconnected::_mcp_all_converged() { # `oc wait --for=condition=Updated=True` alone is not enough: right after an # apply the MCO has not rendered the new config yet, so the condition is still # True from the previous rollout and the wait returns immediately. With the -# snapshot taken before the change, wait until the MCO picks it up first. +# snapshot taken before the change, wait until the MCO picks it up first. A +# config rendered while the rollout is still running is covered: each pool's spec +# moves to the newest render, and convergence compares status against it. # -# Warns and continues on timeout. +# Warns and continues on rollout timeout. # Args: -# $1 - snapshot from disconnected::snapshot_mcp_rendered_configs (optional) -disconnected::wait_mcp_updated() { - local before_snapshot=${1:-} +# $1 - snapshot from disconnected::snapshot_machineconfigpool_rendered_configs +disconnected::wait_machineconfigpool_updated() { + local before_snapshot=$1 local render_timeout=180 local rollout_timeout=1200 - if [[ -n "${before_snapshot}" ]]; then - log::info "Waiting for MachineConfigPool to pick up the new rendered config (up to ${render_timeout}s)..." - local rendered=0 - local render_start=$SECONDS - while ((SECONDS - render_start < render_timeout)); do - if [[ "$(disconnected::snapshot_mcp_rendered_configs)" != "${before_snapshot}" ]]; then - rendered=1 - break - fi - sleep 5 - done - if ((rendered != 1)); then - # Re-applying identical content (e.g. an unchanged IDMS) does not roll - # the nodes, so there is nothing to wait for. - log::warn "No MachineConfigPool rendered a new config within ${render_timeout}s -- nothing to roll, continuing" - return 0 + if [[ -z "${before_snapshot}" ]]; then + log::error "wait_machineconfigpool_updated needs the pre-change MachineConfigPool snapshot" + return 1 + fi + + log::info "Waiting for MachineConfigPool to pick up the new rendered config (up to ${render_timeout}s)..." + local rendered=0 render_start=$SECONDS current="" + while ((SECONDS - render_start < render_timeout)); do + # A failed read is not a change; retry on the next iteration. + if current=$(disconnected::snapshot_machineconfigpool_rendered_configs) \ + && [[ "${current}" != "${before_snapshot}" ]]; then + rendered=1 + break fi - log::success "MachineConfigPool picked up the new rendered config" + sleep 5 + done + if ((rendered != 1)); then + # Re-applying identical content (e.g. an unchanged IDMS) does not roll + # the nodes, so there is nothing to wait for. + log::warn "No MachineConfigPool rendered a new config within ${render_timeout}s -- nothing to roll, continuing" + return 0 fi + log::success "MachineConfigPool picked up the new rendered config" log::info "Waiting for MachineConfigPool updates to complete (up to ${rollout_timeout}s)..." local rollout_start=$SECONDS while ((SECONDS - rollout_start < rollout_timeout)); do - if disconnected::_mcp_all_converged; then + if disconnected::_machineconfigpool_all_converged; then log::success "All MachineConfigPools are Updated" return 0 fi @@ -331,5 +349,5 @@ disconnected::wait_mcp_updated() { done log::warn "MachineConfigPool wait timed out -- proceeding anyway" - oc get machineconfigpool 2>&1 | tee -a "${ARTIFACT_DIR}/disconnected-mcp-status.txt" || true + oc get machineconfigpool 2>&1 | tee -a "${ARTIFACT_DIR}/disconnected-machineconfigpool-status.txt" || true } diff --git a/.ci/pipelines/lib/disconnected/namespace.sh b/.ci/pipelines/lib/disconnected/namespace.sh index 0c70c81ea7..dd786b8cf0 100644 --- a/.ci/pipelines/lib/disconnected/namespace.sh +++ b/.ci/pipelines/lib/disconnected/namespace.sh @@ -44,7 +44,8 @@ disconnected::create_plugin_registry_auth_secret() { # Ensure the mirror registry CA is trusted cluster-wide via # image.config.openshift.io/cluster additionalTrustedCA so OLM v1 catalogd # can pull ClusterCatalog images (x509 unknown authority otherwise). -# Triggers an MCP update; callers should run disconnected::wait_mcp_updated after. +# Triggers a MachineConfigPool update; callers should run +# disconnected::wait_machineconfigpool_updated after. disconnected::ensure_mirror_registry_ca() { local registry_host="${MIRROR_REGISTRY_URL%%/*}" # OpenShift ConfigMap keys use ".." in place of ":" for host:port. From c5c17fe6c00e2afad23a9bcd63e13dd2aabaffea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 27 Oct 2026 09:33:35 +0100 Subject: [PATCH 14/16] refactor(ci): derive smoke plugin checks from the plugin list The smoke set was spelled out in six places: the array, the case in _smoke_plugin_package_var, two require_vars lists and two templates. A template entry the array did not know about rendered an empty package: and only failed at pod runtime. Render both templates through one helper that maps the array to the package variables, requires them, and rejects a package: left empty by envsubst. Assisted-by: OpenCode --- .ci/pipelines/jobs/ocp-disconnected-helm.sh | 9 +---- .ci/pipelines/lib/disconnected.sh | 1 + .ci/pipelines/lib/disconnected/plugins.sh | 43 +++++++++++++++++---- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-helm.sh b/.ci/pipelines/jobs/ocp-disconnected-helm.sh index 4756e43c4f..fab8e5604b 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-helm.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-helm.sh @@ -214,13 +214,8 @@ handle_ocp_disconnected_helm() { # to the rendered Deployment, avoiding the Helm "array clobber" pitfall. local post_renderer="${DIR}/resources/disconnected/helm-post-renderer.sh" local helm_values="${DISCONNECTED_TMPDIR}/values_disconnected-smoke.yaml" - common::require_vars HOMEPAGE_PLUGIN_PACKAGE APP_AUTH_PLUGIN_PACKAGE \ - APP_DEFAULTS_PLUGIN_PACKAGE APP_INTEGRATIONS_PLUGIN_PACKAGE \ - GUEST_PROVIDER_PLUGIN_PACKAGE || return 1 - envsubst < "${DIR}/value_files/values_disconnected-smoke.yaml" > "${helm_values}" || { - log::error "Failed to render disconnected Helm values" - return 1 - } + disconnected::render_smoke_plugins_template \ + "${DIR}/value_files/values_disconnected-smoke.yaml" "${helm_values}" || return 1 cp "${helm_values}" "${ARTIFACT_DIR}/values_disconnected-smoke.yaml" 2> /dev/null || true helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \ diff --git a/.ci/pipelines/lib/disconnected.sh b/.ci/pipelines/lib/disconnected.sh index ed750fac6f..734addcdbc 100644 --- a/.ci/pipelines/lib/disconnected.sh +++ b/.ci/pipelines/lib/disconnected.sh @@ -70,6 +70,7 @@ export -f disconnected::apply_plugin_mirror_configmap export -f disconnected::write_digest_plugin_list export -f disconnected::write_smoke_plugin_list export -f disconnected::set_smoke_plugin_packages_from_list +export -f disconnected::render_smoke_plugins_template export -f disconnected::_smoke_plugin_package_var export -f disconnected::copy_catalog_index_to_mirror export -f disconnected::_catalog_index_source_ref diff --git a/.ci/pipelines/lib/disconnected/plugins.sh b/.ci/pipelines/lib/disconnected/plugins.sh index d9c2839816..3b5883e151 100644 --- a/.ci/pipelines/lib/disconnected/plugins.sh +++ b/.ci/pipelines/lib/disconnected/plugins.sh @@ -200,6 +200,41 @@ disconnected::_smoke_plugin_package_var() { esac } +# Render a smoke plugin template, mapping DISCONNECTED_SMOKE_PLUGIN_NAMES_STR to +# the _PLUGIN_PACKAGE variables. Fails when a package is unset or a +# template entry renders without one, so a template/array mismatch surfaces here +# instead of at pod runtime. +# Args: +# $1 - template path +# $2 - output path +disconnected::render_smoke_plugins_template() { + local template=$1 output=$2 name var + local -a package_vars=() + + if [[ -z "${DISCONNECTED_SMOKE_PLUGIN_NAMES_STR:-}" ]]; then + log::error "DISCONNECTED_SMOKE_PLUGIN_NAMES_STR is empty; cannot render ${template}" + return 1 + fi + + for name in ${DISCONNECTED_SMOKE_PLUGIN_NAMES_STR}; do + var=$(disconnected::_smoke_plugin_package_var "${name}") || { + log::error "No package variable mapped for plugin ${name}" + return 1 + } + package_vars+=("${var}") + done + common::require_vars "${package_vars[@]}" || return 1 + + envsubst < "${template}" > "${output}" || { + log::error "Failed to render ${template}" + return 1 + } + if grep -qE '^[[:space:]-]*package:[[:space:]]*$' "${output}"; then + log::error "${template} has a plugin entry without a package after rendering" + return 1 + fi +} + # Export one _PLUGIN_PACKAGE per smoke plugin from the digest plugin list. # Adds !name if the list line has no OCI subpath (install-dynamic-plugins needs it). # Args: @@ -374,18 +409,12 @@ disconnected::resolve_catalog_index_image() { # Args: # $1 - namespace disconnected::create_smoke_plugins_configmap() { - common::require_vars HOMEPAGE_PLUGIN_PACKAGE APP_AUTH_PLUGIN_PACKAGE \ - APP_DEFAULTS_PLUGIN_PACKAGE APP_INTEGRATIONS_PLUGIN_PACKAGE \ - GUEST_PROVIDER_PLUGIN_PACKAGE || return 1 local namespace=$1 local cm_name="dynamic-plugins-disconnected-smoke" local template="${DIR}/resources/disconnected/dynamic-plugins-smoke.yaml" local tmp_yaml="${DISCONNECTED_TMPDIR}/dynamic-plugins-disconnected-smoke.yaml" - envsubst < "${template}" > "${tmp_yaml}" || { - log::error "Failed to render ${template}" - return 1 - } + disconnected::render_smoke_plugins_template "${template}" "${tmp_yaml}" || return 1 oc create configmap "${cm_name}" \ --from-file="dynamic-plugins.yaml=${tmp_yaml}" \ From 4802563ae506915ecb55996ef4fb03abdd017eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 27 Oct 2026 09:34:01 +0100 Subject: [PATCH 15/16] docs(ci): correct disconnected comments - the CI mirror serves TLS on port 5000, not plain HTTP: the openshift/release step curls https:///v2/ with the vault CA - nameOverride pins the chart fullname for the registry-auth secret; the egress NetworkPolicy selects on pod labels - the smoke app config is a subset of dynamic-plugins-config.yaml, not a mirror - the Backstage CR installs the minimal NFS plugin set, not only the homepage Assisted-by: OpenCode --- .ci/pipelines/lib/disconnected/namespace.sh | 12 ++++++------ .../app-config-rhdh-disconnected-smoke.yaml | 2 +- .../rhdh-operator/rhdh-start-disconnected-smoke.yaml | 5 +++-- .../value_files/values_disconnected-smoke.yaml | 7 ++++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.ci/pipelines/lib/disconnected/namespace.sh b/.ci/pipelines/lib/disconnected/namespace.sh index dd786b8cf0..1ba61c90bf 100644 --- a/.ci/pipelines/lib/disconnected/namespace.sh +++ b/.ci/pipelines/lib/disconnected/namespace.sh @@ -120,10 +120,10 @@ disconnected::ensure_olm_mirror_pull_secret() { # Allow backend egress to the mirror registry. The chart and the Operator both # default-deny backend egress except DNS, HTTPS, Redis, and PostgreSQL, while CI -# mirrors serve plain HTTP on a non-443 port: the install-dynamic-plugins init -# container (skopeo catalog/plugin pulls) times out without this rule. Kubelet -# pulls are unaffected. NetworkPolicies are additive and separate selectors cover -# the Helm chart and the Operator pod labels (same split as +# mirrors serve TLS on a non-443 port: the install-dynamic-plugins init container +# (skopeo catalog/plugin pulls) times out without this rule. Kubelet pulls are +# unaffected. NetworkPolicies are additive and separate selectors cover the Helm +# chart and the Operator pod labels (same split as # resources/network_policy/netpol-ci-allow-backend-egress.yaml). # # The rule is scoped by port only. MIRROR_REGISTRY_URL is the bastion's public @@ -144,14 +144,14 @@ disconnected::allow_mirror_registry_egress() { fi if [[ -z "${registry_port}" || "${registry_port}" == "443" ]]; then - log::info "Mirror registry ${registry_host} is served on HTTPS; skipping dedicated egress rule" + log::info "Mirror registry ${registry_host} is served on port 443; skipping dedicated egress rule" return 0 fi local policy_file="${DISCONNECTED_TMPDIR}/netpol-mirror-registry-egress.yaml" cat > "${policy_file}" << EOF # Allow backend egress to the CI mirror registry (${registry_host}). -# install-dynamic-plugins pulls the catalog index and plugins over plain HTTP. +# install-dynamic-plugins pulls the catalog index and plugins from this port. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml b/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml index 047861bd9a..cc1fc217da 100644 --- a/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml +++ b/.ci/pipelines/resources/disconnected/app-config-rhdh-disconnected-smoke.yaml @@ -1,4 +1,4 @@ -# Minimal app-config for disconnected operator smoke (guest login only). +# Minimal app-config for disconnected operator smoke (guest login + homepage). app: title: Red Hat Developer Hub # New Frontend System: the homepage plugin owns page:homepage and its layout; diff --git a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml index ce83841da6..4a606dd9d9 100644 --- a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml +++ b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml @@ -1,8 +1,9 @@ # Minimal Backstage CR for disconnected operator smoke test. # Rendered via envsubst (requires MIRROR_REGISTRY_URL, RELEASE_NAME). # -# Guest login + homepage only. flavours: [] disables default flavours -# (lightspeed) to avoid extra sidecar images. +# Guest login + homepage smoke test; installs the minimal NFS plugin set. +# flavours: [] disables default flavours (lightspeed) to avoid extra sidecar +# images. # # Hub image: CSV RELATED_IMAGE_backstage from the newest CI bundle of the # branch's operator stream. prepare + oc-mirror mirror that ref and the generated diff --git a/.ci/pipelines/value_files/values_disconnected-smoke.yaml b/.ci/pipelines/value_files/values_disconnected-smoke.yaml index 8f6d45a692..c54445ad5d 100644 --- a/.ci/pipelines/value_files/values_disconnected-smoke.yaml +++ b/.ci/pipelines/value_files/values_disconnected-smoke.yaml @@ -11,8 +11,9 @@ # the catalog. ref:// needs includes and cannot be used. The plugin set is # explained in .ci/pipelines/lib/disconnected/plugins.sh. # -# Pins the chart fullname (-developer-hub). The disconnected job creates -# the registry-auth secret and the backend egress NetworkPolicy for that name. +# Pins the chart fullname (-developer-hub, also the chart default): the +# job creates the registry-auth secret under that name. The egress NetworkPolicy +# selects on pod labels and does not depend on this. nameOverride: developer-hub dynamicPlugins: includes: [] @@ -50,7 +51,7 @@ appConfig: app: # New Frontend System: the homepage plugin owns page:homepage and its # layout; disable the community home page so only one Home destination is - # registered. Mirrors resources/config_map/dynamic-plugins-config.yaml. + # registered. Subset of resources/config_map/dynamic-plugins-config.yaml. extensions: - page:home: false - page:homepage: From 7e9d963c0180aa20378110bda3bd658f57fb4e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Tue, 27 Oct 2026 09:46:14 +0100 Subject: [PATCH 16/16] fix(ci): keep the published catalog for GA verification A GA verification run (--image-registry registry.redhat.io with a released x.y.z chart) should pair with the published Red Hat catalog, and the existing image and chart overrides already select it. Restore that branch. It cannot keep the version filter: the published catalog declares defaultChannel fast, and oc-mirror rejects a channels list that leaves the default channel out, which is why GA runs failed at mirroring before. Pass --filter-versions '*' for the published catalog (prepare then mirrors every rhdh channel, still scoped to the rhdh package) and keep the branch-stream filter for the CI catalog, so the nightlies are unchanged. Assisted-by: OpenCode --- .../jobs/ocp-disconnected-operator.sh | 83 +++++++++++-------- .../rhdh-start-disconnected-smoke.yaml | 7 +- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/.ci/pipelines/jobs/ocp-disconnected-operator.sh b/.ci/pipelines/jobs/ocp-disconnected-operator.sh index e0e632f56f..f4dc6f3cb6 100644 --- a/.ci/pipelines/jobs/ocp-disconnected-operator.sh +++ b/.ci/pipelines/jobs/ocp-disconnected-operator.sh @@ -55,15 +55,6 @@ handle_ocp_disconnected_operator() { # # OLM version: leave default (auto). On OCP 4.21+ this selects OLM v1. # - # The operator stream follows the checkout branch: main mirrors the 2.2 - # bundles, release-2.1 the 2.1 ones. TAG_NAME, IMAGE_* and CHART_VERSION do - # not change it: the hub image comes from the bundle, so only the newest - # bundle of the branch's stream can be tested. An unfiltered catalog would - # pull every historical version. - local filter_versions - filter_versions=$(common::get_release_stream) || return 1 - log::info "filter-versions=${filter_versions} (from RELEASE_BRANCH_NAME=${RELEASE_BRANCH_NAME})" - # CONTAINER_PLATFORM_VERSION is set by e2e-tests/container-init.sh from the # live cluster (oc version → major.minor). local ocp_version="${CONTAINER_PLATFORM_VERSION:-}" @@ -71,26 +62,48 @@ handle_ocp_disconnected_operator() { log::error "CONTAINER_PLATFORM_VERSION is unset or invalid ('${ocp_version}'); expected OpenShift major.minor" return 1 fi - # rhdh-operator publishes one catalog per stream and OCP minor, e.g. - # quay.io/rhdh/iib:2.2-v4.21-x86_64 (amd64 only; the jobs run on amd64), built - # from the same commit as the hub and plugin catalogs. The tag follows - # filter_versions, so the tag and --filter-versions cannot disagree, and a - # stream without a published catalog fails here instead of in oc-mirror. - local index_image="quay.io/rhdh/iib:${filter_versions}-v${ocp_version}-x86_64" - skopeo inspect --raw "docker://${index_image}" > /dev/null || { - log::error "RHDH CI operator catalog ${index_image} not found or unreadable" - return 1 - } - log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}" - log::info "Operator and hub come from the newest ${filter_versions} bundle in ${index_image}; TAG_NAME/IMAGE_* do not apply" - - # The CI bundle's RELATED_IMAGE refs point at registry.redhat.io/rhdh/* - # (plus the stage/access aliases), whose digests exist on quay.io/rhdh only. - # Rewrite those pulls for the oc-mirror run via CONTAINERS_REGISTRIES_CONF. - # Images in other repositories (e.g. registry.redhat.io/rhel10/postgresql-18) - # keep their original source and are pulled with the CI credentials. - local registries_conf="${DISCONNECTED_TMPDIR}/oc-mirror-registries.conf" - cat > "${registries_conf}" << 'EOF' + + # A GA verification run (registry.redhat.io with a released x.y.z chart) pairs + # with the published Red Hat catalog. Every other run mirrors the CI catalog + # built from the same commit as the hub and plugin catalogs. + local filter_versions="" + local index_image="" + local registries_conf="" + if [[ "${IMAGE_REGISTRY}" == "registry.redhat.io" || "${CHART_VERSION:-}" != *-CI ]]; then + # The published catalog declares defaultChannel fast, and oc-mirror rejects a + # channels list that leaves the default channel out, so mirror every version + # of the rhdh package instead of filtering. + filter_versions="*" + index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}" + log::warn "GA verification: ${index_image} without a version filter mirrors every RHDH operator version" + else + # The operator stream follows the checkout branch: main mirrors the 2.2 + # bundles, release-2.1 the 2.1 ones. TAG_NAME, IMAGE_* and CHART_VERSION do + # not change it: the hub image comes from the bundle, so only the newest + # bundle of the branch's stream can be tested. An unfiltered catalog would + # pull every historical version. + filter_versions=$(common::get_release_stream) || return 1 + log::info "filter-versions=${filter_versions} (from RELEASE_BRANCH_NAME=${RELEASE_BRANCH_NAME})" + + # rhdh-operator publishes one catalog per stream and OCP minor, e.g. + # quay.io/rhdh/iib:2.2-v4.21-x86_64 (amd64 only; the jobs run on amd64). The + # tag follows filter_versions, so the tag and --filter-versions cannot + # disagree, and a stream without a published catalog fails here instead of in + # oc-mirror. + index_image="quay.io/rhdh/iib:${filter_versions}-v${ocp_version}-x86_64" + skopeo inspect --raw "docker://${index_image}" > /dev/null || { + log::error "RHDH CI operator catalog ${index_image} not found or unreadable" + return 1 + } + log::info "Operator and hub come from the newest ${filter_versions} bundle in ${index_image}; TAG_NAME/IMAGE_* do not apply" + + # The CI bundle's RELATED_IMAGE refs point at registry.redhat.io/rhdh/* + # (plus the stage/access aliases), whose digests exist on quay.io/rhdh only. + # Rewrite those pulls for the oc-mirror run via CONTAINERS_REGISTRIES_CONF. + # Images in other repositories (e.g. registry.redhat.io/rhel10/postgresql-18) + # keep their original source and are pulled with the CI credentials. + registries_conf="${DISCONNECTED_TMPDIR}/oc-mirror-registries.conf" + cat > "${registries_conf}" << 'EOF' [[registry]] prefix = "registry.redhat.io/rhdh" location = "quay.io/rhdh" @@ -103,6 +116,8 @@ location = "quay.io/rhdh" prefix = "registry.access.redhat.com/rhdh" location = "quay.io/rhdh" EOF + fi + log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}; filter-versions ${filter_versions}" # Local connected OCP: OCP_INTERNAL sentinel (same as omit-on-OCP). CI bastion # keeps the external MIRROR_REGISTRY_URL. Route URL stays in MIRROR_REGISTRY_URL @@ -145,10 +160,12 @@ EOF fi local -a prepare_cmd=(bash "${prepare_script_path}" "${prepare_args[@]}") - # Scoped to this call: CATALOG_INDEX_IMAGE and the plugin pulls keep their own - # registry configuration. - prepare_cmd=(env CONTAINERS_REGISTRIES_CONF="${registries_conf}" "${prepare_cmd[@]}") - log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" + if [[ -n "${registries_conf}" ]]; then + # Scoped to this call: CATALOG_INDEX_IMAGE and the plugin pulls keep their + # own registry configuration. + prepare_cmd=(env CONTAINERS_REGISTRIES_CONF="${registries_conf}" "${prepare_cmd[@]}") + log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}" + fi machineconfigpool_snapshot=$(disconnected::snapshot_machineconfigpool_rendered_configs) || return 1 diff --git a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml index 4a606dd9d9..9246c9416d 100644 --- a/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml +++ b/.ci/pipelines/resources/rhdh-operator/rhdh-start-disconnected-smoke.yaml @@ -5,9 +5,10 @@ # flavours: [] disables default flavours (lightspeed) to avoid extra sidecar # images. # -# Hub image: CSV RELATED_IMAGE_backstage from the newest CI bundle of the -# branch's operator stream. prepare + oc-mirror mirror that ref and the generated -# IDMS rewrites the kubelet pull. +# Hub image: CSV RELATED_IMAGE_backstage. CI runs install the CI catalog, whose +# bundle points at the newest stream build; GA runs install the published +# catalog. Either way the CSV ref is mirrored by prepare + oc-mirror and the +# generated IDMS rewrites the kubelet pull. # # CATALOG_INDEX_IMAGE is injected by deploy_rhdh_operator from the mirrored # catalog digest so install-dynamic-plugins does not pull the hub profile