Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions .ci/pipelines/install-methods/operator.sh
Original file line number Diff line number Diff line change
Expand Up @@ -212,14 +212,23 @@ deploy_rhdh_operator() {
# Verify Backstage CRD is available
k8s_wait::crd "backstages.rhdh.redhat.com" 60 5 || return 1

rendered_yaml=$(envsubst < "$backstage_crd_path")
rendered_yaml=$(envsubst < "$backstage_crd_path") || {
log::error "Failed to render Backstage CR from $backstage_crd_path"
return 1
}
if [[ -n "${CATALOG_INDEX_IMAGE:-}" ]]; then
# Dynamically inject CATALOG_INDEX_IMAGE environment variable if specified
rendered_yaml=$(echo "$rendered_yaml" | yq eval '.spec.application.extraEnvs.envs += [{"name": "CATALOG_INDEX_IMAGE", "value": "'"$CATALOG_INDEX_IMAGE"'", "containers": ["install-dynamic-plugins"]}]' -)
rendered_yaml=$(echo "$rendered_yaml" | yq eval '.spec.application.extraEnvs.envs += [{"name": "CATALOG_INDEX_IMAGE", "value": "'"$CATALOG_INDEX_IMAGE"'", "containers": ["install-dynamic-plugins"]}]' -) || {
log::error "Failed to inject CATALOG_INDEX_IMAGE into the Backstage CR"
return 1
}
fi
log::info "Applying Backstage CR from: $backstage_crd_path"
log::debug "$rendered_yaml"
echo "$rendered_yaml" | oc apply -f - -n "$namespace"
echo "$rendered_yaml" | oc apply -f - -n "$namespace" || {
log::error "Failed to apply Backstage CR from $backstage_crd_path in $namespace"
return 1
}

# Wait for the operator to create the Backstage deployment (5 minutes max)
if ! common::poll_until \
Expand Down
32 changes: 19 additions & 13 deletions .ci/pipelines/jobs/ocp-disconnected-helm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ handle_ocp_disconnected_helm() {

disconnected::patch_idms "${OC_MIRROR_IDMS_FILE}"

local machineconfigpool_snapshot
machineconfigpool_snapshot=$(disconnected::snapshot_machineconfigpool_rendered_configs) || return 1

oc apply -f "${OC_MIRROR_IDMS_FILE}" || {
log::error "Failed to apply IDMS — aborting"
return 1
Expand All @@ -138,29 +141,35 @@ handle_ocp_disconnected_helm() {
# IDMS/ITMS changes trigger a MachineConfig update which rolls worker nodes
# (drain → apply config → reboot). Wait for all MachineConfigPools to finish
# before deploying workloads, otherwise pods get evicted mid-startup.
disconnected::wait_mcp_updated
disconnected::wait_machineconfigpool_updated "${machineconfigpool_snapshot}" || return 1

log::section "Plugin Mirroring"
disconnected::mirror_plugins || return 1
# Pin CATALOG_INDEX_IMAGE to the digest actually pushed (hub profile default
# digest is often absent from the mirror). Homepage package is the digest-pinned
# OCI exported as HOMEPAGE_PLUGIN_PACKAGE (ref:// cannot resolve with includes: []).
# digest is often absent from the mirror). mirror_plugins exported the
# digest-pinned OCI package for each smoke plugin (ref:// cannot resolve with
# includes: []).
disconnected::resolve_catalog_index_image || return 1

log::section "Namespace and Secrets"

namespace::configure "${NAME_SPACE}"
disconnected::allow_mirror_registry_egress "${NAME_SPACE}" || return 1
if [[ "${LOCAL_DISCONNECTED:-}" == "1" ]]; then
disconnected::ensure_local_image_pull_access "${NAME_SPACE}" || return 1
fi
disconnected::apply_plugin_mirror_configmap "${NAME_SPACE}" || return 1

# Mirror CA + registry auth for install-dynamic-plugins (skopeo).
# Post-renderer mounts CA at /etc/containers/certs.d/<registry>/ca.crt;
# the chart mounts ${RELEASE_NAME}-dynamic-plugins-registry-auth at
# /opt/app-root/src/.config/containers.
# Mirror CA + registry auth for install-dynamic-plugins (skopeo). The
# post-renderer mounts the CA at /etc/containers/certs.d/<registry>/ca.crt; the
# chart mounts <fullname>-dynamic-plugins-registry-auth (optional: true) at
# /opt/app-root/src/.config/containers, where fullname is <release>-<nameOverride>
# and nameOverride is pinned to developer-hub in the values file. A differently
# named secret leaves the init container without auth.json and the mirror pull
# fails with "authentication required".
disconnected::create_mirror_registry_ca_configmap "${NAME_SPACE}" || return 1
disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" || return 1
disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" \
"${RELEASE_NAME}-developer-hub-dynamic-plugins-registry-auth" || return 1

log::section "Helm Deployment"

Expand Down Expand Up @@ -205,11 +214,8 @@ handle_ocp_disconnected_helm() {
# to the rendered Deployment, avoiding the Helm "array clobber" pitfall.
local post_renderer="${DIR}/resources/disconnected/helm-post-renderer.sh"
local helm_values="${DISCONNECTED_TMPDIR}/values_disconnected-smoke.yaml"
common::require_vars HOMEPAGE_PLUGIN_PACKAGE || return 1
envsubst < "${DIR}/value_files/values_disconnected-smoke.yaml" > "${helm_values}" || {
log::error "Failed to render disconnected Helm values"
return 1
}
disconnected::render_smoke_plugins_template \
"${DIR}/value_files/values_disconnected-smoke.yaml" "${helm_values}" || return 1
cp "${helm_values}" "${ARTIFACT_DIR}/values_disconnected-smoke.yaml" 2> /dev/null || true

helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \
Expand Down
122 changes: 87 additions & 35 deletions .ci/pipelines/jobs/ocp-disconnected-operator.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ handle_ocp_disconnected_operator() {
log::section "Operator Mirroring and Installation"

# Fetch prepare-restricted-environment.sh from RELEASE_BRANCH_NAME (default
# main). rhdh-operator#3259 (native oc-mirror cc-* catalogs for OLM v1) is
# merged on main.
# main).
Comment thread
zdrapela marked this conversation as resolved.
Resolved
local prepare_script_path="${DISCONNECTED_TMPDIR}/prepare-restricted-environment.sh"
disconnected::fetch_operator_repo_script "prepare-restricted-environment.sh" \
"${prepare_script_path}" || {
Expand All @@ -51,41 +50,74 @@ handle_ocp_disconnected_operator() {
# default skopeo/umoci/podman-build path. Nested Podman in this CI pod cannot
# initialize storage (newuidmap / VFS chown both fail under hostUsers: false).
# CATALOG_INDEX_IMAGE is the plugin catalog index — do not pass it as
# --index-image (OLM operator catalog). Used only for mirror-plugins.sh below,
# then cleared so deploy does not inject :next defaults onto the GA hub.
# --index-image (OLM operator catalog). It is consumed by mirror-plugins.sh and
# injected into the Backstage CR by deploy_rhdh_operator.
#
# OLM version: leave default (auto). On OCP 4.21+ this selects OLM v1.
local filter_versions="${RELEASE_VERSION}"
if [[ "${filter_versions}" == "next" || "${filter_versions}" == "*" ]]; then
filter_versions="*"
fi

# Mirroring every historical RHDH operator version (* / next) is unnecessary
# for smoke and flakes on quay CDN EOF (CI) or overwhelms the integrated
# registry (local). Pin to the chart major.minor (e.g. 2.0-80-CI → 2.0).
# Release branches already pass a concrete version (e.g. 1.9) and skip this.
if [[ "${filter_versions}" == "*" ]]; then
local chart_mm=""
if [[ -n "${CHART_VERSION:-}" && "${CHART_VERSION}" =~ ^([0-9]+\.[0-9]+) ]]; then
chart_mm="${BASH_REMATCH[1]}"
fi
if [[ -z "${chart_mm}" ]]; then
log::error "Cannot derive --filter-versions from CHART_VERSION='${CHART_VERSION:-unset}'; expected major.minor prefix (e.g. 2.0-80-CI)"
return 1
fi
filter_versions="${chart_mm}"
log::info "filter-versions=${filter_versions} (from CHART_VERSION=${CHART_VERSION})"
fi

#
# CONTAINER_PLATFORM_VERSION is set by e2e-tests/container-init.sh from the
# live cluster (oc version → major.minor).
local ocp_version="${CONTAINER_PLATFORM_VERSION:-}"
if [[ -z "${ocp_version}" || "${ocp_version}" == "unknown" || ! "${ocp_version}" =~ ^[0-9]+\.[0-9]+$ ]]; then
log::error "CONTAINER_PLATFORM_VERSION is unset or invalid ('${ocp_version}'); expected OpenShift major.minor"
return 1
fi
local index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}"
log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}"

# A GA verification run (registry.redhat.io with a released x.y.z chart) pairs
# with the published Red Hat catalog. Every other run mirrors the CI catalog
# built from the same commit as the hub and plugin catalogs.
local filter_versions=""
local index_image=""
local registries_conf=""
if [[ "${IMAGE_REGISTRY}" == "registry.redhat.io" || "${CHART_VERSION:-}" != *-CI ]]; then
# The published catalog declares defaultChannel fast, and oc-mirror rejects a
# channels list that leaves the default channel out, so mirror every version
# of the rhdh package instead of filtering.
filter_versions="*"
index_image="registry.redhat.io/redhat/redhat-operator-index:v${ocp_version}"
log::warn "GA verification: ${index_image} without a version filter mirrors every RHDH operator version"
else
# The operator stream follows the checkout branch: main mirrors the 2.2
# bundles, release-2.1 the 2.1 ones. TAG_NAME, IMAGE_* and CHART_VERSION do
# not change it: the hub image comes from the bundle, so only the newest
# bundle of the branch's stream can be tested. An unfiltered catalog would
# pull every historical version.
filter_versions=$(common::get_release_stream) || return 1
log::info "filter-versions=${filter_versions} (from RELEASE_BRANCH_NAME=${RELEASE_BRANCH_NAME})"

# rhdh-operator publishes one catalog per stream and OCP minor, e.g.
# quay.io/rhdh/iib:2.2-v4.21-x86_64 (amd64 only; the jobs run on amd64). The
# tag follows filter_versions, so the tag and --filter-versions cannot
# disagree, and a stream without a published catalog fails here instead of in
# oc-mirror.
index_image="quay.io/rhdh/iib:${filter_versions}-v${ocp_version}-x86_64"
skopeo inspect --raw "docker://${index_image}" > /dev/null || {
log::error "RHDH CI operator catalog ${index_image} not found or unreadable"
return 1
}
log::info "Operator and hub come from the newest ${filter_versions} bundle in ${index_image}; TAG_NAME/IMAGE_* do not apply"

# The CI bundle's RELATED_IMAGE refs point at registry.redhat.io/rhdh/*
# (plus the stage/access aliases), whose digests exist on quay.io/rhdh only.
# Rewrite those pulls for the oc-mirror run via CONTAINERS_REGISTRIES_CONF.
# Images in other repositories (e.g. registry.redhat.io/rhel10/postgresql-18)
# keep their original source and are pulled with the CI credentials.
registries_conf="${DISCONNECTED_TMPDIR}/oc-mirror-registries.conf"
cat > "${registries_conf}" << 'EOF'
[[registry]]
prefix = "registry.redhat.io/rhdh"
location = "quay.io/rhdh"

[[registry]]
prefix = "registry.stage.redhat.io/rhdh"
location = "quay.io/rhdh"

[[registry]]
prefix = "registry.access.redhat.com/rhdh"
location = "quay.io/rhdh"
EOF
fi
log::info "Using OCP ${ocp_version} (CONTAINER_PLATFORM_VERSION); index image ${index_image}; filter-versions ${filter_versions}"

# Local connected OCP: OCP_INTERNAL sentinel (same as omit-on-OCP). CI bastion
# keeps the external MIRROR_REGISTRY_URL. Route URL stays in MIRROR_REGISTRY_URL
Expand Down Expand Up @@ -115,18 +147,32 @@ handle_ocp_disconnected_operator() {
# external registries. Catalogd must trust the mirror CA and authenticate
# before ClusterCatalog can reach Serving=True. Skip on LOCAL_DISCONNECTED —
# OCP_INTERNAL path configures internal registry trust itself.
local machineconfigpool_snapshot
if [[ "${LOCAL_DISCONNECTED:-}" != "1" ]]; then
# Snapshot before the additionalTrustedCA patch: it also rolls the nodes.
machineconfigpool_snapshot=$(disconnected::snapshot_machineconfigpool_rendered_configs) || return 1

disconnected::ensure_olm_mirror_pull_secret || return 1
disconnected::ensure_mirror_registry_ca || return 1
disconnected::wait_mcp_updated
disconnected::wait_machineconfigpool_updated "${machineconfigpool_snapshot}" || return 1
else
log::info "LOCAL_DISCONNECTED=1: skipping external bastion CA/pull-secret pre-prepare helpers"
fi

local -a prepare_cmd=(bash "${prepare_script_path}" "${prepare_args[@]}")
if [[ -n "${registries_conf}" ]]; then
# Scoped to this call: CATALOG_INDEX_IMAGE and the plugin pulls keep their
# own registry configuration.
prepare_cmd=(env CONTAINERS_REGISTRIES_CONF="${registries_conf}" "${prepare_cmd[@]}")
log::info "Redirecting registry.redhat.io/rhdh to quay.io/rhdh via ${registries_conf}"
fi

machineconfigpool_snapshot=$(disconnected::snapshot_machineconfigpool_rendered_configs) || return 1

log::info "Running prepare-restricted-environment.sh with: ${prepare_args[*]}"
if ! disconnected::retry_on_local_registry 5 \
disconnected::with_unset_registry_auth_file \
bash "${prepare_script_path}" "${prepare_args[@]}"; then
"${prepare_cmd[@]}"; then
log::error "prepare-restricted-environment.sh failed — aborting"
return 1
fi
Expand Down Expand Up @@ -162,7 +208,7 @@ handle_ocp_disconnected_operator() {
# prepare-restricted-environment.sh applies IDMS/CatalogSource which triggers
# a MachineConfig update and node rolling. Wait for completion before deploying
# workloads, same as the Helm path.
disconnected::wait_mcp_updated
disconnected::wait_machineconfigpool_updated "${machineconfigpool_snapshot}" || return 1

if [[ "${LOCAL_DISCONNECTED:-}" == "1" ]]; then
# Catalogd needs the registry + image-puller grants above before Serving=True.
Expand Down Expand Up @@ -193,13 +239,15 @@ handle_ocp_disconnected_operator() {
disconnected::mirror_plugins || return 1

# Pin CATALOG_INDEX_IMAGE to the digest actually pushed (hub profile default
# digest is often absent from the mirror). Homepage package is the digest-pinned
# OCI exported as HOMEPAGE_PLUGIN_PACKAGE (ref:// cannot resolve with includes: []).
# digest is often absent from the mirror). mirror_plugins exported the
# digest-pinned OCI package for each smoke plugin (ref:// cannot resolve with
# includes: []).
disconnected::resolve_catalog_index_image || return 1

log::section "Namespace and Secrets"

namespace::configure "${NAME_SPACE}"
disconnected::allow_mirror_registry_egress "${NAME_SPACE}" || return 1
if [[ "${LOCAL_DISCONNECTED:-}" == "1" ]]; then
disconnected::ensure_local_image_pull_access "${NAME_SPACE}" || return 1
fi
Expand All @@ -208,7 +256,7 @@ handle_ocp_disconnected_operator() {
# from the mirror (registries.conf alone is not enough — TLS verify fails).
disconnected::create_mirror_registry_ca_configmap "${NAME_SPACE}" || return 1
disconnected::create_plugin_registry_auth_secret "${NAME_SPACE}" || return 1
disconnected::create_homepage_plugins_configmap "${NAME_SPACE}" || return 1
disconnected::create_smoke_plugins_configmap "${NAME_SPACE}" || return 1

# Operator mounts one volume per extraFiles ConfigMap name. policy.json cannot
# share rhdh-plugin-mirror-conf or reconcile fails with duplicate volume keys.
Expand Down Expand Up @@ -240,7 +288,11 @@ handle_ocp_disconnected_operator() {

cp "${cr_temp}" "${ARTIFACT_DIR}/disconnected-backstage-cr.yaml" 2> /dev/null || true

deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}"
deploy_rhdh_operator "${NAME_SPACE}" "${cr_temp}" || {
Comment thread
zdrapela marked this conversation as resolved.
Resolved
log::error "deploy_rhdh_operator failed in ${NAME_SPACE} — dumping OLM v1 status"
disconnected::dump_olm_v1_status "rhdh-operator"
return 1
}
log::success "Backstage CR deployed in ${NAME_SPACE}"

log::section "Smoke Test"
Expand Down
2 changes: 1 addition & 1 deletion .ci/pipelines/jobs/upgrade.sh
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ handle_ocp_helm_upgrade() {

# Resolve the previous release's chart and values for the baseline deployment.
local current_release_version previous_release_version
current_release_version=$(helm::get_chart_stream)
current_release_version=$(common::get_release_stream)
if [[ -z "$current_release_version" ]]; then
log::error "Failed to determine current release version. Exiting."
save_overall_result 1
Expand Down
3 changes: 2 additions & 1 deletion .ci/pipelines/lib/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ Functions: `log::info`, `log::warn`, `log::error`, `log::success`, `log::debug`,

Common utilities.

Functions: `common::oc_login`, `common::sed_inplace`, `common::get_previous_release_version`
Functions: `common::oc_login`, `common::sed_inplace`, `common::get_release_stream`,
`common::get_previous_release_version`

### `k8s-wait.sh`

Expand Down
47 changes: 46 additions & 1 deletion .ci/pipelines/lib/common.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env bash

# Common utility functions for pipeline scripts
# Dependencies: oc, kubectl, lib/log.sh
# Dependencies: oc, kubectl, jq, lib/log.sh

if [[ -n "${COMMON_LIB_SOURCED:-}" ]]; then
return 0
Expand Down Expand Up @@ -72,6 +72,51 @@ common::sed_inplace() {
return $?
}

# Get the release stream this run belongs to: a major.minor pair like "1.10",
# not a bare major. 'main' takes it from package.json; 'release-x.y' from the
# branch name.
# Args:
# $1 - (optional) version_override: Specific stream to use (e.g., "1.8" for upgrade base)
# Returns:
# Prints the stream (e.g., "1.10")
common::get_release_stream() {
local version_override=${1:-}

if [[ -n "$version_override" ]]; then
echo "$version_override"
return 0
fi

if [[ -z "${RELEASE_BRANCH_NAME:-}" ]]; then
log::error "RELEASE_BRANCH_NAME is not set"
return 1
fi

if [[ "$RELEASE_BRANCH_NAME" == "main" ]]; then
# main carries no version in its name. The published tags cannot answer
# this either - they hold every stream anyone pushed, so "highest on quay"
# follows whichever stream is ahead, not this checkout's.
local package_json="${DIR}/../../package.json"
if [[ ! -f "$package_json" ]]; then
log::error "Cannot determine the release stream: ${package_json} not found"
return 1
fi

local release_stream
release_stream=$(jq -r '.version // empty' "$package_json" | grep -oE '^[0-9]+\.[0-9]+')
if [[ -z "$release_stream" ]]; then
log::error "Cannot determine the release stream: no usable .version in ${package_json}"
return 1
fi
echo "$release_stream"
elif echo "$RELEASE_BRANCH_NAME" | grep -qE '^release-[0-9]+\.[0-9]+$'; then
echo "$RELEASE_BRANCH_NAME" | grep -oE '[0-9]+\.[0-9]+'
else
log::error "Invalid RELEASE_BRANCH_NAME: $RELEASE_BRANCH_NAME (expected 'main' or 'release-x.y')"
return 1
fi
}

# Find the highest existing release branch below the current version.
# Usage: prev=$(common::get_previous_release_version "1.6") # Returns: "1.5"
# prev=$(common::get_previous_release_version "2.0") # Returns: "1.10"
Expand Down
Loading
Loading