diff --git a/.rhdh/docs/installing-ci-builds.adoc b/.rhdh/docs/installing-ci-builds.adoc index 2ffd3455e..a543873f8 100644 --- a/.rhdh/docs/installing-ci-builds.adoc +++ b/.rhdh/docs/installing-ci-builds.adoc @@ -44,6 +44,8 @@ chmod +x install-rhdh-catalog-source.sh # install catalog source and operator subscription, for the latest downstream stable, RC or GA build from the release-1.yy branch ./install-rhdh-catalog-source.sh --latest --install-operator rhdh +# add --log-level debug for verbose output + # OR, install catalog source and operator subscription, for the next donwstream CI build from the main branch ./install-rhdh-catalog-source.sh --next --install-operator rhdh ---- diff --git a/.rhdh/scripts/install-rhdh-catalog-source.sh b/.rhdh/scripts/install-rhdh-catalog-source.sh index 8172e970d..b2ca7ecdc 100755 --- a/.rhdh/scripts/install-rhdh-catalog-source.sh +++ b/.rhdh/scripts/install-rhdh-catalog-source.sh @@ -6,10 +6,9 @@ set -euo pipefail +LOG_LEVEL="${LOG_LEVEL:-info}" NC='\033[0m' - IS_OPENSHIFT="" - NAMESPACE_SUBSCRIPTION="rhdh-operator" if [[ -n "${CI_X_VERSION:-}" && -n "${CI_Y_VERSION:-}" ]]; then OLM_CHANNEL="fast-${CI_X_VERSION}.${CI_Y_VERSION}" @@ -34,6 +33,30 @@ function logf() { local msg=$3 local fullMsg="[${prefix}] ${msg}" + local msg_rank=2 + case "${prefix}" in + ERROR) msg_rank=0 ;; + WARN) msg_rank=1 ;; + INFO) msg_rank=2 ;; + DEBUG) msg_rank=3 ;; + *) + ;; + esac + + local current_rank=2 + case "${LOG_LEVEL}" in + error) current_rank=0 ;; + warn) current_rank=1 ;; + info) current_rank=2 ;; + debug) current_rank=3 ;; + *) + ;; + esac + + if [[ "${msg_rank}" -gt "${current_rank}" ]]; then + return 0 + fi + if [[ "$TERM" == *"color"* ]]; then echo -e "${color}${fullMsg}${NC}" else @@ -57,6 +80,24 @@ function errorf() { logf "ERROR" "\033[0;31m" "$1" } +# Run a command. At debug, stream it. Otherwise hide stdout/stderr, +# and print the captured output only if the command fails. +function run_cmd() { + local out rc=0 + if [[ "${LOG_LEVEL}" == "debug" ]]; then + "$@" + return $? + fi + out="$(mktemp)" + "$@" >"${out}" 2>&1 || rc=$? + if [[ "${rc}" -ne 0 ]]; then + errorf "Command failed (${rc})" + cat "${out}" >&2 + fi + rm -f "${out}" + return "${rc}" +} + function usage() { echo " This script streamlines testing IIB images by configuring an OpenShift or Kubernetes cluster to enable it to use the specified IIB image @@ -80,6 +121,7 @@ Options: --install-operator : Install operator named \$NAME after creating CatalogSource --install-plan-approval : Specify the install plan strategy for the subscription (default: Automatic) --olm-version v0|v1|auto : Force OLM version for catalog/operator resources (default: auto-detect) + --log-level error|warn|info|debug : Log verbosity (default: info) Examples: $0 \\ @@ -160,6 +202,12 @@ function resolve_olm_version() { fi } +function cleanup() { + rm -fr "${TMPDIR}" || true + jobs -p | xargs -r kill 2>/dev/null || true + wait 2>/dev/null || true +} + # On failure, print ClusterCatalog / ClusterExtension status so CI logs show why Serving/Installed never became True. function dump_olm_v1_diagnostics() { local catalog_name="${1:-}" @@ -266,7 +314,19 @@ function render_iib() { mkdir -p "${TMPDIR}/rhdh/rhdh" debugf "Rendering IIB $UPSTREAM_IIB as a local file..." - opm render "$UPSTREAM_IIB" --output=yaml > "${TMPDIR}/rhdh/rhdh/render.yaml" + local render_err + render_err="$(mktemp)" + if [[ "${LOG_LEVEL}" == "debug" ]]; then + opm render "$UPSTREAM_IIB" --output=yaml > "${TMPDIR}/rhdh/rhdh/render.yaml" + else + if ! opm render "$UPSTREAM_IIB" --output=yaml > "${TMPDIR}/rhdh/rhdh/render.yaml" 2>"${render_err}"; then + errorf "opm render failed" + cat "${render_err}" >&2 + rm -f "${render_err}" + return 1 + fi + fi + rm -f "${render_err}" if [ ! -s "${TMPDIR}/rhdh/rhdh/render.yaml" ]; then errorf " [ERROR] 'opm render $UPSTREAM_IIB' returned an empty output, which likely means that this IIB Image does not contain any operators in it. @@ -324,7 +384,7 @@ function process_bundle() { local pull_attempt local pull_ok=false for pull_attempt in $(seq 1 "$max_retries"); do - if skopeo copy "docker://$bundleImg" "oci:./${bundle_dir}/src:latest" 2>"${bundle_dir}/copy.err"; then + if run_cmd skopeo copy "docker://$bundleImg" "oci:./${bundle_dir}/src:latest"; then pull_ok=true break fi @@ -332,12 +392,12 @@ function process_bundle() { sleep "$retry_delay" done if [[ "$pull_ok" != "true" ]]; then - debugf "bundle #${bundle_id}: skopeo copy failed after ${max_retries} attempts, skipping (see ${bundle_dir}/copy.err)" >&2 + debugf "bundle #${bundle_id}: skopeo copy failed after ${max_retries} attempts, skipping" >&2 return 0 fi debugf "bundle #${bundle_id}: pulled ${bundleImg}" >&2 - umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless + run_cmd umoci unpack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" --rootless for folder in manifests metadata; do for file in "./${bundle_dir}/unpacked/rootfs/${folder}"/*; do @@ -349,13 +409,13 @@ function process_bundle() { done done - umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" + run_cmd umoci repack --image "./${bundle_dir}/src:latest" "./${bundle_dir}/unpacked" local newBundleImage="${my_registry}/rhdh/rhdh-operator-bundle:${digest}" local push_attempt local push_ok=false for push_attempt in $(seq 1 "$max_retries"); do - if skopeo copy --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}" 2>"${bundle_dir}/push.err"; then + if run_cmd skopeo copy --dest-tls-verify=false "oci:./${bundle_dir}/src:latest" "docker://${newBundleImage}"; then push_ok=true break fi @@ -363,7 +423,7 @@ function process_bundle() { sleep "$retry_delay" done if [[ "$push_ok" != "true" ]]; then - errorf "bundle #${bundle_id}: push to registry failed after ${max_retries} attempts (see ${bundle_dir}/push.err)" >&2 + errorf "bundle #${bundle_id}: push to registry failed after ${max_retries} attempts" >&2 return 1 fi debugf "bundle #${bundle_id}: pushed to ${newBundleImage}" >&2 @@ -442,7 +502,7 @@ function update_refs_in_iib_bundles() { fi debugf "Regenerating IIB Dockerfile with updated refs..." >&2 - opm generate dockerfile rhdh/rhdh + run_cmd opm generate dockerfile rhdh/rhdh } # Enabling the route does not mean that the router and registry backends are ready. @@ -494,6 +554,7 @@ function ocp_install() { if ! my_registry=$(wait_for_ocp_registry); then return 1 fi + run_cmd skopeo login -u kubeadmin -p "$(oc whoami -t)" --tls-verify=false "$my_registry" oc -n openshift-marketplace delete secret internal-reg-auth-for-rhdh --ignore-not-found >&2 oc -n openshift-marketplace delete secret internal-reg-ext-auth-for-rhdh --ignore-not-found >&2 oc -n openshift-marketplace create secret docker-registry internal-reg-ext-auth-for-rhdh \ @@ -526,10 +587,10 @@ function ocp_install() { debugf "Submitting in-cluster build request for the updated IIB..." >&2 if ! oc -n rhdh get buildconfig.build.openshift.io/iib >& /dev/null; then - oc -n rhdh new-build --strategy docker --binary --name iib >&2 + run_cmd oc -n rhdh new-build --strategy docker --binary --name iib fi oc -n rhdh patch buildconfig.build.openshift.io/iib -p '{"spec": {"strategy": {"dockerStrategy": {"dockerfilePath": "rhdh.Dockerfile"}}}}' >&2 - oc -n rhdh start-build iib --wait --follow --from-dir=rhdh >&2 + run_cmd oc -n rhdh start-build iib --wait --follow --from-dir=rhdh local imageStreamWithTag="rhdh/iib:${IIB_TAG}" oc tag rhdh/iib:latest "${imageStreamWithTag}" >&2 @@ -689,7 +750,7 @@ EOF debugf "Port-forwarding from localhost:${portFwdLocalPort} to the cluster registry..." >&2 local internal_registry_url internal_registry_url="localhost:${registrySvcNodePort}" - skopeo login -u "${username}" -p "${password}" --tls-verify=false "localhost:$portFwdLocalPort" >&2 + run_cmd skopeo login -u "${username}" -p "${password}" --tls-verify=false "localhost:$portFwdLocalPort" local kaniko_internal_registry_url kaniko_internal_registry_url="${registry_name}.${namespace}.svc.cluster.local:5000" @@ -789,15 +850,20 @@ EOF return 1 fi debugf "Waiting for Kaniko pod $kanikoPod to be ready..." >&2 - invoke_cluster_cli -n "${namespace}" wait --for=condition=Ready "pod/$kanikoPod" --timeout=60s >&2 - invoke_cluster_cli -n "${namespace}" logs -f "${kanikoPod}" >&2 & + + run_cmd invoke_cluster_cli -n "${namespace}" wait --for=condition=Ready "pod/$kanikoPod" --timeout=60s + if [[ "${LOG_LEVEL}" == "debug" ]]; then + invoke_cluster_cli -n "${namespace}" logs -f "${kanikoPod}" >&2 & + fi localContext=context.tar.gz tar -czf "${localContext}" -C rhdh . >&2 invoke_cluster_cli -n "${namespace}" cp "${localContext}" "${kanikoPod}:/workspace/${localContext}" >&2 - invoke_cluster_cli -n "${namespace}" wait --for=condition=complete "job/${kanikoJobName}" --timeout=300s >&2 - + if ! run_cmd invoke_cluster_cli -n "${namespace}" wait --for=condition=complete "job/${kanikoJobName}" --timeout=300s; then + invoke_cluster_cli -n "${namespace}" logs "${kanikoPod}" >&2 + return 1 + fi debugf "IIB built and pushed to internal cluster registry: $result..." >&2 printf "%s" "${result}" } @@ -810,6 +876,83 @@ if [[ "$#" -lt 1 ]]; then exit 0 fi +IIB_STREAM="latest" +TO_INSTALL="" + +while [[ "$#" -gt 0 ]]; do + case $1 in + '--install-operator') + TO_INSTALL="$2" + shift 1 + ;; + '--next'|'--latest') + IIB_STREAM="${1/--/}" + ;; + '-v') + IIB_STREAM="${2}" + OLM_CHANNEL="fast-${2}" + shift 1 + ;; + '--catalog-source') + UPSTREAM_IIB_OVERRIDE="$2" + shift 1 + ;; + '--install-plan-approval') + if [[ "$2" != "Manual" && "$2" != "Automatic" ]]; then + errorf "Unknown parameter used: $2. Must be Manual or Automatic." + usage + exit 1 + fi + INSTALL_PLAN_APPROVAL="$2" + shift 1 + ;; + '--olm-version') + if [[ $# -lt 2 ]]; then + errorf "--olm-version requires a value: v0, v1, or auto." + usage + exit 1 + fi + case "$2" in + v0|v1|auto) ;; + *) + errorf "Unknown OLM version: $2. Must be v0, v1, or auto." + usage + exit 1 + ;; + esac + OLM_VERSION="$2" + shift 1 + ;; + '--log-level') + if [[ $# -lt 2 ]]; then + errorf "--log-level requires a value: error, warn, info, or debug." + usage + exit 1 + fi + case "$2" in + error|warn|info|debug) ;; + *) + errorf "Unknown log level: $2. Must be error, warn, info, or debug." + usage + exit 1 + ;; + esac + LOG_LEVEL="$2" + shift 1 + ;; + '-h'|'--help') + usage + exit 0 + ;; + *) + errorf "Unknown parameter is used: $1." + usage + exit 1 + ;; + esac + shift 1 +done + # minimum requirements if ! command -v jq &> /dev/null; then errorf "Please install jq 1.2+ from an RPM or https://pypi.org/project/jq/" @@ -824,8 +967,7 @@ TMPDIR=$(mktemp -d) pushd "${TMPDIR}" > /dev/null debugf ">>> WORKING DIR: $TMPDIR <<<" -# shellcheck disable=SC2064 -trap "rm -fr '$TMPDIR' || true; jobs -p | xargs -r kill 2>/dev/null; wait 2>/dev/null" EXIT +trap cleanup EXIT trap "exit 1" INT TERM detect_ocp_and_set_env_var @@ -872,67 +1014,8 @@ if [[ "${IS_OPENSHIFT}" = "true" ]]; then fi fi -# if logged in, this should return something like latest-v4.12-x86_64 -IIB_TAG="latest-${OCP_VER}-${OCP_ARCH}" -TO_INSTALL="" - -while [[ "$#" -gt 0 ]]; do - case $1 in - '--install-operator') - TO_INSTALL="$2" - shift 1 - ;; - '--next'|'--latest') - # if logged in, this should return something like latest-v4.12-x86_64 or next-v4.12-x86_64 - IIB_TAG="${1/--/}-${OCP_VER}-$OCP_ARCH" - ;; - '-v') - IIB_TAG="${2}-${OCP_VER}-$OCP_ARCH" - OLM_CHANNEL="fast-${2}" - shift 1 - ;; - '--catalog-source') - UPSTREAM_IIB_OVERRIDE="$2" - shift 1 - ;; - '--install-plan-approval') - if [[ "$2" != "Manual" && "$2" != "Automatic" ]]; then - errorf "Unknown parameter used: $2. Must be Manual or Automatic." - usage - exit 1 - fi - INSTALL_PLAN_APPROVAL="$2" - shift 1 - ;; - '--olm-version') - if [[ $# -lt 2 ]]; then - errorf "--olm-version requires a value: v0, v1, or auto." - usage - exit 1 - fi - case "$2" in - v0|v1|auto) ;; - *) - errorf "Unknown OLM version: $2. Must be v0, v1, or auto." - usage - exit 1 - ;; - esac - OLM_VERSION="$2" - shift 1 - ;; - '-h'|'--help') - usage - exit 0 - ;; - *) - errorf "Unknown parameter is used: $1." - usage - exit 1 - ;; - esac - shift 1 -done +# if logged in, this should return something like latest-v4.22-x86_64 +IIB_TAG="${IIB_STREAM}-${OCP_VER}-${OCP_ARCH}" if [[ $UPSTREAM_IIB_OVERRIDE ]]; then UPSTREAM_IIB="$UPSTREAM_IIB_OVERRIDE"