From 239926223788f01d6b5feea85db5aa9a2ae8d010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Wed, 11 Nov 2026 16:06:01 +0100 Subject: [PATCH 1/4] chore(ci): replace Crunchy Postgres operator with RHEL PostgreSQL StatefulSet The Crunchy Postgres operator is not available in the OCP 5.0 certified-operators catalog (crunchy-postgres-operator is published for OCP 4.15-4.22 only), so every OCP job fails before the tests start. On top of that, Crunchy announced that the Developer Program images our PostgresCluster pulls will stop receiving updates on 2026-12-31. Deploy the external PostgreSQL used by showcase-rbac as a plain StatefulSet running registry.redhat.io/rhel9/postgresql-18 instead: - TLS server certificate generated by the CI script (CA + cert with SANs for the Service DNS names), served from the same postgress-external-db-cluster-cert secret the RBAC values mount. - The start hook restricts network connections to TLS and creates the janus-idp SUPERUSER from postgress-external-db-pguser-janus-idp. - The registry.redhat.io pull secret is created from the existing REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON CI secret. The Crunchy operator install, the PostgresCluster CRD waits and the OLM Subscription helpers (only used for Crunchy) are removed, together with the now empty cluster_setup_ocp_* hooks. Also rename the external DB TLS spec and update the CI docs accordingly. Jira: https://redhat.atlassian.net/browse/RHDHBUGS-4113 Assisted-by: OpenCode --- .ci/pipelines/install-methods/operator.sh | 19 +- .ci/pipelines/jobs/ocp-fips-helm.sh | 2 - .ci/pipelines/jobs/ocp-localization.sh | 1 - .ci/pipelines/jobs/ocp-nightly.sh | 2 - .ci/pipelines/jobs/ocp-operator.sh | 4 - .ci/pipelines/jobs/ocp-pull.sh | 1 - .ci/pipelines/jobs/upgrade.sh | 2 - .ci/pipelines/lib/README.md | 5 +- .ci/pipelines/lib/operators.sh | 63 ----- .ci/pipelines/lib/schema-mode-env.sh | 16 +- .../resources/postgres-db/postgres.yaml | 221 +++++++++++++----- .ci/pipelines/utils.sh | 194 ++++++++------- .../diff-values_showcase-rbac_AKS.yaml | 2 +- .../diff-values_showcase-rbac_EKS.yaml | 2 +- .../diff-values_showcase-rbac_GKE.yaml | 2 +- docs/e2e-tests/CI-medic-guide.md | 5 +- docs/e2e-tests/layer-migration-matrix.md | 2 +- e2e-tests/playwright.config.ts | 10 +- ...tls-config-with-external-postgres.spec.ts} | 2 +- .../e2e/plugin-division-mode-schema/README.md | 10 +- 20 files changed, 279 insertions(+), 286 deletions(-) rename e2e-tests/playwright/e2e/external-database/{verify-tls-config-with-external-crunchy.spec.ts => verify-tls-config-with-external-postgres.spec.ts} (93%) diff --git a/.ci/pipelines/install-methods/operator.sh b/.ci/pipelines/install-methods/operator.sh index 52f1d06bb9..8f46df304d 100755 --- a/.ci/pipelines/install-methods/operator.sh +++ b/.ci/pipelines/install-methods/operator.sh @@ -193,17 +193,6 @@ prepare_operator() { k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || return 1 } -# Waits for the Crunchy Data PostgreSQL Operator's PostgresCluster CRD to become available. -# Must be called after the Crunchy DB CRD is created and before RHDH is deployed -# with internal DB disabled and configured to use Crunchy DB as the external PostgreSQL database. -wait_for_crunchy_crd() { - log::info "Verifying PostgresCluster CRD is available before deploying Backstage CR..." - k8s_wait::crd "postgresclusters.postgres-operator.crunchydata.com" 60 5 || { - log::error "PostgresCluster CRD not available - operator won't be able to create internal database" - return 1 - } -} - deploy_rhdh_operator() { local namespace=$1 local backstage_crd_path=$2 @@ -244,11 +233,9 @@ deploy_rhdh_operator() { return 0 fi - # Wait for the operator to create the database resource (5 minutes max) - # The operator can create either PostgresCluster (Crunchy) or StatefulSet (built-in) + # Wait for the operator to create the database StatefulSet (5 minutes max) if ! common::poll_until \ - "oc get postgrescluster -n '$namespace' --no-headers 2>/dev/null | grep -q 'backstage-psql' || \ - oc get statefulset -n '$namespace' --no-headers 2>/dev/null | grep -q 'backstage-psql'" \ + "oc get statefulset -n '$namespace' --no-headers 2>/dev/null | grep -q 'backstage-psql'" \ 60 5 "Database resource created by operator"; then log::error "Database resource not created after 5 minutes" _operator_debug_info "$namespace" @@ -267,8 +254,6 @@ _operator_debug_info() { oc logs -n "${OPERATOR_MANAGER:-rhdh-operator}" -l control-plane=controller-manager --tail=50 || true log::info "Checking for StatefulSet..." oc get statefulset -n "$namespace" || true - log::info "Checking for PostgresCluster..." - oc get postgrescluster -n "$namespace" 2> /dev/null || echo "No PostgresCluster CRD or resources found" } delete_rhdh_operator() { diff --git a/.ci/pipelines/jobs/ocp-fips-helm.sh b/.ci/pipelines/jobs/ocp-fips-helm.sh index 27b00a1c50..7fbd0cee6c 100644 --- a/.ci/pipelines/jobs/ocp-fips-helm.sh +++ b/.ci/pipelines/jobs/ocp-fips-helm.sh @@ -21,8 +21,6 @@ handle_ocp_fips_helm() { fips_configure_custom_ca_ingress - cluster_setup_ocp_helm - fips_deployment "${PW_PROJECT_SHOWCASE_FIPS}" deploy_test_backstage_customization_provider "${NAME_SPACE}" diff --git a/.ci/pipelines/jobs/ocp-localization.sh b/.ci/pipelines/jobs/ocp-localization.sh index 6b620daa39..79762afeb4 100644 --- a/.ci/pipelines/jobs/ocp-localization.sh +++ b/.ci/pipelines/jobs/ocp-localization.sh @@ -24,7 +24,6 @@ handle_ocp_localization() { K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') export K8S_CLUSTER_ROUTER_BASE - cluster_setup_ocp_helm base_deployment "${PW_PROJECT_SHOWCASE}" deploy_test_backstage_customization_provider "${NAME_SPACE}" diff --git a/.ci/pipelines/jobs/ocp-nightly.sh b/.ci/pipelines/jobs/ocp-nightly.sh index ddea0ea049..700550bd05 100644 --- a/.ci/pipelines/jobs/ocp-nightly.sh +++ b/.ci/pipelines/jobs/ocp-nightly.sh @@ -33,8 +33,6 @@ handle_ocp_nightly() { K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') export K8S_CLUSTER_ROUTER_BASE - cluster_setup_ocp_helm - if [[ "${JOB_NAME}" == *osd-gcp* ]]; then log::info "Detected OSD-GCP job, using OSD-GCP specific deployment" initiate_deployments_osd_gcp "${PW_PROJECT_SHOWCASE}" "${PW_PROJECT_SHOWCASE_RBAC}" diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index 539565a29e..4bf2efabef 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -40,7 +40,6 @@ initiate_operator_deployments() { apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}" config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml" oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" - wait_for_crunchy_crd || return 1 deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" } @@ -68,7 +67,6 @@ initiate_operator_deployments_osd_gcp() { common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "/tmp/configmap-dynamic-plugins-rbac.yaml" oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" - wait_for_crunchy_crd || return 1 deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" } @@ -102,8 +100,6 @@ handle_ocp_operator() { local url="https://backstage-${RELEASE_NAME}-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" local rbac_url="https://backstage-${RELEASE_NAME_RBAC}-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" - cluster_setup_ocp_operator - if [[ "${JOB_NAME}" =~ osd-gcp ]]; then # OSD-GCP internal registry is unreliable under parallel load; reduce # concurrent skopeo pushes and allow retries (RHDHBUGS-1136). diff --git a/.ci/pipelines/jobs/ocp-pull.sh b/.ci/pipelines/jobs/ocp-pull.sh index a738763678..fcb953435d 100644 --- a/.ci/pipelines/jobs/ocp-pull.sh +++ b/.ci/pipelines/jobs/ocp-pull.sh @@ -28,7 +28,6 @@ handle_ocp_pull() { K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') export K8S_CLUSTER_ROUTER_BASE - cluster_setup_ocp_helm initiate_deployments "${PW_PROJECT_SHOWCASE}" "${PW_PROJECT_SHOWCASE_RBAC}" deploy_test_backstage_customization_provider "${NAME_SPACE}" local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" diff --git a/.ci/pipelines/jobs/upgrade.sh b/.ci/pipelines/jobs/upgrade.sh index 997a309de5..32d274eb2c 100644 --- a/.ci/pipelines/jobs/upgrade.sh +++ b/.ci/pipelines/jobs/upgrade.sh @@ -138,8 +138,6 @@ handle_ocp_helm_upgrade() { K8S_CLUSTER_ROUTER_BASE=$(oc get route console -n openshift-console -o=jsonpath='{.spec.host}' | sed 's/^[^.]*\.//') export K8S_CLUSTER_ROUTER_BASE - cluster_setup_ocp_helm - local url="https://${RELEASE_NAME}-developer-hub-${NAME_SPACE}.${K8S_CLUSTER_ROUTER_BASE}" UPGRADE_PHASE="baseline-deploy" initiate_upgrade_base_deployments \ diff --git a/.ci/pipelines/lib/README.md b/.ci/pipelines/lib/README.md index 9f3d429cf3..a2c046ea24 100644 --- a/.ci/pipelines/lib/README.md +++ b/.ci/pipelines/lib/README.md @@ -28,9 +28,8 @@ Functions: `k8s_wait::deployment`, `k8s_wait::job`, `k8s_wait::service`, `k8s_wa Operator and OLM installations. -Functions: `operator::install_subscription`, `operator::check_status`, -`operator::install_postgres_ocp`, `operator::install_postgres_k8s`, `operator::install_olm`, -`operator::uninstall_olm` +Functions: `operator::install_subscription`, `operator::check_status`, `operator::install_tekton`, +`operator::install_olm`, `operator::uninstall_olm` ### `helm.sh` diff --git a/.ci/pipelines/lib/operators.sh b/.ci/pipelines/lib/operators.sh index fcda542fe4..f14b830445 100644 --- a/.ci/pipelines/lib/operators.sh +++ b/.ci/pipelines/lib/operators.sh @@ -8,69 +8,6 @@ if [[ -n "${OPERATORS_LIB_SOURCED:-}" ]]; then fi readonly OPERATORS_LIB_SOURCED=1 -# Constants -readonly OPERATOR_STATUS_SUCCEEDED="Succeeded" -readonly OPERATOR_NAMESPACE="openshift-operators" - -# Create OpenShift Operator subscription -# Args: name, namespace, channel, package, source_name, source_namespace -operator::install_subscription() { - local name=$1 - local namespace=$2 - local channel=$3 - local package=$4 - local source_name=$5 - local source_namespace=$6 - - oc apply -f - << EOD -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: $name - namespace: $namespace -spec: - channel: $channel - installPlanApproval: Automatic - name: $package - source: $source_name - sourceNamespace: $source_namespace -EOD - return $? -} - -# Monitor operator status until expected phase is reached -# Args: timeout_seconds, namespace, operator_name, expected_status (default: "Succeeded") -operator::check_status() { - local timeout=${1:-300} - local namespace=$2 - local operator_name=$3 - local expected_status=${4:-${OPERATOR_STATUS_SUCCEEDED}} - - log::info "Checking the status of operator '${operator_name}' in namespace '${namespace}' with a timeout of ${timeout} seconds." - log::info "Expected status: ${expected_status}" - - timeout "${timeout}" bash -c " - while true; do - CURRENT_PHASE=\$(oc get csv -n '${namespace}' -o jsonpath='{.items[?(@.spec.displayName==\"${operator_name}\")].status.phase}') - echo \"Operator '${operator_name}' current phase: \${CURRENT_PHASE}\" - [[ \"\${CURRENT_PHASE}\" == \"${expected_status}\" ]] && echo \"Operator '${operator_name}' is now in '${expected_status}' phase.\" && break - sleep 10 - done - " || log::error "Timed out after ${timeout} seconds. Operator '${operator_name}' did not reach '${expected_status}' phase." - return $? -} - -# Install Crunchy Postgres Operator from OpenShift Marketplace -operator::install_postgres_ocp() { - operator::install_subscription crunchy-postgres-operator "${OPERATOR_NAMESPACE}" v5 crunchy-postgres-operator certified-operators openshift-marketplace - operator::check_status 300 "${OPERATOR_NAMESPACE}" "Crunchy Postgres for Kubernetes" "${OPERATOR_STATUS_SUCCEEDED}" || return 1 - - # Wait for PostgresCluster CRD to be registered - log::info "Waiting for PostgresCluster CRD to be registered..." - k8s_wait::crd "postgresclusters.postgres-operator.crunchydata.com" 120 5 || return 1 - return 0 -} - # Install Tekton Pipelines (alternative to OpenShift Pipelines for Kubernetes) operator::install_tekton() { local display_name="tekton-pipelines-webhook" diff --git a/.ci/pipelines/lib/schema-mode-env.sh b/.ci/pipelines/lib/schema-mode-env.sh index 5959bc335e..7f4957f5f9 100755 --- a/.ci/pipelines/lib/schema-mode-env.sh +++ b/.ci/pipelines/lib/schema-mode-env.sh @@ -65,24 +65,24 @@ configure_schema_mode_runtime_env() { done else local pdb="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}" - local crunchy_cluster="${SCHEMA_MODE_CRUNCHY_CLUSTER_NAME:-postgress-external-db}" + local external_db="${SCHEMA_MODE_EXTERNAL_DB_NAME:-postgress-external-db}" if oc get svc postgress-external-db-primary -n "${pdb}" &> /dev/null; then forward_namespace="${pdb}" - log::info "Schema-mode (${install_method}): no in-cluster Postgres Service in ${runtime_namespace}; using Crunchy cluster in ${pdb}" - local crunchy_admin_secret="${crunchy_cluster}-pguser-janus-idp" - if oc get secret "${crunchy_admin_secret}" -n "${pdb}" &> /dev/null; then - admin_password=$(oc get secret "${crunchy_admin_secret}" -n "${pdb}" -o jsonpath='{.data.password}' 2> /dev/null | base64 -d || true) + log::info "Schema-mode (${install_method}): no in-cluster Postgres Service in ${runtime_namespace}; using external PostgreSQL in ${pdb}" + local external_db_secret="${external_db}-pguser-janus-idp" + if oc get secret "${external_db_secret}" -n "${pdb}" &> /dev/null; then + admin_password=$(oc get secret "${external_db_secret}" -n "${pdb}" -o jsonpath='{.data.password}' 2> /dev/null | base64 -d || true) fi if [[ -z "${admin_password}" ]]; then - log::warn "Schema-mode (${install_method}): could not read ${crunchy_admin_secret} password in ${pdb}; schema tests remain opt-in." + log::warn "Schema-mode (${install_method}): could not read ${external_db_secret} password in ${pdb}; schema tests remain opt-in." return 1 fi postgres_service=$(oc get pods -n "${pdb}" \ - -l "postgres-operator.crunchydata.com/cluster=${crunchy_cluster},postgres-operator.crunchydata.com/data=postgres" \ + -l "app.kubernetes.io/name=${external_db}" \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2> /dev/null) if [[ -z "${postgres_service}" ]]; then - log::warn "Schema-mode (${install_method}): no Running Postgres pod in ${pdb} for cluster ${crunchy_cluster}; schema tests remain opt-in." + log::warn "Schema-mode (${install_method}): no Running Postgres pod in ${pdb} for ${external_db}; schema tests remain opt-in." return 1 fi forward_via_pod=1 diff --git a/.ci/pipelines/resources/postgres-db/postgres.yaml b/.ci/pipelines/resources/postgres-db/postgres.yaml index 87a804e619..7d77fc714f 100644 --- a/.ci/pipelines/resources/postgres-db/postgres.yaml +++ b/.ci/pipelines/resources/postgres-db/postgres.yaml @@ -1,74 +1,167 @@ -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster +# External PostgreSQL used by the showcase-rbac tests. Deployed by +# configure_external_postgres_db in utils.sh, which also creates the +# postgress-external-db-cluster-cert, postgress-external-db-pguser-janus-idp +# and rh-pull-secret secrets referenced below. +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgress-external-db-config +data: + ssl.conf: | + ssl = on + ssl_cert_file = '/opt/app-root/src/certs/tls.crt' + ssl_key_file = '/opt/app-root/src/certs/tls.key' +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgress-external-db-start +data: + rhdh-setup.sh: | + # Sourced by the image's run-postgresql after the server starts locally. + # Only accept TLS connections over the network. + sed -i -E 's/^host([[:space:]]+all[[:space:]]+all[[:space:]]+all[[:space:]]+md5)/hostssl\1/' "${PGDATA}/pg_hba.conf" + psql -v ON_ERROR_STOP=1 -v user="${RHDH_DB_USER}" -v password="${RHDH_DB_PASSWORD}" << 'EOSQL' + SELECT format('CREATE ROLE %I', :'user') WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'user') \gexec + ALTER ROLE :"user" WITH LOGIN SUPERUSER PASSWORD :'password'; + EOSQL + unset RHDH_DB_USER RHDH_DB_PASSWORD +--- +apiVersion: v1 +kind: Service +metadata: + name: postgress-external-db-primary + labels: + app.kubernetes.io/name: postgress-external-db +spec: + selector: + app.kubernetes.io/name: postgress-external-db + ports: + - name: postgres + port: 5432 + targetPort: postgres +--- +apiVersion: apps/v1 +kind: StatefulSet metadata: name: postgress-external-db + labels: + app.kubernetes.io/name: postgress-external-db spec: - image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-18.1-2547 - postgresVersion: 18 - instances: - - name: instance1 - dataVolumeClaimSpec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: 2Gi - resources: - limits: - cpu: 300m - requests: - cpu: 200m - sidecars: - replicaCertCopy: + serviceName: postgress-external-db-primary + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgress-external-db + template: + metadata: + labels: + app.kubernetes.io/name: postgress-external-db + spec: + imagePullSecrets: + - name: rh-pull-secret + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + initContainers: + # PostgreSQL refuses a private key that is readable by others, so copy + # the certificates to a volume owned by the container user. + - name: copy-certs + image: registry.redhat.io/rhel9/postgresql-18:latest + command: + - /bin/sh + - -c + - cp /tls/ca.crt /tls/tls.crt /tls/tls.key /certs/ && chmod 0600 /certs/tls.key + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL resources: - limits: - cpu: 300m requests: - cpu: 200m - backups: - pgbackrest: - image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2547 - global: - # Save backups for 7 days, this means 1 full backups with 6 differential ones in between - repo1-retention-full: "1" - repo1-retention-full-type: count - repoHost: - resources: - limits: - cpu: 300m - requests: - cpu: 200m - repos: - - name: repo1 - schedules: - # Every sunday at 01:00 full backup - full: "0 1 * * 0" - # Monday through saturday at 01:00 differential backup - differential: "0 1 * * 1-6" - volume: - volumeClaimSpec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: 2Gi - sidecars: - pgbackrest: - resources: + cpu: 10m + memory: 32Mi limits: - cpu: 200m - requests: cpu: 100m - pgbackrestConfig: + memory: 64Mi + volumeMounts: + - name: tls-secret + mountPath: /tls + readOnly: true + - name: certs + mountPath: /certs + containers: + - name: postgres + image: registry.redhat.io/rhel9/postgresql-18:latest + ports: + - name: postgres + containerPort: 5432 + env: + - name: POSTGRESQL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: postgress-external-db-pguser-janus-idp + key: postgres-password + - name: RHDH_DB_USER + valueFrom: + secretKeyRef: + name: postgress-external-db-pguser-janus-idp + key: user + - name: RHDH_DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgress-external-db-pguser-janus-idp + key: password + readinessProbe: + exec: + command: + - /usr/libexec/check-container + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL resources: - limits: - cpu: 300m requests: cpu: 200m - users: - - name: janus-idp - options: "SUPERUSER" - - name: sonarqube - databases: - - sonarqube - options: "NOSUPERUSER" + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: data + mountPath: /var/lib/pgsql/data + - name: certs + mountPath: /opt/app-root/src/certs + readOnly: true + - name: config + mountPath: /opt/app-root/src/postgresql-cfg + readOnly: true + - name: start + mountPath: /opt/app-root/src/postgresql-start + readOnly: true + volumes: + - name: tls-secret + secret: + secretName: postgress-external-db-cluster-cert + - name: certs + emptyDir: {} + - name: config + configMap: + name: postgress-external-db-config + - name: start + configMap: + name: postgress-external-db-start + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/.ci/pipelines/utils.sh b/.ci/pipelines/utils.sh index 6dd5ebf698..63a9ac0257 100755 --- a/.ci/pipelines/utils.sh +++ b/.ci/pipelines/utils.sh @@ -160,37 +160,6 @@ save_all_pod_logs() { set -e } -# ============================================================================== -# Operator Functions - Delegate to lib/operators.sh -# ============================================================================== -install_subscription() { operator::install_subscription "$@"; } - -check_operator_status() { operator::check_status "$@"; } - -# Installs the Crunchy Postgres Operator -# Args: platform ("ocp" or "k8s", default: "ocp") -install_crunchy_postgres_operator() { - local platform=${1:-ocp} - install_subscription crunchy-postgres-operator openshift-operators v5 crunchy-postgres-operator certified-operators openshift-marketplace -} - -# Waits for the Crunchy Postgres Operator to be ready -# Args: platform ("ocp" or "k8s", default: "ocp") -waitfor_crunchy_postgres_operator() { - local platform=${1:-ocp} - local namespace="openshift-operators" - [[ "$platform" == "k8s" ]] && namespace="operators" - - check_operator_status 300 "$namespace" "Crunchy Postgres for Kubernetes" "Succeeded" - k8s_wait::crd "postgresclusters.postgres-operator.crunchydata.com" 120 5 || return 1 -} - -# Backward compatibility shims -install_crunchy_postgres_ocp_operator() { install_crunchy_postgres_operator "ocp"; } -install_crunchy_postgres_k8s_operator() { install_crunchy_postgres_operator "k8s"; } -waitfor_crunchy_postgres_ocp_operator() { waitfor_crunchy_postgres_operator "ocp"; } -waitfor_crunchy_postgres_k8s_operator() { waitfor_crunchy_postgres_operator "k8s"; } - # Create the postgres-cred secret in a target namespace. # Accepts key=value pairs as positional args for additional/override literals. # Usage: create_postgres_cred_secret [extra_key=value ...] @@ -217,90 +186,123 @@ create_postgres_cred_secret() { fi } +# Generate a CA and a server certificate for the external PostgreSQL Service. +# Usage: generate_external_postgres_certs +# Writes ca.crt, tls.crt and tls.key to output_dir. +generate_external_postgres_certs() { + local out_dir=$1 service=$2 namespace=$3 + + openssl ecparam -name prime256v1 -genkey -noout -out "${out_dir}/ca.key" || return 1 + openssl req -x509 -new -key "${out_dir}/ca.key" -sha256 -days 30 \ + -subj "/O=RHDH-CI/CN=RHDH CI external PostgreSQL CA" \ + -out "${out_dir}/ca.crt" || return 1 + + openssl ecparam -name prime256v1 -genkey -noout -out "${out_dir}/tls.key" || return 1 + openssl req -new -key "${out_dir}/tls.key" -subj "/O=RHDH-CI/CN=${service}" \ + -out "${out_dir}/tls.csr" || return 1 + cat > "${out_dir}/tls_ext.cnf" << EOF +basicConstraints = CA:FALSE +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:${service}, DNS:${service}.${namespace}, DNS:${service}.${namespace}.svc, DNS:${service}.${namespace}.svc.cluster.local +EOF + openssl x509 -req -in "${out_dir}/tls.csr" -CA "${out_dir}/ca.crt" -CAkey "${out_dir}/ca.key" \ + -CAcreateserial -days 30 -sha256 -extfile "${out_dir}/tls_ext.cnf" \ + -out "${out_dir}/tls.crt" || return 1 +} + +# Deploy the external PostgreSQL (registry.redhat.io/rhel9/postgresql-18) with TLS +# in NAME_SPACE_POSTGRES_DB and create the connection secrets in the given project. configure_external_postgres_db() { local project=$1 - local max_attempts=60 # 5 minutes total (60 attempts × 5 seconds) - local wait_interval=5 + local db_name="postgress-external-db" + local db_namespace="${NAME_SPACE_POSTGRES_DB}" + local service="${db_name}-primary" + local cert_secret="${db_name}-cluster-cert" + local user_secret="${db_name}-pguser-janus-idp" - log::info "Creating PostgresCluster in namespace ${NAME_SPACE_POSTGRES_DB}..." + common::require_vars "NAME_SPACE_POSTGRES_DB" "REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON" || return 1 - # Validate oc apply command execution - if ! oc apply -f "${DIR}/resources/postgres-db/postgres.yaml" --namespace="${NAME_SPACE_POSTGRES_DB}"; then - log::error "Failed to create PostgresCluster" - return 1 + local cert_dir + cert_dir=$(mktemp -d) + + # The image is pulled from registry.redhat.io, which requires authentication. + namespace::create_dockerconfigjson_secret "${db_namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" || return 1 + + # Reuse existing certificates and credentials so a second call (e.g. upgrade + # jobs) does not break the running database. + if oc get secret "${cert_secret}" -n "${db_namespace}" &> /dev/null; then + log::info "Reusing existing ${cert_secret} secret" + else + log::info "Generating TLS certificates for ${service}.${db_namespace}" + if ! generate_external_postgres_certs "${cert_dir}" "${service}" "${db_namespace}"; then + log::error "Failed to generate PostgreSQL TLS certificates" + return 1 + fi + if ! oc create secret generic "${cert_secret}" \ + --from-file=ca.crt="${cert_dir}/ca.crt" \ + --from-file=tls.crt="${cert_dir}/tls.crt" \ + --from-file=tls.key="${cert_dir}/tls.key" \ + --namespace="${db_namespace}"; then + log::error "Failed to create ${cert_secret} secret" + return 1 + fi fi - # Wait for cluster cert secret (usually created quickly) - log::info "Waiting for cluster certificate secret..." - if ! common::poll_until \ - "oc get secret postgress-external-db-cluster-cert -n '${NAME_SPACE_POSTGRES_DB}'" \ - "$max_attempts" "$wait_interval" \ - "Cluster certificate secret found"; then + if oc get secret "${user_secret}" -n "${db_namespace}" &> /dev/null; then + log::info "Reusing existing ${user_secret} secret" + elif ! oc create secret generic "${user_secret}" \ + --from-literal=user="janus-idp" \ + --from-literal=password="$(openssl rand -hex 24)" \ + --from-literal=postgres-password="$(openssl rand -hex 24)" \ + --namespace="${db_namespace}"; then + log::error "Failed to create ${user_secret} secret" return 1 fi - # Extract cluster certificates - oc get secret postgress-external-db-cluster-cert -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.ca\.crt}' | base64 --decode > postgres-ca || { - log::error "Failed to extract ca.crt" - return 1 - } - oc get secret postgress-external-db-cluster-cert -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.tls\.crt}' | base64 --decode > postgres-tls-crt || { - log::error "Failed to extract tls.crt" + log::info "Deploying external PostgreSQL in namespace ${db_namespace}..." + if ! oc apply -f "${DIR}/resources/postgres-db/postgres.yaml" --namespace="${db_namespace}"; then + log::error "Failed to apply external PostgreSQL resources" return 1 - } - oc get secret postgress-external-db-cluster-cert -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.tls\.key}' | base64 --decode > postgres-tls-key || { - log::error "Failed to extract tls.key" + fi + + if ! oc rollout status "statefulset/${db_name}" -n "${db_namespace}" --timeout=10m; then + log::error "External PostgreSQL did not become ready" + oc get pods -n "${db_namespace}" -o wide || true + oc describe pod "${db_name}-0" -n "${db_namespace}" || true + oc logs "${db_name}-0" -n "${db_namespace}" --all-containers --tail=100 || true return 1 - } + fi - # Validate secret creation - if ! oc create secret generic postgress-external-db-cluster-cert \ - --from-file=ca.crt=postgres-ca \ - --from-file=tls.crt=postgres-tls-crt \ - --from-file=tls.key=postgres-tls-key \ + # Copy the certificates to the RHDH namespace (mounted by the RBAC values and operator CRs) + local file + for file in ca.crt tls.crt tls.key; do + if ! oc get secret "${cert_secret}" -n "${db_namespace}" -o jsonpath="{.data.${file//./\\.}}" | base64 --decode > "${cert_dir}/${file}"; then + log::error "Failed to extract ${file} from ${cert_secret}" + return 1 + fi + done + + if ! oc create secret generic "${cert_secret}" \ + --from-file=ca.crt="${cert_dir}/ca.crt" \ + --from-file=tls.crt="${cert_dir}/tls.crt" \ + --from-file=tls.key="${cert_dir}/tls.key" \ --dry-run=client -o yaml | oc apply -f - --namespace="${project}"; then log::error "Failed to create cluster certificate secret" return 1 fi - # Create postgres-crt secret for Backstage deployment (Helm/Operator) - # This secret is referenced by RBAC deployment values and operator CRs if ! oc create secret generic postgres-crt \ - --from-file=postgres-crt.pem=postgres-ca \ + --from-file=postgres-crt.pem="${cert_dir}/ca.crt" \ --dry-run=client -o yaml | oc apply -f - --namespace="${project}"; then log::error "Failed to create postgres-crt secret" return 1 fi + rm -rf "${cert_dir}" - # Wait for USER secret (this is the critical one that causes CI failures!) - log::info "Waiting for PostgreSQL user secret 'postgress-external-db-pguser-janus-idp'..." - log::info "This secret is created by the Crunchy Postgres operator after the database is ready" - if ! common::poll_until \ - "oc get secret postgress-external-db-pguser-janus-idp -n '${NAME_SPACE_POSTGRES_DB}'" \ - "$max_attempts" "$wait_interval" \ - "PostgreSQL user secret found"; then - log::error "This usually means the Crunchy Postgres operator failed to create the user" - log::info "Checking PostgresCluster status..." - oc describe postgrescluster postgress-external-db -n "${NAME_SPACE_POSTGRES_DB}" || true - log::info "Checking operator logs..." - oc logs -n "${NAME_SPACE_POSTGRES_DB}" -l postgres-operator.crunchydata.com/cluster=postgress-external-db --tail=50 || true - return 1 - fi - - # Wait for PostgreSQL pods to be Running (needed for schema-mode tests) - log::info "Waiting for PostgreSQL pods to be Running..." - if ! common::poll_until \ - "oc get pods -n '${NAME_SPACE_POSTGRES_DB}' -l 'postgres-operator.crunchydata.com/cluster=postgress-external-db,postgres-operator.crunchydata.com/data=postgres' --field-selector=status.phase=Running -o name 2>/dev/null | grep -q pod/" \ - "$max_attempts" "$wait_interval" \ - "PostgreSQL pod is Running"; then - log::warn "PostgreSQL pod not Running yet; schema-mode tests may skip" - # Don't fail here - the database might work fine, just schema tests won't run - fi - - # Now we can safely get the password (decode from base64 since K8s .data is always base64-encoded) local pg_password pg_host - pg_password=$(oc get secret/postgress-external-db-pguser-janus-idp -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.password}' | base64 --decode) - pg_host="postgress-external-db-primary.${NAME_SPACE_POSTGRES_DB}.svc.cluster.local" + pg_password=$(oc get secret "${user_secret}" -n "${db_namespace}" -o jsonpath='{.data.password}' | base64 --decode) + pg_host="${service}.${db_namespace}.svc.cluster.local" create_postgres_cred_secret "${project}" "${pg_host}" "${pg_password}" @@ -399,19 +401,10 @@ uninstall_olm() { operator::uninstall_olm "$@"; } # These functions configure the cluster for different deployment types # ============================================================================== -cluster_setup_ocp_helm() { - operator::install_postgres_ocp -} - -cluster_setup_ocp_operator() { - operator::install_postgres_ocp -} - cluster_setup_k8s_operator() { operator::install_olm # Install Tekton Pipelines for K8s deployments (AKS/EKS/GKE) for topology tests to work operator::install_tekton - # operator::install_postgres_k8s # Works with K8s but disabled in values file } cluster_setup_k8s_helm() { @@ -419,7 +412,6 @@ cluster_setup_k8s_helm() { log::info "Installing Tekton Pipelines for K8s Helm deployment" # operator::install_olm operator::install_tekton - # operator::install_postgres_k8s # Works with K8s but disabled in values file } # ============================================================================== diff --git a/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml b/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml index c67acb4e3a..db82021167 100644 --- a/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase-rbac_AKS.yaml @@ -5,7 +5,7 @@ openshift: route: enabled: false -# The base config points ssl at Crunchy certs that are not mounted here, and the +# The base config points ssl at external PostgreSQL certs that are not mounted here, and the # bundled Fedora postgres has no TLS. The merge drops null keys, removing ssl. appConfig: backend: diff --git a/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml b/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml index b6e008531d..de16851eb2 100644 --- a/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase-rbac_EKS.yaml @@ -5,7 +5,7 @@ openshift: route: enabled: false -# The base config points ssl at Crunchy certs that are not mounted here, and the +# The base config points ssl at external PostgreSQL certs that are not mounted here, and the # bundled Fedora postgres has no TLS. The merge drops null keys, removing ssl. appConfig: backend: diff --git a/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml b/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml index 66b9f6ace6..dcbd4a6bf8 100644 --- a/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml +++ b/.ci/pipelines/value_files/diff-values_showcase-rbac_GKE.yaml @@ -5,7 +5,7 @@ openshift: route: enabled: false -# The base config points ssl at Crunchy certs that are not mounted here, and the +# The base config points ssl at external PostgreSQL certs that are not mounted here, and the # bundled Fedora postgres has no TLS. The merge drops null keys, removing ssl. appConfig: backend: diff --git a/docs/e2e-tests/CI-medic-guide.md b/docs/e2e-tests/CI-medic-guide.md index f70d89b30e..e021e65321 100644 --- a/docs/e2e-tests/CI-medic-guide.md +++ b/docs/e2e-tests/CI-medic-guide.md @@ -315,7 +315,6 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t **What happens**: The [test script](../../.ci/pipelines/openshift-ci-tests.sh) installs required operators and infrastructure (see [operators.sh](../../.ci/pipelines/lib/operators.sh)): - OpenShift Pipelines (Tekton) operator -- Crunchy PostgreSQL operator **What can go wrong**: - Operator installation timeout (OperatorHub/Marketplace issues) @@ -357,7 +356,7 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t - `CrashLoopBackOff` -- pod is crash-looping - `Failed to reach Backstage` -- health check timeout - `helm upgrade` failures -- `Crunchy Postgres operator failed to create the user` -- PostgreSQL setup issue +- `External PostgreSQL did not become ready` -- external PostgreSQL (`postgress-external-db` StatefulSet) setup issue - Check `pod_logs/` for application-level errors **Action**: Check pod logs and events in artifacts. May be a config issue (real bug) or transient infra (re-trigger). @@ -419,7 +418,7 @@ Same as OCP nightly but deploys RHDH using the Operator instead of Helm. See [`o **Key differences**: - Installs RHDH Operator and waits for `backstages.rhdh.redhat.com` CRD (300s timeout) - Uses Backstage CR (`rhdh-start.yaml`) instead of Helm release -- Runtime config tests enabled, including `pluginDivisionMode: schema` tests with external Crunchy PostgreSQL +- Runtime config tests enabled, including `pluginDivisionMode: schema` tests with the external PostgreSQL ### OCP PR Check (`ocp-pull`) diff --git a/docs/e2e-tests/layer-migration-matrix.md b/docs/e2e-tests/layer-migration-matrix.md index 8e61369cf8..75446ce56c 100644 --- a/docs/e2e-tests/layer-migration-matrix.md +++ b/docs/e2e-tests/layer-migration-matrix.md @@ -117,7 +117,7 @@ Legend: ✅ = Layer 3 equivalent **already merged** on `main` via | 17 | `plugins/scaffolder-relation-processor/scaffolder-relation-processor` | showcase | no | yes | GitHub (repo CRUD) | **L4a** | | 18 | `github-happy-path` | none (`blocked/`) | no | yes | GitHub OAuth+API | **L4a / L4b** | | 19 | `plugin-dynamic-loading` | none (moved) | no | no (API) | catalog index image | **L4a** (already cluster-free) | -| 25 | `external-database/...-crunchy` | runtime-db | **yes** | yes | Crunchy PG | **L4b** | +| 25 | `external-database/...-postgres` | runtime-db | **yes** | yes | RHEL PostgreSQL | **L4b** | | 26 | `external-database/...-azure-db` | runtime-db | **yes** | yes | Azure PG (x4) | **L4b** | | 27 | `external-database/...-rds` | runtime-db | **yes** | yes | AWS RDS (x4) | **L4b** | | 28 | `configuration-test/config-map` | showcase | **yes** | yes | ConfigMap reload | **L4b** | diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 459fc596fa..467186b581 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -98,7 +98,7 @@ export default defineConfig({ "**/playwright/seed.spec.ts", "**/playwright/e2e/plugins/rbac/**/*.spec.ts", "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", "**/playwright/e2e/external-database/verify-tls-config-with-external-rds.spec.ts", "**/playwright/e2e/external-database/verify-tls-config-with-external-azure-db.spec.ts", "**/playwright/e2e/external-database/verify-tls-config-with-external-cloudsql.spec.ts", @@ -121,7 +121,7 @@ export default defineConfig({ "**/playwright/e2e/plugins/rbac/**/*.spec.ts", "**/playwright/e2e/**/*-rbac.spec.ts", "**/playwright/e2e/audit-log/**/*.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", "**/playwright/e2e/external-database/verify-tls-config-with-external-rds.spec.ts", "**/playwright/e2e/external-database/verify-tls-config-with-external-azure-db.spec.ts", "**/playwright/e2e/plugin-division-mode-schema/*.spec.ts", @@ -134,7 +134,7 @@ export default defineConfig({ dependencies: [PW_PROJECT.SMOKE_TEST], testMatch: [ "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", ], }, { @@ -146,7 +146,7 @@ export default defineConfig({ "**/playwright/e2e/smoke-test.spec.ts", "**/playwright/e2e/plugins/rbac/**/*.spec.ts", "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", "**/playwright/e2e/plugins/scaffolder-backend-module-annotator/**/*.spec.ts", "**/playwright/e2e/plugins/scaffolder-relation-processor/**/*.spec.ts", "**/playwright/e2e/plugins/ocm.spec.ts", @@ -174,7 +174,7 @@ export default defineConfig({ "**/playwright/seed.spec.ts", "**/playwright/e2e/plugins/rbac/**/*.spec.ts", "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", "**/playwright/e2e/plugins/scaffolder-backend-module-annotator/**/*.spec.ts", "**/playwright/e2e/plugins/scaffolder-relation-processor/**/*.spec.ts", "**/playwright/e2e/audit-log/**/*.spec.ts", diff --git a/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts b/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts similarity index 93% rename from e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts rename to e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts index 72bcb967ae..cb22ddfd9d 100644 --- a/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts +++ b/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts @@ -3,7 +3,7 @@ import { test, expect } from "@support/coverage/test"; import { CatalogBrowsePage } from "../../support/pages/catalog-browse-page"; import { HomePage } from "../../support/pages/home-page"; -test.describe("Verify TLS configuration with external Crunchy Postgres DB", () => { +test.describe("Verify TLS configuration with external PostgreSQL DB", () => { test.beforeAll(() => { test.info().annotations.push( { diff --git a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md index 88e252b03e..a904149b63 100644 --- a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md +++ b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md @@ -47,11 +47,11 @@ On OCP Helm nightly jobs, [`.ci/pipelines/lib/schema-mode-env.sh`](../../../../. The test spec starts `oc port-forward` in `beforeAll` (similar to [`verify-redis-cache.spec.ts`](../../verify-redis-cache.spec.ts)) and stops it in `afterAll`. -When the runtime namespace has no Bitnami `*-postgresql` Service, auto-discovery uses the Crunchy cluster in `NAME_SPACE_POSTGRES_DB`: +When the runtime namespace has no Bitnami `*-postgresql` Service, auto-discovery uses the external PostgreSQL StatefulSet in `NAME_SPACE_POSTGRES_DB` (see [`postgres.yaml`](../../../../.ci/pipelines/resources/postgres-db/postgres.yaml)): -- Admin password from `${SCHEMA_MODE_CRUNCHY_CLUSTER_NAME:-postgress-external-db}-pguser-janus-idp` -- Forward target is a Running postgres pod (the `*-primary` Service has no selector, so forwarding the Service fails) -- Override cluster name with `SCHEMA_MODE_CRUNCHY_CLUSTER_NAME` if your `PostgresCluster` metadata name differs +- Admin password from `${SCHEMA_MODE_EXTERNAL_DB_NAME:-postgress-external-db}-pguser-janus-idp` +- Forward target is the Running pod labeled `app.kubernetes.io/name=${SCHEMA_MODE_EXTERNAL_DB_NAME:-postgress-external-db}` +- Override the name with `SCHEMA_MODE_EXTERNAL_DB_NAME` if your StatefulSet name differs Set `DEBUG_SCHEMA_MODE_PF=1` to log port-forward output. @@ -67,7 +67,7 @@ The tests include built-in retry logic to handle transient port-forward instabil - Automatic retry with exponential backoff for transient connection failures - Automatic port-forward restart when connection is refused (port-forward crashed) -- SSL fallback for Crunchy PostgreSQL clusters requiring encrypted connections +- SSL fallback for PostgreSQL servers requiring encrypted connections (the external PostgreSQL only accepts TLS) ## Local / ClusterBot Runs From df32d323accc756e9415397f4e880f17ce6874de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Thu, 12 Nov 2026 10:06:08 +0100 Subject: [PATCH 2/4] fix(ci): harden external PostgreSQL TLS rules and FIPS compatibility - Replace the pg_hba.conf rewrite in the start hook with declarative pg_hba.rules loaded through hba_file, so TLS-only enforcement no longer depends on the authentication lines generated by the image - Set ssl_groups to prime256v1, since the PostgreSQL 18 default (X25519:prime256v1) is not FIPS-approved - Extract the certificates with oc extract and fail on empty passwords instead of publishing empty secrets - Propagate configure_external_postgres_db failures and drop the redundant StatefulSet wait in rbac_deployment - Drop the unused SCHEMA_MODE_EXTERNAL_DB_NAME override and refresh the CI docs, the pr-triage skill and the pgdb catalog entry Assisted-by: OpenCode --- .agents/skills/pr-triage/SKILL.md | 18 ++++----- .ci/pipelines/lib/README.md | 3 +- .ci/pipelines/lib/schema-mode-env.sh | 5 ++- .../resources/postgres-db/postgres.yaml | 19 ++++++++-- .ci/pipelines/utils.sh | 37 +++++++++---------- catalog-entities/resources/pgdb.yaml | 8 ++-- docs/e2e-tests/CI-medic-guide.md | 17 ++++----- .../e2e/plugin-division-mode-schema/README.md | 8 ++-- 8 files changed, 62 insertions(+), 53 deletions(-) diff --git a/.agents/skills/pr-triage/SKILL.md b/.agents/skills/pr-triage/SKILL.md index c83496b63d..4fc2729500 100644 --- a/.agents/skills/pr-triage/SKILL.md +++ b/.agents/skills/pr-triage/SKILL.md @@ -151,13 +151,13 @@ Use the failure classification from the CI Medic Guide to categorize each failur - "CrashLoopBackOff" - "Failed to reach Backstage" - "helm upgrade.*failed" -- "Crunchy Postgres operator failed to create the user" +- "External PostgreSQL did not become ready" - "ImagePullBackOff" - "FailedScheduling" ``` **Classification:** `DEPLOY_FAILED` with subcategories: -- `DEPLOY_POSTGRES` - PostgreSQL operator issues +- `DEPLOY_POSTGRES` - External PostgreSQL issues - `DEPLOY_HELM` - Helm chart failures - `DEPLOY_OPERATOR` - Operator CRD/deployment issues - `DEPLOY_HEALTH_CHECK` - Health check timeouts @@ -218,7 +218,7 @@ Create a fingerprint for each failure to detect duplicates: **Example:** ``` -DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly:showcase-rbac +DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly:showcase-rbac ``` ### Similarity Detection @@ -231,9 +231,9 @@ Compare fingerprints across all analyzed PRs: **Output:** ``` -PR #4537: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4418: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4501: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-nightly +PR #4537: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4418: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4501: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-nightly → Identical: #4537, #4418 → Related: #4501 (same error, different namespace) @@ -330,8 +330,8 @@ For each detected failure, extract keywords for Jira searching: # Example error: "Timed out waiting for Docker image :pr-4537-9c77192a" # Keywords: "Docker image", "timeout", "waiting" - # Example error: "Crunchy Postgres operator failed to create the user" - # Keywords: "PostgreSQL operator", "create user", "failed" + # Example error: "External PostgreSQL did not become ready" + # Keywords: "External PostgreSQL", "not ready", "StatefulSet" ``` 2. **Build Jira search URLs**: @@ -533,7 +533,7 @@ Reference the CI Medic Guide for known patterns: | Pattern | Classification | Jira Label | Action | |---------|---------------|------------|--------| -| `Crunchy Postgres operator failed to create the user` | DEPLOY_POSTGRES | `postgresql-operator` | Check operator version, re-trigger | +| `External PostgreSQL did not become ready` | DEPLOY_POSTGRES | `external-postgres` | Check StatefulSet pod events/logs, re-trigger | | `Failed to reach Backstage after N attempts` | DEPLOY_HEALTH_CHECK | `health-check-timeout` | Check pod logs, resource limits | | `CrashLoopBackOff` | DEPLOY_POD_CRASH | `deployment-crash` | Investigate pod logs | | `ImagePullBackOff` | DEPLOY_IMAGE_PULL | `image-registry` | Check registry, rate limits | diff --git a/.ci/pipelines/lib/README.md b/.ci/pipelines/lib/README.md index a2c046ea24..74fdb875e6 100644 --- a/.ci/pipelines/lib/README.md +++ b/.ci/pipelines/lib/README.md @@ -28,8 +28,7 @@ Functions: `k8s_wait::deployment`, `k8s_wait::job`, `k8s_wait::service`, `k8s_wa Operator and OLM installations. -Functions: `operator::install_subscription`, `operator::check_status`, `operator::install_tekton`, -`operator::install_olm`, `operator::uninstall_olm` +Functions: `operator::install_tekton`, `operator::install_olm`, `operator::uninstall_olm` ### `helm.sh` diff --git a/.ci/pipelines/lib/schema-mode-env.sh b/.ci/pipelines/lib/schema-mode-env.sh index 7f4957f5f9..258d434a27 100755 --- a/.ci/pipelines/lib/schema-mode-env.sh +++ b/.ci/pipelines/lib/schema-mode-env.sh @@ -65,8 +65,9 @@ configure_schema_mode_runtime_env() { done else local pdb="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}" - local external_db="${SCHEMA_MODE_EXTERNAL_DB_NAME:-postgress-external-db}" - if oc get svc postgress-external-db-primary -n "${pdb}" &> /dev/null; then + # Name and labels deployed by resources/postgres-db/postgres.yaml + local external_db="postgress-external-db" + if oc get svc "${external_db}-primary" -n "${pdb}" &> /dev/null; then forward_namespace="${pdb}" log::info "Schema-mode (${install_method}): no in-cluster Postgres Service in ${runtime_namespace}; using external PostgreSQL in ${pdb}" local external_db_secret="${external_db}-pguser-janus-idp" diff --git a/.ci/pipelines/resources/postgres-db/postgres.yaml b/.ci/pipelines/resources/postgres-db/postgres.yaml index 7d77fc714f..e42e5d5db7 100644 --- a/.ci/pipelines/resources/postgres-db/postgres.yaml +++ b/.ci/pipelines/resources/postgres-db/postgres.yaml @@ -7,10 +7,23 @@ kind: ConfigMap metadata: name: postgress-external-db-config data: + # Merged into the image-generated postgresql.conf on every start. ssl.conf: | ssl = on ssl_cert_file = '/opt/app-root/src/certs/tls.crt' ssl_key_file = '/opt/app-root/src/certs/tls.key' + # The PostgreSQL 18 default (X25519:prime256v1) is not FIPS-approved + ssl_groups = 'prime256v1' + hba_file = '/opt/app-root/src/postgresql-cfg/pg_hba.rules' + # Replaces the image-generated pg_hba.conf instead of rewriting it, so the + # TLS-only rules do not depend on the image's authentication defaults. + # Not named *.conf on purpose: the image loads every *.conf in this + # directory as a settings file. + pg_hba.rules: | + local all all trust + host all all 127.0.0.1/32 trust + host all all ::1/128 trust + hostssl all all all scram-sha-256 --- apiVersion: v1 kind: ConfigMap @@ -18,9 +31,9 @@ metadata: name: postgress-external-db-start data: rhdh-setup.sh: | - # Sourced by the image's run-postgresql after the server starts locally. - # Only accept TLS connections over the network. - sed -i -E 's/^host([[:space:]]+all[[:space:]]+all[[:space:]]+all[[:space:]]+md5)/hostssl\1/' "${PGDATA}/pg_hba.conf" + # Sourced by the image's run-postgresql on every start, while the server is + # listening on the unix socket only. TLS and authentication are configured + # by the pg_hba.rules/ssl.conf keys of postgress-external-db-config. psql -v ON_ERROR_STOP=1 -v user="${RHDH_DB_USER}" -v password="${RHDH_DB_PASSWORD}" << 'EOSQL' SELECT format('CREATE ROLE %I', :'user') WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'user') \gexec ALTER ROLE :"user" WITH LOGIN SUPERUSER PASSWORD :'password'; diff --git a/.ci/pipelines/utils.sh b/.ci/pipelines/utils.sh index 63a9ac0257..4e059c0b3b 100755 --- a/.ci/pipelines/utils.sh +++ b/.ci/pipelines/utils.sh @@ -229,8 +229,10 @@ configure_external_postgres_db() { # The image is pulled from registry.redhat.io, which requires authentication. namespace::create_dockerconfigjson_secret "${db_namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" || return 1 - # Reuse existing certificates and credentials so a second call (e.g. upgrade - # jobs) does not break the running database. + # Reuse existing certificates and credentials so a second call in the same job + # (release-1.10 reuses the database created for the RBAC instance when setting + # up the runtime namespace) does not rotate the credentials of a running + # database. if oc get secret "${cert_secret}" -n "${db_namespace}" &> /dev/null; then log::info "Reusing existing ${cert_secret} secret" else @@ -275,13 +277,10 @@ configure_external_postgres_db() { fi # Copy the certificates to the RHDH namespace (mounted by the RBAC values and operator CRs) - local file - for file in ca.crt tls.crt tls.key; do - if ! oc get secret "${cert_secret}" -n "${db_namespace}" -o jsonpath="{.data.${file//./\\.}}" | base64 --decode > "${cert_dir}/${file}"; then - log::error "Failed to extract ${file} from ${cert_secret}" - return 1 - fi - done + if ! oc extract "secret/${cert_secret}" -n "${db_namespace}" --to="${cert_dir}" --confirm > /dev/null; then + log::error "Failed to extract certificates from ${cert_secret}" + return 1 + fi if ! oc create secret generic "${cert_secret}" \ --from-file=ca.crt="${cert_dir}/ca.crt" \ @@ -301,10 +300,14 @@ configure_external_postgres_db() { rm -rf "${cert_dir}" local pg_password pg_host - pg_password=$(oc get secret "${user_secret}" -n "${db_namespace}" -o jsonpath='{.data.password}' | base64 --decode) + if ! pg_password=$(oc get secret "${user_secret}" -n "${db_namespace}" -o go-template='{{.data.password | base64decode}}') \ + || [[ -z "${pg_password}" ]]; then + log::error "Failed to read the password from ${user_secret}" + return 1 + fi pg_host="${service}.${db_namespace}.svc.cluster.local" - create_postgres_cred_secret "${project}" "${pg_host}" "${pg_password}" + create_postgres_cred_secret "${project}" "${pg_host}" "${pg_password}" || return 1 log::success "External PostgreSQL database configured successfully!" } @@ -444,14 +447,8 @@ rbac_deployment() { namespace::configure "${NAME_SPACE_POSTGRES_DB}" namespace::configure "${NAME_SPACE_RBAC}" - configure_external_postgres_db "${NAME_SPACE_RBAC}" - - # Wait for PostgreSQL to be fully ready before deploying RBAC instance - log::info "Waiting for external PostgreSQL to be ready..." - if ! k8s_wait::deployment "${NAME_SPACE_POSTGRES_DB}" "postgress-external-db" 10 10; then - log::error "PostgreSQL deployment failed to become ready" - return 1 - fi + # Waits for the StatefulSet rollout, so the RBAC instance can connect immediately + configure_external_postgres_db "${NAME_SPACE_RBAC}" || return 1 # Initiate rbac instance deployment. local rbac_rhdh_base_url="https://${RELEASE_NAME_RBAC}-developer-hub-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" @@ -556,7 +553,7 @@ rbac_deployment_osd_gcp() { namespace::configure "${NAME_SPACE_POSTGRES_DB}" namespace::configure "${NAME_SPACE_RBAC}" - configure_external_postgres_db "${NAME_SPACE_RBAC}" + configure_external_postgres_db "${NAME_SPACE_RBAC}" || return 1 # Initiate rbac instance deployment. local rbac_rhdh_base_url="https://${RELEASE_NAME_RBAC}-developer-hub-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" diff --git a/catalog-entities/resources/pgdb.yaml b/catalog-entities/resources/pgdb.yaml index 1c9132a14e..3eb2b01198 100644 --- a/catalog-entities/resources/pgdb.yaml +++ b/catalog-entities/resources/pgdb.yaml @@ -3,12 +3,12 @@ kind: Resource metadata: name: pgdb title: PostgreSQL cluster - description: PostgreSQL is a powerful, open source object-relational database system. PostgreSQL cluster is created by Crunchy Postgres for Kubernetes operator. We use Postgres as the databse for this showcase. + description: PostgreSQL is a powerful, open source object-relational database system. The cluster runs as a StatefulSet using the RHEL PostgreSQL image. We use Postgres as the database for this showcase. links: - - url: https://www.crunchydata.com/products/crunchy-high-availability-postgresql - title: Crunchy product pages + - url: https://www.postgresql.org/about/ + title: PostgreSQL product pages icon: web - - url: https://access.crunchydata.com/documentation/postgres-operator/v5/ + - url: https://github.com/sclorg/postgresql-container title: Documentation icon: web spec: diff --git a/docs/e2e-tests/CI-medic-guide.md b/docs/e2e-tests/CI-medic-guide.md index e021e65321..96a70e1be9 100644 --- a/docs/e2e-tests/CI-medic-guide.md +++ b/docs/e2e-tests/CI-medic-guide.md @@ -311,22 +311,21 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t **Action**: Usually transient -- re-trigger. If the Dockerfile or GitHub Actions workflow changed recently, check the [workflow runs](https://github.com/redhat-developer/rhdh/actions/workflows/push-e2e-runner.yaml) to verify the image was built and pushed successfully. -### Phase 3: Cluster Setup (Operators and Prerequisites) +### Phase 3: Cluster Setup (Prerequisites) -**What happens**: The [test script](../../.ci/pipelines/openshift-ci-tests.sh) installs required operators and infrastructure (see [operators.sh](../../.ci/pipelines/lib/operators.sh)): -- OpenShift Pipelines (Tekton) operator +**What happens**: The [test script](../../.ci/pipelines/openshift-ci-tests.sh) prepares the cluster before deployment: +- OCP jobs install no operators. The external PostgreSQL used by `showcase-rbac` is a plain StatefulSet (see [postgres.yaml](../../.ci/pipelines/resources/postgres-db/postgres.yaml)) +- AKS/EKS/GKE jobs install Tekton Pipelines for the topology tests (see [operators.sh](../../.ci/pipelines/lib/operators.sh)) **What can go wrong**: -- Operator installation timeout (OperatorHub/Marketplace issues) -- CRD not becoming available -- Tekton webhook deployment not ready +- Tekton webhook deployment not ready (Kubernetes jobs only) +- External PostgreSQL StatefulSet not becoming ready (image pull, TLS secret, PVC) **How to tell**: Search `build-log.txt` for: -- `Failed to install subscription` -- Timeout waiting for operator CRDs - `Tekton` or `pipeline` related errors early in the log +- `External PostgreSQL did not become ready` -**Action**: Usually infrastructure -- re-trigger. If operators were recently upgraded, investigate compatibility. +**Action**: Usually infrastructure -- re-trigger. If a recent change touched the database setup, investigate compatibility. ### Phase 4: RHDH Deployment diff --git a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md index a904149b63..b6fd8fea51 100644 --- a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md +++ b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md @@ -49,9 +49,9 @@ The test spec starts `oc port-forward` in `beforeAll` (similar to [`verify-redis When the runtime namespace has no Bitnami `*-postgresql` Service, auto-discovery uses the external PostgreSQL StatefulSet in `NAME_SPACE_POSTGRES_DB` (see [`postgres.yaml`](../../../../.ci/pipelines/resources/postgres-db/postgres.yaml)): -- Admin password from `${SCHEMA_MODE_EXTERNAL_DB_NAME:-postgress-external-db}-pguser-janus-idp` -- Forward target is the Running pod labeled `app.kubernetes.io/name=${SCHEMA_MODE_EXTERNAL_DB_NAME:-postgress-external-db}` -- Override the name with `SCHEMA_MODE_EXTERNAL_DB_NAME` if your StatefulSet name differs +- Admin password from `postgress-external-db-pguser-janus-idp` +- Forward target is the Running pod labeled `app.kubernetes.io/name=postgress-external-db` +- The names match the StatefulSet deployed by `postgres.yaml`; there is no override Set `DEBUG_SCHEMA_MODE_PF=1` to log port-forward output. @@ -67,7 +67,7 @@ The tests include built-in retry logic to handle transient port-forward instabil - Automatic retry with exponential backoff for transient connection failures - Automatic port-forward restart when connection is refused (port-forward crashed) -- SSL fallback for PostgreSQL servers requiring encrypted connections (the external PostgreSQL only accepts TLS) +- SSL fallback for PostgreSQL servers requiring encrypted connections (the external PostgreSQL requires TLS for every network connection) ## Local / ClusterBot Runs From d40d5f3d2d165648a30cec1a29daae28092f778a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Thu, 12 Nov 2026 10:13:47 +0100 Subject: [PATCH 3/4] chore(ci): rename the external PostgreSQL admin user to rhdh The janus-idp user name is a leftover from the Crunchy operator user spec and no longer describes the database used by CI. Rename the user and the secret that carries its credentials. Assisted-by: OpenCode --- .ci/pipelines/lib/schema-mode-env.sh | 4 ++-- .ci/pipelines/resources/postgres-db/postgres.yaml | 8 ++++---- .ci/pipelines/utils.sh | 6 +++--- docs/e2e-tests/CI-medic-guide.md | 2 +- .../playwright/e2e/plugin-division-mode-schema/README.md | 2 +- e2e-tests/playwright/utils/runtime-deploy.ts | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.ci/pipelines/lib/schema-mode-env.sh b/.ci/pipelines/lib/schema-mode-env.sh index 258d434a27..ccedd1bb4d 100755 --- a/.ci/pipelines/lib/schema-mode-env.sh +++ b/.ci/pipelines/lib/schema-mode-env.sh @@ -70,7 +70,7 @@ configure_schema_mode_runtime_env() { if oc get svc "${external_db}-primary" -n "${pdb}" &> /dev/null; then forward_namespace="${pdb}" log::info "Schema-mode (${install_method}): no in-cluster Postgres Service in ${runtime_namespace}; using external PostgreSQL in ${pdb}" - local external_db_secret="${external_db}-pguser-janus-idp" + local external_db_secret="${external_db}-pguser-rhdh" if oc get secret "${external_db_secret}" -n "${pdb}" &> /dev/null; then admin_password=$(oc get secret "${external_db_secret}" -n "${pdb}" -o jsonpath='{.data.password}' 2> /dev/null | base64 -d || true) fi @@ -109,7 +109,7 @@ configure_schema_mode_runtime_env() { export SCHEMA_MODE_PORT_FORWARD_RESOURCE="${pf_target}" if [[ "${forward_via_pod}" -eq 1 ]]; then - export SCHEMA_MODE_DB_ADMIN_USER="${SCHEMA_MODE_DB_ADMIN_USER:-janus-idp}" + export SCHEMA_MODE_DB_ADMIN_USER="${SCHEMA_MODE_DB_ADMIN_USER:-rhdh}" else export SCHEMA_MODE_DB_ADMIN_USER="${SCHEMA_MODE_DB_ADMIN_USER:-postgres}" fi diff --git a/.ci/pipelines/resources/postgres-db/postgres.yaml b/.ci/pipelines/resources/postgres-db/postgres.yaml index e42e5d5db7..c8acdab60b 100644 --- a/.ci/pipelines/resources/postgres-db/postgres.yaml +++ b/.ci/pipelines/resources/postgres-db/postgres.yaml @@ -1,6 +1,6 @@ # External PostgreSQL used by the showcase-rbac tests. Deployed by # configure_external_postgres_db in utils.sh, which also creates the -# postgress-external-db-cluster-cert, postgress-external-db-pguser-janus-idp +# postgress-external-db-cluster-cert, postgress-external-db-pguser-rhdh # and rh-pull-secret secrets referenced below. apiVersion: v1 kind: ConfigMap @@ -114,17 +114,17 @@ spec: - name: POSTGRESQL_ADMIN_PASSWORD valueFrom: secretKeyRef: - name: postgress-external-db-pguser-janus-idp + name: postgress-external-db-pguser-rhdh key: postgres-password - name: RHDH_DB_USER valueFrom: secretKeyRef: - name: postgress-external-db-pguser-janus-idp + name: postgress-external-db-pguser-rhdh key: user - name: RHDH_DB_PASSWORD valueFrom: secretKeyRef: - name: postgress-external-db-pguser-janus-idp + name: postgress-external-db-pguser-rhdh key: password readinessProbe: exec: diff --git a/.ci/pipelines/utils.sh b/.ci/pipelines/utils.sh index 4e059c0b3b..a95648d9ea 100755 --- a/.ci/pipelines/utils.sh +++ b/.ci/pipelines/utils.sh @@ -175,7 +175,7 @@ create_postgres_cred_secret() { if ! oc create secret generic postgres-cred \ --from-literal=POSTGRES_PASSWORD="${password}" \ --from-literal=POSTGRES_PORT="5432" \ - --from-literal=POSTGRES_USER="janus-idp" \ + --from-literal=POSTGRES_USER="rhdh" \ --from-literal=POSTGRES_HOST="${host}" \ --from-literal=PGSSLMODE="require" \ --from-literal=NODE_EXTRA_CA_CERTS="/opt/app-root/src/postgres-crt.pem" \ @@ -219,7 +219,7 @@ configure_external_postgres_db() { local db_namespace="${NAME_SPACE_POSTGRES_DB}" local service="${db_name}-primary" local cert_secret="${db_name}-cluster-cert" - local user_secret="${db_name}-pguser-janus-idp" + local user_secret="${db_name}-pguser-rhdh" common::require_vars "NAME_SPACE_POSTGRES_DB" "REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON" || return 1 @@ -254,7 +254,7 @@ configure_external_postgres_db() { if oc get secret "${user_secret}" -n "${db_namespace}" &> /dev/null; then log::info "Reusing existing ${user_secret} secret" elif ! oc create secret generic "${user_secret}" \ - --from-literal=user="janus-idp" \ + --from-literal=user="rhdh" \ --from-literal=password="$(openssl rand -hex 24)" \ --from-literal=postgres-password="$(openssl rand -hex 24)" \ --namespace="${db_namespace}"; then diff --git a/docs/e2e-tests/CI-medic-guide.md b/docs/e2e-tests/CI-medic-guide.md index 96a70e1be9..a26ea6013b 100644 --- a/docs/e2e-tests/CI-medic-guide.md +++ b/docs/e2e-tests/CI-medic-guide.md @@ -349,7 +349,7 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t - Helm chart errors (invalid values, missing CRDs) - Pod stuck in `CrashLoopBackOff` (bad config, missing secrets, image pull failure) - Health check timeout (`Failed to reach Backstage after N attempts`) -- PostgreSQL operator fails to create user secret (`postgress-external-db-pguser-janus-idp`) +- External PostgreSQL user secret is missing (`postgress-external-db-pguser-rhdh`) **How to tell**: Search `build-log.txt` for: - `CrashLoopBackOff` -- pod is crash-looping diff --git a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md index b6fd8fea51..fbe53cb814 100644 --- a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md +++ b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md @@ -49,7 +49,7 @@ The test spec starts `oc port-forward` in `beforeAll` (similar to [`verify-redis When the runtime namespace has no Bitnami `*-postgresql` Service, auto-discovery uses the external PostgreSQL StatefulSet in `NAME_SPACE_POSTGRES_DB` (see [`postgres.yaml`](../../../../.ci/pipelines/resources/postgres-db/postgres.yaml)): -- Admin password from `postgress-external-db-pguser-janus-idp` +- Admin password from `postgress-external-db-pguser-rhdh` - Forward target is the Running pod labeled `app.kubernetes.io/name=postgress-external-db` - The names match the StatefulSet deployed by `postgres.yaml`; there is no override diff --git a/e2e-tests/playwright/utils/runtime-deploy.ts b/e2e-tests/playwright/utils/runtime-deploy.ts index dfff5789e3..aca6db11f9 100644 --- a/e2e-tests/playwright/utils/runtime-deploy.ts +++ b/e2e-tests/playwright/utils/runtime-deploy.ts @@ -89,7 +89,7 @@ async function createPlaceholderSecrets(kubeClient: KubeClient, namespace: strin data: { POSTGRES_PASSWORD: base64Encode("tmp"), POSTGRES_PORT: base64Encode("5432"), - POSTGRES_USER: base64Encode("janus-idp"), + POSTGRES_USER: base64Encode("rhdh"), POSTGRES_HOST: base64Encode("tmp"), // internal DB has no TLS PGSSLMODE: base64Encode("disable"), From d6b9712f49cb0b42c72915e4934e29b884d46e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbyn=C4=9Bk=20Dr=C3=A1pela?= Date: Thu, 12 Nov 2026 14:56:40 +0100 Subject: [PATCH 4/4] fix(ci): require TLS on every TCP connection to the external PostgreSQL pg_hba trusted connections from 127.0.0.1 and ::1, so anything entering over loopback skipped both TLS and authentication. oc port-forward arrives over loopback, which means the showcase-rbac schema-mode discovery could reach the database in plaintext, contrary to the "requires TLS for every network connection" contract the docs and the RBAC values rely on. Keep the unix socket trusted for the image's start hooks and let every TCP client fall through to the hostssl scram-sha-256 rule. The readiness probe stays healthy: pg_isready counts a password request as "server up". Verified in podman with the unchanged start hook: plaintext loopback is rejected, TLS + password connects, the socket still authenticates, and check-container keeps returning 0. Also correct the docs the previous commits touched: the external PostgreSQL is deployed in Phase 4 with the RBAC instance, not during cluster setup; the operator job's schema tests use the operator-managed database, not the external one; and the cert secret comment no longer claims the operator CRs mount it. Assisted-by: OpenCode --- .ci/pipelines/resources/postgres-db/postgres.yaml | 9 +++++---- .ci/pipelines/utils.sh | 2 +- docs/e2e-tests/CI-medic-guide.md | 10 ++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.ci/pipelines/resources/postgres-db/postgres.yaml b/.ci/pipelines/resources/postgres-db/postgres.yaml index c8acdab60b..32151a0c64 100644 --- a/.ci/pipelines/resources/postgres-db/postgres.yaml +++ b/.ci/pipelines/resources/postgres-db/postgres.yaml @@ -19,11 +19,12 @@ data: # TLS-only rules do not depend on the image's authentication defaults. # Not named *.conf on purpose: the image loads every *.conf in this # directory as a settings file. + # The unix socket is trusted for the image's start hooks; every TCP client, + # including oc port-forward (which arrives over loopback), needs TLS and a + # password. The readiness probe counts a password request as "server up". pg_hba.rules: | - local all all trust - host all all 127.0.0.1/32 trust - host all all ::1/128 trust - hostssl all all all scram-sha-256 + local all all trust + hostssl all all all scram-sha-256 --- apiVersion: v1 kind: ConfigMap diff --git a/.ci/pipelines/utils.sh b/.ci/pipelines/utils.sh index a95648d9ea..242012329c 100755 --- a/.ci/pipelines/utils.sh +++ b/.ci/pipelines/utils.sh @@ -276,7 +276,7 @@ configure_external_postgres_db() { return 1 fi - # Copy the certificates to the RHDH namespace (mounted by the RBAC values and operator CRs) + # Copy the certificates to the RHDH namespace (mounted by the showcase-rbac Helm values) if ! oc extract "secret/${cert_secret}" -n "${db_namespace}" --to="${cert_dir}" --confirm > /dev/null; then log::error "Failed to extract certificates from ${cert_secret}" return 1 diff --git a/docs/e2e-tests/CI-medic-guide.md b/docs/e2e-tests/CI-medic-guide.md index a26ea6013b..e9c0dd0016 100644 --- a/docs/e2e-tests/CI-medic-guide.md +++ b/docs/e2e-tests/CI-medic-guide.md @@ -314,18 +314,16 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t ### Phase 3: Cluster Setup (Prerequisites) **What happens**: The [test script](../../.ci/pipelines/openshift-ci-tests.sh) prepares the cluster before deployment: -- OCP jobs install no operators. The external PostgreSQL used by `showcase-rbac` is a plain StatefulSet (see [postgres.yaml](../../.ci/pipelines/resources/postgres-db/postgres.yaml)) +- OCP jobs need no cluster setup. The RHDH operator (operator jobs only) is installed later, in Phase 4 - AKS/EKS/GKE jobs install Tekton Pipelines for the topology tests (see [operators.sh](../../.ci/pipelines/lib/operators.sh)) **What can go wrong**: - Tekton webhook deployment not ready (Kubernetes jobs only) -- External PostgreSQL StatefulSet not becoming ready (image pull, TLS secret, PVC) **How to tell**: Search `build-log.txt` for: - `Tekton` or `pipeline` related errors early in the log -- `External PostgreSQL did not become ready` -**Action**: Usually infrastructure -- re-trigger. If a recent change touched the database setup, investigate compatibility. +**Action**: Usually infrastructure -- re-trigger. If a recent change touched the operator installation, investigate compatibility. ### Phase 4: RHDH Deployment @@ -349,7 +347,7 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t - Helm chart errors (invalid values, missing CRDs) - Pod stuck in `CrashLoopBackOff` (bad config, missing secrets, image pull failure) - Health check timeout (`Failed to reach Backstage after N attempts`) -- External PostgreSQL user secret is missing (`postgress-external-db-pguser-rhdh`) +- External PostgreSQL for `showcase-rbac` not ready (image pull from `registry.redhat.io`, PVC, TLS secret) **How to tell**: Search `build-log.txt` for: - `CrashLoopBackOff` -- pod is crash-looping @@ -417,7 +415,7 @@ Same as OCP nightly but deploys RHDH using the Operator instead of Helm. See [`o **Key differences**: - Installs RHDH Operator and waits for `backstages.rhdh.redhat.com` CRD (300s timeout) - Uses Backstage CR (`rhdh-start.yaml`) instead of Helm release -- Runtime config tests enabled, including `pluginDivisionMode: schema` tests with the external PostgreSQL +- Runtime config tests enabled, including `pluginDivisionMode: schema` tests against the PostgreSQL the operator deploys in the runtime namespace (not the `showcase-rbac` external DB) ### OCP PR Check (`ocp-pull`)