diff --git a/.ci/pipelines/install-methods/operator.sh b/.ci/pipelines/install-methods/operator.sh index d6f3df5b58..c467c08831 100755 --- a/.ci/pipelines/install-methods/operator.sh +++ b/.ci/pipelines/install-methods/operator.sh @@ -53,17 +53,6 @@ prepare_operator() { k8s_wait::crd "backstages.rhdh.redhat.com" 300 10 || return 1 } -# Waits for the Crunchy Data PostgreSQL Operator's PostgresCluster CRD to become available. -# Must be called after the Crunchy DB CRD is created and before RHDH is deployed -# with internal DB disabled and configured to use Crunchy DB as the external PostgreSQL database. -wait_for_crunchy_crd() { - log::info "Verifying PostgresCluster CRD is available before deploying Backstage CR..." - k8s_wait::crd "postgresclusters.postgres-operator.crunchydata.com" 60 5 || { - log::error "PostgresCluster CRD not available - operator won't be able to create internal database" - return 1 - } -} - deploy_rhdh_operator() { local namespace=$1 local backstage_crd_path=$2 @@ -95,11 +84,9 @@ deploy_rhdh_operator() { return 0 fi - # Wait for the operator to create the database resource (5 minutes max) - # The operator can create either PostgresCluster (Crunchy) or StatefulSet (built-in) + # Wait for the operator to create the database StatefulSet (5 minutes max) if ! common::poll_until \ - "oc get postgrescluster -n '$namespace' --no-headers 2>/dev/null | grep -q 'backstage-psql' || \ - oc get statefulset -n '$namespace' --no-headers 2>/dev/null | grep -q 'backstage-psql'" \ + "oc get statefulset -n '$namespace' --no-headers 2>/dev/null | grep -q 'backstage-psql'" \ 60 5 "Database resource created by operator"; then log::error "Database resource not created after 5 minutes" _operator_debug_info "$namespace" @@ -118,8 +105,6 @@ _operator_debug_info() { oc logs -n "${OPERATOR_MANAGER:-rhdh-operator}" -l control-plane=controller-manager --tail=50 || true log::info "Checking for StatefulSet..." oc get statefulset -n "$namespace" || true - log::info "Checking for PostgresCluster..." - oc get postgrescluster -n "$namespace" 2> /dev/null || echo "No PostgresCluster CRD or resources found" } delete_rhdh_operator() { diff --git a/.ci/pipelines/jobs/ocp-operator.sh b/.ci/pipelines/jobs/ocp-operator.sh index 1187d73914..349a8aaaad 100644 --- a/.ci/pipelines/jobs/ocp-operator.sh +++ b/.ci/pipelines/jobs/ocp-operator.sh @@ -35,7 +35,6 @@ initiate_operator_deployments() { apply_yaml_files "${DIR}" "${NAME_SPACE_RBAC}" "${rbac_rhdh_base_url}" config::create_dynamic_plugins_config "${DIR}/value_files/${HELM_CHART_RBAC_VALUE_FILE_NAME}" "/tmp/configmap-dynamic-plugins-rbac.yaml" oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" - wait_for_crunchy_crd || return 1 deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" } @@ -63,20 +62,19 @@ initiate_operator_deployments_osd_gcp() { common::save_artifact "${PW_PROJECT_SHOWCASE_OPERATOR_RBAC}" "/tmp/configmap-dynamic-plugins-rbac.yaml" oc apply -f /tmp/configmap-dynamic-plugins-rbac.yaml -n "${NAME_SPACE_RBAC}" - wait_for_crunchy_crd || return 1 deploy_rhdh_operator "${NAME_SPACE_RBAC}" "${DIR}/resources/rhdh-operator/rhdh-start-rbac.yaml" } run_operator_runtime_config_change_tests() { # Deploy `showcase-runtime` to run tests that require configuration changes at runtime. - # Uses enableLocalDb=false with external Crunchy PostgreSQL for both runtime and schema-mode tests. + # Uses enableLocalDb=false with the external PostgreSQL for both runtime and schema-mode tests. namespace::configure "${NAME_SPACE_RUNTIME}" local runtime_url="https://backstage-${RELEASE_NAME}-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}" local postgres_ready postgres_ready=false - # Set up real external PostgreSQL (Crunchy) instead of placeholder secrets. + # Set up the real external PostgreSQL instead of placeholder secrets. # Creates postgres-cred and postgres-crt secrets in NAME_SPACE_RUNTIME. # IMPORTANT: Must be called AFTER namespace is created but BEFORE operator deployment. namespace::configure "${NAME_SPACE_POSTGRES_DB}" diff --git a/.ci/pipelines/lib/README.md b/.ci/pipelines/lib/README.md index fb8252b2a6..9bbcffe765 100644 --- a/.ci/pipelines/lib/README.md +++ b/.ci/pipelines/lib/README.md @@ -27,9 +27,8 @@ Functions: `k8s_wait::deployment`, `k8s_wait::job`, `k8s_wait::service`, `k8s_wa Operator and OLM installations. -Functions: `operator::install_subscription`, `operator::check_status`, -`operator::install_postgres_ocp`, `operator::install_postgres_k8s`, `operator::install_pipelines`, -`operator::install_olm`, `operator::uninstall_olm` +Functions: `operator::install_subscription`, `operator::install_pipelines`, +`operator::install_tekton`, `operator::install_olm`, `operator::uninstall_olm` ### `helm.sh` diff --git a/.ci/pipelines/lib/operators.sh b/.ci/pipelines/lib/operators.sh index cfec0949b9..0630f977d2 100644 --- a/.ci/pipelines/lib/operators.sh +++ b/.ci/pipelines/lib/operators.sh @@ -10,7 +10,6 @@ fi readonly OPERATORS_LIB_SOURCED=1 # Constants -readonly OPERATOR_STATUS_SUCCEEDED="Succeeded" readonly OPERATOR_NAMESPACE="openshift-operators" # Create OpenShift Operator subscription @@ -39,39 +38,6 @@ EOD return $? } -# Monitor operator status until expected phase is reached -# Args: timeout_seconds, namespace, operator_name, expected_status (default: "Succeeded") -operator::check_status() { - local timeout=${1:-300} - local namespace=$2 - local operator_name=$3 - local expected_status=${4:-${OPERATOR_STATUS_SUCCEEDED}} - - log::info "Checking the status of operator '${operator_name}' in namespace '${namespace}' with a timeout of ${timeout} seconds." - log::info "Expected status: ${expected_status}" - - timeout "${timeout}" bash -c " - while true; do - CURRENT_PHASE=\$(oc get csv -n '${namespace}' -o jsonpath='{.items[?(@.spec.displayName==\"${operator_name}\")].status.phase}') - echo \"Operator '${operator_name}' current phase: \${CURRENT_PHASE}\" - [[ \"\${CURRENT_PHASE}\" == \"${expected_status}\" ]] && echo \"Operator '${operator_name}' is now in '${expected_status}' phase.\" && break - sleep 10 - done - " || log::error "Timed out after ${timeout} seconds. Operator '${operator_name}' did not reach '${expected_status}' phase." - return $? -} - -# Install Crunchy Postgres Operator from OpenShift Marketplace -operator::install_postgres_ocp() { - operator::install_subscription crunchy-postgres-operator "${OPERATOR_NAMESPACE}" v5 crunchy-postgres-operator certified-operators openshift-marketplace - operator::check_status 300 "${OPERATOR_NAMESPACE}" "Crunchy Postgres for Kubernetes" "${OPERATOR_STATUS_SUCCEEDED}" || return 1 - - # Wait for PostgresCluster CRD to be registered - log::info "Waiting for PostgresCluster CRD to be registered..." - k8s_wait::crd "postgresclusters.postgres-operator.crunchydata.com" 120 5 || return 1 - return 0 -} - # Install Red Hat OpenShift Pipelines operator if not present operator::install_pipelines() { local display_name="Red Hat OpenShift Pipelines" diff --git a/.ci/pipelines/lib/schema-mode-env.sh b/.ci/pipelines/lib/schema-mode-env.sh index ceae6eae2a..552b3723ec 100755 --- a/.ci/pipelines/lib/schema-mode-env.sh +++ b/.ci/pipelines/lib/schema-mode-env.sh @@ -58,24 +58,25 @@ configure_schema_mode_runtime_env() { done else local pdb="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}" - local crunchy_cluster="${SCHEMA_MODE_CRUNCHY_CLUSTER_NAME:-postgress-external-db}" - if oc get svc postgress-external-db-primary -n "${pdb}" &> /dev/null; then + # Name and labels deployed by resources/postgres-db/postgres.yaml + local external_db="postgress-external-db" + if oc get svc "${external_db}-primary" -n "${pdb}" &> /dev/null; then forward_namespace="${pdb}" - log::info "Schema-mode (${install_method}): no in-cluster Postgres Service in ${runtime_namespace}; using Crunchy cluster in ${pdb}" - local crunchy_admin_secret="${crunchy_cluster}-pguser-janus-idp" - if oc get secret "${crunchy_admin_secret}" -n "${pdb}" &> /dev/null; then - admin_password=$(oc get secret "${crunchy_admin_secret}" -n "${pdb}" -o jsonpath='{.data.password}' 2> /dev/null | base64 -d || true) + log::info "Schema-mode (${install_method}): no in-cluster Postgres Service in ${runtime_namespace}; using external PostgreSQL in ${pdb}" + local external_db_secret="${external_db}-pguser-rhdh" + if oc get secret "${external_db_secret}" -n "${pdb}" &> /dev/null; then + admin_password=$(oc get secret "${external_db_secret}" -n "${pdb}" -o jsonpath='{.data.password}' 2> /dev/null | base64 -d || true) fi if [[ -z "${admin_password}" ]]; then - log::warn "Schema-mode (${install_method}): could not read ${crunchy_admin_secret} password in ${pdb}; schema tests remain opt-in." + log::warn "Schema-mode (${install_method}): could not read ${external_db_secret} password in ${pdb}; schema tests remain opt-in." return 1 fi postgres_service=$(oc get pods -n "${pdb}" \ - -l "postgres-operator.crunchydata.com/cluster=${crunchy_cluster},postgres-operator.crunchydata.com/data=postgres" \ + -l "app.kubernetes.io/name=${external_db}" \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].metadata.name}' 2> /dev/null) if [[ -z "${postgres_service}" ]]; then - log::warn "Schema-mode (${install_method}): no Running Postgres pod in ${pdb} for cluster ${crunchy_cluster}; schema tests remain opt-in." + log::warn "Schema-mode (${install_method}): no Running Postgres pod in ${pdb} for ${external_db}; schema tests remain opt-in." return 1 fi forward_via_pod=1 @@ -101,7 +102,7 @@ configure_schema_mode_runtime_env() { export SCHEMA_MODE_PORT_FORWARD_RESOURCE="${pf_target}" if [[ "${forward_via_pod}" -eq 1 ]]; then - export SCHEMA_MODE_DB_ADMIN_USER="${SCHEMA_MODE_DB_ADMIN_USER:-janus-idp}" + export SCHEMA_MODE_DB_ADMIN_USER="${SCHEMA_MODE_DB_ADMIN_USER:-rhdh}" else export SCHEMA_MODE_DB_ADMIN_USER="${SCHEMA_MODE_DB_ADMIN_USER:-postgres}" fi diff --git a/.ci/pipelines/resources/postgres-db/postgres.yaml b/.ci/pipelines/resources/postgres-db/postgres.yaml index e7e94b8ee2..4e15c1f35a 100644 --- a/.ci/pipelines/resources/postgres-db/postgres.yaml +++ b/.ci/pipelines/resources/postgres-db/postgres.yaml @@ -1,74 +1,180 @@ -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster +# External PostgreSQL used by the showcase-rbac tests. Deployed by +# configure_external_postgres_db in utils.sh, which also creates the +# postgress-external-db-cluster-cert, postgress-external-db-pguser-rhdh +# and rh-pull-secret secrets referenced below. +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgress-external-db-config +data: + # Merged into the image-generated postgresql.conf on every start. + ssl.conf: | + ssl = on + ssl_cert_file = '/opt/app-root/src/certs/tls.crt' + ssl_key_file = '/opt/app-root/src/certs/tls.key' + ssl_ecdh_curve = 'prime256v1' + hba_file = '/opt/app-root/src/postgresql-cfg/pg_hba.rules' + # Replaces the image-generated pg_hba.conf instead of rewriting it, so the + # TLS-only rules do not depend on the image's authentication defaults. + # Not named *.conf on purpose: the image loads every *.conf in this + # directory as a settings file. + # The unix socket is trusted for the image's start hooks; every TCP client, + # including oc port-forward (which arrives over loopback), needs TLS and a + # password. The readiness probe counts a password request as "server up". + pg_hba.rules: | + local all all trust + hostssl all all all scram-sha-256 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgress-external-db-start +data: + rhdh-setup.sh: | + # Sourced by the image's run-postgresql on every start, while the server is + # listening on the unix socket only. TLS and authentication are configured + # by the pg_hba.rules/ssl.conf keys of postgress-external-db-config. + psql -v ON_ERROR_STOP=1 -v user="${RHDH_DB_USER}" -v password="${RHDH_DB_PASSWORD}" << 'EOSQL' + SELECT format('CREATE ROLE %I', :'user') WHERE NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'user') \gexec + ALTER ROLE :"user" WITH LOGIN SUPERUSER PASSWORD :'password'; + EOSQL + unset RHDH_DB_USER RHDH_DB_PASSWORD +--- +apiVersion: v1 +kind: Service +metadata: + name: postgress-external-db-primary + labels: + app.kubernetes.io/name: postgress-external-db +spec: + selector: + app.kubernetes.io/name: postgress-external-db + ports: + - name: postgres + port: 5432 + targetPort: postgres +--- +apiVersion: apps/v1 +kind: StatefulSet metadata: name: postgress-external-db + labels: + app.kubernetes.io/name: postgress-external-db spec: - image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.7-2547 - postgresVersion: 17 - instances: - - name: instance1 - dataVolumeClaimSpec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: 2Gi - resources: - limits: - cpu: 300m - requests: - cpu: 200m - sidecars: - replicaCertCopy: + serviceName: postgress-external-db-primary + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgress-external-db + template: + metadata: + labels: + app.kubernetes.io/name: postgress-external-db + spec: + imagePullSecrets: + - name: rh-pull-secret + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + initContainers: + # PostgreSQL refuses a private key that is readable by others, so copy + # the certificates to a volume owned by the container user. + - name: copy-certs + image: registry.redhat.io/rhel9/postgresql-16:latest + command: + - /bin/sh + - -c + - cp /tls/ca.crt /tls/tls.crt /tls/tls.key /certs/ && chmod 0600 /certs/tls.key + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL resources: - limits: - cpu: 300m requests: - cpu: 200m - backups: - pgbackrest: - image: registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi9-2.56.0-2547 - global: - # Save backups for 7 days, this means 1 full backups with 6 differential ones in between - repo1-retention-full: "1" - repo1-retention-full-type: count - repoHost: - resources: - limits: - cpu: 300m - requests: - cpu: 200m - repos: - - name: repo1 - schedules: - # Every sunday at 01:00 full backup - full: "0 1 * * 0" - # Monday through saturday at 01:00 differential backup - differential: "0 1 * * 1-6" - volume: - volumeClaimSpec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: 2Gi - sidecars: - pgbackrest: - resources: + cpu: 10m + memory: 32Mi limits: - cpu: 200m - requests: cpu: 100m - pgbackrestConfig: + memory: 64Mi + volumeMounts: + - name: tls-secret + mountPath: /tls + readOnly: true + - name: certs + mountPath: /certs + containers: + - name: postgres + image: registry.redhat.io/rhel9/postgresql-16:latest + ports: + - name: postgres + containerPort: 5432 + env: + - name: POSTGRESQL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: postgress-external-db-pguser-rhdh + key: postgres-password + - name: RHDH_DB_USER + valueFrom: + secretKeyRef: + name: postgress-external-db-pguser-rhdh + key: user + - name: RHDH_DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgress-external-db-pguser-rhdh + key: password + readinessProbe: + exec: + command: + - /usr/libexec/check-container + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL resources: - limits: - cpu: 300m requests: cpu: 200m - users: - - name: janus-idp - options: "SUPERUSER" - - name: sonarqube - databases: - - sonarqube - options: "NOSUPERUSER" + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: data + mountPath: /var/lib/pgsql/data + - name: certs + mountPath: /opt/app-root/src/certs + readOnly: true + - name: config + mountPath: /opt/app-root/src/postgresql-cfg + readOnly: true + - name: start + mountPath: /opt/app-root/src/postgresql-start + readOnly: true + volumes: + - name: tls-secret + secret: + secretName: postgress-external-db-cluster-cert + - name: certs + emptyDir: {} + - name: config + configMap: + name: postgress-external-db-config + - name: start + configMap: + name: postgress-external-db-start + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/.ci/pipelines/utils.sh b/.ci/pipelines/utils.sh index 227869f203..639afeac21 100755 --- a/.ci/pipelines/utils.sh +++ b/.ci/pipelines/utils.sh @@ -156,37 +156,6 @@ save_all_pod_logs() { set -e } -# ============================================================================== -# Operator Functions - Delegate to lib/operators.sh -# ============================================================================== -install_subscription() { operator::install_subscription "$@"; } - -check_operator_status() { operator::check_status "$@"; } - -# Installs the Crunchy Postgres Operator -# Args: platform ("ocp" or "k8s", default: "ocp") -install_crunchy_postgres_operator() { - local platform=${1:-ocp} - install_subscription crunchy-postgres-operator openshift-operators v5 crunchy-postgres-operator certified-operators openshift-marketplace -} - -# Waits for the Crunchy Postgres Operator to be ready -# Args: platform ("ocp" or "k8s", default: "ocp") -waitfor_crunchy_postgres_operator() { - local platform=${1:-ocp} - local namespace="openshift-operators" - [[ "$platform" == "k8s" ]] && namespace="operators" - - check_operator_status 300 "$namespace" "Crunchy Postgres for Kubernetes" "Succeeded" - k8s_wait::crd "postgresclusters.postgres-operator.crunchydata.com" 120 5 || return 1 -} - -# Backward compatibility shims -install_crunchy_postgres_ocp_operator() { install_crunchy_postgres_operator "ocp"; } -install_crunchy_postgres_k8s_operator() { install_crunchy_postgres_operator "k8s"; } -waitfor_crunchy_postgres_ocp_operator() { waitfor_crunchy_postgres_operator "ocp"; } -waitfor_crunchy_postgres_k8s_operator() { waitfor_crunchy_postgres_operator "k8s"; } - # Create the postgres-cred secret in a target namespace. # Accepts key=value pairs as positional args for additional/override literals. # Usage: create_postgres_cred_secret [extra_key=value ...] @@ -202,7 +171,7 @@ create_postgres_cred_secret() { if ! oc create secret generic postgres-cred \ --from-literal=POSTGRES_PASSWORD="${password}" \ --from-literal=POSTGRES_PORT="5432" \ - --from-literal=POSTGRES_USER="janus-idp" \ + --from-literal=POSTGRES_USER="rhdh" \ --from-literal=POSTGRES_HOST="${host}" \ --from-literal=PGSSLMODE="require" \ --from-literal=NODE_EXTRA_CA_CERTS="/opt/app-root/src/postgres-crt.pem" \ @@ -213,47 +182,106 @@ create_postgres_cred_secret() { fi } +# Generate a CA and a server certificate for the external PostgreSQL Service. +# Usage: generate_external_postgres_certs +# Writes ca.crt, tls.crt and tls.key to output_dir. +generate_external_postgres_certs() { + local out_dir=$1 service=$2 namespace=$3 + + openssl ecparam -name prime256v1 -genkey -noout -out "${out_dir}/ca.key" || return 1 + openssl req -x509 -new -key "${out_dir}/ca.key" -sha256 -days 30 \ + -subj "/O=RHDH-CI/CN=RHDH CI external PostgreSQL CA" \ + -out "${out_dir}/ca.crt" || return 1 + + openssl ecparam -name prime256v1 -genkey -noout -out "${out_dir}/tls.key" || return 1 + openssl req -new -key "${out_dir}/tls.key" -subj "/O=RHDH-CI/CN=${service}" \ + -out "${out_dir}/tls.csr" || return 1 + cat > "${out_dir}/tls_ext.cnf" << EOF +basicConstraints = CA:FALSE +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth +subjectAltName = DNS:${service}, DNS:${service}.${namespace}, DNS:${service}.${namespace}.svc, DNS:${service}.${namespace}.svc.cluster.local +EOF + openssl x509 -req -in "${out_dir}/tls.csr" -CA "${out_dir}/ca.crt" -CAkey "${out_dir}/ca.key" \ + -CAcreateserial -days 30 -sha256 -extfile "${out_dir}/tls_ext.cnf" \ + -out "${out_dir}/tls.crt" || return 1 +} + +# Deploy the external PostgreSQL (registry.redhat.io/rhel9/postgresql-16) with TLS +# in NAME_SPACE_POSTGRES_DB and create the connection secrets in the given project. configure_external_postgres_db() { local project=$1 - local max_attempts=60 # 5 minutes total (60 attempts × 5 seconds) - local wait_interval=5 - - log::info "Creating PostgresCluster in namespace ${NAME_SPACE_POSTGRES_DB}..." + local db_name="postgress-external-db" + local db_namespace="${NAME_SPACE_POSTGRES_DB}" + local service="${db_name}-primary" + local cert_secret="${db_name}-cluster-cert" + local user_secret="${db_name}-pguser-rhdh" + + common::require_vars "NAME_SPACE_POSTGRES_DB" "REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON" || return 1 + + local cert_dir + cert_dir=$(mktemp -d) + + # The image is pulled from registry.redhat.io, which requires authentication. + namespace::create_dockerconfigjson_secret "${db_namespace}" "rh-pull-secret" "${REGISTRY_REDHAT_IO_SERVICE_ACCOUNT_DOCKERCONFIGJSON}" || return 1 + + # Reuse existing certificates and credentials so a second call in the same job + # (release-1.10 reuses the database created for the RBAC instance when setting + # up the runtime namespace) does not rotate the credentials of a running + # database. + if oc get secret "${cert_secret}" -n "${db_namespace}" &> /dev/null; then + log::info "Reusing existing ${cert_secret} secret" + else + log::info "Generating TLS certificates for ${service}.${db_namespace}" + if ! generate_external_postgres_certs "${cert_dir}" "${service}" "${db_namespace}"; then + log::error "Failed to generate PostgreSQL TLS certificates" + return 1 + fi + if ! oc create secret generic "${cert_secret}" \ + --from-file=ca.crt="${cert_dir}/ca.crt" \ + --from-file=tls.crt="${cert_dir}/tls.crt" \ + --from-file=tls.key="${cert_dir}/tls.key" \ + --namespace="${db_namespace}"; then + log::error "Failed to create ${cert_secret} secret" + return 1 + fi + fi - # Validate oc apply command execution - if ! oc apply -f "${DIR}/resources/postgres-db/postgres.yaml" --namespace="${NAME_SPACE_POSTGRES_DB}"; then - log::error "Failed to create PostgresCluster" + if oc get secret "${user_secret}" -n "${db_namespace}" &> /dev/null; then + log::info "Reusing existing ${user_secret} secret" + elif ! oc create secret generic "${user_secret}" \ + --from-literal=user="rhdh" \ + --from-literal=password="$(openssl rand -hex 24)" \ + --from-literal=postgres-password="$(openssl rand -hex 24)" \ + --namespace="${db_namespace}"; then + log::error "Failed to create ${user_secret} secret" return 1 fi - # Wait for cluster cert secret (usually created quickly) - log::info "Waiting for cluster certificate secret..." - if ! common::poll_until \ - "oc get secret postgress-external-db-cluster-cert -n '${NAME_SPACE_POSTGRES_DB}'" \ - "$max_attempts" "$wait_interval" \ - "Cluster certificate secret found"; then + log::info "Deploying external PostgreSQL in namespace ${db_namespace}..." + if ! oc apply -f "${DIR}/resources/postgres-db/postgres.yaml" --namespace="${db_namespace}"; then + log::error "Failed to apply external PostgreSQL resources" return 1 fi - # Extract cluster certificates - oc get secret postgress-external-db-cluster-cert -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.ca\.crt}' | base64 --decode > postgres-ca || { - log::error "Failed to extract ca.crt" + if ! oc rollout status "statefulset/${db_name}" -n "${db_namespace}" --timeout=10m; then + log::error "External PostgreSQL did not become ready" + oc get pods -n "${db_namespace}" -o wide || true + oc describe pod "${db_name}-0" -n "${db_namespace}" || true + oc logs "${db_name}-0" -n "${db_namespace}" --all-containers --tail=100 || true return 1 - } - oc get secret postgress-external-db-cluster-cert -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.tls\.crt}' | base64 --decode > postgres-tls-crt || { - log::error "Failed to extract tls.crt" - return 1 - } - oc get secret postgress-external-db-cluster-cert -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.tls\.key}' | base64 --decode > postgres-tls-key || { - log::error "Failed to extract tls.key" + fi + + # Copy the certificates to the RHDH namespace (mounted by the showcase-rbac Helm values) + if ! oc extract "secret/${cert_secret}" -n "${db_namespace}" --to="${cert_dir}" --confirm > /dev/null; then + log::error "Failed to extract certificates from ${cert_secret}" return 1 - } + fi - # Validate secret creation - if ! oc create secret generic postgress-external-db-cluster-cert \ - --from-file=ca.crt=postgres-ca \ - --from-file=tls.crt=postgres-tls-crt \ - --from-file=tls.key=postgres-tls-key \ + if ! oc create secret generic "${cert_secret}" \ + --from-file=ca.crt="${cert_dir}/ca.crt" \ + --from-file=tls.crt="${cert_dir}/tls.crt" \ + --from-file=tls.key="${cert_dir}/tls.key" \ --dry-run=client -o yaml | oc apply -f - --namespace="${project}"; then log::error "Failed to create cluster certificate secret" return 1 @@ -262,43 +290,22 @@ configure_external_postgres_db() { # Create postgres-crt secret for Backstage deployment (Helm/Operator) # This secret is referenced by rhdh-start-runtime.yaml and values-showcase-postgres.yaml if ! oc create secret generic postgres-crt \ - --from-file=postgres-crt.pem=postgres-ca \ + --from-file=postgres-crt.pem="${cert_dir}/ca.crt" \ --dry-run=client -o yaml | oc apply -f - --namespace="${project}"; then log::error "Failed to create postgres-crt secret" return 1 fi + rm -rf "${cert_dir}" - # Wait for USER secret (this is the critical one that causes CI failures!) - log::info "Waiting for PostgreSQL user secret 'postgress-external-db-pguser-janus-idp'..." - log::info "This secret is created by the Crunchy Postgres operator after the database is ready" - if ! common::poll_until \ - "oc get secret postgress-external-db-pguser-janus-idp -n '${NAME_SPACE_POSTGRES_DB}'" \ - "$max_attempts" "$wait_interval" \ - "PostgreSQL user secret found"; then - log::error "This usually means the Crunchy Postgres operator failed to create the user" - log::info "Checking PostgresCluster status..." - oc describe postgrescluster postgress-external-db -n "${NAME_SPACE_POSTGRES_DB}" || true - log::info "Checking operator logs..." - oc logs -n "${NAME_SPACE_POSTGRES_DB}" -l postgres-operator.crunchydata.com/cluster=postgress-external-db --tail=50 || true + local pg_password pg_host + if ! pg_password=$(oc get secret "${user_secret}" -n "${db_namespace}" -o go-template='{{.data.password | base64decode}}') \ + || [[ -z "${pg_password}" ]]; then + log::error "Failed to read the password from ${user_secret}" return 1 fi + pg_host="${service}.${db_namespace}.svc.cluster.local" - # Wait for PostgreSQL pods to be Running (needed for schema-mode tests) - log::info "Waiting for PostgreSQL pods to be Running..." - if ! common::poll_until \ - "oc get pods -n '${NAME_SPACE_POSTGRES_DB}' -l 'postgres-operator.crunchydata.com/cluster=postgress-external-db,postgres-operator.crunchydata.com/data=postgres' --field-selector=status.phase=Running -o name 2>/dev/null | grep -q pod/" \ - "$max_attempts" "$wait_interval" \ - "PostgreSQL pod is Running"; then - log::warn "PostgreSQL pod not Running yet; schema-mode tests may skip" - # Don't fail here - the database might work fine, just schema tests won't run - fi - - # Now we can safely get the password (decode from base64 since K8s .data is always base64-encoded) - local pg_password pg_host - pg_password=$(oc get secret/postgress-external-db-pguser-janus-idp -n "${NAME_SPACE_POSTGRES_DB}" -o jsonpath='{.data.password}' | base64 --decode) - pg_host="postgress-external-db-primary.${NAME_SPACE_POSTGRES_DB}.svc.cluster.local" - - create_postgres_cred_secret "${project}" "${pg_host}" "${pg_password}" + create_postgres_cred_secret "${project}" "${pg_host}" "${pg_password}" || return 1 log::success "External PostgreSQL database configured successfully!" } @@ -398,8 +405,6 @@ cluster_setup_ocp_helm() { log::info "Waiting for OpenShift Pipelines to be ready..." k8s_wait::deployment "${OPERATOR_NAMESPACE}" "pipelines" 30 10 || return 1 k8s_wait::endpoint "${PIPELINES_OPERATOR_WEBHOOK}" "openshift-pipelines" 1800 10 || return 1 - - operator::install_postgres_ocp } cluster_setup_ocp_operator() { @@ -409,15 +414,12 @@ cluster_setup_ocp_operator() { log::info "Waiting for OpenShift Pipelines to be ready..." k8s_wait::deployment "${OPERATOR_NAMESPACE}" "pipelines" 30 10 || return 1 k8s_wait::endpoint "${PIPELINES_OPERATOR_WEBHOOK}" "openshift-pipelines" 1800 10 || return 1 - - operator::install_postgres_ocp } cluster_setup_k8s_operator() { operator::install_olm # Install Tekton Pipelines for K8s deployments (AKS/EKS/GKE) for topology tests to work operator::install_tekton - # operator::install_postgres_k8s # Works with K8s but disabled in values file } cluster_setup_k8s_helm() { @@ -425,7 +427,6 @@ cluster_setup_k8s_helm() { log::info "Installing Tekton Pipelines for K8s Helm deployment" # operator::install_olm operator::install_tekton - # operator::install_postgres_k8s # Works with K8s but disabled in values file } # ============================================================================== @@ -459,14 +460,8 @@ rbac_deployment() { namespace::configure "${NAME_SPACE_POSTGRES_DB}" namespace::configure "${NAME_SPACE_RBAC}" - configure_external_postgres_db "${NAME_SPACE_RBAC}" - - # Wait for PostgreSQL to be fully ready before deploying RBAC instance - log::info "Waiting for external PostgreSQL to be ready..." - if ! k8s_wait::deployment "${NAME_SPACE_POSTGRES_DB}" "postgress-external-db" 10 10; then - log::error "PostgreSQL deployment failed to become ready" - return 1 - fi + # Waits for the StatefulSet rollout, so the RBAC instance can connect immediately + configure_external_postgres_db "${NAME_SPACE_RBAC}" || return 1 # Initiate rbac instance deployment. local rbac_rhdh_base_url="https://${RELEASE_NAME_RBAC}-developer-hub-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" @@ -571,7 +566,7 @@ rbac_deployment_osd_gcp() { namespace::configure "${NAME_SPACE_POSTGRES_DB}" namespace::configure "${NAME_SPACE_RBAC}" - configure_external_postgres_db "${NAME_SPACE_RBAC}" + configure_external_postgres_db "${NAME_SPACE_RBAC}" || return 1 # Initiate rbac instance deployment. local rbac_rhdh_base_url="https://${RELEASE_NAME_RBAC}-developer-hub-${NAME_SPACE_RBAC}.${K8S_CLUSTER_ROUTER_BASE}" diff --git a/.claude/commands/pr-triage.md b/.claude/commands/pr-triage.md index 85e3060d52..cb2399be41 100644 --- a/.claude/commands/pr-triage.md +++ b/.claude/commands/pr-triage.md @@ -152,13 +152,13 @@ Use the failure classification from the CI Medic Guide to categorize each failur - "CrashLoopBackOff" - "Failed to reach Backstage" - "helm upgrade.*failed" -- "Crunchy Postgres operator failed to create the user" +- "External PostgreSQL did not become ready" - "ImagePullBackOff" - "FailedScheduling" ``` **Classification:** `DEPLOY_FAILED` with subcategories: -- `DEPLOY_POSTGRES` - PostgreSQL operator issues +- `DEPLOY_POSTGRES` - External PostgreSQL issues - `DEPLOY_HELM` - Helm chart failures - `DEPLOY_OPERATOR` - Operator CRD/deployment issues - `DEPLOY_HEALTH_CHECK` - Health check timeouts @@ -219,7 +219,7 @@ Create a fingerprint for each failure to detect duplicates: **Example:** ``` -DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly:showcase-rbac +DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly:showcase-rbac ``` ### Similarity Detection @@ -232,9 +232,9 @@ Compare fingerprints across all analyzed PRs: **Output:** ``` -PR #4537: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4418: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4501: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-nightly +PR #4537: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4418: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4501: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-nightly → Identical: #4537, #4418 → Related: #4501 (same error, different namespace) @@ -331,8 +331,8 @@ For each detected failure, extract keywords for Jira searching: # Example error: "Timed out waiting for Docker image :pr-4537-9c77192a" # Keywords: "Docker image", "timeout", "waiting" - # Example error: "Crunchy Postgres operator failed to create the user" - # Keywords: "PostgreSQL operator", "create user", "failed" + # Example error: "External PostgreSQL did not become ready" + # Keywords: "External PostgreSQL", "not ready", "StatefulSet" ``` 2. **Build Jira search URLs**: @@ -534,7 +534,7 @@ Reference the CI Medic Guide for known patterns: | Pattern | Classification | Jira Label | Action | |---------|---------------|------------|--------| -| `Crunchy Postgres operator failed to create the user` | DEPLOY_POSTGRES | `postgresql-operator` | Check operator version, re-trigger | +| `External PostgreSQL did not become ready` | DEPLOY_POSTGRES | `external-postgres` | Check StatefulSet pod events/logs, re-trigger | | `Failed to reach Backstage after N attempts` | DEPLOY_HEALTH_CHECK | `health-check-timeout` | Check pod logs, resource limits | | `CrashLoopBackOff` | DEPLOY_POD_CRASH | `deployment-crash` | Investigate pod logs | | `ImagePullBackOff` | DEPLOY_IMAGE_PULL | `image-registry` | Check registry, rate limits | diff --git a/.cursor/commands/pr-triage.md b/.cursor/commands/pr-triage.md index 3f07d665da..20a7c2a8bd 100644 --- a/.cursor/commands/pr-triage.md +++ b/.cursor/commands/pr-triage.md @@ -149,13 +149,13 @@ Use the failure classification from the CI Medic Guide to categorize each failur - "CrashLoopBackOff" - "Failed to reach Backstage" - "helm upgrade.*failed" -- "Crunchy Postgres operator failed to create the user" +- "External PostgreSQL did not become ready" - "ImagePullBackOff" - "FailedScheduling" ``` **Classification:** `DEPLOY_FAILED` with subcategories: -- `DEPLOY_POSTGRES` - PostgreSQL operator issues +- `DEPLOY_POSTGRES` - External PostgreSQL issues - `DEPLOY_HELM` - Helm chart failures - `DEPLOY_OPERATOR` - Operator CRD/deployment issues - `DEPLOY_HEALTH_CHECK` - Health check timeouts @@ -216,7 +216,7 @@ Create a fingerprint for each failure to detect duplicates: **Example:** ``` -DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly:showcase-rbac +DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly:showcase-rbac ``` ### Similarity Detection @@ -229,9 +229,9 @@ Compare fingerprints across all analyzed PRs: **Output:** ``` -PR #4537: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4418: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4501: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-nightly +PR #4537: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4418: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4501: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-nightly → Identical: #4537, #4418 → Related: #4501 (same error, different namespace) @@ -328,8 +328,8 @@ For each detected failure, extract keywords for Jira searching: # Example error: "Timed out waiting for Docker image :pr-4537-9c77192a" # Keywords: "Docker image", "timeout", "waiting" - # Example error: "Crunchy Postgres operator failed to create the user" - # Keywords: "PostgreSQL operator", "create user", "failed" + # Example error: "External PostgreSQL did not become ready" + # Keywords: "External PostgreSQL", "not ready", "StatefulSet" ``` 2. **Build Jira search URLs**: @@ -531,7 +531,7 @@ Reference the CI Medic Guide for known patterns: | Pattern | Classification | Jira Label | Action | |---------|---------------|------------|--------| -| `Crunchy Postgres operator failed to create the user` | DEPLOY_POSTGRES | `postgresql-operator` | Check operator version, re-trigger | +| `External PostgreSQL did not become ready` | DEPLOY_POSTGRES | `external-postgres` | Check StatefulSet pod events/logs, re-trigger | | `Failed to reach Backstage after N attempts` | DEPLOY_HEALTH_CHECK | `health-check-timeout` | Check pod logs, resource limits | | `CrashLoopBackOff` | DEPLOY_POD_CRASH | `deployment-crash` | Investigate pod logs | | `ImagePullBackOff` | DEPLOY_IMAGE_PULL | `image-registry` | Check registry, rate limits | diff --git a/.opencode/command/pr-triage.md b/.opencode/command/pr-triage.md index 85e3060d52..cb2399be41 100644 --- a/.opencode/command/pr-triage.md +++ b/.opencode/command/pr-triage.md @@ -152,13 +152,13 @@ Use the failure classification from the CI Medic Guide to categorize each failur - "CrashLoopBackOff" - "Failed to reach Backstage" - "helm upgrade.*failed" -- "Crunchy Postgres operator failed to create the user" +- "External PostgreSQL did not become ready" - "ImagePullBackOff" - "FailedScheduling" ``` **Classification:** `DEPLOY_FAILED` with subcategories: -- `DEPLOY_POSTGRES` - PostgreSQL operator issues +- `DEPLOY_POSTGRES` - External PostgreSQL issues - `DEPLOY_HELM` - Helm chart failures - `DEPLOY_OPERATOR` - Operator CRD/deployment issues - `DEPLOY_HEALTH_CHECK` - Health check timeouts @@ -219,7 +219,7 @@ Create a fingerprint for each failure to detect duplicates: **Example:** ``` -DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly:showcase-rbac +DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly:showcase-rbac ``` ### Similarity Detection @@ -232,9 +232,9 @@ Compare fingerprints across all analyzed PRs: **Output:** ``` -PR #4537: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4418: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4501: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-nightly +PR #4537: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4418: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4501: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-nightly → Identical: #4537, #4418 → Related: #4501 (same error, different namespace) @@ -331,8 +331,8 @@ For each detected failure, extract keywords for Jira searching: # Example error: "Timed out waiting for Docker image :pr-4537-9c77192a" # Keywords: "Docker image", "timeout", "waiting" - # Example error: "Crunchy Postgres operator failed to create the user" - # Keywords: "PostgreSQL operator", "create user", "failed" + # Example error: "External PostgreSQL did not become ready" + # Keywords: "External PostgreSQL", "not ready", "StatefulSet" ``` 2. **Build Jira search URLs**: @@ -534,7 +534,7 @@ Reference the CI Medic Guide for known patterns: | Pattern | Classification | Jira Label | Action | |---------|---------------|------------|--------| -| `Crunchy Postgres operator failed to create the user` | DEPLOY_POSTGRES | `postgresql-operator` | Check operator version, re-trigger | +| `External PostgreSQL did not become ready` | DEPLOY_POSTGRES | `external-postgres` | Check StatefulSet pod events/logs, re-trigger | | `Failed to reach Backstage after N attempts` | DEPLOY_HEALTH_CHECK | `health-check-timeout` | Check pod logs, resource limits | | `CrashLoopBackOff` | DEPLOY_POD_CRASH | `deployment-crash` | Investigate pod logs | | `ImagePullBackOff` | DEPLOY_IMAGE_PULL | `image-registry` | Check registry, rate limits | diff --git a/.rulesync/commands/pr-triage.md b/.rulesync/commands/pr-triage.md index c99f35f141..f3a4f32d60 100644 --- a/.rulesync/commands/pr-triage.md +++ b/.rulesync/commands/pr-triage.md @@ -153,13 +153,13 @@ Use the failure classification from the CI Medic Guide to categorize each failur - "CrashLoopBackOff" - "Failed to reach Backstage" - "helm upgrade.*failed" -- "Crunchy Postgres operator failed to create the user" +- "External PostgreSQL did not become ready" - "ImagePullBackOff" - "FailedScheduling" ``` **Classification:** `DEPLOY_FAILED` with subcategories: -- `DEPLOY_POSTGRES` - PostgreSQL operator issues +- `DEPLOY_POSTGRES` - External PostgreSQL issues - `DEPLOY_HELM` - Helm chart failures - `DEPLOY_OPERATOR` - Operator CRD/deployment issues - `DEPLOY_HEALTH_CHECK` - Health check timeouts @@ -220,7 +220,7 @@ Create a fingerprint for each failure to detect duplicates: **Example:** ``` -DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly:showcase-rbac +DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly:showcase-rbac ``` ### Similarity Detection @@ -233,9 +233,9 @@ Compare fingerprints across all analyzed PRs: **Output:** ``` -PR #4537: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4418: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-rbac-nightly -PR #4501: DEPLOY_POSTGRES:Crunchy Postgres operator failed:showcase-nightly +PR #4537: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4418: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-rbac-nightly +PR #4501: DEPLOY_POSTGRES:External PostgreSQL did not become ready:showcase-nightly → Identical: #4537, #4418 → Related: #4501 (same error, different namespace) @@ -332,8 +332,8 @@ For each detected failure, extract keywords for Jira searching: # Example error: "Timed out waiting for Docker image :pr-4537-9c77192a" # Keywords: "Docker image", "timeout", "waiting" - # Example error: "Crunchy Postgres operator failed to create the user" - # Keywords: "PostgreSQL operator", "create user", "failed" + # Example error: "External PostgreSQL did not become ready" + # Keywords: "External PostgreSQL", "not ready", "StatefulSet" ``` 2. **Build Jira search URLs**: @@ -535,7 +535,7 @@ Reference the CI Medic Guide for known patterns: | Pattern | Classification | Jira Label | Action | |---------|---------------|------------|--------| -| `Crunchy Postgres operator failed to create the user` | DEPLOY_POSTGRES | `postgresql-operator` | Check operator version, re-trigger | +| `External PostgreSQL did not become ready` | DEPLOY_POSTGRES | `external-postgres` | Check StatefulSet pod events/logs, re-trigger | | `Failed to reach Backstage after N attempts` | DEPLOY_HEALTH_CHECK | `health-check-timeout` | Check pod logs, resource limits | | `CrashLoopBackOff` | DEPLOY_POD_CRASH | `deployment-crash` | Investigate pod logs | | `ImagePullBackOff` | DEPLOY_IMAGE_PULL | `image-registry` | Check registry, rate limits | diff --git a/catalog-entities/resources/pgdb.yaml b/catalog-entities/resources/pgdb.yaml index 1c9132a14e..3eb2b01198 100644 --- a/catalog-entities/resources/pgdb.yaml +++ b/catalog-entities/resources/pgdb.yaml @@ -3,12 +3,12 @@ kind: Resource metadata: name: pgdb title: PostgreSQL cluster - description: PostgreSQL is a powerful, open source object-relational database system. PostgreSQL cluster is created by Crunchy Postgres for Kubernetes operator. We use Postgres as the databse for this showcase. + description: PostgreSQL is a powerful, open source object-relational database system. The cluster runs as a StatefulSet using the RHEL PostgreSQL image. We use Postgres as the database for this showcase. links: - - url: https://www.crunchydata.com/products/crunchy-high-availability-postgresql - title: Crunchy product pages + - url: https://www.postgresql.org/about/ + title: PostgreSQL product pages icon: web - - url: https://access.crunchydata.com/documentation/postgres-operator/v5/ + - url: https://github.com/sclorg/postgresql-container title: Documentation icon: web spec: diff --git a/docs/e2e-tests/CI-medic-guide.md b/docs/e2e-tests/CI-medic-guide.md index d038b0caf1..93f4566453 100644 --- a/docs/e2e-tests/CI-medic-guide.md +++ b/docs/e2e-tests/CI-medic-guide.md @@ -288,20 +288,18 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t ### Phase 3: Cluster Setup (Operators and Prerequisites) **What happens**: The [test script](../../.ci/pipelines/openshift-ci-tests.sh) installs required operators and infrastructure (see [operators.sh](../../.ci/pipelines/lib/operators.sh)): -- OpenShift Pipelines (Tekton) operator -- Crunchy PostgreSQL operator +- OCP jobs install the OpenShift Pipelines (Tekton) operator +- AKS/EKS/GKE jobs install Tekton Pipelines for the topology tests **What can go wrong**: - Operator installation timeout (OperatorHub/Marketplace issues) -- CRD not becoming available - Tekton webhook deployment not ready **How to tell**: Search `build-log.txt` for: - `Failed to install subscription` -- Timeout waiting for operator CRDs - `Tekton` or `pipeline` related errors early in the log -**Action**: Usually infrastructure -- re-trigger. If operators were recently upgraded, investigate compatibility. +**Action**: Usually infrastructure -- re-trigger. If a recent change touched the operator installation, investigate compatibility. ### Phase 4: RHDH Deployment @@ -325,13 +323,13 @@ Open `index.html` in a browser from the GCS artifacts. The report contains per-t - Helm chart errors (invalid values, missing CRDs) - Pod stuck in `CrashLoopBackOff` (bad config, missing secrets, image pull failure) - Health check timeout (`Failed to reach Backstage after N attempts`) -- PostgreSQL operator fails to create user secret (`postgress-external-db-pguser-janus-idp`) +- External PostgreSQL for `showcase-rbac` and `showcase-runtime` not ready (image pull from `registry.redhat.io`, PVC, TLS secret) **How to tell**: Search `build-log.txt` for: - `CrashLoopBackOff` -- pod is crash-looping - `Failed to reach Backstage` -- health check timeout - `helm upgrade` failures -- `Crunchy Postgres operator failed to create the user` -- PostgreSQL setup issue +- `External PostgreSQL did not become ready` -- external PostgreSQL (`postgress-external-db` StatefulSet) setup issue - Check `pod_logs/` for application-level errors **Action**: Check pod logs and events in artifacts. May be a config issue (real bug) or transient infra (re-trigger). @@ -393,7 +391,7 @@ Same as OCP nightly but deploys RHDH using the Operator instead of Helm. See [`o **Key differences**: - Installs RHDH Operator and waits for `backstages.rhdh.redhat.com` CRD (300s timeout) - Uses Backstage CR (`rhdh-start.yaml`) instead of Helm release -- Runtime config tests enabled, including `pluginDivisionMode: schema` tests with external Crunchy PostgreSQL +- Runtime config tests enabled, including `pluginDivisionMode: schema` tests with the external PostgreSQL (`enableLocalDb=false`) ### OCP PR Check (`ocp-pull`) diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 096ce26bbc..8f13496439 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -99,7 +99,7 @@ export default defineConfig({ "**/playwright/seed.spec.ts", "**/playwright/e2e/plugins/rbac/**/*.spec.ts", "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", "**/playwright/e2e/auth-providers/**/*.spec.ts", "**/playwright/e2e/external-database/verify-tls-config-with-external-rds.spec.ts", "**/playwright/e2e/external-database/verify-tls-config-with-external-azure-db.spec.ts", @@ -112,7 +112,7 @@ export default defineConfig({ dependencies: [PW_PROJECT.SMOKE_TEST], testMatch: [ "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", ], }, { @@ -134,7 +134,7 @@ export default defineConfig({ "**/playwright/e2e/smoke-test.spec.ts", "**/playwright/e2e/plugins/rbac/**/*.spec.ts", "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", "**/playwright/e2e/auth-providers/**/*.spec.ts", "**/playwright/e2e/plugins/scaffolder-backend-module-annotator/**/*.spec.ts", "**/playwright/e2e/plugins/scaffolder-relation-processor/**/*.spec.ts", @@ -160,7 +160,7 @@ export default defineConfig({ "**/playwright/seed.spec.ts", "**/playwright/e2e/plugins/rbac/**/*.spec.ts", "**/playwright/e2e/**/*-rbac.spec.ts", - "**/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts", + "**/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts", "**/playwright/e2e/auth-providers/**/*.spec.ts", "**/playwright/e2e/plugins/scaffolder-backend-module-annotator/**/*.spec.ts", "**/playwright/e2e/plugins/scaffolder-relation-processor/**/*.spec.ts", diff --git a/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts b/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts similarity index 92% rename from e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts rename to e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts index a9f7023220..dc5338aa20 100644 --- a/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-crunchy.spec.ts +++ b/e2e-tests/playwright/e2e/external-database/verify-tls-config-with-external-postgres.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from "@playwright/test"; import { UIhelper } from "../../utils/ui-helper"; import { Common } from "../../utils/common"; -test.describe("Verify TLS configuration with external Crunchy Postgres DB", () => { +test.describe("Verify TLS configuration with external PostgreSQL DB", () => { test.beforeAll(async () => { test.info().annotations.push( { diff --git a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md index 88e252b03e..fbe53cb814 100644 --- a/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md +++ b/e2e-tests/playwright/e2e/plugin-division-mode-schema/README.md @@ -47,11 +47,11 @@ On OCP Helm nightly jobs, [`.ci/pipelines/lib/schema-mode-env.sh`](../../../../. The test spec starts `oc port-forward` in `beforeAll` (similar to [`verify-redis-cache.spec.ts`](../../verify-redis-cache.spec.ts)) and stops it in `afterAll`. -When the runtime namespace has no Bitnami `*-postgresql` Service, auto-discovery uses the Crunchy cluster in `NAME_SPACE_POSTGRES_DB`: +When the runtime namespace has no Bitnami `*-postgresql` Service, auto-discovery uses the external PostgreSQL StatefulSet in `NAME_SPACE_POSTGRES_DB` (see [`postgres.yaml`](../../../../.ci/pipelines/resources/postgres-db/postgres.yaml)): -- Admin password from `${SCHEMA_MODE_CRUNCHY_CLUSTER_NAME:-postgress-external-db}-pguser-janus-idp` -- Forward target is a Running postgres pod (the `*-primary` Service has no selector, so forwarding the Service fails) -- Override cluster name with `SCHEMA_MODE_CRUNCHY_CLUSTER_NAME` if your `PostgresCluster` metadata name differs +- Admin password from `postgress-external-db-pguser-rhdh` +- Forward target is the Running pod labeled `app.kubernetes.io/name=postgress-external-db` +- The names match the StatefulSet deployed by `postgres.yaml`; there is no override Set `DEBUG_SCHEMA_MODE_PF=1` to log port-forward output. @@ -67,7 +67,7 @@ The tests include built-in retry logic to handle transient port-forward instabil - Automatic retry with exponential backoff for transient connection failures - Automatic port-forward restart when connection is refused (port-forward crashed) -- SSL fallback for Crunchy PostgreSQL clusters requiring encrypted connections +- SSL fallback for PostgreSQL servers requiring encrypted connections (the external PostgreSQL requires TLS for every network connection) ## Local / ClusterBot Runs