cloud-e2e-engine.yml

name: Cloud E2E Engine

on:
  workflow_dispatch:
    inputs:
      TEST_ENGINES:
        description: "The specify version of GO (e.g. mysql,postgresql,redis)"
        type: string
        required: false
        default: 'mysql,postgresql,redis'
      CLOUD_ENV_NAME:
        description: "The cloud env name of test (e.g. idc) "
        type: string
        required: false
        default: 'idc'
      CLOUD_BRANCH:
        description: "The cloud branch name (e.g. main) "
        type: string
        required: false
        default: 'main'
      APECD_REF:
        description: "The branch name of apecloud-cd"
        type: string
        required: false
        default: 'main'
      CURRENT_VERSION:
        description: "The current release version (e.g. v2.2) "
        type: string
        required: false
        default: 'v2.2'
      TEST_INDEX:
        description: "Test index"
        type: string
        required: false
        default: ''
      CLOUD_TEST_ENV:
        description: "The cloud test env"
        type: string
        required: false
        default: 'idc'
      CLOUD_TEST_KB_VERSION:
        description: "The cloud test kubeblocks version"
        type: string
        required: false
        default: '1.0'
      TEST_ENGINES_BATCH:
        description: "The batched test engines (comma separated) for run-name display"
        type: string
        required: false
        default: ''
  workflow_call:
    inputs:
      TEST_ENGINES:
        description: "The specify version of GO (e.g. mysql,postgresql,redis)"
        type: string
        required: false
        default: 'mysql,postgresql,redis'
      CLOUD_ENV_NAME:
        description: "The cloud env name of test (e.g. v2.2) "
        type: string
        required: false
        default: 'idc'
      CLOUD_BRANCH:
        description: "The cloud branch name (e.g. main) "
        type: string
        required: false
        default: 'main'
      APECD_REF:
        description: "The branch name of apecloud-cd"
        type: string
        required: false
        default: 'main'
      CURRENT_VERSION:
        description: "The current release version (e.g. v2.2) "
        type: string
        required: false
        default: 'v2.2'
      TEST_INDEX:
        description: "Test index"
        type: string
        required: false
        default: ''
      CLOUD_TEST_ENV:
        description: "The cloud test env"
        type: string
        required: false
        default: 'idc'
      CLOUD_TEST_KB_VERSION:
        description: "The cloud test kubeblocks version"
        type: string
        required: false
        default: '1.0'
      TEST_ENGINES_BATCH:
        description: "The batched test engines (comma separated) for run-name display"
        type: string
        required: false
        default: ''

run-name: Cloud E2E Engine Test on Env:${{ inputs.CLOUD_ENV_NAME }}:${{ inputs.CLOUD_TEST_ENV }}:${{ inputs.CLOUD_TEST_KB_VERSION }} ${{ inputs.TEST_ENGINES_BATCH != '' && inputs.TEST_ENGINES_BATCH || inputs.TEST_ENGINES }} Ref:${{ inputs.CLOUD_BRANCH }}

env:
  ACK_KUBECONFIG_DEV: ${{ secrets.ACK_KUBECONFIG_DEV }}
  IDC_KUBECONFIG: ${{ secrets.IDC_KUBECONFIG }}
  IDC_KUBECONFIG_1: ${{ secrets.IDC_KUBECONFIG_1 }}
  IDC_KUBECONFIG_2: ${{ secrets.IDC_KUBECONFIG_2 }}
  IDC_KUBECONFIG_4: ${{ secrets.IDC_KUBECONFIG_4 }}
  GH_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}


jobs:
  get-test-engine:
    runs-on: ubuntu-latest
    outputs:
      test-engines: ${{ steps.get_test_engine.outputs.test-engines }}
      test-engines-all: ${{ steps.get_test_engine.outputs.test-engines-all }}
      cloud-branch: ${{ steps.get_test_engine.outputs.cloud-branch }}
      cloud-env-name: ${{ steps.get_test_engine.outputs.cloud-env-name }}
      current-version: ${{ steps.get_test_engine.outputs.current-version }}
    steps:
      - name: Get test engine
        id: get_test_engine
        run: |
          TEST_INDEX="${{ inputs.TEST_INDEX }}"
          TEST_ENGINES="${{ inputs.TEST_ENGINES }}"
          if [[ -z "$TEST_ENGINES" ]]; then
              TEST_ENGINES="mysql"
          fi
          TEST_ENGINES_BATCH="${{ inputs.TEST_ENGINES_BATCH }}"
          if [[ -n "${TEST_ENGINES_BATCH}" ]]; then
              echo "Use TEST_ENGINES_BATCH as test engines: ${TEST_ENGINES_BATCH}"
              if [[ -n "${TEST_INDEX}" ]]; then
                  test_job_index_tmp=$(printf "%02d" ${TEST_INDEX})
              else
                  test_job_index_tmp="01"
              fi
              test_engines="{\"test-engine\":\"${TEST_ENGINES_BATCH}\",\"test-job-index\":\"${test_job_index_tmp}\"}"
              test_engines_all="${TEST_ENGINES_BATCH}|${test_job_index_tmp}"
          else
              test_engines=""
              test_engines_all=""
              test_engines_tmp=""
              engine_index=0
              test_job_index=0
              for engine_tmp in $(echo "${TEST_ENGINES}" | sed 's/,/ /g' ); do
                  engine_index=$(( engine_index + 1 ))
                  if [[ -z "${test_engines_tmp}" ]]; then
                      test_engines_tmp="${engine_tmp}"
                  else
                      test_engines_tmp="${test_engines_tmp},${engine_tmp}"
                  fi
          
                  KB_CLOUD_TEST_ENGINES_PRE="${{ vars.KB_CLOUD_TEST_ENGINES_PRE }}"
                  if [[ -z "${KB_CLOUD_TEST_ENGINES_PRE}" ]]; then
                      KB_CLOUD_TEST_ENGINES_PRE=5
                  fi
                  if [[ $(( engine_index % ${KB_CLOUD_TEST_ENGINES_PRE} )) == 0 ]]; then
                      test_job_index=$(( test_job_index + 1 ))
                      if [[ -n "${TEST_INDEX}" && "${TEST_INDEX}" != "${test_job_index}" ]]; then
                          echo "TEST_INDEX:${TEST_INDEX} not equal to test job index:${test_job_index}, Skip test."
                          test_engines_tmp=""
                          continue
                      fi
          
                      test_job_index_tmp=$(printf "%02d" $test_job_index)
                      if [[ -z "${test_engines}" ]]; then
                          test_engines_all="${test_engines_tmp}|${test_job_index_tmp}"
                          test_engines="{\"test-engine\":\"${test_engines_tmp}\",\"test-job-index\":\"${test_job_index_tmp}\"}"
                      else
                          test_engines_all="${test_engines_all}##${test_engines_tmp}|${test_job_index_tmp}"
                          test_engines="${test_engines},{\"test-engine\":\"${test_engines_tmp}\",\"test-job-index\":\"${test_job_index_tmp}\"}"
                      fi
                      test_engines_tmp=""
                  fi
              done
              if [[ -n "${test_engines_tmp}" ]]; then
                  test_job_index=$(( test_job_index + 1 ))
                  if [[ -n "${TEST_INDEX}" && "${TEST_INDEX}" != "${test_job_index}" ]]; then
                      echo "TEST_INDEX:${TEST_INDEX} not equal to test job index:${test_job_index}, Skip test."
                  else
                      test_job_index_tmp=$(printf "%02d" $test_job_index)
                      if [[ -z "${test_engines}" ]]; then
                          test_engines_all="${test_engines_tmp}|${test_job_index_tmp}"
                          test_engines="{\"test-engine\":\"${test_engines_tmp}\",\"test-job-index\":\"${test_job_index_tmp}\"}"
                      else
                          test_engines_all="${test_engines_all}##${test_engines_tmp}|${test_job_index_tmp}"
                          test_engines="${test_engines},{\"test-engine\":\"${test_engines_tmp}\",\"test-job-index\":\"${test_job_index_tmp}\"}"
                    fi
                  fi
              fi
          fi
          echo "${test_engines}"
          echo "${test_engines_all}"
          echo "test-engines={\"include\":[${test_engines}]}" >> $GITHUB_OUTPUT
          echo test-engines-all="${test_engines_all}" >> $GITHUB_OUTPUT
          
          CLOUD_BRANCH="${{ inputs.CLOUD_BRANCH }}"
          if [[ -z "$CLOUD_BRANCH" ]]; then
              CLOUD_BRANCH="main"
          fi
          echo cloud-branch="$CLOUD_BRANCH" >> $GITHUB_OUTPUT
          
          CLOUD_ENV_NAME="${{ inputs.CLOUD_ENV_NAME }}"
          if [[ -z "$CLOUD_ENV_NAME" ]]; then
              CLOUD_ENV_NAME="dev"
          fi
          echo cloud-env-name="$CLOUD_ENV_NAME" >> $GITHUB_OUTPUT
          
          CURRENT_VERSION="${{ inputs.CURRENT_VERSION }}"
          if [[ -n "${CURRENT_VERSION}" ]]; then
              if [[ "${CURRENT_VERSION}" != "v"* ]]; then
                  CURRENT_VERSION="v${CURRENT_VERSION}"
              fi
          fi
          echo current-version="$CURRENT_VERSION" >> $GITHUB_OUTPUT

  e2e-cluster-clear:
    needs: [ get-test-engine ]
    uses: ./.github/workflows/cloud-cluster-clear.yml
    with:
      cloud_env_name: "${{ inputs.CLOUD_ENV_NAME }}"
      cloud_org_name: "e2e-test"
      cloud_version: "${{ needs.get-test-engine.outputs.cloud-branch }}"
      delete_env_name: "${{ inputs.CLOUD_TEST_ENV }}"
    secrets: inherit

  e2e-engine-run:
    if: ${{ always() }}
    needs: [ get-test-engine, e2e-cluster-clear ]
    strategy:
      max-parallel: 1
      fail-fast: false
      matrix: ${{ fromJSON(needs.get-test-engine.outputs.test-engines) }}
    outputs:
      pod-name-1: ${{ steps.wait_cloud_e2e.outputs.pod-name-1 }}
      pod-name-2: ${{ steps.wait_cloud_e2e.outputs.pod-name-2 }}
      pod-name-3: ${{ steps.wait_cloud_e2e.outputs.pod-name-3 }}
      pod-name-4: ${{ steps.wait_cloud_e2e.outputs.pod-name-4 }}
      pod-name-5: ${{ steps.wait_cloud_e2e.outputs.pod-name-5 }}
      pod-name-6: ${{ steps.wait_cloud_e2e.outputs.pod-name-6 }}
      pod-name-7: ${{ steps.wait_cloud_e2e.outputs.pod-name-7 }}
      check-completed-1: ${{ steps.wait_cloud_e2e.outputs.check-completed-1 }}
      check-completed-2: ${{ steps.wait_cloud_e2e.outputs.check-completed-2 }}
      check-completed-3: ${{ steps.wait_cloud_e2e.outputs.check-completed-3 }}
      check-completed-4: ${{ steps.wait_cloud_e2e.outputs.check-completed-4 }}
      check-completed-5: ${{ steps.wait_cloud_e2e.outputs.check-completed-5 }}
      check-completed-6: ${{ steps.wait_cloud_e2e.outputs.check-completed-6 }}
      check-completed-7: ${{ steps.wait_cloud_e2e.outputs.check-completed-7 }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout testinfra Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/testinfra
          path: ./
          token: ${{ env.GH_TOKEN }}

      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          path: ./apecloud-cd
          ref: ${{ inputs.APECD_REF }}

      - name: Checkout apecloud Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud
          path: ./apecloud
          token: ${{ env.GH_TOKEN }}
          ref: "${{ needs.get-test-engine.outputs.cloud-branch }}"
          fetch-depth: 0

      - name: Configure ACK Context ${{ inputs.CLOUD_ENV_NAME }}
        id: cloud_env
        run: |
          mkdir -p $HOME/.kube
          touch $HOME/.kube/config
          CLOUD_ENV_NAME="${{ inputs.CLOUD_ENV_NAME }}"
          if [[ -z "$CLOUD_ENV_NAME" ]]; then
              CLOUD_ENV_NAME="dev"
          fi
          echo CLOUD_ENV_NAME="$CLOUD_ENV_NAME" >> $GITHUB_ENV
          
          E2E_ENV_VARS=""
          case "$CLOUD_ENV_NAME" in
              dev)
                  echo '${{ env.ACK_KUBECONFIG_DEV }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_DEV }}"
              ;;
              idc)
                  echo '${{ env.IDC_KUBECONFIG }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC }}"
              ;;
              idc1)
                  echo '${{ env.IDC_KUBECONFIG_1 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC1 }}"
              ;;
              idc2)
                  echo '${{ env.IDC_KUBECONFIG_2 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC2 }}"
              ;;
              idc4)
                  echo '${{ env.IDC_KUBECONFIG_4 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC4 }}"
              ;;
          esac
          chmod 600 $HOME/.kube/config
          echo E2E_ENV_VARS="$E2E_ENV_VARS" >> $GITHUB_ENV

      - name: install python dateutil
        run: |
          pip3 install python-dateutil

      - name: run ${{ matrix.test-engine }}
        run: |
          run_file_log="run_e2e_test_result.log"
          touch ${run_file_log} 
          
          for env_vars in $(echo "${E2E_ENV_VARS}" | sed 's/|/ /g'); do
              eval_cmd="export $env_vars"
              echo "$eval_cmd"
              eval "$eval_cmd"
          done
          
          CLOUD_TEST_ENV="${{ inputs.CLOUD_TEST_ENV }}"
          if [[ -n "${CLOUD_TEST_ENV}" ]]; then
              echo "export KB_CLOUD_TEST_ENVIRONMENT=${CLOUD_TEST_ENV}"
              export KB_CLOUD_TEST_ENVIRONMENT="${CLOUD_TEST_ENV}"
              echo "export KB_CLOUD_TEST_NODE=node3"
              export KB_CLOUD_TEST_NODE="node3"
          fi
          CLOUD_TEST_KB_VERSION="${{ inputs.CLOUD_TEST_KB_VERSION }}"
          echo "export KB_CLOUD_TEST_KUBEBLOCKS_VERSION=${CLOUD_TEST_KB_VERSION}"
          export KB_CLOUD_TEST_KUBEBLOCKS_VERSION="${CLOUD_TEST_KB_VERSION}"
          
          TEST_ENGINES="${{ matrix.test-engine }}"
          TEST_JOB_INDEX="${{ matrix.test-job-index }}"
          echo "export KB_CLOUD_TEST_ENGINES=${TEST_ENGINES}"
          export KB_CLOUD_TEST_ENGINES=${TEST_ENGINES}
          
          # run cloud e2e test
          set +e
          apply_ok="false"
          for i in $(seq 1 10); do
              bash test/cloud/test_cloud_e2e.sh \
                  --test-type "engine-test" \
                  --pod-index "${TEST_JOB_INDEX}" \
                  --release-version "${{ needs.get-test-engine.outputs.cloud-branch }}" \
                  --env "${CLOUD_TEST_ENV}" | tee -a ${run_file_log}

              CLOUD_E2E_TEST_POD_NAME="$(cat ${run_file_log} | (grep '\[CLOUD-E2E-TEST-POD-NAME\]' || true) | tail -n 1)"
              if [[ -n "${CLOUD_E2E_TEST_POD_NAME}" && "$CLOUD_E2E_TEST_POD_NAME" == "[CLOUD-E2E-TEST-POD-NAME]"* ]]; then
                  CLOUD_E2E_TEST_POD_NAME=${CLOUD_E2E_TEST_POD_NAME#*"[CLOUD-E2E-TEST-POD-NAME]"}
              fi
              if [[ -n "${CLOUD_E2E_TEST_POD_NAME}" ]] && kubectl get pod -n default "${CLOUD_E2E_TEST_POD_NAME}" >/dev/null 2>&1; then
                  apply_ok="true"
                  break
              fi
              echo "create cloud e2e test pod failed, retry ${i}/10"
              sleep 1
          done
          set -e
          if [[ "${apply_ok}" != "true" ]]; then
              echo "create cloud e2e test pod failed"
              exit 1
          fi
          echo CLOUD_E2E_TEST_POD_NAME="$CLOUD_E2E_TEST_POD_NAME" >> $GITHUB_ENV

      - name: Waiting for Cloud E2E to be Completed
        id: wait_cloud_e2e
        run: |
          TEST_JOB_INDEX="${{ matrix.test-job-index }}"
          case "${TEST_JOB_INDEX}" in
              01)
                  echo "pod-name-1=${CLOUD_E2E_TEST_POD_NAME}" >> $GITHUB_OUTPUT
              ;;
              02)
                  echo "pod-name-2=${CLOUD_E2E_TEST_POD_NAME}" >> $GITHUB_OUTPUT
              ;;
              03)
                  echo "pod-name-3=${CLOUD_E2E_TEST_POD_NAME}" >> $GITHUB_OUTPUT
              ;;
              04)
                  echo "pod-name-4=${CLOUD_E2E_TEST_POD_NAME}" >> $GITHUB_OUTPUT
              ;;
              05)
                  echo "pod-name-5=${CLOUD_E2E_TEST_POD_NAME}" >> $GITHUB_OUTPUT
              ;;
              06)
                  echo "pod-name-6=${CLOUD_E2E_TEST_POD_NAME}" >> $GITHUB_OUTPUT
              ;;
              07)
                  echo "pod-name-7=${CLOUD_E2E_TEST_POD_NAME}" >> $GITHUB_OUTPUT
              ;;
          esac
          
          set +e
          set -o nounset

          START_TIME=$(date +%s)
          MAX_WAIT_SECONDS=18000
          CHECK_COMPLETED="false"

          sleep 10
          while true; do
              if kubectl get pods -n default | grep "${CLOUD_E2E_TEST_POD_NAME}" | egrep "Completed|Error" ; then
                  echo "Cloud E2E Test is Done"
                  CHECK_COMPLETED="true"
                  break
              fi

              ELAPSED=$(($(date +%s) - START_TIME))
              if [[ ${ELAPSED} -gt ${MAX_WAIT_SECONDS} ]]; then
                  echo "Reached max wait time in e2e-engine-run, handing over to result job"
                  break
              fi
              if kubectl get pods -n default | grep "${CLOUD_E2E_TEST_POD_NAME}" | egrep "Running|Error" ; then
                  kubectl logs -n default ${CLOUD_E2E_TEST_POD_NAME} --tail=5
              else
                  echo "Waiting for Cloud E2E to be Running..."
              fi
              sleep 1
          done

          case "${TEST_JOB_INDEX}" in
              01)
                  echo "check-completed-1=${CHECK_COMPLETED}" >> $GITHUB_OUTPUT
              ;;
              02)
                  echo "check-completed-2=${CHECK_COMPLETED}" >> $GITHUB_OUTPUT
              ;;
              03)
                  echo "check-completed-3=${CHECK_COMPLETED}" >> $GITHUB_OUTPUT
              ;;
              04)
                  echo "check-completed-4=${CHECK_COMPLETED}" >> $GITHUB_OUTPUT
              ;;
              05)
                  echo "check-completed-5=${CHECK_COMPLETED}" >> $GITHUB_OUTPUT
              ;;
              06)
                  echo "check-completed-6=${CHECK_COMPLETED}" >> $GITHUB_OUTPUT
              ;;
              07)
                  echo "check-completed-7=${CHECK_COMPLETED}" >> $GITHUB_OUTPUT
              ;;
          esac

  e2e-engine-test:
    needs: [ get-test-engine, e2e-engine-run ]
    if: ${{ always() }}
    name: ${{ matrix.test-engine }}
    strategy:
      max-parallel: 1
      fail-fast: false
      matrix: ${{ fromJSON(needs.get-test-engine.outputs.test-engines) }}
    outputs:
      test-result-1: ${{ steps.get_test_result.outputs.test-result-1 }}
      test-result-2: ${{ steps.get_test_result.outputs.test-result-2 }}
      test-result-3: ${{ steps.get_test_result.outputs.test-result-3 }}
      test-result-4: ${{ steps.get_test_result.outputs.test-result-4 }}
      test-result-5: ${{ steps.get_test_result.outputs.test-result-5 }}
      test-result-6: ${{ steps.get_test_result.outputs.test-result-6 }}
      test-result-7: ${{ steps.get_test_result.outputs.test-result-7 }}
      test-summary-1: ${{ steps.get_test_result.outputs.test-summary-1 }}
      test-summary-2: ${{ steps.get_test_result.outputs.test-summary-2 }}
      test-summary-3: ${{ steps.get_test_result.outputs.test-summary-3 }}
      test-summary-4: ${{ steps.get_test_result.outputs.test-summary-4 }}
      test-summary-5: ${{ steps.get_test_result.outputs.test-summary-5 }}
      test-summary-6: ${{ steps.get_test_result.outputs.test-summary-6 }}
      test-summary-7: ${{ steps.get_test_result.outputs.test-summary-7 }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout testinfra Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/testinfra
          path: ./
          token: ${{ env.GH_TOKEN }}

      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          path: ./apecloud-cd
          ref: ${{ inputs.APECD_REF }}

      - name: Configure ACK Context ${{ inputs.CLOUD_ENV_NAME }}
        id: cloud_env
        run: |
          mkdir -p $HOME/.kube
          touch $HOME/.kube/config
          CLOUD_ENV_NAME="${{ inputs.CLOUD_ENV_NAME }}"
          if [[ -z "$CLOUD_ENV_NAME" ]]; then
              CLOUD_ENV_NAME="dev"
          fi
          echo CLOUD_ENV_NAME="$CLOUD_ENV_NAME" >> $GITHUB_ENV

          E2E_ENV_VARS=""
          case "$CLOUD_ENV_NAME" in
              dev)
                  echo '${{ env.ACK_KUBECONFIG_DEV }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_DEV }}"
              ;;
              idc)
                  echo '${{ env.IDC_KUBECONFIG }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC }}"
              ;;
              idc1)
                  echo '${{ env.IDC_KUBECONFIG_1 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC1 }}"
              ;;
              idc2)
                  echo '${{ env.IDC_KUBECONFIG_2 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC2 }}"
              ;;
              idc4)
                  echo '${{ env.IDC_KUBECONFIG_4 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC4 }}"
              ;;
          esac
          chmod 600 $HOME/.kube/config
          echo E2E_ENV_VARS="$E2E_ENV_VARS" >> $GITHUB_ENV

      - name: Continue Waiting for Cloud E2E
        run: |
          TEST_JOB_INDEX="${{ matrix.test-job-index }}"
          CLOUD_E2E_TEST_POD_NAME=""
          CHECK_COMPLETED=""
          case "${TEST_JOB_INDEX}" in
              01)
                  CLOUD_E2E_TEST_POD_NAME="${{ needs.e2e-engine-run.outputs.pod-name-1 }}"
                  CHECK_COMPLETED="${{ needs.e2e-engine-run.outputs.check-completed-1 }}"
              ;;
              02)
                  CLOUD_E2E_TEST_POD_NAME="${{ needs.e2e-engine-run.outputs.pod-name-2 }}"
                  CHECK_COMPLETED="${{ needs.e2e-engine-run.outputs.check-completed-2 }}"
              ;;
              03)
                  CLOUD_E2E_TEST_POD_NAME="${{ needs.e2e-engine-run.outputs.pod-name-3 }}"
                  CHECK_COMPLETED="${{ needs.e2e-engine-run.outputs.check-completed-3 }}"
              ;;
              04)
                  CLOUD_E2E_TEST_POD_NAME="${{ needs.e2e-engine-run.outputs.pod-name-4 }}"
                  CHECK_COMPLETED="${{ needs.e2e-engine-run.outputs.check-completed-4 }}"
              ;;
              05)
                  CLOUD_E2E_TEST_POD_NAME="${{ needs.e2e-engine-run.outputs.pod-name-5 }}"
                  CHECK_COMPLETED="${{ needs.e2e-engine-run.outputs.check-completed-5 }}"
              ;;
              06)
                  CLOUD_E2E_TEST_POD_NAME="${{ needs.e2e-engine-run.outputs.pod-name-6 }}"
                  CHECK_COMPLETED="${{ needs.e2e-engine-run.outputs.check-completed-6 }}"
              ;;
              07)
                  CLOUD_E2E_TEST_POD_NAME="${{ needs.e2e-engine-run.outputs.pod-name-7 }}"
                  CHECK_COMPLETED="${{ needs.e2e-engine-run.outputs.check-completed-7 }}"
              ;;
          esac
          echo CLOUD_E2E_TEST_POD_NAME="$CLOUD_E2E_TEST_POD_NAME" >> $GITHUB_ENV
          
          set +e
          set -o nounset

          if [[ -z "${CLOUD_E2E_TEST_POD_NAME}" ]]; then
              echo "cloud-e2e-test-pod-name is empty, cannot continue waiting"
              exit 1
          fi

          if [[ "${CHECK_COMPLETED}" == "true" ]]; then
              echo "cloud-e2e-run is already completed waiting for pod ${CLOUD_E2E_TEST_POD_NAME}"
          
          else
              START_TIME=$(date +%s)
              MAX_WAIT_SECONDS=18000
    
              while true; do
                  if kubectl get pods -n default | grep "${CLOUD_E2E_TEST_POD_NAME}" | egrep "Completed|Error" ; then
                      echo "Cloud E2E Test is Done"
                      break
                  fi
    
                  ELAPSED=$(($(date +%s) - START_TIME))
                  if [[ ${ELAPSED} -gt ${MAX_WAIT_SECONDS} ]]; then
                      echo "Timeout waiting for Cloud E2E in result job"
                      exit 1
                  fi
    
                  kubectl logs -n default ${CLOUD_E2E_TEST_POD_NAME} --tail=5
                  sleep 1
              done
          fi

      - name: Get Test Result
        if: ${{ always() }}
        id: get_test_result
        run: |
          file_log="test_result.log"
          test_file_log_path="${{ github.workspace }}/${file_log}"
          touch ${test_file_log_path}
          
          set +e
          set -o nounset
          for i in $(seq 1 10); do
              kubectl logs -n default ${CLOUD_E2E_TEST_POD_NAME} > ${test_file_log_path}
              log_ret=$?
              if [[ $log_ret -eq 0 ]]; then
                  break
              fi
              sleep 1
          done
          
          for i in $(seq 1 10); do
              kubectl delete pod -n default ${CLOUD_E2E_TEST_POD_NAME} --force
              delete_ret=$?
              if [[ $delete_ret -eq 0 ]]; then
                  break
              fi
              sleep 1
          done
          
          set -e
         
          cat ${test_file_log_path}
          test_ret="$( grep "Test Suite Failed" ${test_file_log_path} || true )"
          TEST_RESULT_ALL="$(cat ${test_file_log_path} | (egrep 'SUCCESS!|FAIL!' | grep -- '--' || true))"
          echo "test result all:${TEST_RESULT_ALL}"
          TEST_RESULT_ALL=$(python3 ${{ github.workspace }}/apecloud-cd/.github/utils/remove_ansi.py --ansi-str "$TEST_RESULT_ALL")
          TEST_RESULT=$(bash ${{ github.workspace }}/apecloud-cd/.github/utils/utils.sh --type 41 --test-result "${TEST_RESULT_ALL}")
          echo "test result total:${TEST_RESULT}"
          
          if [[ -z "$TEST_RESULT" ]]; then
              TEST_RESULT="$(cat ${test_file_log_path} | (egrep 'SUCCESS!|FAIL!' | grep -- '--' || true) | tail -n 1)"
              TEST_RESULT=$(python3 ${{ github.workspace }}/apecloud-cd/.github/utils/remove_ansi.py --ansi-str "$TEST_RESULT")
              echo "test result remove ansi:${TEST_RESULT}"
          fi
          
          if [[ -z "$TEST_RESULT" ]]; then
              TEST_RESULT="[PASSED]"
              if [[ -n "$test_ret" || -z "${TEST_RESULT_ALL}" ]]; then
                  TEST_RESULT="[FAILED]"
              fi
              echo "test result PASSED or FAILED:${TEST_RESULT}"
          fi
          
          file_log_tmp="test_result_tmp.log"
          test_file_log_tmp_path="${{ github.workspace }}/${file_log_tmp}"
          touch ${test_file_log_tmp_path}
          E2E_TEST_RESULT_LOG=$(python3 ${{ github.workspace }}/apecloud-cd/.github/utils/remove_ansi_file.py ${test_file_log_path}) 
          echo "${E2E_TEST_RESULT_LOG}" > ${test_file_log_tmp_path}
          
          ENGINE_SUMMARY=$(python3 test/report/parse_ginkgo.py summary2 ${test_file_log_tmp_path})
          TEST_JOB_INDEX="${{ matrix.test-job-index }}"
          case "${TEST_JOB_INDEX}" in
              01)
                  echo "TEST_JOB_INDEX:${TEST_JOB_INDEX}"
                  echo test-result-1="${TEST_RESULT}" >> $GITHUB_OUTPUT
                  echo test-summary-1="${ENGINE_SUMMARY}" >> $GITHUB_OUTPUT
              ;; 
              02)
                  echo "TEST_JOB_INDEX:${TEST_JOB_INDEX}"
                  echo test-result-2="${TEST_RESULT}" >> $GITHUB_OUTPUT
                  echo test-summary-2="${ENGINE_SUMMARY}" >> $GITHUB_OUTPUT
              ;; 
              03)
                  echo "TEST_JOB_INDEX:${TEST_JOB_INDEX}"
                  echo test-result-3="${TEST_RESULT}" >> $GITHUB_OUTPUT
                  echo test-summary-3="${ENGINE_SUMMARY}" >> $GITHUB_OUTPUT
              ;; 
              04)
                  echo "TEST_JOB_INDEX:${TEST_JOB_INDEX}"
                  echo test-result-4="${TEST_RESULT}" >> $GITHUB_OUTPUT
                  echo test-summary-4="${ENGINE_SUMMARY}" >> $GITHUB_OUTPUT
              ;; 
              05)
                  echo "TEST_JOB_INDEX:${TEST_JOB_INDEX}"
                  echo test-result-5="${TEST_RESULT}" >> $GITHUB_OUTPUT
                  echo test-summary-5="${ENGINE_SUMMARY}" >> $GITHUB_OUTPUT
              ;; 
              06)
                  echo "TEST_JOB_INDEX:${TEST_JOB_INDEX}"
                  echo test-result-6="${TEST_RESULT}" >> $GITHUB_OUTPUT
                  echo test-summary-6="${ENGINE_SUMMARY}" >> $GITHUB_OUTPUT
              ;; 
              07)
                  echo "TEST_JOB_INDEX:${TEST_JOB_INDEX}"
                  echo test-result-7="${TEST_RESULT}" >> $GITHUB_OUTPUT
                  echo test-summary-7="${ENGINE_SUMMARY}" >> $GITHUB_OUTPUT
              ;; 
          esac
          echo "TEST_RESULT:${TEST_RESULT}"
          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
          
          if [[ -n "$test_ret" ]]; then
              exit 1
          fi

  send-message:
    needs: [ get-test-engine, e2e-engine-test ]
    runs-on: ubuntu-latest
    if: ${{ always() }}
    steps:
      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          ref: ${{ inputs.APECD_REF }}

      - name: send test result message
        run: |
          export TZ='Asia/Shanghai'
          date_ret=$(date +%Y-%m-%d-%T)
          CLOUD_BRANCH="${{ needs.get-test-engine.outputs.cloud-branch }}"
          CLOUD_ENV_NAME="${{ needs.get-test-engine.outputs.cloud-env-name }}"
          TEST_ENGINES_ALL="${{ needs.get-test-engine.outputs.test-engines-all }}"
          APECLOUD_CURRENT_VERSION="${{ needs.get-test-engine.outputs.current-version }}"
          TEST_RESULT_ALL=""
          for test_engines_all in $(echo "${TEST_ENGINES_ALL}" | sed 's/##/ /g'); do
              test_engines=${test_engines_all%%|*}
              test_job_index=${test_engines_all#*|}
              case "${test_job_index}" in
                  01)
                      echo "test-job-index:${test_job_index}"
                      TEST_RESULT="${{ needs.e2e-engine-test.outputs.test-result-1 }}"
                      TEST_RESULT_ALL="${TEST_RESULT_ALL}##${test_engines}|${TEST_RESULT}"
                  ;;
                  02)
                      echo "test-job-index:${test_job_index}"
                      TEST_RESULT="${{ needs.e2e-engine-test.outputs.test-result-2 }}"
                      TEST_RESULT_ALL="${TEST_RESULT_ALL}##${test_engines}|${TEST_RESULT}"
                  ;;
                  03)
                      echo "test-job-index:${test_job_index}"
                      TEST_RESULT="${{ needs.e2e-engine-test.outputs.test-result-3 }}"
                      TEST_RESULT_ALL="${TEST_RESULT_ALL}##${test_engines}|${TEST_RESULT}"
                  ;;
                  04)
                      echo "test-job-index:${test_job_index}"
                      TEST_RESULT="${{ needs.e2e-engine-test.outputs.test-result-4 }}"
                      TEST_RESULT_ALL="${TEST_RESULT_ALL}##${test_engines}|${TEST_RESULT}"                     
                  ;;
                  05)
                      echo "test-job-index:${test_job_index}"
                      TEST_RESULT="${{ needs.e2e-engine-test.outputs.test-result-5 }}"
                      TEST_RESULT_ALL="${TEST_RESULT_ALL}##${test_engines}|${TEST_RESULT}"                     
                  ;;
                  06)
                      echo "test-job-index:${test_job_index}"
                      TEST_RESULT="${{ needs.e2e-engine-test.outputs.test-result-6 }}"
                      TEST_RESULT_ALL="${TEST_RESULT_ALL}##${test_engines}|${TEST_RESULT}"                     
                  ;;
                  07)
                      echo "test-job-index:${test_job_index}"
                      TEST_RESULT="${{ needs.e2e-engine-test.outputs.test-result-7 }}"
                      TEST_RESULT_ALL="${TEST_RESULT_ALL}##${test_engines}|${TEST_RESULT}"                     
                  ;;
              esac
          done
          
          echo "TEST_RESULT_ALL:${TEST_RESULT_ALL}"
          TEST_RESULT_ALL=$( bash .github/utils/utils.sh --type 40 \
              --github-repo "${{ github.repository }}" \
              --github-token "${{ env.GH_TOKEN }}" \
              --test-result "${TEST_RESULT_ALL}" \
              --run-id "$GITHUB_RUN_ID" )
          echo "TEST_RESULT_ALL:${TEST_RESULT_ALL}"
          
          CLOUD_ENV_NAME_TMP="${CLOUD_ENV_NAME}"
          CLOUD_TEST_ENV="${{ inputs.CLOUD_TEST_ENV }}"
          if [[ -n "${CLOUD_TEST_ENV}" ]]; then
              CLOUD_ENV_NAME_TMP="${CLOUD_TEST_ENV}"
          fi
          
          TEST_TITLE="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E Engine Test [${date_ret}]"
          
          python3 .github/utils/send_mesage.py \
              --send-type ginkgo \
              --url "${{ vars.CLOUD_E2E_BOT_WEBHOOK }}" \
              --title "$TEST_TITLE" \
              --result "${TEST_RESULT_ALL}"
          
          
          for test_engines_all in $(echo "${TEST_ENGINES_ALL}" | sed 's/##/ /g'); do
              test_engines=${test_engines_all%%|*}
              test_job_index=${test_engines_all#*|}
              case "${test_job_index}" in
                  01)
                      echo "test-job-index:${test_job_index}"
                      ENGINE_SUMMARY="${{ needs.e2e-engine-test.outputs.test-summary-1 }}"
                      TEST_TITLE_ENGINE_SUMMARY="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E ${test_engines} API Summary [${date_ret}]"
                      if [[ -n "${ENGINE_SUMMARY}" ]]; then
                          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
                          ENGINE_SUMMARY_TMP="$(bash .github/utils/utils.sh \
                              --type 45 \
                              --test-result "${TEST_RESULT_ALL}" \
                              --coverage-result "${ENGINE_SUMMARY}")"
                          echo "ENGINE_SUMMARY_TMP:${ENGINE_SUMMARY_TMP}"
          
                          python3 .github/utils/send_mesage.py \
                              --send-type "engine-summary2" \
                              --url "${{ vars.REPORT_BOT_WEBHOOK }}" \
                              --title "$TEST_TITLE_ENGINE_SUMMARY" \
                              --result "$ENGINE_SUMMARY_TMP"
                      fi
                  ;;
                  02)
                      echo "test-job-index:${test_job_index}"
                      ENGINE_SUMMARY="${{ needs.e2e-engine-test.outputs.test-summary-2 }}"
                      TEST_TITLE_ENGINE_SUMMARY="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E ${test_engines} API Summary [${date_ret}]"
                      if [[ -n "${ENGINE_SUMMARY}" ]]; then
                          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
                          ENGINE_SUMMARY_TMP="$(bash .github/utils/utils.sh \
                              --type 45 \
                              --test-result "${TEST_RESULT_ALL}" \
                              --coverage-result "${ENGINE_SUMMARY}")"
                          echo "ENGINE_SUMMARY_TMP:${ENGINE_SUMMARY_TMP}"
          
                          python3 .github/utils/send_mesage.py \
                              --send-type "engine-summary2" \
                              --url "${{ vars.REPORT_BOT_WEBHOOK }}" \
                              --title "$TEST_TITLE_ENGINE_SUMMARY" \
                              --result "$ENGINE_SUMMARY_TMP"
                      fi
                  ;;
                  03)
                      echo "test-job-index:${test_job_index}"
                      ENGINE_SUMMARY="${{ needs.e2e-engine-test.outputs.test-summary-3 }}"
                      TEST_TITLE_ENGINE_SUMMARY="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E ${test_engines} API Summary [${date_ret}]"
                      if [[ -n "${ENGINE_SUMMARY}" ]]; then
                          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
                          ENGINE_SUMMARY_TMP="$(bash .github/utils/utils.sh \
                              --type 45 \
                              --test-result "${TEST_RESULT_ALL}" \
                              --coverage-result "${ENGINE_SUMMARY}")"
                          echo "ENGINE_SUMMARY_TMP:${ENGINE_SUMMARY_TMP}"
          
                          python3 .github/utils/send_mesage.py \
                              --send-type "engine-summary2" \
                              --url "${{ vars.REPORT_BOT_WEBHOOK }}" \
                              --title "$TEST_TITLE_ENGINE_SUMMARY" \
                              --result "$ENGINE_SUMMARY_TMP"
                      fi
                  ;;
                  04)
                      echo "test-job-index:${test_job_index}"
                      ENGINE_SUMMARY="${{ needs.e2e-engine-test.outputs.test-summary-4 }}"
                      TEST_TITLE_ENGINE_SUMMARY="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E ${test_engines} API Summary [${date_ret}]"
                      if [[ -n "${ENGINE_SUMMARY}" ]]; then
                          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
                          ENGINE_SUMMARY_TMP="$(bash .github/utils/utils.sh \
                              --type 45 \
                              --test-result "${TEST_RESULT_ALL}" \
                              --coverage-result "${ENGINE_SUMMARY}")"
                          echo "ENGINE_SUMMARY_TMP:${ENGINE_SUMMARY_TMP}"
          
                          python3 .github/utils/send_mesage.py \
                              --send-type "engine-summary2" \
                              --url "${{ vars.REPORT_BOT_WEBHOOK }}" \
                              --title "$TEST_TITLE_ENGINE_SUMMARY" \
                              --result "$ENGINE_SUMMARY_TMP"
                      fi
                  ;;
                  05)
                      echo "test-job-index:${test_job_index}"
                      ENGINE_SUMMARY="${{ needs.e2e-engine-test.outputs.test-summary-5 }}"
                      TEST_TITLE_ENGINE_SUMMARY="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E ${test_engines} API Summary [${date_ret}]"
                      if [[ -n "${ENGINE_SUMMARY}" ]]; then
                          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
                          ENGINE_SUMMARY_TMP="$(bash .github/utils/utils.sh \
                              --type 45 \
                              --test-result "${TEST_RESULT_ALL}" \
                              --coverage-result "${ENGINE_SUMMARY}")"
                          echo "ENGINE_SUMMARY_TMP:${ENGINE_SUMMARY_TMP}"
          
                          python3 .github/utils/send_mesage.py \
                              --send-type "engine-summary2" \
                              --url "${{ vars.REPORT_BOT_WEBHOOK }}" \
                              --title "$TEST_TITLE_ENGINE_SUMMARY" \
                              --result "$ENGINE_SUMMARY_TMP"
                      fi
                  ;;
                  06)
                      echo "test-job-index:${test_job_index}"
                      ENGINE_SUMMARY="${{ needs.e2e-engine-test.outputs.test-summary-6 }}"
                      TEST_TITLE_ENGINE_SUMMARY="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E ${test_engines} API Summary [${date_ret}]"
                      if [[ -n "${ENGINE_SUMMARY}" ]]; then
                          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
                          ENGINE_SUMMARY_TMP="$(bash .github/utils/utils.sh \
                              --type 45 \
                              --test-result "${TEST_RESULT_ALL}" \
                              --coverage-result "${ENGINE_SUMMARY}")"
                          echo "ENGINE_SUMMARY_TMP:${ENGINE_SUMMARY_TMP}"
          
                          python3 .github/utils/send_mesage.py \
                              --send-type "engine-summary2" \
                              --url "${{ vars.REPORT_BOT_WEBHOOK }}" \
                              --title "$TEST_TITLE_ENGINE_SUMMARY" \
                              --result "$ENGINE_SUMMARY_TMP"
                      fi
                  ;;
                  07)
                      echo "test-job-index:${test_job_index}"
                      ENGINE_SUMMARY="${{ needs.e2e-engine-test.outputs.test-summary-7 }}"
                      TEST_TITLE_ENGINE_SUMMARY="[${CLOUD_BRANCH}][${CLOUD_ENV_NAME_TMP}] Cloud E2E ${test_engines} API Summary [${date_ret}]"
                      if [[ -n "${ENGINE_SUMMARY}" ]]; then
                          echo "ENGINE_SUMMARY:${ENGINE_SUMMARY}"
                          ENGINE_SUMMARY_TMP="$(bash .github/utils/utils.sh \
                              --type 45 \
                              --test-result "${TEST_RESULT_ALL}" \
                              --coverage-result "${ENGINE_SUMMARY}")"
                          echo "ENGINE_SUMMARY_TMP:${ENGINE_SUMMARY_TMP}"
          
                          python3 .github/utils/send_mesage.py \
                              --send-type "engine-summary2" \
                              --url "${{ vars.REPORT_BOT_WEBHOOK }}" \
                              --title "$TEST_TITLE_ENGINE_SUMMARY" \
                              --result "$ENGINE_SUMMARY_TMP"
                      fi
                  ;;
              esac
          done