WPB-20580: Fix resource unavailavility and make sure a clean build with deploy via CI

.github/workflows/offline.yml

@@ -11,12 +11,14 @@ on:
       - '*.md'
       - '**/*.md'
 jobs:
-  offline:
-    name: Prepare offline package
-    # Useful to skip expensive CI when writing docs
+  # Build default profile and create local assets
+  build-default:
+    name: Build default profile
     if: "!contains(github.event.head_commit.message, 'skip ci')"
     runs-on:
       group: wire-server-deploy
+    outputs:
+      upload_name: ${{ steps.upload_name.outputs.UPLOAD_NAME }}
     steps:
       - uses: actions/checkout@v2
         with:
@@ -32,60 +34,177 @@ jobs:
 
       - name: Get upload name
         id: upload_name
-        run: |
-          # FIXME: Tag with a nice release name using the github tag...
-          # SOURCE_TAG=${GITHUB_REF#refs/tags/}
-          echo ::set-output name=UPLOAD_NAME::$GITHUB_SHA
-          # echo ::set-output name=UPLOAD_NAME::${SOURCE_TAG:-$GITHUB_SHA}
+        run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT
 
-      # deafult profile build
+      # default profile build
       - name: Process the default profile build
         run: ./offline/default-build/build.sh
         env:
           GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
           DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
 
-      - name: Copy default build assets tarball to S3 and clean up
+      # Upload the assets to be shared with other jobs
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: default-build-assets
+          path: offline/default-build/output/assets.tgz
+          retention-days: 1
+
+  # Upload to S3 in parallel with deployment
+  upload-s3:
+    name: Upload default build to S3
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    needs: build-default
+    runs-on:
+      group: wire-server-deploy
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: default-build-assets
+          path: offline/default-build/output/
+
+      - name: Copy default build assets tarball to S3
         run: |
-          # Upload tarball for each profile by specifying their OUTPUT_TAR path
-          aws s3 cp offline/default-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz
-          echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz"
-          # removing everything except assets.tgz as it is not required anymore in the further builds
-          find offline/default-build/output/ -mindepth 1 -maxdepth 1 ! -name 'assets.tgz' -exec rm -r {} +
+          aws s3 cp offline/default-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-${{ needs.build-default.outputs.upload_name }}.tgz
+          echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-${{ needs.build-default.outputs.upload_name }}.tgz"
         env:
           AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
           AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
           AWS_REGION: "eu-west-1"
 
+  # Deploy to Hetzner in parallel with S3 upload
+  deploy-hetzner:
+    name: Deploy default build to Hetzner
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    needs: build-default
+    runs-on:
+      group: wire-server-deploy
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v27
+      - uses: cachix/cachix-action@v15
+        with:
+          name: wire-server
+          signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
+
+      - name: Install nix environment
+        run: nix-env -f default.nix -iA env
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: default-build-assets
+          path: offline/default-build/output/
+
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: "^1.3.7"
+          terraform_wrapper: false
+
+      - name: Deploy offline environment to hetzner
+        run: ./offline/cd-with-retry.sh
+        env:
+          HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}'
+
+  # Build container in parallel
+  build-container:
+    name: Build container
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    needs: build-default
+    runs-on:
+      group: wire-server-deploy
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v27
+      - uses: cachix/cachix-action@v15
+        with:
+          name: wire-server
+          signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
+
       - name: Build and upload wire-server-deploy container
         run: |
           container_image=$(nix-build --no-out-link -A container)
           skopeo copy --retry-times 10 --dest-creds "$DOCKER_LOGIN" \
             docker-archive:"$container_image" \
-            "docker://quay.io/wire/wire-server-deploy:${{ steps.upload_name.outputs.UPLOAD_NAME }}"
+            "docker://quay.io/wire/wire-server-deploy:${{ needs.build-default.outputs.upload_name }}"
         env:
           DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
 
-      # demo profile build
+  # Build demo profile
+  build-demo:
+    name: Build demo profile
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    runs-on:
+      group: wire-server-deploy
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v27
+      - uses: cachix/cachix-action@v15
+        with:
+          name: wire-server
+          signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
+
+      - name: Install nix environment
+        run: nix-env -f default.nix -iA env
+
+      - name: Get upload name
+        id: upload_name
+        run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT
+
       - name: Process the demo profile build
         run: ./offline/demo-build/build.sh
         env:
           GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
           DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
 
-      - name: Copy demo build assets tarball to S3 and clean up
+      - name: Copy demo build assets tarball to S3
         run: |
-          # Upload tarball for each profile by specifying their OUTPUT_TAR path
           aws s3 cp offline/demo-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-demo-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz
           echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-demo-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz"
-          # remove the assets from the build to optimize the space on the server
-          rm -rf offline/demo-build/output/*
         env:
           AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
           AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
           AWS_REGION: "eu-west-1"
 
-      # min profile build
+      - name: Cleanup demo build assets
+        run: rm -rf offline/demo-build/output/
+
+  # Build min profile
+  build-min:
+    name: Build min profile
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    runs-on:
+      group: wire-server-deploy
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v27
+      - uses: cachix/cachix-action@v15
+        with:
+          name: wire-server
+          signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
+
+      - name: Install nix environment
+        run: nix-env -f default.nix -iA env
+
+      - name: Get upload name
+        id: upload_name
+        run: echo "UPLOAD_NAME=$GITHUB_SHA" >> $GITHUB_OUTPUT
+
       - name: Process the min profile build
         run: ./offline/min-build/build.sh
         env:
@@ -94,30 +213,12 @@ jobs:
 
       - name: Copy min build assets tarball to S3
         run: |
-          # Upload tarball for each profile by specifying their OUTPUT_TAR path
           aws s3 cp offline/min-build/output/assets.tgz s3://public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz
           echo "Uploaded to: https://s3-$AWS_REGION.amazonaws.com/public.wire.com/artifacts/wire-server-deploy-static-min-${{ steps.upload_name.outputs.UPLOAD_NAME }}.tgz"
-          # remove the archives from the build to optimize the space on the server
-          rm -rf offline/min-build/output/*
         env:
           AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
           AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
           AWS_REGION: "eu-west-1"
 
-      - name: Install terraform
-        uses: hashicorp/setup-terraform@v3
-        with:
-          terraform_version: "^1.3.7"
-          terraform_wrapper: false
-
-      - name: Deploy offline environment to hetzner
-        run: |
-          ./offline/cd.sh
-        env:
-          HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}'
-
-      #- name: Clean up hetzner environment; just in case
-      #  if: always()
-      #  run: (cd terraform/examples/wire-server-deploy-offline-hetzner ; terraform init && terraform destroy -auto-approve)
-      #  env:
-      #    HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}'
+      - name: Cleanup min build assets
+        run: rm -rf offline/min-build/output/

changelog.d/3-deploy-builds/optimize-default-build-deploy-process

@@ -0,0 +1,5 @@
+Changed: Optimize Wire offline deployment pipeline with parallel job execution and S3 direct downloads
+Added: Retry logic with progressive server type fallbacks for Hetzner Cloud resource availability issues
+Changed: Implement parallel terraform operations (15 parallelism) and fast SSH connection multiplexing
+Changed: Move ansible execution directly to adminhost for faster private network connectivity
+Changed: Reduce CI deployment time from 60+ minutes to ~30-40 minutes through parallel builds and optimized deployment process

nix/scripts/list-helm-containers.sh

@@ -5,7 +5,7 @@
 # those.
 # In cases where no container image tag has been specified, it'll use `latest`.
 # The list is sorted and deduplicated, then printed to stdout.
-set -eou pipefail
+set -euo pipefail
 
 VALUES_DIR=""
 HELM_IMAGE_TREE_FILE=""
@@ -74,14 +74,51 @@ function optionally_complain() {
 images=""
 # For each helm chart passed in from stdin, use the example values to
 # render the charts, and assemble the list of images this would fetch.
+chart_count=0
 while IFS= read -r chart; do
-  echo "Running helm template on chart ${chart}…" >&2
-  current_images=$(helm template --debug "${chart}" \
-    --set federate.dtls.tls.key=emptyString \
-    --set federate.dtls.tls.crt=emptyString \
-    $( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-values.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-values.example.yaml" ) \
-    $( [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-secrets.example.yaml ]] && echo "-f ${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-secrets.example.yaml" ) \
-    | yq -r '..|.image? | select(.)' | optionally_complain | sort -u)
+  chart_count=$((chart_count + 1))
+  echo "[$chart_count] Running helm template on chart ${chart}…" >&2
+  set +e  # Temporarily disable exit on error
+  # Determine values file to use (prod first, then demo as fallback)
+  values_file=""
+  if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-values.example.yaml ]]; then
+    values_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-values.example.yaml"
+  elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-values.example.yaml ]]; then
+    values_file="${VALUES_DIR}/$(basename "${chart}")/demo-values.example.yaml"
+    echo "Using demo values for $(basename $chart) (no ${VALUES_TYPE} values found)" >&2
+  fi
+
+  # Determine secrets file to use
+  secrets_file=""
+  if [[ -f "${VALUES_DIR}"/$(basename "${chart}")/"${VALUES_TYPE}"-secrets.example.yaml ]]; then
+    secrets_file="${VALUES_DIR}/$(basename "${chart}")/${VALUES_TYPE}-secrets.example.yaml"
+  elif [[ -f "${VALUES_DIR}"/$(basename "${chart}")/demo-secrets.example.yaml ]]; then
+    secrets_file="${VALUES_DIR}/$(basename "${chart}")/demo-secrets.example.yaml"
+  fi
+
+  raw_images=$(helm template "${chart}" \
+    $( [[ -n "$values_file" ]] && echo "-f $values_file" ) \
+    $( [[ -n "$secrets_file" ]] && echo "-f $secrets_file" ) \
+    2>&1 | yq -r '..|.image?' | grep -v "^null$" | grep -v "^---$" | grep -v "^$" || true)
+
+  helm_exit_code=$?
+  set -e  # Re-enable exit on error
+
+  if [[ $helm_exit_code -ne 0 ]]; then
+    echo "ERROR: Failed to process chart $(basename $chart)" >&2
+    echo "Chart path: $chart" >&2
+    echo "Values file: ${values_file:-none}" >&2
+    echo "Secrets file: ${secrets_file:-none}" >&2
+    echo "Try running: helm template $chart $([ -n "$values_file" ] && echo "-f $values_file") $([ -n "$secrets_file" ] && echo "-f $secrets_file")" >&2
+    raw_images=""
+  fi
+
+  # Process extracted images
+  if [[ -n "$raw_images" ]]; then
+    current_images=$(echo "$raw_images" | grep -v "^$" | optionally_complain | sort -u)
+  else
+    current_images=""
+  fi
 
   images+="$current_images\n"
   if [[ -n "$current_images" ]]; then
@@ -90,4 +127,4 @@ while IFS= read -r chart; do
     append_chart_entry "$(basename $chart)" "$image_array" "${HELM_IMAGE_TREE_FILE}"
   fi
 done
-echo -e "$images" | grep . | sort -u 
+echo -e "$images" | grep . | sort -u || true