ci: enable all terraform tests

.github/workflows/lambda.yml

@@ -34,7 +34,7 @@ jobs:
           persist-credentials: false
 
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
+        run: yarn install --immutable --mode=skip-build
 
       - name: Run prettier
         run: yarn format-check
@@ -77,24 +77,16 @@ jobs:
         uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
 
       - name: Build scale-set service image
-        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
-        with:
-          context: .
-          file: ./lambdas/services/scale-set/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: false
-          cache-from: type=gha,scope=scale-set-service
-          cache-to: type=gha,mode=max,scope=scale-set-service
-
-      - name: Build scale-set service image for smoke test
         uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: .
           file: ./lambdas/services/scale-set/Dockerfile
           platforms: linux/amd64
+          push: false
           load: true
           tags: scale-set-service:smoke-test
           cache-from: type=gha,scope=scale-set-service
+          cache-to: type=gha,mode=max,scope=scale-set-service
 
       - name: Run scale-set service image smoke test
-        run: ./tests/scale-set-container-smoke-test.sh
+        run: ./tests/scale-set-container-smoke-test.sh

.github/workflows/packer-build.yml

@@ -18,33 +18,75 @@ permissions:
 
 env:
   AWS_REGION: eu-west-1
-  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   verify_packer:
     name: Verify packer
     runs-on: ubuntu-latest
     container:
       image: index.docker.io/hashicorp/packer@sha256:12c441b8a3994e7df9f0e2692d9298f14c387e70bcc06139420977dbf80a137b # 1.11.2
-    strategy:
-      matrix:
-        image: ["linux-al2023", "windows-core-2019", "windows-core-2022", "ubuntu-focal", "ubuntu-jammy", "ubuntu-jammy-arm64"]
-    defaults:
-      run:
-        working-directory: images/${{ matrix.image }}
+    env:
+      PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Harden the runner (Audit all outbound calls)
         uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
         with:
           egress-policy: audit
 
-      - name: "Checkout"
+      - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+
+      - name: Discover Packer template directories
+        run: |
+          images="$(
+            find images \
+              \( -type d \( -name .git -o -name .terraform \) -prune \) -o \
+              \( -type f \( -name '*.pkr.hcl' -o -name '*.pkr.json' \) -print \) |
+            while IFS= read -r template; do
+              dirname "${template}"
+            done |
+            sort -u
+          )"
+
+          if [ -z "${images}" ]; then
+            echo "::error::No Packer templates found under images/"
+            exit 1
+          fi
+
+          {
+            printf 'IMAGES<<EOF\n'
+            printf '%s\n' "${images}"
+            printf 'EOF\n'
+          } >> "${GITHUB_ENV}"
+
       - name: packer init
-        run: packer init .
-      - name: check packer formatting
-        run: packer fmt -recursive -check=true .
-      - name: packer validate
-        run: packer validate -evaluate-datasources .
+        run: |
+          printf '%s\n' "${IMAGES}" | while IFS= read -r image; do
+            [ -z "${image}" ] && continue
+
+            echo "::group::Running packer init for image: ${image}"
+            (cd "${image}" && packer init .)
+            echo "::endgroup::"
+          done
+
+      - name: Check packer formatting
+        run: |
+          printf '%s\n' "${IMAGES}" | while IFS= read -r image; do
+            [ -z "${image}" ] && continue
+
+            echo "::group::Checking packer formatting for image: ${image}"
+            (cd "${image}" && packer fmt -recursive -check=true .)
+            echo "::endgroup::"
+          done
+
+      - name: Validate packer
+        run: |
+          printf '%s\n' "${IMAGES}" | while IFS= read -r image; do
+            [ -z "${image}" ] && continue
+
+            echo "::group::Validating image: ${image}"
+            (cd "${image}" && packer validate -evaluate-datasources .)
+            echo "::endgroup::"
+          done

.github/workflows/terraform.yml

@@ -16,47 +16,6 @@ permissions:
 
 env:
   AWS_REGION: eu-west-1
-  MODULES: |
-    .
-    modules/ami-housekeeper
-    modules/compute-providers/aws/ec2
-    modules/compute-providers/aws/ec2/trust-policy
-    modules/download-lambda
-    modules/lambda
-    modules/multi-runner
-    modules/orchestration-providers/scale-set
-    modules/orchestration-providers/webhook
-    modules/orchestration-providers/webhook/job-retry
-    modules/orchestration-providers/webhook/pool
-    modules/orchestration-providers/webhook/scale-runners
-    modules/runner-binaries-syncer
-    modules/runner-config
-    modules/runner-config/ssm-housekeeper
-    modules/runners
-    modules/runners/job-retry
-    modules/runners/pool
-    modules/setup-iam-permissions
-    modules/storage-providers/aws/ssm
-    modules/termination-watcher
-    modules/termination-watcher/notification
-    modules/termination-watcher/termination
-    modules/webhook
-    modules/webhook/direct
-    modules/webhook/eventbridge
-    modules/webhook-github-app
-  EXAMPLES: |
-    default
-    prebuilt
-    ephemeral
-    termination-watcher
-    multi-runner
-    multi-runner-v2
-    multi-runner-scale-set
-    external-managed-ssm-secrets
-  TEST_MODULES: |
-    modules/runners
-    modules/multi-runner
-    modules/orchestration-providers/scale-set
 
 jobs:
   verify_modules:
@@ -127,6 +86,32 @@ jobs:
           tofu_version: ${{ matrix.iac.version }}
           tofu_wrapper: false
 
+      - name: Discover Terraform modules
+        shell: bash
+        run: |
+          modules="$(
+            {
+              printf '.\n'
+              find modules \
+                \( -type d \( -name .git -o -name .terraform -o -name tests \) -prune \) -o \
+                \( -type f -name '*.tf' -print \) |
+              while IFS= read -r tf_file; do
+                dirname "${tf_file}"
+              done
+            } | sort -u
+          )"
+
+          if [ -z "${modules}" ]; then
+            echo "::error::No Terraform modules found"
+            exit 1
+          fi
+
+          {
+            printf 'MODULES<<EOF\n'
+            printf '%s\n' "${modules}"
+            printf 'EOF\n'
+          } >> "${GITHUB_ENV}"
+
       - name: ${{ matrix.iac.name }} init
         env:
           IAC_COMMAND: ${{ matrix.iac.command }}
@@ -254,58 +239,83 @@ jobs:
           tofu_version: ${{ matrix.iac.version }}
           tofu_wrapper: false
 
+      - name: Discover examples
+        shell: bash
+        run: |
+          examples="$(
+            find examples \
+              \( -type d \( -name .git -o -name .terraform \) -prune \) -o \
+              \( -type f -name '*.tf' -print \) |
+            while IFS= read -r tf_file; do
+              dirname "${tf_file}"
+            done |
+            sed 's#^examples/##' |
+            sort -u
+          )"
+
+          if [ -z "${examples}" ]; then
+            echo "::error::No Terraform examples found"
+            exit 1
+          fi
+
+          {
+            printf 'EXAMPLES<<EOF\n'
+            printf '%s\n' "${examples}"
+            printf 'EOF\n'
+          } >> "${GITHUB_ENV}"
+
       - name: Select ${{ matrix.iac.name }} lockfile
         if: matrix.iac.command == 'tofu'
         env:
           IAC_LOCK_FILE: ${{ matrix.iac.lockfile }}
         run: |
-          printf '%s\n' "${EXAMPLES}" | while IFS= read -r example; do
+          while IFS= read -r example; do
             [ -z "${example}" ] && continue
 
             echo "::group::Selecting lockfile for example: ${example}"
             cp "examples/${example}/${IAC_LOCK_FILE}" \
               "examples/${example}/.terraform.lock.hcl"
             echo "::endgroup::"
-          done
+          done <<< "${EXAMPLES}"
 
       - name: ${{ matrix.iac.name }} init
         env:
           IAC_COMMAND: ${{ matrix.iac.command }}
         run: |
-          printf '%s\n' "${EXAMPLES}" | while IFS= read -r example; do
+          while IFS= read -r example; do
             [ -z "${example}" ] && continue
 
             echo "::group::Running $IAC_COMMAND init for example: ${example}"
             $IAC_COMMAND -chdir="examples/${example}" init \
               -get -backend=false -input=false -lockfile=readonly
             echo "::endgroup::"
-          done
+          done <<< "${EXAMPLES}"
 
       - name: Check ${{ matrix.iac.name }} formatting
         env:
           IAC_COMMAND: ${{ matrix.iac.command }}
         run: |
-          printf '%s\n' "${EXAMPLES}" | while IFS= read -r example; do
+          while IFS= read -r example; do
             [ -z "${example}" ] && continue
 
             echo "::group::Checking $IAC_COMMAND formatting for example: ${example}"
             $IAC_COMMAND -chdir="examples/${example}" fmt \
               -recursive -check=true -write=false
             echo "::endgroup::"
-          done
+          done <<< "${EXAMPLES}"
         continue-on-error: ${{ matrix.iac.version == 'latest' }}
 
       - name: Validate ${{ matrix.iac.name }}
         env:
           IAC_COMMAND: ${{ matrix.iac.command }}
         run: |
-          printf '%s\n' "${EXAMPLES}" | while IFS= read -r example; do
+          while IFS= read -r example; do
             [ -z "${example}" ] && continue
 
             echo "::group::Validating example: ${example}"
             $IAC_COMMAND -chdir="examples/${example}" validate
             echo "::endgroup::"
-          done
+          done <<< "${EXAMPLES}"
 
       - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         name: Cache TFLint plugin dir
@@ -322,7 +332,7 @@ jobs:
         run: |
           tflint --init -c ${GITHUB_WORKSPACE}/.tflint.hcl
 
-          printf '%s\n' "${EXAMPLES}" | while IFS= read -r example; do
+          while IFS= read -r example; do
             [ -z "${example}" ] && continue
 
             echo "::group::Running TFLint for example: ${example}"
@@ -331,7 +341,7 @@ jobs:
               --var-file ${GITHUB_WORKSPACE}/.github/lint/tflint.tfvars \
               --chdir "examples/${example}"
             echo "::endgroup::"
-          done
+          done <<< "${EXAMPLES}"
 
   terraform_test:
     name: ${{ matrix.iac.name }} test
@@ -387,27 +397,52 @@ jobs:
           tofu_version: ${{ matrix.iac.version }}
           tofu_wrapper: false
 
+      - name: Discover Terraform test modules
+        shell: bash
+        run: |
+          test_modules="$(
+            find . \
+              \( -type d \( -name .git -o -name .terraform \) -prune \) -o \
+              \( -type f -name '*.tftest.hcl' -print \) |
+            while IFS= read -r test_file; do
+              test_dir=$(dirname "${test_file}")
+              printf '%s\n' "${test_dir%/tests}"
+            done |
+            sort -u
+          )"
+
+          if [ -z "${test_modules}" ]; then
+            echo "::error::No Terraform test modules found"
+            exit 1
+          fi
+
+          {
+            printf 'TEST_MODULES<<EOF\n'
+            printf '%s\n' "${test_modules}"
+            printf 'EOF\n'
+          } >> "${GITHUB_ENV}"
+
       - name: ${{ matrix.iac.name }} init
         env:
           IAC_COMMAND: ${{ matrix.iac.command }}
         run: |
-          printf '%s\n' "${TEST_MODULES}" | while IFS= read -r module; do
+          while IFS= read -r module; do
             [ -z "${module}" ] && continue
 
             echo "::group::Running $IAC_COMMAND init for test module: ${module}"
             $IAC_COMMAND -chdir="${module}" init \
               -backend=false -input=false
             echo "::endgroup::"
-          done
+          done <<< "${TEST_MODULES}"
 
       - name: ${{ matrix.iac.name }} test
         env:
           IAC_COMMAND: ${{ matrix.iac.command }}
         run: |
-          printf '%s\n' "${TEST_MODULES}" | while IFS= read -r module; do
+          while IFS= read -r module; do
             [ -z "${module}" ] && continue
 
             echo "::group::Running $IAC_COMMAND test for module: ${module}"
             $IAC_COMMAND -chdir="${module}" test -test-directory=tests -compact-warnings
             echo "::endgroup::"
-          done
+          done <<< "${TEST_MODULES}"