feat(microvm): add lifecycle hook service

.ci/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update \
 FROM build as lambdas
 COPY lambdas /lambdas
 RUN --mount=type=cache,target=/work/node_modules,id=lambdas \
-        yarn install --frozen-lockfile && yarn dist && \
+        yarn install --immutable --mode=skip-build && yarn dist && \
         find . -name "*.zip" -exec cp {} . \;
 
 FROM scratch as final

.github/workflows/lambda.yml

@@ -34,8 +34,7 @@ jobs:
           persist-credentials: false
 
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
-
+        run: yarn install --immutable --mode=skip-build
       - name: Run prettier
         run: yarn format-check
 
@@ -58,43 +57,33 @@ jobs:
           retention-days: 5
 
   scale-set-container:
-    name: Build scale-set service container
-    runs-on: ubuntu-latest
-    steps:
-      - name: Harden the runner (Audit all outbound calls)
-        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
-        with:
-          egress-policy: audit
-
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          persist-credentials: false
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
-
-      - name: Build scale-set service image
-        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
-        with:
-          context: .
-          file: ./lambdas/services/scale-set/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: false
-          cache-from: type=gha,scope=scale-set-service
-          cache-to: type=gha,mode=max,scope=scale-set-service
-
-      - name: Build scale-set service image for smoke test
-        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
-        with:
-          context: .
-          file: ./lambdas/services/scale-set/Dockerfile
-          platforms: linux/amd64
-          load: true
-          tags: scale-set-service:smoke-test
-          cache-from: type=gha,scope=scale-set-service
-
-      - name: Run scale-set service image smoke test
-        run: ./tests/scale-set-container-smoke-test.sh
+      name: Build scale-set service container
+      runs-on: ubuntu-latest
+      steps:
+        - name: Harden the runner (Audit all outbound calls)
+          uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+          with:
+            egress-policy: audit
+
+        - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+          with:
+            persist-credentials: false
+        - name: Set up QEMU
+          uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+
+        - name: Set up Docker Buildx
+          uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
+
+        - name: Build scale-set service image
+          uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
+          with:
+            context: .
+            file: ./lambdas/services/scale-set/Dockerfile
+            platforms: linux/amd64
+            push: false
+            load: true
+            cache-from: type=gha,scope=scale-set-service
+            cache-to: type=gha,mode=max,scope=scale-set-service
+
+        - name: Run scale-set service image smoke test
+          run: ./tests/scale-set-container-smoke-test.sh

.github/workflows/ministack.yml

@@ -53,7 +53,6 @@ jobs:
           - ephemeral
           - microvm-foundation
           - multi-runner
-          - multi-runner-scale-set
           - migration-test
           - multi-runner
           - termination-watcher

.github/workflows/release.yml

@@ -37,7 +37,21 @@ jobs:
           persist-credentials: false
       - name: Build dist
         working-directory: lambdas
-        run: yarn install --frozen-lockfile && yarn run test && yarn dist
+        run: yarn install --immutable --mode=skip-build && yarn run test && yarn dist
+
+      - name: Build MicroVM lifecycle hook
+        working-directory: lambdas
+        run: yarn workspace @aws-github-runner/microvm-lifecycle-hooks build
+
+      - name: Verify MicroVM lifecycle hook distribution
+        working-directory: lambdas
+        run: |
+          test -s services/microvm-lifecycle-hooks/dist/server.js
+          test -f services/microvm-lifecycle-hooks/dist/package.json
+
+      - name: Package MicroVM lifecycle hook
+        working-directory: lambdas/services/microvm-lifecycle-hooks
+        run: (cd dist && zip -r ../microvm-lifecycle-hooks.zip .)
       - name: Get installation token
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
         id: token
@@ -61,35 +75,48 @@ jobs:
         uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
         with:
           subject-path: '${{ github.workspace }}/lambdas/functions/**/*.zip'
+
+      - name: Attest MicroVM lifecycle hook
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        id: lifecycle-hook-attest
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-path: '${{ github.workspace }}/lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip'
+
       - name: Update release notes with attestation
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          VERSION: ${{ github.event.inputs.version }}
           TAG_NAME: ${{ steps.release.outputs.tag_name }}
           ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }}
+          LIFECYCLE_HOOK_ATTESTATION_URL: ${{ steps.lifecycle-hook-attest.outputs.attestation-url }}
           REPOSITORY: ${{ github.repository }}
         run: |
-          version="${VERSION}"
           tag_name="${TAG_NAME}"
           attestation_url="${ATTESTATION_URL}"
+          lifecycle_hook_attestation_url="${LIFECYCLE_HOOK_ATTESTATION_URL}"
           repository="${REPOSITORY}"
-          gh release view $version --json body -q '.body' > new-release-notes.md
+          gh release view "$tag_name" --json body -q '.body' > new-release-notes.md
           echo "## Attestation" >> new-release-notes.md
-          echo "Attestation url: $attestation_url" >> new-release-notes.md
+          echo "Lambda attestation url: $attestation_url" >> new-release-notes.md
+          echo "MicroVM lifecycle hook attestation url: $lifecycle_hook_attestation_url" >> new-release-notes.md
           echo "Verify the artifacts by running \`gh attestation verify <name_of_artifact> --repo ${repository}\`" >> new-release-notes.md
-          gh release edit $tag_name -F new-release-notes.md -t $tag_name
+          gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name"
+
       - name: Upload release assets
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TAG_NAME: ${{ steps.release.outputs.tag_name }}
         run: |
           tag_name="${TAG_NAME}"
-          for f in $(find . -name '*.zip'); do
-            gh release upload $tag_name $f
-          done
-      - name: Attach attestation
+          while IFS= read -r -d '' f; do
+            gh release upload "$tag_name" "$f"
+          done < <(find lambdas/functions -name '*.zip' -print0)
+          gh release upload "$tag_name" \
+            "lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip"
+
+      - name: Attach Lambda attestation
         if: ${{ steps.release.outputs.releases_created == 'true' }}
         env:
           ATTESTATION_BUNDLE: ${{ steps.attest.outputs.bundle-path }}
@@ -99,13 +126,26 @@ jobs:
         run: |
           # rename attest bundle to github-aws-runners-terraform-aws-github-runner-attestation-$attestation-id.sigstore
           # OpenSSF expects the attestation bundle to be named in this format (*.sigstore)
-          SIGSTORE_BUNDLE=$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.sigstore
-          INTOTO_BUNDLE=$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.intoto.jsonl
-          mv ${ATTESTATION_BUNDLE} $SIGSTORE_BUNDLE
-          if [ -z "$SIGSTORE_BUNDLE" ]; then
-            echo "No attestation bundle found, skipping attachment."
-            exit 0
-          fi
-          gh release upload $TAG_NAME "$SIGSTORE_BUNDLE"
-          cat ${SIGSTORE_BUNDLE} | jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' | base64 -d | jq .> ${INTOTO_BUNDLE}
-          gh release upload $TAG_NAME "${INTOTO_BUNDLE}"
+          sigstore_bundle="$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.sigstore"
+          intoto_bundle="$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.intoto.jsonl"
+          cp "$ATTESTATION_BUNDLE" "$sigstore_bundle"
+          gh release upload "$TAG_NAME" "$sigstore_bundle"
+          jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' "$sigstore_bundle" \
+            | base64 --decode > "$intoto_bundle"
+          gh release upload "$TAG_NAME" "$intoto_bundle"
+
+      - name: Attach MicroVM lifecycle hook attestation
+        if: ${{ steps.release.outputs.releases_created == 'true' }}
+        env:
+          ATTESTATION_BUNDLE: ${{ steps.lifecycle-hook-attest.outputs.bundle-path }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG_NAME: ${{ steps.release.outputs.tag_name }}
+          ATTESTATION_ID: ${{ steps.lifecycle-hook-attest.outputs.attestation-id }}
+        run: |
+          sigstore_bundle="$RUNNER_TEMP/github-aws-runners-microvm-lifecycle-hooks-attestation-${ATTESTATION_ID}.sigstore"
+          intoto_bundle="$RUNNER_TEMP/github-aws-runners-microvm-lifecycle-hooks-attestation-${ATTESTATION_ID}.intoto.jsonl"
+          cp "$ATTESTATION_BUNDLE" "$sigstore_bundle"
+          gh release upload "$TAG_NAME" "$sigstore_bundle"
+          jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' "$sigstore_bundle" \
+            | base64 --decode > "$intoto_bundle"
+          gh release upload "$TAG_NAME" "$intoto_bundle"

.github/workflows/smoke-tests.yml

@@ -2,7 +2,13 @@ name: "Smoke Tests"
 
 on:
   pull_request:
-    paths: ["**/*.tf", "**/*.hcl", ".github/workflows/smoke-tests.yml"]
+    paths:
+      - "**/*.tf"
+      - "**/*.hcl"
+      - "images/microvm-ubuntu/**"
+      - "lambdas/**"
+      - "tests/ministack/**"
+      - ".github/workflows/smoke-tests.yml"
   workflow_dispatch:
 
 concurrency:
@@ -23,74 +29,13 @@ env:
   TF_INPUT: "false"
 
 jobs:
-  control_plane_smoke:
-    name: Run webhook and pool lifecycle smoke test against MiniStack
+  ministack_smoke:
+    name: Run webhook EC2/MicroVM and scale-set EC2 smoke tests against MiniStack
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 120
     services:
       ministack:
-        image: ghcr.io/ministackorg/ministack:1.5.13@sha256:ce3c906f2866ff953ce4c56f06b1fa3e453bc32e41c00de17b5f5a8672c5a42c
-        ports:
-          - 4566:4566
-        options: --add-host=host.docker.internal:host-gateway
-        env:
-          MINISTACK_ACCOUNT_ID: "000000000000"
-          MINISTACK_REGION: eu-west-1
-    steps:
-      - name: Harden the runner (Audit all outbound calls)
-        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
-        with:
-          egress-policy: audit
-
-      - name: Checkout
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          persist-credentials: false
-
-      - name: Setup Node.js
-        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
-        with:
-          node-version-file: lambdas/.nvmrc
-          package-manager-cache: false
-
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
-        with:
-          terraform_version: latest
-          terraform_wrapper: false
-
-      - name: Install Lambda dependencies
-        working-directory: lambdas
-        run: yarn install --frozen-lockfile
-
-      - name: Build smoke-test Lambda distributions
-        working-directory: lambdas
-        run: |
-          yarn workspace @aws-github-runner/webhook dist
-          yarn workspace @aws-github-runner/control-plane dist
-
-      - name: Start MockServer
-        id: mockserver
-        uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0
-        with:
-          version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290'
-          port: '1080'
-          startup-timeout: '60'
-
-      - name: Run webhook and pool lifecycle smoke test
-        env:
-          MINISTACK_GITHUB_MOCK_HOST: host.docker.internal
-          MINISTACK_GITHUB_MOCK_PORT: "1080"
-          MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
-        run: sh tests/ministack/run-smoke.sh
-
-  scale_set_integration_smoke:
-    name: Run scale-set ECS smoke test against MiniStack and MockServer
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    services:
-      ministack:
-        image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0
+        image: ghcr.io/ministackorg/ministack:1.5.16@sha256:9813da34285a0760477c761c1c03717e0290d259213c0ca97f551fefd87b292d
         ports:
           - 4566:4566
         # MiniStack launches nested service containers for the integration smoke test.
@@ -101,6 +46,11 @@ jobs:
         env:
           MINISTACK_ACCOUNT_ID: "000000000000"
           MINISTACK_REGION: eu-west-1
+      mockserver:
+        image: mockserver/mockserver:7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290
+        ports:
+          - 1080:1080
+        options: --network-alias=host.docker.internal
     steps:
       - name: Harden the runner (Audit all outbound calls)
         uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
@@ -124,39 +74,17 @@ jobs:
           terraform_version: latest
           terraform_wrapper: false
 
-      - name: Install Lambda dependencies
-        working-directory: lambdas
-        run: yarn install --frozen-lockfile
-
-      - name: Build smoke-test Lambda distributions
-        working-directory: lambdas
+      - name: Build Lambda distributions for smoke tests
         run: |
-          yarn workspace @aws-github-runner/webhook dist
-          yarn workspace @aws-github-runner/control-plane dist
+          ./.ci/build.sh
 
-      - name: Start MockServer
-        id: mockserver
-        uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0
-        with:
-          version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290'
-          port: '1080'
-          startup-timeout: '60'
-
-      - name: Connect MockServer to MiniStack network
-        shell: bash
-        run: |
-          set -euo pipefail
-          ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')"
-          network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')"
-          docker network connect --alias mockserver "$network" mockserver
+      - name: Install boto3
+        run: python3 -m pip install --upgrade boto3 botocore
 
-      - name: Mark repository as safe
-        shell: sh
-        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Set up ARM64 emulation
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
+        with:
+          platforms: arm64
 
-      - name: Run scale-set ECS/MockServer smoke test
-        env:
-          MINISTACK_GITHUB_MOCK_HOST: mockserver
-          MINISTACK_GITHUB_MOCK_PORT: "1080"
-          MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }}
-        run: sh tests/ministack/run-scale-set-integration.sh        
+      - name: Run combined MiniStack smoke test
+        run: python3 tests/ministack/run-ministack-smoke.py

.github/workflows/terraform.yml

@@ -53,7 +53,7 @@ env:
     termination-watcher
     microvm-foundation
     multi-runner
-    multi-runner-scale-set
+    multi-runner-orchestration
     external-managed-ssm-secrets
   TEST_MODULES: |
     modules/runners

.gitignore

@@ -29,3 +29,7 @@ secrets.auto.tfvars
 
 node_modules/
 site/
+
+__pycache__/
+ministack-smoke-checklist.txt
+ministack-smoke.log