diff --git a/.ci/Dockerfile b/.ci/Dockerfile index 3566cc1251..aa6d0e8128 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update \ FROM build as lambdas COPY lambdas /lambdas RUN --mount=type=cache,target=/work/node_modules,id=lambdas \ - yarn install --frozen-lockfile && yarn dist && \ + yarn install --immutable --mode=skip-build && yarn dist && \ find . -name "*.zip" -exec cp {} . \; FROM scratch as final diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index 8537e4beaa..b306cae0b3 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -34,8 +34,7 @@ jobs: persist-credentials: false - name: Install dependencies - run: yarn install --frozen-lockfile - + run: yarn install --immutable --mode=skip-build - name: Run prettier run: yarn format-check @@ -58,43 +57,33 @@ jobs: retention-days: 5 scale-set-container: - name: Build scale-set service container - runs-on: ubuntu-latest - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Set up QEMU - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - - name: Build scale-set service image - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: . - file: ./lambdas/services/scale-set/Dockerfile - platforms: linux/amd64,linux/arm64 - push: false - cache-from: type=gha,scope=scale-set-service - cache-to: type=gha,mode=max,scope=scale-set-service - - - name: Build scale-set service image for smoke test - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: . - file: ./lambdas/services/scale-set/Dockerfile - platforms: linux/amd64 - load: true - tags: scale-set-service:smoke-test - cache-from: type=gha,scope=scale-set-service - - - name: Run scale-set service image smoke test - run: ./tests/scale-set-container-smoke-test.sh + name: Build scale-set service container + runs-on: ubuntu-latest + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up QEMU + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build scale-set service image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64 + push: false + load: true + cache-from: type=gha,scope=scale-set-service + cache-to: type=gha,mode=max,scope=scale-set-service + + - name: Run scale-set service image smoke test + run: ./tests/scale-set-container-smoke-test.sh \ No newline at end of file diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 09aac4132a..1af03c71b9 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -53,7 +53,6 @@ jobs: - ephemeral - microvm-foundation - multi-runner - - multi-runner-scale-set - migration-test - multi-runner - termination-watcher diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68f5a38341..b5789d50dd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,21 @@ jobs: persist-credentials: false - name: Build dist working-directory: lambdas - run: yarn install --frozen-lockfile && yarn run test && yarn dist + run: yarn install --immutable --mode=skip-build && yarn run test && yarn dist + + - name: Build MicroVM lifecycle hook + working-directory: lambdas + run: yarn workspace @aws-github-runner/microvm-lifecycle-hooks build + + - name: Verify MicroVM lifecycle hook distribution + working-directory: lambdas + run: | + test -s services/microvm-lifecycle-hooks/dist/server.js + test -f services/microvm-lifecycle-hooks/dist/package.json + + - name: Package MicroVM lifecycle hook + working-directory: lambdas/services/microvm-lifecycle-hooks + run: (cd dist && zip -r ../microvm-lifecycle-hooks.zip .) - name: Get installation token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 id: token @@ -61,24 +75,34 @@ jobs: uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: '${{ github.workspace }}/lambdas/functions/**/*.zip' + + - name: Attest MicroVM lifecycle hook + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: lifecycle-hook-attest + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: '${{ github.workspace }}/lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip' + - name: Update release notes with attestation if: ${{ steps.release.outputs.releases_created == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ github.event.inputs.version }} TAG_NAME: ${{ steps.release.outputs.tag_name }} ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }} + LIFECYCLE_HOOK_ATTESTATION_URL: ${{ steps.lifecycle-hook-attest.outputs.attestation-url }} REPOSITORY: ${{ github.repository }} run: | - version="${VERSION}" tag_name="${TAG_NAME}" attestation_url="${ATTESTATION_URL}" + lifecycle_hook_attestation_url="${LIFECYCLE_HOOK_ATTESTATION_URL}" repository="${REPOSITORY}" - gh release view $version --json body -q '.body' > new-release-notes.md + gh release view "$tag_name" --json body -q '.body' > new-release-notes.md echo "## Attestation" >> new-release-notes.md - echo "Attestation url: $attestation_url" >> new-release-notes.md + echo "Lambda attestation url: $attestation_url" >> new-release-notes.md + echo "MicroVM lifecycle hook attestation url: $lifecycle_hook_attestation_url" >> new-release-notes.md echo "Verify the artifacts by running \`gh attestation verify --repo ${repository}\`" >> new-release-notes.md - gh release edit $tag_name -F new-release-notes.md -t $tag_name + gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name" + - name: Upload release assets if: ${{ steps.release.outputs.releases_created == 'true' }} env: @@ -86,10 +110,13 @@ jobs: TAG_NAME: ${{ steps.release.outputs.tag_name }} run: | tag_name="${TAG_NAME}" - for f in $(find . -name '*.zip'); do - gh release upload $tag_name $f - done - - name: Attach attestation + while IFS= read -r -d '' f; do + gh release upload "$tag_name" "$f" + done < <(find lambdas/functions -name '*.zip' -print0) + gh release upload "$tag_name" \ + "lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip" + + - name: Attach Lambda attestation if: ${{ steps.release.outputs.releases_created == 'true' }} env: ATTESTATION_BUNDLE: ${{ steps.attest.outputs.bundle-path }} @@ -99,13 +126,26 @@ jobs: run: | # rename attest bundle to github-aws-runners-terraform-aws-github-runner-attestation-$attestation-id.sigstore # OpenSSF expects the attestation bundle to be named in this format (*.sigstore) - SIGSTORE_BUNDLE=$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.sigstore - INTOTO_BUNDLE=$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.intoto.jsonl - mv ${ATTESTATION_BUNDLE} $SIGSTORE_BUNDLE - if [ -z "$SIGSTORE_BUNDLE" ]; then - echo "No attestation bundle found, skipping attachment." - exit 0 - fi - gh release upload $TAG_NAME "$SIGSTORE_BUNDLE" - cat ${SIGSTORE_BUNDLE} | jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' | base64 -d | jq .> ${INTOTO_BUNDLE} - gh release upload $TAG_NAME "${INTOTO_BUNDLE}" + sigstore_bundle="$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.sigstore" + intoto_bundle="$RUNNER_TEMP/github-aws-runners-terraform-aws-github-runner-attestation-${ATTESTATION_ID}.intoto.jsonl" + cp "$ATTESTATION_BUNDLE" "$sigstore_bundle" + gh release upload "$TAG_NAME" "$sigstore_bundle" + jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' "$sigstore_bundle" \ + | base64 --decode > "$intoto_bundle" + gh release upload "$TAG_NAME" "$intoto_bundle" + + - name: Attach MicroVM lifecycle hook attestation + if: ${{ steps.release.outputs.releases_created == 'true' }} + env: + ATTESTATION_BUNDLE: ${{ steps.lifecycle-hook-attest.outputs.bundle-path }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ steps.release.outputs.tag_name }} + ATTESTATION_ID: ${{ steps.lifecycle-hook-attest.outputs.attestation-id }} + run: | + sigstore_bundle="$RUNNER_TEMP/github-aws-runners-microvm-lifecycle-hooks-attestation-${ATTESTATION_ID}.sigstore" + intoto_bundle="$RUNNER_TEMP/github-aws-runners-microvm-lifecycle-hooks-attestation-${ATTESTATION_ID}.intoto.jsonl" + cp "$ATTESTATION_BUNDLE" "$sigstore_bundle" + gh release upload "$TAG_NAME" "$sigstore_bundle" + jq -r '.dsseEnvelope | select(.payloadType == "application/vnd.in-toto+json").payload' "$sigstore_bundle" \ + | base64 --decode > "$intoto_bundle" + gh release upload "$TAG_NAME" "$intoto_bundle" diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index 85ca7b44c0..0efaf6f416 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -2,7 +2,13 @@ name: "Smoke Tests" on: pull_request: - paths: ["**/*.tf", "**/*.hcl", ".github/workflows/smoke-tests.yml"] + paths: + - "**/*.tf" + - "**/*.hcl" + - "images/microvm-ubuntu/**" + - "lambdas/**" + - "tests/ministack/**" + - ".github/workflows/smoke-tests.yml" workflow_dispatch: concurrency: @@ -23,74 +29,13 @@ env: TF_INPUT: "false" jobs: - control_plane_smoke: - name: Run webhook and pool lifecycle smoke test against MiniStack + ministack_smoke: + name: Run webhook EC2/MicroVM and scale-set EC2 smoke tests against MiniStack runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 120 services: ministack: - image: ghcr.io/ministackorg/ministack:1.5.13@sha256:ce3c906f2866ff953ce4c56f06b1fa3e453bc32e41c00de17b5f5a8672c5a42c - ports: - - 4566:4566 - options: --add-host=host.docker.internal:host-gateway - env: - MINISTACK_ACCOUNT_ID: "000000000000" - MINISTACK_REGION: eu-west-1 - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version-file: lambdas/.nvmrc - package-manager-cache: false - - - name: Setup Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: latest - terraform_wrapper: false - - - name: Install Lambda dependencies - working-directory: lambdas - run: yarn install --frozen-lockfile - - - name: Build smoke-test Lambda distributions - working-directory: lambdas - run: | - yarn workspace @aws-github-runner/webhook dist - yarn workspace @aws-github-runner/control-plane dist - - - name: Start MockServer - id: mockserver - uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 - with: - version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' - port: '1080' - startup-timeout: '60' - - - name: Run webhook and pool lifecycle smoke test - env: - MINISTACK_GITHUB_MOCK_HOST: host.docker.internal - MINISTACK_GITHUB_MOCK_PORT: "1080" - MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} - run: sh tests/ministack/run-smoke.sh - - scale_set_integration_smoke: - name: Run scale-set ECS smoke test against MiniStack and MockServer - runs-on: ubuntu-latest - timeout-minutes: 30 - services: - ministack: - image: ghcr.io/ministackorg/ministack:1.5.12@sha256:41fe1ce2e666c6cc410c6047a9db8bf1df69cd0028ebc0a6c6e5517c3a83d6e0 + image: ghcr.io/ministackorg/ministack:1.5.16@sha256:9813da34285a0760477c761c1c03717e0290d259213c0ca97f551fefd87b292d ports: - 4566:4566 # MiniStack launches nested service containers for the integration smoke test. @@ -101,6 +46,11 @@ jobs: env: MINISTACK_ACCOUNT_ID: "000000000000" MINISTACK_REGION: eu-west-1 + mockserver: + image: mockserver/mockserver:7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290 + ports: + - 1080:1080 + options: --network-alias=host.docker.internal steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -124,39 +74,17 @@ jobs: terraform_version: latest terraform_wrapper: false - - name: Install Lambda dependencies - working-directory: lambdas - run: yarn install --frozen-lockfile - - - name: Build smoke-test Lambda distributions - working-directory: lambdas + - name: Build Lambda distributions for smoke tests run: | - yarn workspace @aws-github-runner/webhook dist - yarn workspace @aws-github-runner/control-plane dist + ./.ci/build.sh - - name: Start MockServer - id: mockserver - uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 - with: - version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' - port: '1080' - startup-timeout: '60' - - - name: Connect MockServer to MiniStack network - shell: bash - run: | - set -euo pipefail - ministack_container="$(docker ps --format '{{.ID}} {{.Image}}' | awk '$2 ~ /ministack/ {print $1; exit}')" - network="$(docker inspect --format '{{range $name, $_ := .NetworkSettings.Networks}}{{println $name}}{{end}}' "$ministack_container" | sed -n '1p')" - docker network connect --alias mockserver "$network" mockserver + - name: Install boto3 + run: python3 -m pip install --upgrade boto3 botocore - - name: Mark repository as safe - shell: sh - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Set up ARM64 emulation + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + with: + platforms: arm64 - - name: Run scale-set ECS/MockServer smoke test - env: - MINISTACK_GITHUB_MOCK_HOST: mockserver - MINISTACK_GITHUB_MOCK_PORT: "1080" - MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} - run: sh tests/ministack/run-scale-set-integration.sh + - name: Run combined MiniStack smoke test + run: python3 tests/ministack/run-ministack-smoke.py diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 9c16821875..4be16fa2c6 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -53,7 +53,7 @@ env: termination-watcher microvm-foundation multi-runner - multi-runner-scale-set + multi-runner-orchestration external-managed-ssm-secrets TEST_MODULES: | modules/runners diff --git a/.gitignore b/.gitignore index 276fe10733..e95da29855 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ secrets.auto.tfvars node_modules/ site/ + +__pycache__/ +ministack-smoke-checklist.txt +ministack-smoke.log diff --git a/docs/configuration.md b/docs/configuration.md index 2a8534e162..4604afdee9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -367,7 +367,7 @@ The scale-set lane itself defaults to `runner.group_name = "Default"`, name, runner owner, and GitHub App SSM references in the lane; credential values are not placed in the controller manifest. -The [multi-runner scale-set example](multi-runner-scale-set.md) shows how these +The [multi-runner orchestration example](examples/multi-runner-orchestration.md) shows how these provider-specific settings coexist with webhook lanes in the same v2 `multi_runner_config` map. diff --git a/docs/examples/index.md b/docs/examples/index.md index a4a3e7e1a7..260a2775d6 100644 --- a/docs/examples/index.md +++ b/docs/examples/index.md @@ -5,8 +5,7 @@ Examples are located in the [examples](https://github.com/github-aws-runners/ter - _[Default](default.md)_: The default example of the module - _[Ephemeral](ephemeral.md)_: Example usages of ephemeral runners based on the default example. - _[Multi Runner](multi-runner.md)_ : Example usage of creating a multi runner which creates multiple runners/ configurations with a single deployment. The examples including: "arm64", "windows", and "ubuntu" runners. -- _[Multi Runner Webhook](multi-runner-webhook.md)_: Example usage of one webhook deployment serving EC2 and Lambda MicroVM runner lanes. -- _[Multi Runner scale-set](multi-runner-scale-set.md)_ : Example usage of a v2 deployment combining webhook lanes with an experimental GitHub Actions scale-set lane. +- _[Multi-runner orchestration](multi-runner-orchestration.md)_: Example deployment with webhook and scale-set orchestration lanes across EC2 and MicroVM compute. - _[Permissions boundary](permissions-boundary.md)_: Example usages of permissions boundaries. - _[Prebuilt Images](prebuilt.md)_: Example usages of deploying runners with a custom prebuilt image. - _[Termination watcher](termination-watcher.md)_: Example usages of termination watcher. diff --git a/docs/examples/multi-runner-orchestration.md b/docs/examples/multi-runner-orchestration.md new file mode 100644 index 0000000000..319778a148 --- /dev/null +++ b/docs/examples/multi-runner-orchestration.md @@ -0,0 +1 @@ +--8<-- "examples/multi-runner-orchestration/README.md" diff --git a/docs/examples/multi-runner-scale-set.md b/docs/examples/multi-runner-scale-set.md deleted file mode 100644 index 3aefbd3056..0000000000 --- a/docs/examples/multi-runner-scale-set.md +++ /dev/null @@ -1,14 +0,0 @@ -# Multi-runner scale-set example - -This example combines ordinary webhook-managed lanes with one experimental -GitHub Actions runner scale-set lane. It demonstrates that v2 keeps the -deployment-wide defaults in `global_config*` and places orchestration and -compute-provider settings inside each `multi_runner_config` lane. - -The source example is available at -[examples/multi-runner-scale-set](https://github.com/github-aws-runners/terraform-aws-github-runner/tree/main/examples/multi-runner-scale-set). -Read its README before applying: the GitHub App values are sensitive, the -scale-set controller image must be supplied explicitly, and the GitHub scale -set/runner group must be authorized for the selected GitHub scope. - ---8<-- "examples/multi-runner-scale-set/README.md" diff --git a/docs/examples/multi-runner-webhook.md b/docs/examples/multi-runner-webhook.md deleted file mode 100644 index 19d24d5994..0000000000 --- a/docs/examples/multi-runner-webhook.md +++ /dev/null @@ -1 +0,0 @@ ---8<-- "examples/multi-runner-webhook/README.md" diff --git a/docs/microvm-runners.md b/docs/microvm-runners.md index 872dd52de0..fd9b003d3a 100644 --- a/docs/microvm-runners.md +++ b/docs/microvm-runners.md @@ -10,7 +10,7 @@ Lambda MicroVMs. The runner control plane receives demand, obtains the one-time runner configuration, starts a MicroVM from a published image, and passes the runtime execution role to the MicroVM. -The repository includes a combined [multi-runner webhook example](examples/multi-runner-webhook.md) +The repository includes a combined [multi-runner orchestration example](examples/multi-runner-orchestration.md) that places EC2 and Lambda MicroVM lanes behind one webhook endpoint. The provider-specific lifecycle checks are shared where possible, so the same deployment can validate both providers. @@ -82,9 +82,34 @@ The lifecycle-hook server is part of the image artifact. Updating the hook server therefore requires building/releasing the artifact and publishing a new compatible image before deploying that image version to the runner lane. +## Image boot and runner lifecycle + +The image starts its processes in two layers: + +1. Docker starts the S6 overlay (`/init`). The image configures internal + services, including the CloudWatch Agent, as S6 services. The internal + services startup script validates the requested service names, exposes the + MicroVM ID and runner-configuration SSM path to S6, and starts the configured + services. +2. The image command runs `image-entrypoint.sh`, which executes the Actions + runner's Node binary with `/opt/microvm/server.js`. That Node process is the + HTTP endpoint for the AWS Lambda MicroVM lifecycle hooks. + +When AWS sends the `run` hook, the server validates the request and passes the +MicroVM ID and `runHookPayload` to the lifecycle handler. The payload identifies +the SSM-backed runner configuration; the hook consumes the one-time JIT runner +configuration from SSM, then starts `/opt/actions-runner/run.sh` with that +configuration. The runner starts as the unprivileged `runner` user, and the +hook waits for the process launch handoff before acknowledging the request. + +The `terminate` hook stops the runner process. The server also handles the +runtime's readiness, validation, resume, and suspend hooks. After the runner +exits, the hook cleans up and the Node server shuts down, allowing the MicroVM +to finish its lifecycle. + ## Combined EC2 and MicroVM deployment -The [multi-runner webhook example](examples/multi-runner-webhook.md) accepts +The [multi-runner orchestration example](examples/multi-runner-orchestration.md) accepts explicit `runners_lambda_zip` and `webhook_lambda_zip` inputs and configures both compute providers behind one webhook. Its MicroVM settings require a published image: @@ -125,5 +150,5 @@ build the foundation, lifecycle-hook artifact, or MicroVM image for you. - [MicroVM foundation](examples/microvm-foundation.md) - [MicroVM image build README](https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/images/microvm-ubuntu/README.md) - [Lifecycle-hook service README](https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/services/microvm-lifecycle-hooks/README.md) -- [Multi-runner webhook](examples/multi-runner-webhook.md) +- [Multi-runner orchestration](examples/multi-runner-orchestration.md) - [MicroVM foundation module](modules/public/microvm-foundation.md) diff --git a/examples/microvm-foundation/README.md b/examples/microvm-foundation/README.md index c52c872d47..b7cee53971 100644 --- a/examples/microvm-foundation/README.md +++ b/examples/microvm-foundation/README.md @@ -32,7 +32,7 @@ The deployment order is: outputs and the released lifecycle-hook ZIP. The image builder uses the **build role**. 4. Deploy the runner control plane, such as - `examples/multi-runner-webhook`, with the published image ARN/version. The + `examples/multi-runner-orchestration`, with the published image ARN/version. The control plane resolves the **execution role** from the runner configuration and passes it to `RunMicrovm` when it starts a job. diff --git a/examples/multi-runner-webhook/.terraform.lock.hcl b/examples/multi-runner-orchestration/.terraform.lock.hcl similarity index 100% rename from examples/multi-runner-webhook/.terraform.lock.hcl rename to examples/multi-runner-orchestration/.terraform.lock.hcl diff --git a/examples/multi-runner-webhook/.terraform.lock.hcl.tofu b/examples/multi-runner-orchestration/.terraform.lock.hcl.tofu similarity index 100% rename from examples/multi-runner-webhook/.terraform.lock.hcl.tofu rename to examples/multi-runner-orchestration/.terraform.lock.hcl.tofu diff --git a/examples/multi-runner-webhook/README.md b/examples/multi-runner-orchestration/README.md similarity index 74% rename from examples/multi-runner-webhook/README.md rename to examples/multi-runner-orchestration/README.md index 1d63205c7c..af04218adf 100644 --- a/examples/multi-runner-webhook/README.md +++ b/examples/multi-runner-orchestration/README.md @@ -1,14 +1,13 @@ -# Multi-runner webhook example +# Multi-runner orchestration example -This example exercises the shared experimental multi-runner v2 webhook path -with EC2 and Lambda MicroVM compute. The runner lanes, webhook orchestration, -Lambda artifacts, and GitHub configuration are common; provider-owned inputs -are grouped under `compute_provider`. +This example exercises the experimental multi-runner v2 webhook path with EC2 +and Lambda MicroVM compute, plus a GitHub Actions scale-set lane. The webhook +and scale-set lanes share the same deployment, VPC, and GitHub App configuration. +Provider-owned inputs are grouped under `compute_provider`. -The example creates both an EC2 lane and a Lambda MicroVM lane behind the same -webhook endpoint. The MiniStack smoke test sends matching jobs to each lane in -sequence, so adding another provider means adding another lane and provider -specific lifecycle assertions to the same deployment. +The example creates webhook-managed EC2 and Lambda MicroVM lanes behind one +webhook endpoint, and an EC2 GitHub Actions scale-set lane. The MiniStack smoke +test exercises all three lanes in one deployment. The runner-control and webhook Lambda archives are explicit inputs: @@ -18,6 +17,12 @@ terraform apply \ -var='webhook_lambda_zip=/path/to/webhook.zip' ``` +## Scale-set configuration + +Set `scale_set.name` and provide an immutable controller image through +`scale_set.container.image`. The GitHub App installation must be authorized for +the configured runner owner and group. + ## MicroVM prerequisites The MicroVM lane expects an image that has already been built and published in @@ -85,9 +90,10 @@ or execution-role setup steps. | [aws\_region](#input\_aws\_region) | AWS Region where the runner control plane and compute provider resources are deployed. | `string` | `"eu-west-1"` | no | | [compute\_provider](#input\_compute\_provider) | Provider-specific settings for the EC2 and MicroVM runner lanes. |
object({
aws = object({
ec2 = object({
instance_types = list(string)
ami = object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter = optional(object({
arn = string
}), null)
kms_key = optional(object({
arn = string
}), null)
})
})
microvm = object({
image_arn = string
image_version = optional(string, null)
ingress_network_connectors = optional(list(string), [])
egress_network_connectors = list(string)
})
})
})
| n/a | yes | | [environment](#input\_environment) | Name prefix for the example resources. | `string` | n/a | yes | -| [github\_app](#input\_github\_app) | GitHub App credentials used by the webhook orchestration provider. |
object({
id = string
key_base64 = string
webhook_secret = string
})
| n/a | yes | -| [github\_enterprise\_server](#input\_github\_enterprise\_server) | Optional GitHub Enterprise Server endpoint used by the smoke-test API mock. |
object({
url = string
ssl_verify = bool
})
| `null` | no | +| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. |
object({
url = optional(string, null)
ssl_verify = optional(bool, true)
runner_owner = optional(string, null)
registration_level = optional(string, "organization")
})
| `{}` | no | +| [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. |
object({
id = string
key_base64 = string
installation_id = optional(string, null)
webhook_secret = string
})
| n/a | yes | | [runners\_lambda\_zip](#input\_runners\_lambda\_zip) | Local ZIP file for the runner-control Lambda. | `string` | n/a | yes | +| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. |
object({
name = string
runner_group_name = optional(string, "Default")
min_runners = optional(number, 0)
container = optional(object({
image = optional(string, null)
}), {})
})
| n/a | yes | | [webhook\_lambda\_zip](#input\_webhook\_lambda\_zip) | Local ZIP file for the webhook Lambda. | `string` | n/a | yes | ## Outputs diff --git a/examples/multi-runner-webhook/main.tf b/examples/multi-runner-orchestration/main.tf similarity index 75% rename from examples/multi-runner-webhook/main.tf rename to examples/multi-runner-orchestration/main.tf index 5b1b8b94c6..bf8ccfa62a 100644 --- a/examples/multi-runner-webhook/main.tf +++ b/examples/multi-runner-orchestration/main.tf @@ -26,11 +26,18 @@ module "runners" { global_config_github = { app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = var.github_app.webhook_secret + key_base64 = var.github_app.key_base64 + id = var.github_app.id + installation_id = var.github_app.installation_id + webhook_secret = var.github_app.webhook_secret } - enterprise_server = var.github_enterprise_server + enterprise_server = { + url = var.github.url + ssl_verify = var.github.ssl_verify + } + runner_owner = var.github.runner_owner + runner_registration_level = var.github.registration_level + } global_config_lambda = { @@ -83,6 +90,16 @@ module "runners" { } } } + scale_set = { + grouping = { + strategy = "runner_config" + } + container = var.scale_set.container + network = { + vpc_id = module.base.vpc.vpc_id + subnet_ids = module.base.vpc.private_subnets + } + } } global_config_storage_provider = { @@ -148,6 +165,33 @@ module "runners" { } } } + ec2_scalet_set = { + runner = { + os = "linux" + architecture = "x64" + name_prefix = "ec2_scalet_set-" + extra_labels = ["self-hosted", "linux", "x64", "ec2", "scale-set"] + group_name = var.scale_set.runner_group_name + } + orchestration_provider = { + scale_set = { + name = var.scale_set.name + runner = { + min_runners = var.scale_set.min_runners + max_runners = 10 + boot_time_in_minutes = 10 + } + } + } + compute_provider = { + aws = { + ec2 = { + instance_types = var.compute_provider.aws.ec2.instance_types + ami = var.compute_provider.aws.ec2.ami + } + } + } + } microvm = { runner = { diff --git a/examples/multi-runner-webhook/microvm.tf b/examples/multi-runner-orchestration/microvm.tf similarity index 100% rename from examples/multi-runner-webhook/microvm.tf rename to examples/multi-runner-orchestration/microvm.tf diff --git a/examples/multi-runner-webhook/outputs.tf b/examples/multi-runner-orchestration/outputs.tf similarity index 100% rename from examples/multi-runner-webhook/outputs.tf rename to examples/multi-runner-orchestration/outputs.tf diff --git a/examples/multi-runner-webhook/providers.tf b/examples/multi-runner-orchestration/providers.tf similarity index 100% rename from examples/multi-runner-webhook/providers.tf rename to examples/multi-runner-orchestration/providers.tf diff --git a/examples/multi-runner-webhook/variables.tf b/examples/multi-runner-orchestration/variables.tf similarity index 62% rename from examples/multi-runner-webhook/variables.tf rename to examples/multi-runner-orchestration/variables.tf index 7e19d64c1a..df763c4d7c 100644 --- a/examples/multi-runner-webhook/variables.tf +++ b/examples/multi-runner-orchestration/variables.tf @@ -10,23 +10,28 @@ variable "environment" { } variable "github_app" { - description = "GitHub App credentials used by the webhook orchestration provider." - sensitive = true + description = "GitHub App ID, base64-encoded private key, and installation ID." type = object({ - id = string - key_base64 = string - webhook_secret = string + id = string + key_base64 = string + installation_id = optional(string, null) + webhook_secret = string }) + sensitive = true } -variable "github_enterprise_server" { - description = "Optional GitHub Enterprise Server endpoint used by the smoke-test API mock." +variable "github" { + description = "Optional GitHub endpoint and scale-set ownership settings." + type = object({ - url = string - ssl_verify = bool + url = optional(string, null) + ssl_verify = optional(bool, true) + runner_owner = optional(string, null) + registration_level = optional(string, "organization") }) - default = null + + default = {} } variable "runners_lambda_zip" { @@ -66,3 +71,16 @@ variable "compute_provider" { }) }) } + +variable "scale_set" { + description = "GitHub Actions scale-set configuration." + + type = object({ + name = string + runner_group_name = optional(string, "Default") + min_runners = optional(number, 0) + container = optional(object({ + image = optional(string, null) + }), {}) + }) +} \ No newline at end of file diff --git a/examples/multi-runner-webhook/versions.tf b/examples/multi-runner-orchestration/versions.tf similarity index 100% rename from examples/multi-runner-webhook/versions.tf rename to examples/multi-runner-orchestration/versions.tf diff --git a/examples/multi-runner-scale-set/.terraform.lock.hcl b/examples/multi-runner-scale-set/.terraform.lock.hcl deleted file mode 100644 index c96d2b19bf..0000000000 --- a/examples/multi-runner-scale-set/.terraform.lock.hcl +++ /dev/null @@ -1,93 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "6.63.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:9cre7jh1lSs/9igpgAcENMUAUlYW3HCtkav3up4oit0=", - "h1:dRlYHkc+r6fgzF57WC7Zjcmb6sF/6TTGDEgwGK+LAZY=", - "zh:005d56736afd17d963998c405cee6f434dbc23a415109f9435ff1542879ae611", - "zh:026ef126321a86ad7080b5d858e2527f96f5289678cbcd8856296e229c43339d", - "zh:06e0b58b2d1eddb5137fc86bee7ad2d07953c0bc3f57cccfc5ae0d2456068a3a", - "zh:07221735d61ababed84734e5ffcfc5bd59d01f29f029166ba5f2175895dceed1", - "zh:1a72db00583112bdb8c19b213a78a3f5de754fffc08f07e061f4e326289fab7d", - "zh:32968e74a53b03e97a084dc7050c22ef661fb5b3ea8a44f5a63e47bc45ad0e7c", - "zh:4b357dfe4b820e3e4acd2881cff8288b2186491e63416751f0d12692ba478ceb", - "zh:81e30884d7de686265e7d87bb92527e802878c65a378470ede2a1e9f4e40ccc9", - "zh:82e137297f6a5a08b9ce2138f7aabea245ad99495d9d9eff502f752d6ca90dbd", - "zh:8eb83b67099f0ea9df238a979dff933ff50ce06a2e3ff05a48556a10f10dd204", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:d0ba30886cbe41850fee689f51ef9088578f323cfd21817bb409951d43c465eb", - "zh:dd48e7089784454bc03d713e9057f5ca0ea1613bd402125054a51894957b7925", - "zh:f250fa81e54cf60fcb0e9c0fc4ac043f1ecc2ac24967f628b3609364fcab3d04", - "zh:f38fc09fc25a8d2cf89a4d4cd6a5ef7cb1aad72798dbdcad58b8876b6a551a54", - "zh:f7c7380fdf126e1901f2084588dbfd724c76cb131ccfa795a541219111103c06", - ] -} - -provider "registry.terraform.io/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.0" - hashes = [ - "h1:9rBZCMNpxKwMlRbWH2QpwD3kqUCAejdOZQ/aiiDObXQ=", - "h1:m24fjcInWvTVZ1XSo2MaNuKPe+X/gfG8SIi09rA7a7M=", - "zh:0baa4566cf77f1ff52f4293d1c8536202dd23edc197c3196413a28343c3ac3a0", - "zh:16b5559c3c07088ddad11a9bb9e9c0799999363c2958e9a5be2bcbbf2cd9ca64", - "zh:197c79015a10d1cce904a8ea722cbc750c42aeae2da53f44a6a0751d9fd1aa90", - "zh:29d0b03e5343a80677ebfeb2e2c31cbe4b1f65e736e53417454a4277fec2544c", - "zh:4896bfa6cf1d2fd562b47ef2e87f47862ae92a04f8ad5d764380f0c6653473b8", - "zh:531f8529cbca49f681883e57761a05a8398afaef6d1ab0d205d26bf12f4428e8", - "zh:6aaf5011d83161c86d2bfb80c0923ec934e578288758da2f37acb7aec129004b", - "zh:7430275253d3d3c40aa6179e0ec0d63212874dbbc06c5a51b9d07ec590f9756c", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:be17dc611e95e26cdf6cad79dfccf1064f0e32032a2efeb939a9bbe7fb1cbfe9", - "zh:f0e3b0aa644202e1d79d2000dca91f6019425da71e9800fa23f27e51c034f195", - "zh:f62bae4519e4ead49182ddc8afe8cf61e2a4c3ba3973b0fbba967736a2696aa3", - "zh:fcafa360a5b0b96244f26f4e3a6d642b716a376557142c2442ff2fb12d11da18", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.3.1" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:TuxJq10DVnRP7c5HBZPyyvQGcckNVfijyU1eXEu5e4M=", - "h1:m5FqidbIgh+E9OigiZh8/xbkvpUQFSj3hZo/jqNLCLQ=", - "zh:08c59776542ea16e5a8545752787b17ff412922182b4cfabe16139197be8ac44", - "zh:123109cc7e5ed6d515787fbc212f2a3fd5e75647bb24ab7c801ccd4d4ed42451", - "zh:14b3fa4372754b54844b41d5dbd4671a292d8d6828b90169061feb4d7b15dd05", - "zh:56a4daaa3212f57b764bf3d1f333141c6610c5f21abb240e0111221f7c7fa4d4", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7e888a026dbacd2474a42264227ae35f639780f0f0c613529d10a95cd61988b3", - "zh:85a53646267e87d600df7124e4767ffde9bba3b6356d45d961618bdd68131cc7", - "zh:8ffa0e9c7c39b2ab0905b472465d6e35ef0b776b3f6273bb34c150340b61bff1", - "zh:9846510a1841530d4403f4818e233f91e3b3bade7441047599fbf800742f65be", - "zh:afa98d44860875f037c6def0a7e6ff208e042712ba771f620482b143cd336891", - "zh:bdca130d9ef27488ae0b13bc8fd8019e8bbdd4f2ceff29da066bd333165d68c5", - "zh:cb3b94cbca88210dd0d1f11e2b8a89333f48c3857faf8f70f589072ce7c28610", - "zh:f0c0ba87925fe32f84b80f7513b1efb1b0866f51f899ba825e95ad59ff09b018", - ] -} - -provider "registry.terraform.io/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", - "h1:UlBuNVuCGJ39tTv2c5gz2NRZnQbXfbIWbTzWcth5o74=", - "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", - "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", - "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", - "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", - "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", - "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", - "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", - "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", - "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", - "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", - "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", - ] -} diff --git a/examples/multi-runner-scale-set/.terraform.lock.hcl.tofu b/examples/multi-runner-scale-set/.terraform.lock.hcl.tofu deleted file mode 100644 index 577519c08d..0000000000 --- a/examples/multi-runner-scale-set/.terraform.lock.hcl.tofu +++ /dev/null @@ -1,150 +0,0 @@ -# This file is maintained automatically by "tofu init". -# Manual edits may be lost in future updates. - -provider "registry.opentofu.org/hashicorp/aws" { - version = "6.63.0" - constraints = ">= 5.0.0, >= 6.21.0, >= 6.33.0" - hashes = [ - "h1:1jhQJPHOPu2mzDG/ke3tK8PNcEqQHA4vhF05WWlM/yg=", - "h1:3+pvT0KN/bkJ6TBuExj+gxptEozhnpo80Ztblwq85eo=", - "h1:5aTequ87wZS7Mh4dEIayDGKcFdaFgHtw74NtqY5Idi0=", - "h1:AMRlrrM3z1SmrslOtotqKq02zapxLKtXaSN9Jbs0Oho=", - "h1:OTjECFWTDxsjcUfOKCNBp75Z5lGrW/KplRDsjTZYT2g=", - "h1:b8LORLOKMOOl+nK1M2UhCjELSjjziClJuAv6hYuySHs=", - "h1:bUfTX1giRLOyfDbBvsDbwR3tJmsTFRWcOTQdj2npDWA=", - "h1:dzs4kwx+itVGAH7yEOyeoWcE3LNRMnWtlt4ROgyAa0M=", - "h1:lnjou+SiwpYJ+j9PXWozXPHSPlhxIZb0RqpsSEBzfGw=", - "h1:pqzUeHAQj9NctgkwaynaF2aB+3QiZXcoslzMGjT743w=", - "h1:qTXEWOWxA6sfUpC29UXrsbHnNzWH7+j1RTUVG4YCm+U=", - "h1:qdHKOKt/ISn9RLjUe22OZBpN3F7H2DFeHJL/CSc2x8E=", - "h1:tpNzIZBzzUW7/kLU3BhYf3jhdO5uNwYfNmgC9B8kvMM=", - "h1:uVVlFgjg6GyxJLbCsTO1+R5fTNbZ73mLpVpSd0mMrFk=", - "h1:xGJsV5IFf7c11cXzJrsY40hiJCghp4odT0eJyTyAUYY=", - "zh:039a03e920e55f14a691feb67216a2d142bfee603128e15f9c5138f9ecd85016", - "zh:14e060b7f46ca7b0fa009b91aef419c58cbdff854de96e9a1d853166f8d902fd", - "zh:18803e8fe2c291c8db5526c71b3287ff7c81453f10ca6d8e69cdf9c535b00783", - "zh:1b83fce6e31a6095e932d80a7c3f47ac04252653a2de2b98ec6204563310fcba", - "zh:2add7bc976ceebb1a94d84598762c9b9cf281ca52ec83deeb4e95e90aa200a12", - "zh:2f22cd5372408f11937fa5513a7b960d3cebc334c5ec65fc5322c3bac1c1f664", - "zh:41c5e857dacfd83b7ca12a435204957ff6ca8830b9efefd0d381ad4d63b19779", - "zh:4eace6246e46999782d219bc4f50f83d19ef9156bacf5ca1528da12da4918015", - "zh:5e1c1281c3f929399e2ed3dbdce03426fd57a9ec55cd36e04acf1712aa5954ba", - "zh:608272b1f5d75ead123c9d933aa1fed7dc832cedd1506019046b4c8fdcc91dce", - "zh:6b3680f8a2f7be2c171953aba89d639fb2624b9cf52ec304e16434874566601d", - "zh:99aa1006f2141f3341a02020e1c91abfb02280e57c77e0415c98b8d900353d88", - "zh:9ad235bef34a89a8dd9943f9fa9f05cc729bb52a4e0dc926a31bb13cb0ae2418", - "zh:e0e3ac361e04748a4ca0c1cdbb6abab2aa817f4ad67e1692817d16e370161d59", - "zh:f60962c982a41fde956e796425e7194b4311741c179c060c1c8b5e16a557d635", - ] -} - -provider "registry.opentofu.org/hashicorp/local" { - version = "2.9.0" - constraints = "~> 2.0" - hashes = [ - "h1:1dtKYW/5a1qob3yneL6WzOlnSGfYtJ6a2XeejCk9yb4=", - "h1:5NseXq5wU8O20ersTtV4ocrLYFFtgFr7n0pRLO1W2Rw=", - "h1:5d22ZPPK4iiygPbwRz/PJF5Es/0axVpMlPRpCR0Padw=", - "h1:AnwyolirmIlBMjH6+tV8bKkvT+5axJNYxi2y2IguiX4=", - "h1:PBp+HeseY021Fw3sLznCG27idgwPoff4cBuNmKgPL2w=", - "h1:VDxIhe4GbzdOCdmt7mQaqdwERQW6GSI7Roonts42Gr0=", - "h1:ZO6eWWnf8LjjV1q/JNeL9WLtZ6fwIttOnyN5LjCNSEo=", - "h1:dPIAf8oUAz+vW2E0iZunMvpuPddRZIztRsPSY1u+VnY=", - "h1:fwTDVG9AhFVKQZIb1EXkHv4FqzsZNlLWgkyPGDmZZEE=", - "h1:kDc465XPC7/6XFCjrMC4mTqhA9ef0FHKuJ3ZgfGNfeg=", - "h1:kGbjxrI2P8MHeyVtE1U3Q1TbyF71ExnHxtkrE+Aj6UU=", - "h1:kcoK6Afbsj54u9zaEqpecWAFKytqjBijtguCNwV3d4M=", - "h1:rxomJjDwOo+YZ+WIPc25FqEgsz9orh/2MCyUcZmFjvw=", - "h1:t0CMn/Rkwquw8l2yQ+O4ApzbMZfY2UazbsDnZygzACA=", - "h1:tJwgm2BS4xCGlElCDQEFXQoefY9Y4t0JdSKTtsPBbBo=", - "zh:13ef7ecd1e397ec5b20ea588508dd3e3b8d6c50d809ae76b079abf9dd8d02e4b", - "zh:2190c9325980076489ce02b0f5dd2c0b91fc8711cefa99e714d8619a32827ad1", - "zh:2a0cfc5600730093705071707e4a4e4e953e7d9091859e0f66b46daa1060dd5d", - "zh:2ff53eac1af43ab9a2248a0e53c963d46e19cf04bc4c3f323591cfcebb218252", - "zh:4ebc3dee700f60af9da29970052fd02fa947813162b224716862dc9d7f1f7542", - "zh:5fe6dab84ceeaa8eb3f1567c5f05578333370c472240ca5c5bfc25e92d4d5586", - "zh:66bbec16367bbf440045502c9779b11f4ac5b022c8d8d17afe12d431950838b5", - "zh:7641e5c2e4b529e869cde29ab5b1de2fd1091489eb745b19ac2709bd7f4dfd84", - "zh:855bfba0756d17ce07595ff57d7cf664443d1495127cb88fb063362734b8b22a", - "zh:aaec10f237921d60c581d1b7a66f0a8a8019d9802dc04af11b5b981f6682e01d", - "zh:e460835a38ffa1e74f6929904bfd14ef473d217fd537b7ce834abe5ce5e2ce07", - "zh:ecc4295215db0e4aea3c9329611c31e09a853e1ae207d56742403bd4f5516703", - "zh:ee6d9fae63a612072e00402894e14826af7a3351c235b9c5b423b7629a77ca29", - "zh:f2b5c8db74aa7ebcf7cd423672358437d42401675069ef67b01ff910054e49d5", - "zh:f5aff74d3eb96d4592c7bca5cd3ea89b469e84efbf382944bd0f844a57059c09", - ] -} - -provider "registry.opentofu.org/hashicorp/null" { - version = "3.3.1" - constraints = "~> 3.0, ~> 3.2" - hashes = [ - "h1:2wld81FnmHW0WVgy081sIfokCr2+NuatS8yjeLEet7Y=", - "h1:AClQjJ6X22V4qcRgcYSxiXCMmp2pz0G8WVQC7wAx66o=", - "h1:AY3XQbuviNd2X5VhHYEbhNta1m/CG3JD2BKFKhCt1Y4=", - "h1:CUOZUd7H11lsU+4tISlnYIiP5BqnX8IDwFCVqfLJyAg=", - "h1:JIfV0nA/pLWnIFGscvTfuavQCn2NeHxJBeb6UUg/joA=", - "h1:RejAh+nyCwqDGExGln2Kb4Ro5LyHak0eJe0P9g8CHPc=", - "h1:SHOuTZjYymsmy4asuRq6NC3yW+zdVZOOt4f5nrb+EPM=", - "h1:WwPat/gT4gO8GvvKNdSkkXWVD65JppLJfqKOt9HhOqQ=", - "h1:Z3hXVLrOyaRiiLmmL5UCOdcRMguwjN1x5TYNdmBDgls=", - "h1:dd78Ad5HdfPzPts7A9qIxfitXhAriV/qza38fr2ukjk=", - "h1:dyVb++KwDdybzLTE6bf7GZiVQ31iWsgKPWmhTQ8G42k=", - "h1:gD8ZH6WWe+5gg5+y8SpLWGPUDzSxcQ3HKP8IDM/wW3I=", - "h1:juXCww0zRQKFTDZoKqYR0+Sn1lu99oeL6pr0Jh6LWx0=", - "h1:kFAySmtsshyNV7IhIrEdASzVcvwy68eeZCVC66P7yNk=", - "h1:nS5azDopRisB2NInwDx3Hrfg2FdVt8Gw0gTQzC0rd70=", - "zh:164eb061d84e01759f391265865fb31828083d0a06b25f7af7e094cbdb18c799", - "zh:1bb9b669a82b52c0cba2860c71e9ee6699ef302f28cb8ed06f572d39bc6c7c4f", - "zh:1ea9b31a8f29302122c1e8d673693f3ac270336dae560af803cd1117265a469a", - "zh:238bd463cb0154fb935dc331da40c0a9cbe5db9cee615ae5f35ccad5eed7dc41", - "zh:30ef2b7384cf7e20f33fe75754b54cf669d59816f3ad4fc73bfb2b26fb6735e9", - "zh:35b5cded16e4b57c207d03ee0979b14baf486fa520e6edb7a2eecf18f1b85471", - "zh:3dc840d13a50cd215c7540573f27e2b61f739ba90aee5b7c3846079aa0ab5534", - "zh:3f9309a18db608f975d5691fcb47a6e14d77199156a52e9c39dcafe3737f2b07", - "zh:44263a219f7dbd1848b545d080110b4f7d0495e77b71cd3c7a0b5ec52a09accb", - "zh:4dec54aa5f445eeea035bbd4839bcded5e47ecd07cba0e70c5a09e9272cb592f", - "zh:5e8fb319d7c6d6c4566a18b9d0c91580b4901a96acd7fdc476bfc79f074368e2", - "zh:b0e8b6d41834b57fcfbb5ca00da52ccb757e1a95b6a2d546c0dae8bfbeca1cdf", - "zh:bbde4c3a1dcc1718027a61a4cdf661619d17af1b58df1038fe27bcf43c3dc29b", - "zh:c4140fff9f692baf29236557f706f9515f93229413438527d764023a82301da3", - "zh:f8e9d83184e4bbeb97c6f0d569833007c48ba5a7ff334def201df4991d03a962", - ] -} - -provider "registry.opentofu.org/hashicorp/random" { - version = "3.9.0" - constraints = "~> 3.0" - hashes = [ - "h1:8EQU5KSxezcjo/phRSe69rDOI0lk4pSaggj7FsskYp8=", - "h1:Lw9im2VBBJQ3RyAbHPQ0rcvcmmcZWm3x+kIOpN+Tv9s=", - "h1:U8KXqGCoNI9/guYbTvzgdtVk3fRthoG0UXwm1JoEpIs=", - "h1:YXaVd4p6qXPPVaxIBaIDNXmBwT02ZqDn0qD+tYpw8sA=", - "h1:cOpc03fphEt/G9Rfc4jLL/fW0D7tgvlXqiDKPF4vuww=", - "h1:g09RR7T1xWkeGrZwWvWMT9ncJrFGr1k3CBD585UmO7w=", - "h1:gGDdPPibmw2EWROx+sh1RGLjR5+nPwZyrf6/N9jXfeM=", - "h1:haE7/nXCOhXKP4oXeEnER3t5CaVQWqujz4nBnpeTUv4=", - "h1:ieSVpfZS2lKuMr05ph0QsOVpCzg7uk3cgKBaXR+Ikug=", - "h1:ig2s1IS9IzehorRjvVAnKIsUUj8fkgyxct1L/kswcc4=", - "h1:j3lS+ZEERFnoab8t1ppDrScGVP/cgWbzlCrEYKTCXYw=", - "h1:lxezrKmOiQIySHAM+os8qLVq7hqufDr8h3Hpzvsk+78=", - "h1:lzRqBJAG+NETxHbEZUJ/YP3RMEjZBinTX7VmgH3lw60=", - "h1:tdSNWK5ApqUsgbdYieyeYLTu6nIZUV3hR1oFqUfAuGo=", - "h1:xedet8yH/zI2CfdxsGlK0nlFWc/Bp61yrWsEa3fHB8g=", - "zh:03f1114cc20b8913523735ab76e0f0a2b16ce13c92923a53304bf85f07fc0dbc", - "zh:105b678ee72322a3067f105d7e05e940f6143238f377f6e87ff4ec909246ac2a", - "zh:55f3bbf13ea18cbace61a706566a80f25f33fe2b1780b6f3d7b582af2a05b6d2", - "zh:63adf996db48f082f7a6351eb485e219cd88795fc71e6ec60a837263ab0d2cb1", - "zh:7e99550738a4e3cc68b8a467714b0d69371025fe95e3326d5323d026d55653e9", - "zh:8342b54af3a18a37e075eeae61be57f4de2ba71b35d95c5075d402dd2c1f289d", - "zh:83ee18e32ac9dd5fc91298554b7c4cfa4c3a1db50f4c797945637cc93c0844ae", - "zh:993ecc0adbf6bd535a59fbc9b735d8c33950e6f6eb5e621d750da9b71d65d80a", - "zh:ad722bc59d4edbf1415e827fc007c0efe6e0e9462d5568bae20b34be1058a261", - "zh:ae9448e1f87b2f9a6c5197a0e9862162ec6b137cb3a3835e11522995d8939e7c", - "zh:bc9cdd3aac784f759125c6627f6f6416e8726a1c184eb9cf3e55b9edbc94c627", - "zh:c8e35b89572ba1c40a9b20022e033a3395fb8d42e7604d50c900f193ba10382e", - "zh:e2deaa8a9975ef81d9f62baed12c41286918b0a10908e0e031f13f69a3b730a1", - "zh:ee39707557210a0ab1098aa357d2cdfe502e5a312d0dbdffb09d08facc4d3fc5", - "zh:f81afe4eb63e8aa9e0ea71be6c990f0dc69cb360e7191c0742a991f4a5081b64", - ] -} diff --git a/examples/multi-runner-scale-set/README.md b/examples/multi-runner-scale-set/README.md deleted file mode 100644 index a2ca69cff4..0000000000 --- a/examples/multi-runner-scale-set/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# Multi-runner scale-set example - -This example demonstrates the experimental multi-runner v2 interface. Shared -defaults are configured with `global_config*` variables, while -each runner lane uses `multi_runner_config` for its matcher, -runner lifecycle, and compute-provider settings. - -The example creates four lanes from one deployment: - -- Linux ARM64 Amazon Linux runners. -- Ephemeral Linux x64 Amazon Linux runners with job retry enabled. -- Linux x64 runners managed by a GitHub Actions scale set. -- Windows x64 Server Core 2022 runners. - -The v2 interface keeps provider-owned settings inside the selected provider -configuration. For example, VPC and subnet settings are under -`global_config_compute_provider.aws.ec2`, while the per-lane -instance types and AMI filter are under each lane's compute provider block. - -The scale-set lane uses `orchestration_provider.scale_set`. Its controller -network is configured under the global scale-set block and its GitHub -installation ID is provided by `var.github_app`. - -Configure the GitHub App variables before applying: - -```bash -terraform init -terraform apply \ - -var='github_app={id="123456",key_base64="...",installation_id="123456789"}' \ - -var='github={runner_owner="example",registration_level="organization"}' \ - -var='scale_set={name="linux-scale-set",container={image="ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256:"}}' -``` - -The `github_app` value is sensitive and should be supplied through a secure -variable source in real deployments rather than committed to configuration. -The GitHub App must be installed for the configured GitHub account. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.5.6 | -| [aws](#requirement\_aws) | >= 6.33 | -| [local](#requirement\_local) | ~> 2.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [random](#provider\_random) | 3.9.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [base](#module\_base) | ../base | n/a | -| [runners](#module\_runners) | ../../modules/multi-runner | n/a | -| [webhook\_github\_app](#module\_webhook\_github\_app) | ../../modules/webhook-github-app | n/a | - -## Resources - -| Name | Type | -|------|------| -| [random_id.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [ami](#input\_ami) | Optional AMI configuration keyed by runner lane. |
map(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter = optional(object({
arn = string
}), null)
kms_key = optional(object({
arn = string
}), null)
}))
| `{}` | no | -| [aws\_region](#input\_aws\_region) | AWS region to deploy to. | `string` | `"eu-west-1"` | no | -| [environment](#input\_environment) | Environment name, used as prefix. | `string` | n/a | yes | -| [github](#input\_github) | Optional GitHub endpoint and scale-set ownership settings. |
object({
url = optional(string, null)
ssl_verify = optional(bool, true)
runner_owner = optional(string, null)
registration_level = optional(string, "organization")
})
| `{}` | no | -| [github\_app](#input\_github\_app) | GitHub App ID, base64-encoded private key, and installation ID. |
object({
id = string
key_base64 = string
installation_id = optional(string, null)
})
| n/a | yes | -| [runner\_binaries\_enabled](#input\_runner\_binaries\_enabled) | Whether runner binary synchronization is enabled. | `bool` | `true` | no | -| [scale\_set](#input\_scale\_set) | GitHub Actions scale-set configuration. |
object({
name = string
runner_group_name = optional(string, "Default")
min_runners = optional(number, 0)
container = optional(object({
image = optional(string, null)
}), {})
})
| n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [webhook\_endpoint](#output\_webhook\_endpoint) | n/a | -| [webhook\_secret](#output\_webhook\_secret) | n/a | - diff --git a/examples/multi-runner-scale-set/main.tf b/examples/multi-runner-scale-set/main.tf deleted file mode 100644 index 30d70a3a9b..0000000000 --- a/examples/multi-runner-scale-set/main.tf +++ /dev/null @@ -1,215 +0,0 @@ -locals { - environment = var.environment - aws_region = var.aws_region -} - -resource "random_id" "random" { - byte_length = 20 -} - -module "base" { - source = "../base" - - prefix = local.environment - aws_region = local.aws_region -} - -module "runners" { - source = "../../modules/multi-runner" - - prefix = local.environment - aws_region = local.aws_region - - experimental_features = ["multi-runner-v2"] - - global_config = { - tags = { - Example = local.environment - Project = "ProjectX" - } - runner = { - os = "linux" - architecture = "x64" - extra_labels = ["v2"] - } - } - - global_config_github = { - app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - installation_id = var.github_app.installation_id - webhook_secret = random_id.random.hex - } - enterprise_server = { - url = var.github.url - ssl_verify = var.github.ssl_verify - } - runner_owner = var.github.runner_owner - runner_registration_level = var.github.registration_level - } - - global_config_lambda = { - architecture = "arm64" - } - - global_config_orchestration_provider = { - webhook = { - eventbridge = { - enabled = true - accept_events = ["workflow_job"] - } - } - scale_set = { - grouping = { - strategy = "runner_config" - } - container = var.scale_set.container - network = { - vpc_id = module.base.vpc.vpc_id - subnet_ids = module.base.vpc.private_subnets - } - } - } - - global_config_compute_provider = { - aws = { - ec2 = { - vpc_id = module.base.vpc.vpc_id - subnet_ids = module.base.vpc.private_subnets - ssm_enabled = true - runner_binaries = { - enabled = var.runner_binaries_enabled - } - } - } - } - - multi_runner_config = { - linux-arm64 = { - runner = { - architecture = "arm64" - name_prefix = "amazon-arm64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "linux", "arm64", "amazon"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["t4g.large", "c6g.large"] - ami = lookup(var.ami, "linux-arm64", null) - } - } - } - } - - linux-x64 = { - runner = { - name_prefix = "amazon-x64-" - extra_labels = ["amazon"] - } - orchestration_provider = { - webhook = { - runner = { - ephemeral = true - maximum_count = 1 - } - matcherConfig = { - labelMatchers = [["self-hosted", "linux", "x64", "amazon"]] - exactMatch = false - priority = 1 - } - queue = { - delay_webhook_event = 0 - } - job_retry = { - enabled = true - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5a.large", "m5ad.large"] - ami = lookup(var.ami, "linux-x64", null) - } - } - } - } - - linux-scale-set = { - runner = { - name_prefix = "scale-set-" - extra_labels = ["scale-set"] - group_name = var.scale_set.runner_group_name - } - orchestration_provider = { - scale_set = { - name = var.scale_set.name - runner = { - min_runners = var.scale_set.min_runners - max_runners = 10 - boot_time_in_minutes = 10 - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5.large"] - ami = lookup(var.ami, "linux-scale-set", null) - } - } - } - } - - windows-x64 = { - runner = { - os = "windows" - name_prefix = "windows-x64-" - } - orchestration_provider = { - webhook = { - runner = { - boot_time_in_minutes = 20 - maximum_count = 1 - } - matcherConfig = { - exactMatch = true - labelMatchers = [["self-hosted", "windows", "x64", "servercore-2022"]] - } - } - } - compute_provider = { - aws = { - ec2 = { - instance_types = ["m5.large", "c5.large"] - ami = lookup(var.ami, "windows-x64", null) - } - } - } - } - } -} - -module "webhook_github_app" { - source = "../../modules/webhook-github-app" - depends_on = [module.runners] - - github_app = { - key_base64 = var.github_app.key_base64 - id = var.github_app.id - webhook_secret = random_id.random.hex - } - webhook_endpoint = module.runners.webhook.endpoint -} \ No newline at end of file diff --git a/examples/multi-runner-scale-set/outputs.tf b/examples/multi-runner-scale-set/outputs.tf deleted file mode 100644 index 1feaf2e671..0000000000 --- a/examples/multi-runner-scale-set/outputs.tf +++ /dev/null @@ -1,8 +0,0 @@ -output "webhook_endpoint" { - value = module.runners.webhook.endpoint -} - -output "webhook_secret" { - sensitive = true - value = random_id.random.hex -} diff --git a/examples/multi-runner-scale-set/providers.tf b/examples/multi-runner-scale-set/providers.tf deleted file mode 100644 index eca2fe96a7..0000000000 --- a/examples/multi-runner-scale-set/providers.tf +++ /dev/null @@ -1,9 +0,0 @@ -provider "aws" { - region = local.aws_region - - default_tags { - tags = { - Example = local.environment - } - } -} diff --git a/examples/multi-runner-scale-set/variables.tf b/examples/multi-runner-scale-set/variables.tf deleted file mode 100644 index f0f9f66c4d..0000000000 --- a/examples/multi-runner-scale-set/variables.tf +++ /dev/null @@ -1,72 +0,0 @@ -variable "github_app" { - description = "GitHub App ID, base64-encoded private key, and installation ID." - - type = object({ - id = string - key_base64 = string - installation_id = optional(string, null) - }) - sensitive = true -} - -variable "github" { - description = "Optional GitHub endpoint and scale-set ownership settings." - - type = object({ - url = optional(string, null) - ssl_verify = optional(bool, true) - runner_owner = optional(string, null) - registration_level = optional(string, "organization") - }) - - default = {} -} - -variable "scale_set" { - description = "GitHub Actions scale-set configuration." - - type = object({ - name = string - runner_group_name = optional(string, "Default") - min_runners = optional(number, 0) - container = optional(object({ - image = optional(string, null) - }), {}) - }) -} - -variable "environment" { - description = "Environment name, used as prefix." - - type = string -} - -variable "aws_region" { - description = "AWS region to deploy to." - - type = string - default = "eu-west-1" -} - -variable "runner_binaries_enabled" { - description = "Whether runner binary synchronization is enabled." - - type = bool - default = true -} - -variable "ami" { - description = "Optional AMI configuration keyed by runner lane." - - type = map(object({ - filter = optional(map(list(string)), { state = ["available"] }) - owners = optional(list(string), ["amazon"]) - id_ssm_parameter = optional(object({ - arn = string - }), null) - kms_key = optional(object({ - arn = string - }), null) - })) - default = {} -} \ No newline at end of file diff --git a/examples/multi-runner-scale-set/versions.tf b/examples/multi-runner-scale-set/versions.tf deleted file mode 100644 index 6af69ab915..0000000000 --- a/examples/multi-runner-scale-set/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 6.33" - } - local = { - source = "hashicorp/local" - version = "~> 2.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - required_version = ">= 1.5.6" -} diff --git a/images/microvm-ubuntu/output/microvm/microvm-image.json b/images/microvm-ubuntu/output/microvm/microvm-image.json new file mode 100644 index 0000000000..5f8719f3f3 --- /dev/null +++ b/images/microvm-ubuntu/output/microvm/microvm-image.json @@ -0,0 +1,18 @@ +{ + "artifactSha256": "e2a764c8cf009c590cb4cc333de367f573ab2d1642e9f1aa08b03be64b2f40cd", + "artifactUri": "s3://ministack-microvm-artifacts-eu-west-1/lambda-microvms/artifacts/e2a764c8cf009c590cb4cc333de367f573ab2d1642e9f1aa08b03be64b2f40cd.zip", + "egressNetworkConnectorArn": "arn:aws:lambda:eu-west-1:000000000000:network-connector:ministack", + "imageArn": "arn:aws:lambda:eu-west-1:000000000000:microvm-image/micro-ubuntu24", + "imageState": "UPDATED", + "imageVersion": "3", + "logGroup": "/aws/lambda/microvms/ubuntu24", + "logStream": "micro-ubuntu24/ministack-smoke", + "name": "micro-ubuntu24", + "operation": "update", + "region": "eu-west-1", + "releaseVersion": "", + "sourceRevision": "42e753ab1dbd", + "state": "SUCCESSFUL", + "status": "ACTIVE", + "ubuntuBaseImage": "000000000000.dkr.ecr.eu-west-1.amazonaws.com/base-ubuntu24@sha256:11dc1ccb427f0464a2369e645454c272bb0baece7357c892ba69d313b3a332cf" +} diff --git a/lambdas/services/microvm-lifecycle-hooks/README.md b/lambdas/services/microvm-lifecycle-hooks/README.md new file mode 100644 index 0000000000..075de5295c --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/README.md @@ -0,0 +1,137 @@ +# Lambda MicroVM lifecycle hooks + +This service implements the lifecycle-hook HTTP server used to start one ephemeral GitHub Actions runner inside an AWS Lambda MicroVM. Storage-specific reads and one-time consumption are delegated to `@aws-github-runner/storage-providers`; this package owns only payload validation, lifecycle state, and the runner process boundary. + +## Build and run + +From `lambdas/`: + +```bash +yarn nx test @aws-github-runner/microvm-lifecycle-hooks +yarn workspace @aws-github-runner/microvm-lifecycle-hooks build +yarn workspace @aws-github-runner/microvm-lifecycle-hooks start +``` + +`build` uses esbuild to create the self-contained CommonJS server bundle `dist/server.js`. It also writes `dist/package.json` with `type: commonjs` so the bundle remains executable after it is copied outside the Yarn workspace. + +## Build and release with the Lambda artifacts + +The lifecycle-hook server is a deployable image-build artifact, not a service +that is installed separately beside the runner control plane. Build and test it +through the normal Lambda workspace/release process, then provide the resulting +ZIP to the MicroVM image build as `MICROVM_LIFECYCLE_HOOK_ZIP`. Packer embeds +that artifact in the image; every published image used by the MicroVM provider +must contain a compatible hook server. + +The hook server's release lifecycle is therefore separate from the MicroVM +execution role. The image build uses the foundation's build role. When a job +starts, the runner control plane supplies the runtime execution role to +`RunMicrovm`. + +To build before invoking Docker, run the workspace build above. In the existing MicroVM runner Dockerfile, which already installs s6-overlay and the GitHub runner's Node 24 runtime, copy the complete artifact and replace the old hook command with: + +```dockerfile +COPY lambdas/services/microvm-lifecycle-hooks/dist/ /opt/microvm-lifecycle-hooks/ +ENTRYPOINT ["/init"] +CMD ["/command/with-contenv", "/opt/actions-runner/externals/node24/bin/node", "/opt/microvm-lifecycle-hooks/server.js"] +``` + +Alternatively, build the service inside Docker with the repository root as the build context. Add this pinned builder stage: + +```dockerfile +ARG NODE_BUILDER_IMAGE=node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 +FROM ${NODE_BUILDER_IMAGE} AS lifecycle-build +WORKDIR /source +COPY lambdas/ ./lambdas/ +RUN corepack enable \ + && cd lambdas \ + && yarn install --immutable \ + && yarn workspace @aws-github-runner/microvm-lifecycle-hooks build +``` + +Use a clean checkout for that build context, or exclude local `node_modules/`, `coverage/`, and `dist/` directories with `.dockerignore`, so host-built dependencies are not copied into the Linux builder. + +Then copy the builder output into the existing final runner stage and use its supervisor and Node 24 runtime: + +```dockerfile +COPY --from=lifecycle-build \ + /source/lambdas/services/microvm-lifecycle-hooks/dist/ \ + /opt/microvm-lifecycle-hooks/ +ENTRYPOINT ["/init"] +CMD ["/command/with-contenv", "/opt/actions-runner/externals/node24/bin/node", "/opt/microvm-lifecycle-hooks/server.js"] +``` + +For an image without s6-overlay, start the artifact with `node /opt/microvm-lifecycle-hooks/server.js` under that image's process supervisor. The hook binds to `0.0.0.0:8080` by default. Restrict the port to the Lambda MicroVM lifecycle network; the protocol does not add a separate application authentication layer. + +## Run payloads + +AWS sends an outer JSON object whose `runHookPayload` is itself a JSON string. Version 1 remains strict and SSM-specific for backwards compatibility: + +```json +{ + "microvmId": "microvm-bdd2d536-3d87-35e4-8b40-18664608ebc1", + "runHookPayload": "{\"version\":1,\"imageArn\":\"arn:aws:lambda:eu-west-1:166060576821:function:microvm-image\",\"imageVersion\":\"8.0\",\"runnerConfigSsmPath\":\"/github-action-runners/example/config\",\"runnerTokenSsmPath\":\"/github-action-runners/example/token\"}" +} +``` + +Version 1 is translated to the shared allowlisted SSM environment. Version 2 carries the exact environment-variable map under `context.storage`. SSM example: + +```json +{ + "version": 2, + "context": { + "storage": { + "RUNNER_CONFIG_STORAGE_PROVIDER": "aws_ssm", + "SSM_TOKEN_PATH": "/github-action-runners/example/token" + } + } +} +``` + +Both versions reject missing, unknown, or provider-incompatible fields. The SSM storage context accepts only its two keys; AWS credentials, timeout overrides, and arbitrary environment names are rejected. The validated storage map is exported once before the consumer is resolved. A retry may reuse the identical map, but it cannot change storage configuration after initialization. + +`microvmId` is an opaque path-safe `[A-Za-z0-9_.-]{1,256}` value. The resolved storage provider uses it to consume the one-time JIT configuration. Storage context variables are removed from the runner child environment. + +For a rolling upgrade, keep emitting version 1 SSM payloads until every deployed image contains this service. Old images do not understand version 2. + +## Entrypoint contract + +On `/run`, the hook starts `${RUNNER_ROOT:-/opt/actions-runner}/run.sh --jitconfig ` directly without a shell. The JIT configuration is passed only as the `--jitconfig` argument to the runner process: + +```text +run.sh --jitconfig +``` + +The hook waits for a short process-launch handoff before acknowledging `/run`; it does not require a custom readiness pipe. The JIT configuration, storage context, and AWS credential environment variables are not inherited by the runner process. `/terminate` sends `SIGTERM` to the detached process group and escalates to `SIGKILL` after the grace period. + +After the runner entrypoint exits on its own, the hook closes its HTTP server and exits with status `0` only when the runner exited cleanly. In the documented s6-overlay image layout above, that makes the foreground container command exit so s6 can stop the remaining image services and shut down the application container's PID 1. This path does not require `lambda:TerminateMicrovm` in the runner role. AWS documents only explicit termination and maximum duration as MicroVM termination triggers, so retain trusted control-plane cleanup and the maximum duration as failure backstops, and verify the container-exit behavior against a restored MicroVM before relying on it operationally. + +Useful environment variables are: + +| Variable | Default | Purpose | +| --------------------------------- | --------------------- | --------------------------------------------- | +| `HOOK_PORT` | `8080` | Lifecycle-hook HTTP port | +| `RUNNER_ROOT` | `/opt/actions-runner` | GitHub Actions runner installation | +| `RUNNER_USER` | `runner` | Runner process user name | +| `RUNNER_UID` | `1000` | Runner UID when the hook runs as root | +| `RUNNER_GID` | `1000` | Runner GID when the hook runs as root | +| `RUN_HOOK_TIMEOUT_SECONDS` | `55` | Total `/run` budget, bounded to 40–55 seconds | +| `HOOK_HEADERS_TIMEOUT_SECONDS` | `5` | HTTP header receive timeout | +| `HOOK_REQUEST_TIMEOUT_SECONDS` | `10` | HTTP request receive timeout | +| `HOOK_KEEP_ALIVE_TIMEOUT_SECONDS` | `5` | Idle keep-alive timeout | +| `AWS_SDK_CALL_TIMEOUT_SECONDS` | `5` | Individual storage-provider call timeout | +| `RUNNER_CONFIG_TIMEOUT_SECONDS` | `20` | Total runner-configuration polling timeout | +| `RUNNER_CONFIG_POLL_SECONDS` | `2` | Delay between provider polling attempts | +| `RUNNER_CONFIG_DELETE_ATTEMPTS` | `3` | SSM one-time configuration delete attempts | + +The request body is capped at 20 KiB and HTTP headers at 16 KiB. Internal errors are returned generically and secret-bearing provider errors are never logged. + +## Runtime security + +Removing AWS credential and storage variables from the runner child prevents accidental environment inheritance; it is not an IAM boundary. A job can still obtain credentials made available to the runtime role, so scope that role to each lane and treat job code as untrusted. + +- For SSM, grant only `ssm:GetParameter` and `ssm:DeleteParameter` on the lane's token path. Add `kms:Decrypt` only for the customer-managed key that encrypts those parameters. + +## TypeScript API + +The workspace service root is import-safe; importing it does not start the server. It exports the parser, lifecycle, process launcher, storage adapter, and server factories for composition and testing. `src/index.ts` is the executable-only NCC entrypoint. A producer can call `loadRunnerConfigStorageContextFromEnvironment` from `@aws-github-runner/storage-providers/runner-config-consumer` to copy only the selected provider and locator into `context.storage`. diff --git a/lambdas/services/microvm-lifecycle-hooks/build.mjs b/lambdas/services/microvm-lifecycle-hooks/build.mjs new file mode 100644 index 0000000000..4f26c0dcb8 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/build.mjs @@ -0,0 +1,21 @@ +import { mkdir, rm, writeFile } from 'node:fs/promises'; + +import { build } from 'esbuild'; + +await rm('dist', { force: true, recursive: true }); +await mkdir('dist', { recursive: true }); + +await build({ + bundle: true, + entryPoints: ['src/index.ts'], + format: 'cjs', + legalComments: 'eof', + minify: false, + packages: 'bundle', + platform: 'node', + sourcemap: false, + target: 'node24', + outfile: 'dist/server.js', +}); + +await writeFile('dist/package.json', '{\n "type": "commonjs"\n}\n'); diff --git a/lambdas/services/microvm-lifecycle-hooks/package.json b/lambdas/services/microvm-lifecycle-hooks/package.json new file mode 100644 index 0000000000..663194be7e --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/package.json @@ -0,0 +1,45 @@ +{ + "name": "@aws-github-runner/microvm-lifecycle-hooks", + "version": "1.0.0", + "private": true, + "description": "AWS Lambda MicroVM lifecycle hook server for ephemeral GitHub Actions runners", + "main": "src/public.ts", + "exports": { + ".": "./src/public.ts" + }, + "type": "module", + "license": "MIT", + "engines": { + "node": ">=24" + }, + "scripts": { + "start": "node dist/server.js", + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "build": "node build.mjs", + "dist": "yarn build && cd dist && zip ../microvm-lifecycle-hooks.zip *", + "format": "prettier --write \"**/*.{ts,json,md}\"", + "format-check": "prettier --check \"**/*.{ts,json,md}\"", + "all": "yarn build && yarn format && yarn lint && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "esbuild": "^0.27.0" + }, + "dependencies": { + "@aws-github-runner/storage-providers": "*" + }, + "nx": { + "includedScripts": [ + "build", + "dist", + "format", + "format-check", + "lint", + "start", + "watch", + "all" + ] + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/contracts.ts b/lambdas/services/microvm-lifecycle-hooks/src/contracts.ts new file mode 100644 index 0000000000..fbaadeacd1 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/contracts.ts @@ -0,0 +1,41 @@ +import type { RunnerConfigStorageContext } from '@aws-github-runner/storage-providers/runner-config-consumer'; + +export interface RunContext { + imageArn?: string; + imageVersion?: string; + microvmId: string; + storage: RunnerConfigStorageContext; +} + +export interface ConsumeOptions { + deadlineMs: number; + signal: AbortSignal; +} + +export interface RunnerBootstrap { + jitConfig: string; +} + +/** Resolves and consumes a one-time runner configuration without exposing provider details. */ +export interface JitConfigSource { + consume(context: RunContext, options: ConsumeOptions): Promise; +} + +export interface ManagedProcess { + readonly ready: Promise; + readonly exit: Promise; + readonly exited: boolean; + stop(graceMs?: number): Promise; +} + +export interface RunnerLauncher { + launch(bootstrap: RunnerBootstrap, microvmId: string): ManagedProcess; +} + +export interface Logger { + info(message: string, ...values: unknown[]): void; + warn(message: string, ...values: unknown[]): void; + error(message: string, ...values: unknown[]): void; +} + +export const consoleLogger: Logger = console; diff --git a/lambdas/services/microvm-lifecycle-hooks/src/index.ts b/lambdas/services/microvm-lifecycle-hooks/src/index.ts new file mode 100644 index 0000000000..30cde16d7b --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/index.ts @@ -0,0 +1,7 @@ +import { consoleLogger } from './contracts'; +import { main } from './server'; + +void main().catch(() => { + consoleLogger.error('Lambda MicroVM lifecycle hook server failed to start'); + process.exitCode = 1; +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.test.ts new file mode 100644 index 0000000000..9bdf48eadc --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.test.ts @@ -0,0 +1,357 @@ +import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { readFileSync } from 'node:fs'; +import { arch, tmpdir, type } from 'node:os'; +import { join } from 'node:path'; + +import type { JitConfigSource, Logger, ManagedProcess, RunContext, RunnerBootstrap, RunnerLauncher } from './contracts'; +import { RunnerLifecycle } from './lifecycle'; + +const quietLogger: Logger = { + error: () => undefined, + info: () => undefined, + warn: () => undefined, +}; + +const MICROVM_ID = 'microvm-bdd2d536-3d87-35e4-8b40-18664608ebc1'; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function overrideEnvironment(overrides: Record): () => void { + const previous = new Map(); + for (const [name, value] of Object.entries(overrides)) { + previous.set(name, process.env[name]); + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + return (): void => { + for (const [name, value] of previous) { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + } + }; +} + +function runRequest( + payload: object = { + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + runnerConfigSsmPath: '/runner/config', + runnerTokenSsmPath: '/runner/token', + version: 1, + }, +): string { + return JSON.stringify({ + microvmId: MICROVM_ID, + runHookPayload: JSON.stringify(payload), + }); +} + +class DeferredProcess implements ManagedProcess { + public readonly ready = Promise.resolve(); + public readonly exit: Promise; + public exited = false; + private resolveExit!: (code: number | null) => void; + + public constructor() { + this.exit = new Promise((resolve) => { + this.resolveExit = resolve; + }); + } + + public finish(code: number | null): void { + this.exited = true; + this.resolveExit(code); + } + + public async stop(): Promise { + if (!this.exited) { + this.finish(null); + } + } +} + +describe('RunnerLifecycle', () => { + it('writes setup information before launching the runner', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-lifecycle-setup-info-')); + const restoreEnvironment = overrideEnvironment({ ACTIONS_RUNNER_ROOT: directory }); + let setupInfoAtLaunch: unknown; + const launcher: RunnerLauncher = { + launch(): ManagedProcess { + setupInfoAtLaunch = JSON.parse(readFileSync(join(directory, '.setup_info'), 'utf8')); + return new DeferredProcess(); + }, + }; + + try { + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + launcher, + quietLogger, + ); + + await lifecycle.start(runRequest()); + + expect(setupInfoAtLaunch).toEqual([ + { + group: 'Operating System', + detail: `Platform: ${type()}\nArchitecture: ${arch()}`, + }, + { + group: 'Runner Image', + detail: + 'MicroVM image ARN: arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner\nMicroVM image version: 8.0', + }, + { + group: 'Lambda MicroVM', + detail: `MicroVM id: ${MICROVM_ID}`, + }, + ]); + expect((await stat(join(directory, '.setup_info'))).mode & 0o777).toBe(0o644); + } finally { + restoreEnvironment(); + await rm(directory, { force: true, recursive: true }); + } + }); + + it('writes available setup information without image metadata', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-lifecycle-setup-info-')); + const restoreEnvironment = overrideEnvironment({ ACTIONS_RUNNER_ROOT: directory }); + let setupInfoAtLaunch: unknown; + const launcher: RunnerLauncher = { + launch(): ManagedProcess { + setupInfoAtLaunch = JSON.parse(readFileSync(join(directory, '.setup_info'), 'utf8')); + return new DeferredProcess(); + }, + }; + + try { + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + launcher, + quietLogger, + ); + + await lifecycle.start( + runRequest({ + runnerConfigSsmPath: '/runner/config', + runnerTokenSsmPath: '/runner/token', + version: 1, + }), + ); + + expect(setupInfoAtLaunch).toEqual([ + { + group: 'Operating System', + detail: `Platform: ${type()}\nArchitecture: ${arch()}`, + }, + { + group: 'Lambda MicroVM', + detail: `MicroVM id: ${MICROVM_ID}`, + }, + ]); + } finally { + restoreEnvironment(); + await rm(directory, { force: true, recursive: true }); + } + }); + + it('starts only once and waits for terminate cleanup after the runner exits', async () => { + const events: string[] = []; + const processHandle = new DeferredProcess(); + const source: JitConfigSource = { + async consume(context: RunContext): Promise { + events.push(`consume:${context.storage.RUNNER_CONFIG_STORAGE_PROVIDER}:${context.microvmId}`); + return { jitConfig: 'encoded-jit' }; + }, + }; + const launcher: RunnerLauncher = { + launch(bootstrap, id): ManagedProcess { + events.push(`launch:${id}:${bootstrap.jitConfig}`); + return processHandle; + }, + }; + const lifecycle = new RunnerLifecycle(source, launcher, quietLogger); + + await expect(lifecycle.start(runRequest())).resolves.toBe(true); + await expect(lifecycle.start(runRequest())).resolves.toBe(false); + expect(events).toEqual([`consume:aws_ssm:${MICROVM_ID}`, `launch:${MICROVM_ID}:encoded-jit`]); + + processHandle.finish(0); + await expect(lifecycle.completion).resolves.toBe(0); + await lifecycle.stop(); + expect(processHandle.exited).toBe(true); + }); + + it('does not report an externally requested stop as runner self-completion', async () => { + const processHandle = new DeferredProcess(); + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + { launch: () => processHandle }, + quietLogger, + ); + + await lifecycle.start(runRequest()); + await lifecycle.stop(); + + await expect( + Promise.race([ + lifecycle.completion.then(() => 'completed'), + new Promise((resolve) => setImmediate(() => resolve('pending'))), + ]), + ).resolves.toBe('pending'); + }); + + it('reserves the runner startup budget before consuming configuration', async () => { + let consumeDeadline = 0; + const processHandle = new DeferredProcess(); + const lifecycle = new RunnerLifecycle( + { + async consume(_context, options): Promise { + consumeDeadline = options.deadlineMs; + return { jitConfig: 'encoded-jit' }; + }, + }, + { launch: () => processHandle }, + quietLogger, + ); + vi.spyOn(Date, 'now').mockReturnValue(1_000); + + await lifecycle.start(runRequest()); + + expect(consumeDeadline).toBe(21_000); + await lifecycle.stop(); + }); + + it('returns to idle if the GitHub Actions runner cannot launch', async () => { + const consume = vi.fn().mockResolvedValue({ jitConfig: 'encoded-jit' }); + const lifecycle = new RunnerLifecycle( + { consume }, + { + launch(): ManagedProcess { + throw new Error('spawn failed'); + }, + }, + quietLogger, + ); + + await expect(lifecycle.start(runRequest())).rejects.toThrow('spawn failed'); + await expect(lifecycle.start(runRequest())).rejects.toThrow('spawn failed'); + expect(consume).toHaveBeenCalledTimes(2); + }); + + it('logs the startup stage and safe error details without exposing internal messages', async () => { + const messages: unknown[] = []; + const logger: Logger = { + error: (...values) => messages.push(...values), + info: () => undefined, + warn: () => undefined, + }; + const error = new Error('encoded-jit-secret'); + error.name = 'encoded-jit-secret-name'; + Object.assign(error, { code: 'encoded-jit-secret-code' }); + const lifecycle = new RunnerLifecycle( + { + consume: async () => { + throw error; + }, + }, + { launch: () => new DeferredProcess() }, + logger, + ); + + await expect(lifecycle.start(runRequest())).rejects.toBe(error); + + const serializedMessages = JSON.stringify(messages); + expect(serializedMessages).toContain('consume runner configuration'); + expect(serializedMessages).toContain('unknown-error'); + expect(serializedMessages).not.toContain('encoded-jit-secret'); + }); + + it('aborts in-flight consumption when the run-hook deadline elapses', async () => { + let consumedSignal: AbortSignal | undefined; + let launched = false; + let releaseConsume = (): void => undefined; + const consumption = new Promise((resolve) => { + releaseConsume = resolve; + }); + const lifecycle = new RunnerLifecycle( + { + async consume(_context, options): Promise { + consumedSignal = options.signal; + await consumption; + return { jitConfig: 'encoded-jit' }; + }, + }, + { + launch(): ManagedProcess { + launched = true; + return new DeferredProcess(); + }, + }, + quietLogger, + ); + let calls = 0; + vi.spyOn(Date, 'now').mockImplementation(() => (calls++ === 0 ? 1_000 : 61_000)); + + await expect(lifecycle.start(runRequest())).rejects.toThrow('run-hook deadline elapsed'); + releaseConsume(); + await new Promise((resolve) => setImmediate(resolve)); + + expect(consumedSignal?.aborted).toBe(true); + expect(launched).toBe(false); + }); + + it('waits for cleanup when terminate races with the runner launch handoff', async () => { + let finishCleanup = (): void => undefined; + let reportLaunched = (): void => undefined; + let stopCalled = false; + const cleanup = new Promise((resolve) => { + finishCleanup = resolve; + }); + const launched = new Promise((resolve) => { + reportLaunched = resolve; + }); + const processHandle: ManagedProcess = { + ready: new Promise(() => undefined), + exit: new Promise(() => undefined), + exited: false, + async stop(): Promise { + stopCalled = true; + await cleanup; + }, + }; + const lifecycle = new RunnerLifecycle( + { consume: async () => ({ jitConfig: 'encoded-jit' }) }, + { + launch(): ManagedProcess { + reportLaunched(); + return processHandle; + }, + }, + quietLogger, + ); + + const rejectedStart = expect(lifecycle.start(runRequest())).rejects.toThrow('runner start was cancelled'); + await launched; + let terminateSettled = false; + const terminate = lifecycle.stop().then(() => { + terminateSettled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(stopCalled).toBe(true); + expect(terminateSettled).toBe(false); + + finishCleanup(); + await terminate; + await rejectedStart; + expect(terminateSettled).toBe(true); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.ts b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.ts new file mode 100644 index 0000000000..87fe7c02df --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/lifecycle.ts @@ -0,0 +1,235 @@ +import type { JitConfigSource, Logger, ManagedProcess, RunnerBootstrap, RunnerLauncher } from './contracts'; +import { consoleLogger } from './contracts'; +import { parseRunRequest } from './payload'; +import { writeRunnerSetupInfo } from './processes'; +import { beforeDeadline, beforeDeadlineOrAbort } from './timing'; + +type LifecycleState = 'idle' | 'starting' | 'running' | 'stopping' | 'stopped'; + +const SAFE_ERROR_NAMES = new Set([ + 'AccessDeniedException', + 'AbortError', + 'ExpiredTokenException', + 'InternalServerError', + 'InvalidKeyId', + 'KMSInvalidStateException', + 'ParameterNotFound', + 'ResourceNotFoundException', + 'TimeoutError', + 'ThrottlingException', +]); + +const SAFE_ERROR_CODES = new Set(['EACCES', 'EINVAL', 'ENOENT', 'EPERM', 'ETIMEDOUT']); + +const SAFE_ERROR_MESSAGES = new Set([ + 'operation was cancelled', + 'run-hook deadline elapsed', + 'GitHub Actions runner exited before the launch handoff', + 'runner launch was cancelled', + 'runner start was cancelled', +]); + +interface DiagnosticError extends Error { + code?: unknown; + $metadata?: unknown; +} + +function safeErrorDetails(error: unknown): Record { + if (!(error instanceof Error)) { + return { errorType: typeof error }; + } + + const diagnosticError = error as DiagnosticError; + const details: Record = { + errorName: SAFE_ERROR_NAMES.has(error.name) ? error.name : 'unknown-error', + }; + if (SAFE_ERROR_CODES.has(diagnosticError.code as string)) { + details.errorCode = diagnosticError.code; + } + if (SAFE_ERROR_MESSAGES.has(error.message)) { + details.errorMessage = error.message; + } + + if (diagnosticError.$metadata !== null && typeof diagnosticError.$metadata === 'object') { + const metadata = diagnosticError.$metadata as Record; + if (typeof metadata.httpStatusCode === 'number' && Number.isInteger(metadata.httpStatusCode)) { + details.httpStatusCode = metadata.httpStatusCode; + } + if (typeof metadata.attempts === 'number' && Number.isInteger(metadata.attempts)) { + details.awsAttempts = metadata.attempts; + } + if (typeof metadata.totalRetryDelay === 'number' && Number.isInteger(metadata.totalRetryDelay)) { + details.awsRetryDelayMs = metadata.totalRetryDelay; + } + } + + return details; +} + +function boundedNumber(value: string | undefined, fallback: number, minimum: number, maximum: number): number { + const parsed = Number(value); + return Number.isFinite(parsed) ? Math.max(minimum, Math.min(maximum, parsed)) : fallback; +} + +export class RunnerLifecycle { + private readonly runHookBudgetMs = boundedNumber(process.env.RUN_HOOK_TIMEOUT_SECONDS, 55, 40, 55) * 1_000; + // Reserve Lambda's 30-second service readiness window plus five seconds of local margin. + private readonly launchReserveMs = 35_000; + private state: LifecycleState = 'idle'; + private microvmId?: string; + private startAbort?: AbortController; + private startPromise?: Promise; + private runner?: ManagedProcess; + private resolveCompletion!: (exitCode: number | null) => void; + public readonly completion = new Promise((resolve) => { + this.resolveCompletion = resolve; + }); + + public constructor( + private readonly jitConfigSource: JitConfigSource, + private readonly launcher: RunnerLauncher, + private readonly logger: Logger = consoleLogger, + ) {} + + private currentState(): LifecycleState { + return this.state; + } + + public async start(body: string): Promise { + const context = parseRunRequest(body); + const deadlineMs = Date.now() + this.runHookBudgetMs; + + if (this.microvmId === context.microvmId && this.state === 'running') { + return false; + } + if (this.microvmId === context.microvmId && this.state === 'starting') { + if (this.startPromise === undefined) { + throw new Error('runner start state is inconsistent'); + } + await beforeDeadline(this.startPromise, deadlineMs); + if (this.currentState() === 'running') { + return false; + } + throw new Error('the preceding runner start did not succeed'); + } + if (this.state !== 'idle') { + throw new Error('another runner lifecycle is already active in this MicroVM'); + } + + const abort = new AbortController(); + this.state = 'starting'; + this.microvmId = context.microvmId; + this.startAbort = abort; + const startOperation = this.startRunner(context, deadlineMs, abort); + this.startPromise = startOperation; + const clearStartPromise = (): void => { + if (this.startPromise === startOperation) { + this.startPromise = undefined; + } + }; + void startOperation.then(clearStartPromise, clearStartPromise); + try { + await beforeDeadline(startOperation, deadlineMs); + return true; + } catch (error) { + // Cancel the underlying work so a timed-out hook cannot register a runner later. + abort.abort(); + throw error; + } + } + + private async startRunner( + context: ReturnType, + deadlineMs: number, + abort: AbortController, + ): Promise { + let bootstrap: RunnerBootstrap | undefined; + let processHandle: ManagedProcess | undefined; + let stage = 'consume runner configuration'; + try { + this.logger.info('Lifecycle hook consuming runner configuration for MicroVM %s', context.microvmId); + bootstrap = await this.jitConfigSource.consume(context, { + deadlineMs: deadlineMs - this.launchReserveMs, + signal: abort.signal, + }); + if (abort.signal.aborted) { + throw new Error('runner start was cancelled'); + } + + await writeRunnerSetupInfo(context, this.logger); + stage = 'launch GitHub Actions runner'; + this.logger.info('Lifecycle hook launching GitHub Actions runner for MicroVM %s', context.microvmId); + processHandle = this.launcher.launch(bootstrap, context.microvmId); + + stage = 'wait for runner launch handoff'; + this.logger.info('Lifecycle hook waiting for runner launch handoff for MicroVM %s', context.microvmId); + await beforeDeadlineOrAbort(processHandle.ready, deadlineMs, abort.signal); + if (abort.signal.aborted || this.state !== 'starting') { + throw new Error('runner start was cancelled'); + } + + this.runner = processHandle; + this.startAbort = undefined; + this.state = 'running'; + this.logger.info('GitHub Actions runner launch handed off for MicroVM %s', context.microvmId); + void this.monitorRunner(processHandle); + } catch (error) { + this.logger.error('Lifecycle hook runner startup failed', { + microvmId: context.microvmId, + stage, + ...safeErrorDetails(error), + }); + if (processHandle !== undefined) { + await processHandle.stop(); + } + if (this.state === 'stopping') { + this.state = 'stopped'; + } else { + this.state = 'idle'; + this.microvmId = undefined; + } + this.startAbort = undefined; + throw error; + } finally { + // JavaScript strings cannot be zeroized, but release the retained credential promptly. + if (bootstrap !== undefined) { + bootstrap.jitConfig = ''; + } + } + } + + private async monitorRunner(processHandle: ManagedProcess): Promise { + const exitCode = await processHandle.exit; + if (this.runner === processHandle) { + this.runner = undefined; + this.state = 'stopped'; + this.resolveCompletion(exitCode); + } + } + + public async stop(): Promise { + if (this.state === 'idle') { + this.state = 'stopped'; + } else if (this.state === 'starting' || this.state === 'running') { + this.state = 'stopping'; + } + this.startAbort?.abort(); + const starting = this.startPromise; + if (starting !== undefined) { + try { + await starting; + } catch { + // Cancellation is expected when terminate races with /run. + } + } + const running = this.runner; + this.runner = undefined; + await (running?.stop() ?? Promise.resolve()); + this.state = 'stopped'; + } + + public async resume(): Promise { + // Never re-consume a one-time runner configuration on resume. + return true; + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/payload.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/payload.test.ts new file mode 100644 index 0000000000..d357686ffc --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/payload.test.ts @@ -0,0 +1,136 @@ +import { HookRequestError, parseRunRequest } from './payload'; + +const MICROVM_ID = 'microvm-bdd2d536-3d87-35e4-8b40-18664608ebc1'; +const SSM_STORAGE = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', +} as const; +function request( + payload: object = { + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + runnerConfigSsmPath: '/github-action-runners/tenant/config', + runnerTokenSsmPath: '/github-action-runners/tenant/token', + version: 1, + }, + microvmId = MICROVM_ID, +): string { + return JSON.stringify({ + microvmId, + runHookPayload: JSON.stringify(payload), + }); +} + +describe('parseRunRequest', () => { + it('maps the producer version 1 payload to the allowlisted SSM storage environment', () => { + expect(parseRunRequest(request())).toEqual({ + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + microvmId: MICROVM_ID, + storage: SSM_STORAGE, + }); + }); + + it('accepts version 1 payloads without image metadata', () => { + expect( + parseRunRequest( + request({ + runnerConfigSsmPath: '/github-action-runners/tenant/config', + runnerTokenSsmPath: '/github-action-runners/tenant/token', + version: 1, + }), + ), + ).toEqual({ + microvmId: MICROVM_ID, + storage: SSM_STORAGE, + }); + }); + + it('preserves version 1 trailing-slash normalization', () => { + expect( + parseRunRequest( + request({ + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '8.0', + runnerConfigSsmPath: '/github-action-runners/tenant/config/', + runnerTokenSsmPath: '/github-action-runners/tenant/token/', + version: 1, + }), + ).storage, + ).toEqual({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', + }); + }); + + it.each([SSM_STORAGE])('accepts a strict version 2 $RUNNER_CONFIG_STORAGE_PROVIDER storage context', (storage) => { + expect( + parseRunRequest( + request({ + context: { storage }, + version: 2, + }), + ), + ).toEqual({ microvmId: MICROVM_ID, storage }); + }); + + it('accepts opaque path-safe MicroVM identifiers up to 256 characters', () => { + expect(parseRunRequest(request(undefined, 'a'.repeat(256))).microvmId).toHaveLength(256); + expect(parseRunRequest(request(undefined, 'future_id.example-01')).microvmId).toBe('future_id.example-01'); + }); + + it.each([ + ['invalid outer JSON', '{'], + ['an invalid MicroVM identifier', request(undefined, '../vm')], + ['an overlong MicroVM identifier', request(undefined, 'a'.repeat(257))], + ['an unversioned payload', request({ runnerConfigSsmPath: '/runner/config', runnerTokenSsmPath: '/runner/token' })], + [ + 'partial image metadata', + request({ + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + runnerConfigSsmPath: '/runner/config', + runnerTokenSsmPath: '/runner/token', + version: 1, + }), + ], + ['a relative legacy SSM path', request({ runnerConfigSsmPath: 'runner/token', version: 1 })], + ['a root legacy SSM path', request({ runnerConfigSsmPath: '/', version: 1 })], + ['repeated legacy SSM slashes', request({ runnerConfigSsmPath: '/runner//token', version: 1 })], + ['legacy SSM traversal', request({ runnerConfigSsmPath: '/runner/../token', version: 1 })], + [ + 'extra version 1 fields', + request({ encodedJitConfig: 'not-a-real-secret', runnerConfigSsmPath: '/runner/token', version: 1 }), + ], + [ + 'missing version 1 fields', + request({ context: { storage: SSM_STORAGE }, runnerConfigSsmPath: '/runner/token', version: 1 }), + ], + ['missing version 2 context', request({ version: 2 })], + ['missing version 2 storage', request({ context: {}, version: 2 })], + ['extra version 2 fields', request({ context: { storage: SSM_STORAGE }, unexpected: true, version: 2 })], + ['extra version 2 context fields', request({ context: { storage: SSM_STORAGE, unexpected: true }, version: 2 })], + [ + 'an unknown storage provider', + request({ + context: { + storage: { RUNNER_CONFIG_STORAGE_PROVIDER: 'unknown', SSM_TOKEN_PATH: '/runner/token' }, + }, + version: 2, + }), + ], + [ + 'typed provider fields in the environment map', + request({ context: { storage: { provider: 'aws_ssm', tokenPath: '/runner/token' } }, version: 2 }), + ], + [ + 'AWS credential injection', + request({ context: { storage: { ...SSM_STORAGE, AWS_ACCESS_KEY_ID: 'not-a-real-key' } }, version: 2 }), + ], + [ + 'timeout override injection', + request({ context: { storage: { ...SSM_STORAGE, RUNNER_CONFIG_TIMEOUT_SECONDS: '60' } }, version: 2 }), + ], + ])('rejects %s', (_name, body) => { + expect(() => parseRunRequest(body)).toThrow(HookRequestError); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/payload.ts b/lambdas/services/microvm-lifecycle-hooks/src/payload.ts new file mode 100644 index 0000000000..c0f22aeb4c --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/payload.ts @@ -0,0 +1,131 @@ +import { + parseRunnerConfigStorageContext, + type RunnerConfigStorageContext, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; + +import type { RunContext } from './contracts'; + +const MICROVM_ID_PATTERN = /^[A-Za-z0-9_.-]{1,256}$/; +const MICROVM_IMAGE_ARN_PATTERN = /^arn:aws[a-z-]*:lambda:[A-Za-z0-9-]+:[0-9]{12}:microvm-image:[A-Za-z0-9_.-]+$/; +const MICROVM_IMAGE_VERSION_PATTERN = /^[A-Za-z0-9_.-]{1,128}$/; + +export const MAX_REQUEST_BYTES = 20 * 1024; + +export class HookRequestError extends Error { + public constructor(message: string) { + super(message); + this.name = 'HookRequestError'; + } +} + +interface LambdaRunRequest { + microvmId?: unknown; + runHookPayload?: unknown; +} + +interface VersionedRunPayload { + version?: unknown; + imageArn?: unknown; + imageVersion?: unknown; + runnerConfigSsmPath?: unknown; + runnerTokenSsmPath?: unknown; + context?: unknown; +} + +interface VersionTwoContext { + storage?: unknown; +} + +function parseObject(value: string, errorMessage: string): T { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new HookRequestError(errorMessage); + } + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new HookRequestError(errorMessage); + } + return parsed as T; +} + +function hasExactKeys(value: object, expected: readonly string[]): boolean { + const keys = Object.keys(value); + return keys.length === expected.length && keys.every((key) => expected.includes(key)); +} + +function hasOnlyKeys(value: object, allowed: readonly string[]): boolean { + return Object.keys(value).every((key) => allowed.includes(key)); +} + +function isObject(value: unknown): value is object { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parseStorageContext(value: unknown): RunnerConfigStorageContext { + try { + return parseRunnerConfigStorageContext(value); + } catch { + // Storage validation details are deliberately not reflected to the hook caller. + throw new HookRequestError('runner configuration storage context is missing or invalid'); + } +} + +export function parseRunRequest(body: string): RunContext { + const request = parseObject(body, 'request body must be a JSON object'); + if (typeof request.microvmId !== 'string' || !MICROVM_ID_PATTERN.test(request.microvmId)) { + throw new HookRequestError('microvmId is missing or invalid'); + } + if (typeof request.runHookPayload !== 'string') { + throw new HookRequestError('runHookPayload must be a JSON string'); + } + + const payload = parseObject(request.runHookPayload, 'runHookPayload must contain valid JSON'); + if (payload.version === 1) { + if ( + !hasOnlyKeys(payload, ['version', 'imageArn', 'imageVersion', 'runnerConfigSsmPath', 'runnerTokenSsmPath']) || + typeof payload.runnerConfigSsmPath !== 'string' || + typeof payload.runnerTokenSsmPath !== 'string' + ) { + throw new HookRequestError('version 1 runHookPayload contains unsupported or missing fields'); + } + const hasImageMetadata = payload.imageArn !== undefined || payload.imageVersion !== undefined; + if ( + hasImageMetadata && + (typeof payload.imageArn !== 'string' || + payload.imageArn.length > 2_048 || + !MICROVM_IMAGE_ARN_PATTERN.test(payload.imageArn) || + typeof payload.imageVersion !== 'string' || + !MICROVM_IMAGE_VERSION_PATTERN.test(payload.imageVersion)) + ) { + throw new HookRequestError('imageArn and imageVersion must be valid when provided'); + } + return { + ...(hasImageMetadata + ? { + imageArn: payload.imageArn as string, + imageVersion: payload.imageVersion as string, + } + : {}), + microvmId: request.microvmId, + storage: parseStorageContext({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: payload.runnerTokenSsmPath, + }), + }; + } + if (payload.version === 2) { + if (!hasExactKeys(payload, ['version', 'context'])) { + throw new HookRequestError('version 2 runHookPayload contains unsupported or missing fields'); + } + if (!isObject(payload.context) || !hasExactKeys(payload.context, ['storage'])) { + throw new HookRequestError('version 2 context contains unsupported or missing fields'); + } + const context = payload.context as VersionTwoContext; + return { + microvmId: request.microvmId, + storage: parseStorageContext(context.storage), + }; + } + throw new HookRequestError('runHookPayload version must be 1 or 2'); +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/processes.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/processes.test.ts new file mode 100644 index 0000000000..46b46dafce --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/processes.test.ts @@ -0,0 +1,98 @@ +import { chown, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { GitHubRunnerLauncher } from './processes'; + +async function prepareRunnerFixture(directory: string, runner: string): Promise { + if (process.getuid?.() !== 0) { + return; + } + + const uid = Number(process.env.RUNNER_UID ?? 1_000); + const gid = Number(process.env.RUNNER_GID ?? 1_000); + await chown(directory, uid, gid); + await chown(runner, uid, gid); +} + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('GitHubRunnerLauncher', () => { + it('launches run.sh directly with the JIT config and a sanitized environment', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-runner-')); + const output = join(directory, 'output'); + const environmentOutput = join(directory, 'environment-output'); + const runner = join(directory, 'run.sh'); + + await writeFile( + runner, + `#!/bin/sh +set -eu +printf '%s|%s|%s' "$1" "$2" "$MICROVM_ID" > "$TEST_RUNNER_OUTPUT" +printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \ + "\${ENCODED_JIT_CONFIG-unset}" \ + "\${AWS_ACCESS_KEY_ID-unset}" \ + "\${AWS_SESSION_TOKEN-unset}" \ + "\${AWS_CONTAINER_CREDENTIALS_FULL_URI-unset}" \ + "\${AWS_PROFILE-unset}" \ + "\${AWS_DEFAULT_PROFILE-unset}" \ + "\${AWS_CONFIG_FILE-unset}" \ + "\${AWS_SHARED_CREDENTIALS_FILE-unset}" \ + "\${AWS_CREDENTIAL_EXPIRATION-unset}" \ + "\${RUNNER_CONFIG_STORAGE_PROVIDER-unset}" \ + "\${SSM_TOKEN_PATH-unset}" \ + "\${RUNNER_ALLOW_RUNASROOT-unset}" > "$TEST_RUNNER_ENV_OUTPUT" +sleep 0.2 +`, + { mode: 0o700 }, + ); + await prepareRunnerFixture(directory, runner); + + vi.stubEnv('RUNNER_ROOT', directory); + vi.stubEnv('TEST_RUNNER_OUTPUT', output); + vi.stubEnv('TEST_RUNNER_ENV_OUTPUT', environmentOutput); + vi.stubEnv('ENCODED_JIT_CONFIG', 'test-value'); + vi.stubEnv('AWS_ACCESS_KEY_ID', 'test-value'); + vi.stubEnv('AWS_SESSION_TOKEN', 'test-value'); + vi.stubEnv('AWS_CONTAINER_CREDENTIALS_FULL_URI', 'http://127.0.0.1/credentials'); + vi.stubEnv('AWS_PROFILE', 'test-profile'); + vi.stubEnv('AWS_DEFAULT_PROFILE', 'test-profile'); + vi.stubEnv('AWS_CONFIG_FILE', '/tmp/test-config'); + vi.stubEnv('AWS_SHARED_CREDENTIALS_FILE', '/tmp/test-credentials'); + vi.stubEnv('AWS_CREDENTIAL_EXPIRATION', '2099-01-01T00:00:00Z'); + vi.stubEnv('RUNNER_CONFIG_STORAGE_PROVIDER', 'aws_ssm'); + vi.stubEnv('SSM_TOKEN_PATH', '/runner/token'); + vi.stubEnv('RUNNER_ALLOW_RUNASROOT', '1'); + try { + const processHandle = new GitHubRunnerLauncher(30_000, 10).launch({ jitConfig: 'encoded-jit' }, 'mvm-1234'); + + await processHandle.ready; + await expect(processHandle.exit).resolves.toBe(0); + expect(await readFile(output, 'utf8')).toBe('--jitconfig|encoded-jit|mvm-1234'); + expect(await readFile(environmentOutput, 'utf8')).toBe( + 'unset|unset|unset|unset|unset|unset|unset|unset|unset|unset|unset|unset', + ); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); + + it('rejects readiness when run.sh exits before the launch handoff', async () => { + const directory = await mkdtemp(join(tmpdir(), 'microvm-runner-')); + const runner = join(directory, 'run.sh'); + + await writeFile(runner, '#!/bin/sh\nexit 7\n', { mode: 0o700 }); + await prepareRunnerFixture(directory, runner); + vi.stubEnv('RUNNER_ROOT', directory); + try { + const processHandle = new GitHubRunnerLauncher().launch({ jitConfig: 'encoded-jit' }, 'mvm-1234'); + + await expect(processHandle.ready).rejects.toThrow('exited before the launch handoff'); + await expect(processHandle.exit).resolves.toBe(7); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/processes.ts b/lambdas/services/microvm-lifecycle-hooks/src/processes.ts new file mode 100644 index 0000000000..45cadcb7d7 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/processes.ts @@ -0,0 +1,273 @@ +import { type ChildProcess, spawn } from 'node:child_process'; +import { chmod, writeFile } from 'node:fs/promises'; +import { arch, type } from 'node:os'; +import { isAbsolute, join } from 'node:path'; + +import type { Logger, ManagedProcess, RunContext, RunnerBootstrap, RunnerLauncher } from './contracts'; +import { delay } from './timing'; + +function safeErrorName(error: unknown): string { + return error instanceof Error && error.name ? error.name : 'UnknownError'; +} + +/** Writes the runner-visible machine information consumed during job setup. */ +export async function writeRunnerSetupInfo(context: RunContext, logger: Logger): Promise { + const runnerRoot = process.env.ACTIONS_RUNNER_ROOT ?? '/opt/actions-runner'; + const setupInfoPath = join(runnerRoot, '.setup_info'); + const setupInfo = [ + { + group: 'Operating System', + detail: `Platform: ${type()}\nArchitecture: ${arch()}`, + }, + ]; + if (context.imageArn !== undefined && context.imageVersion !== undefined) { + setupInfo.push({ + group: 'Runner Image', + detail: `MicroVM image ARN: ${context.imageArn}\nMicroVM image version: ${context.imageVersion}`, + }); + } + setupInfo.push({ + group: 'Lambda MicroVM', + detail: `MicroVM id: ${context.microvmId}`, + }); + + try { + await writeFile(setupInfoPath, `${JSON.stringify(setupInfo, null, 2)}\n`, { + encoding: 'utf8', + mode: 0o644, + }); + await chmod(setupInfoPath, 0o644); + } catch (error) { + // Setup information is informational and must not strand consumed JIT. + logger.warn('GitHub Actions runner setup information could not be written (%s)', safeErrorName(error)); + } +} + +const LAUNCH_HANDOFF_DELAY_MS = 1_000; +const MAX_POSIX_ID = 2_147_483_647; +const ENVIRONMENT_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +const ALWAYS_DENIED_RUNNER_ENVIRONMENT = new Set([ + 'ACTIONS_RUNNER_INPUT_JITCONFIG', + 'AWS_ACCESS_KEY_ID', + 'AWS_CONFIG_FILE', + 'AWS_CONTAINER_AUTHORIZATION_TOKEN', + 'AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE', + 'AWS_CONTAINER_CREDENTIALS_FULL_URI', + 'AWS_CONTAINER_CREDENTIALS_RELATIVE_URI', + 'AWS_CREDENTIAL_EXPIRATION', + 'AWS_DEFAULT_PROFILE', + 'AWS_PROFILE', + 'AWS_ROLE_ARN', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SECURITY_TOKEN', + 'AWS_SESSION_TOKEN', + 'AWS_SHARED_CREDENTIALS_FILE', + 'AWS_WEB_IDENTITY_TOKEN_FILE', + 'ENCODED_JIT_CONFIG', + 'JIT_CONFIG', + 'MICROVM_RUNNER_ENV_DENYLIST', + 'RUNNER_ALLOW_RUNASROOT', + 'RUNNER_CONFIG_SSM_ARN', + 'RUNNER_CONFIG_SSM_PATH', + 'RUNNER_CONFIG_STORAGE_PROVIDER', + 'RUNNER_TOKEN_SSM_PATH', + 'SSM_TOKEN_PATH', + 'bootstrap_payload', + 'encoded_jit_config', + 'jit_config', +]); + +function signalProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void { + if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null) { + return; + } + try { + process.kill(-child.pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') { + child.kill(signal); + } + } +} + +function parsePosixId(variable: string, fallback: number): number { + const value = process.env[variable] ?? String(fallback); + if (!/^\d+$/.test(value)) { + throw new Error(`${variable} must be a positive integer`); + } + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0 || parsed > MAX_POSIX_ID) { + throw new Error(`${variable} must be a positive integer`); + } + return parsed; +} + +function runnerIdentity(): { gid?: number; uid?: number } { + if (process.getuid?.() !== 0) { + return {}; + } + return { + gid: parsePosixId('RUNNER_GID', 1_000), + uid: parsePosixId('RUNNER_UID', 1_000), + }; +} + +function runnerEnvironmentDenylist(): Set { + const configured = process.env.MICROVM_RUNNER_ENV_DENYLIST; + if (configured === undefined || configured.trim() === '') { + return new Set(ALWAYS_DENIED_RUNNER_ENVIRONMENT); + } + + const denylist = new Set(ALWAYS_DENIED_RUNNER_ENVIRONMENT); + for (const name of configured.split(',')) { + const normalized = name.trim(); + if (!ENVIRONMENT_NAME_PATTERN.test(normalized)) { + throw new Error('MICROVM_RUNNER_ENV_DENYLIST contains an invalid environment name'); + } + denylist.add(normalized); + } + return denylist; +} + +function runnerEnvironment(microvmId: string, denylist: ReadonlySet): NodeJS.ProcessEnv { + const environment = { ...process.env }; + for (const name of denylist) { + delete environment[name]; + } + return { + ...environment, + HOME: process.env.RUNNER_HOME ?? '/home/runner', + LOGNAME: process.env.RUNNER_USER ?? 'runner', + MICROVM_ID: microvmId, + USER: process.env.RUNNER_USER ?? 'runner', + }; +} + +function redactSpawnArguments(child: ChildProcess, arguments_: string[], sensitiveValue: string): void { + for (let index = 0; index < arguments_.length; index += 1) { + if (arguments_[index] === sensitiveValue) { + arguments_[index] = '[redacted]'; + } + } + for (let index = 0; index < child.spawnargs.length; index += 1) { + if (child.spawnargs[index] === sensitiveValue) { + child.spawnargs[index] = '[redacted]'; + } + } +} + +function waitForLaunchHandoff(child: ChildProcess, handoffDelayMs: number): Promise { + return new Promise((resolve, reject) => { + let settled = false; + let handoffTimer: NodeJS.Timeout | undefined; + + const cleanup = (): void => { + child.off('error', onError); + child.off('spawn', onSpawn); + child.off('exit', onExit); + if (handoffTimer !== undefined) { + clearTimeout(handoffTimer); + } + }; + + const fail = (error: Error): void => { + if (settled) { + return; + } + settled = true; + cleanup(); + reject(error); + }; + + const commit = (): void => { + if (settled) { + return; + } + if (child.exitCode !== null || child.signalCode !== null) { + fail(new Error('GitHub Actions runner exited before the launch handoff')); + return; + } + settled = true; + cleanup(); + child.unref(); + resolve(); + }; + + const onError = (error: Error): void => fail(error); + const onExit = (): void => fail(new Error('GitHub Actions runner exited before the launch handoff')); + const onSpawn = (): void => { + handoffTimer = setTimeout(commit, handoffDelayMs); + }; + + child.once('error', onError); + child.once('spawn', onSpawn); + child.once('exit', onExit); + }); +} + +export class NodeManagedProcess implements ManagedProcess { + public readonly ready: Promise; + public readonly exit: Promise; + + public constructor( + private readonly child: ChildProcess, + readiness: Promise, + private readonly defaultStopGraceMs: number, + ) { + this.ready = readiness; + this.exit = new Promise((resolve) => { + child.once('exit', (code) => resolve(code)); + child.once('error', () => resolve(null)); + }); + } + + public get exited(): boolean { + return this.child.exitCode !== null || this.child.signalCode !== null; + } + + public async stop(graceMs = this.defaultStopGraceMs): Promise { + if (this.exited) { + return; + } + signalProcessGroup(this.child, 'SIGTERM'); + const exitedGracefully = await Promise.race([this.exit.then(() => true), delay(graceMs).then(() => false)]); + if (!exitedGracefully && !this.exited) { + signalProcessGroup(this.child, 'SIGKILL'); + await Promise.race([this.exit, delay(5_000)]); + } + } +} + +/** Launches the GitHub Actions runner directly from the image's runner installation. */ +export class GitHubRunnerLauncher implements RunnerLauncher { + private readonly denylist = runnerEnvironmentDenylist(); + private readonly runnerRoot = process.env.RUNNER_ROOT ?? '/opt/actions-runner'; + private readonly identity = runnerIdentity(); + + public constructor( + private readonly stopGraceMs = 30_000, + private readonly handoffDelayMs = LAUNCH_HANDOFF_DELAY_MS, + ) { + if (!isAbsolute(this.runnerRoot)) { + throw new Error('RUNNER_ROOT must be an absolute path'); + } + } + + public launch(bootstrap: RunnerBootstrap, microvmId: string): ManagedProcess { + const runner = join(this.runnerRoot, 'run.sh'); + const arguments_ = ['--jitconfig', bootstrap.jitConfig]; + const child = spawn(runner, arguments_, { + cwd: this.runnerRoot, + detached: true, + env: runnerEnvironment(microvmId, this.denylist), + shell: false, + stdio: ['ignore', 'inherit', 'inherit'], + ...this.identity, + }); + redactSpawnArguments(child, arguments_, bootstrap.jitConfig); + + const ready = waitForLaunchHandoff(child, this.handoffDelayMs); + return new NodeManagedProcess(child, ready, this.stopGraceMs); + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/public.ts b/lambdas/services/microvm-lifecycle-hooks/src/public.ts new file mode 100644 index 0000000000..a82479fecd --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/public.ts @@ -0,0 +1,26 @@ +export type { + ConsumeOptions, + JitConfigSource, + Logger, + ManagedProcess, + RunContext, + RunnerBootstrap, + RunnerLauncher, +} from './contracts'; +export { consoleLogger } from './contracts'; +export { RunnerLifecycle } from './lifecycle'; +export { HookRequestError, MAX_REQUEST_BYTES, parseRunRequest } from './payload'; +export { GitHubRunnerLauncher, NodeManagedProcess } from './processes'; +export { + createHookExitRequester, + createDefaultLifecycle, + createHookServer, + HOOK_PREFIX, + main, + parsePositiveInteger, + shutdownHookServer, + watchRunnerCompletion, +} from './server'; +export type { HookLifecycle, HookServerOptions } from './server'; +export { StorageJitConfigSource } from './storage'; +export type { StorageJitConfigSourceOptions } from './storage'; diff --git a/lambdas/services/microvm-lifecycle-hooks/src/server.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/server.test.ts new file mode 100644 index 0000000000..823955d755 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/server.test.ts @@ -0,0 +1,210 @@ +import type { AddressInfo } from 'node:net'; + +import type { Logger } from './contracts'; +import { + createHookExitRequester, + createHookServer, + type HookLifecycle, + HOOK_PREFIX, + parsePositiveInteger, + shutdownHookServer, + watchRunnerCompletion, +} from './server'; + +const quietLogger: Logger = { + error: () => undefined, + info: () => undefined, + warn: () => undefined, +}; + +const idleLifecycle: HookLifecycle = { + resume: async () => true, + start: async () => true, + stop: async () => undefined, +}; + +async function listen(server: ReturnType): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + +async function close(server: ReturnType): Promise { + await new Promise((resolve, reject) => { + server.close((error) => (error === undefined ? resolve() : reject(error))); + server.closeAllConnections(); + }); +} + +describe('hook server', () => { + it('rejects invalid and out-of-range positive integer values', () => { + expect(parsePositiveInteger(undefined, 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('0', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('-1', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('1.5', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('8080http', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('65536', 8080, 65_535)).toBe(8080); + expect(parsePositiveInteger('9007199254740992', 8080)).toBe(8080); + expect(parsePositiveInteger('9090', 8080, 65_535)).toBe(9090); + }); + + it('configures bounded request, header, connection, and socket limits', () => { + const server = createHookServer(idleLifecycle, quietLogger, { + headersTimeoutMs: 2_000, + keepAliveTimeoutMs: 3_000, + requestTimeoutMs: 4_000, + }); + + expect(server.headersTimeout).toBe(2_000); + expect(server.keepAliveTimeout).toBe(3_000); + expect(server.requestTimeout).toBe(4_000); + expect(server.maxConnections).toBe(128); + expect(server.maxHeadersCount).toBe(64); + expect(server.maxRequestsPerSocket).toBe(100); + }); + + it('acknowledges build hooks without starting a runner', async () => { + const lifecycle: HookLifecycle = { + resume: vi.fn(), + start: vi.fn(), + stop: vi.fn(), + }; + const server = createHookServer(lifecycle, quietLogger); + const baseUrl = await listen(server); + try { + const ready = await fetch(`${baseUrl}${HOOK_PREFIX}/ready`, { method: 'POST' }); + const validate = await fetch(`${baseUrl}${HOOK_PREFIX}/validate`, { method: 'POST' }); + + expect(ready.status).toBe(200); + await expect(ready.json()).resolves.toEqual({ status: 'ready' }); + expect(validate.status).toBe(200); + await expect(validate.json()).resolves.toEqual({ status: 'validated' }); + expect(lifecycle.start).not.toHaveBeenCalled(); + expect(lifecycle.stop).not.toHaveBeenCalled(); + } finally { + await close(server); + } + }); + + it('rejects oversized request bodies before invoking the lifecycle', async () => { + const lifecycle: HookLifecycle = { + ...idleLifecycle, + start: vi.fn(), + }; + const server = createHookServer(lifecycle, quietLogger); + const baseUrl = await listen(server); + try { + const response = await fetch(`${baseUrl}${HOOK_PREFIX}/run`, { + body: 'x'.repeat(20 * 1024 + 1), + method: 'POST', + }); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toEqual({ error: 'request body is too large' }); + expect(lifecycle.start).not.toHaveBeenCalled(); + } finally { + await close(server); + } + }); + + it('does not reflect or log secret-bearing internal errors', async () => { + const messages: unknown[] = []; + const logger: Logger = { + error: (...values) => messages.push(...values), + info: () => undefined, + warn: () => undefined, + }; + const lifecycle: HookLifecycle = { + ...idleLifecycle, + start: async () => { + const error = new Error('encoded-jit-secret'); + error.name = 'encoded-jit-secret'; + throw error; + }, + }; + const server = createHookServer(lifecycle, logger); + const baseUrl = await listen(server); + try { + const response = await fetch(`${baseUrl}${HOOK_PREFIX}/run`, { + body: '{}', + method: 'POST', + }); + + expect(response.status).toBe(500); + await expect(response.json()).resolves.toEqual({ error: 'lifecycle hook failed' }); + expect(JSON.stringify(messages)).not.toContain('encoded-jit-secret'); + } finally { + await close(server); + } + }); + + it('waits for lifecycle cleanup before closing active connections', async () => { + const events: string[] = []; + let finishCleanup = (): void => undefined; + const cleanup = new Promise((resolve) => { + finishCleanup = resolve; + }); + const server = { + close: () => events.push('stop-accepting'), + closeAllConnections: () => events.push('close-connections'), + }; + const lifecycle = { + async stop(): Promise { + events.push('cleanup-started'); + await cleanup; + events.push('cleanup-finished'); + }, + }; + + const shutdown = shutdownHookServer(server, lifecycle); + await new Promise((resolve) => setImmediate(resolve)); + expect(events).toEqual(['stop-accepting', 'cleanup-started']); + + finishCleanup(); + await shutdown; + expect(events).toEqual(['stop-accepting', 'cleanup-started', 'cleanup-finished', 'close-connections']); + }); + + it.each([ + { expectedExitCode: 0, runnerExitCode: 0 }, + { expectedExitCode: 1, runnerExitCode: 7 }, + { expectedExitCode: 1, runnerExitCode: null }, + ])('requests hook exit $expectedExitCode after runner status $runnerExitCode', async (testCase) => { + const requestExit = vi.fn(); + + watchRunnerCompletion({ completion: Promise.resolve(testCase.runnerExitCode) }, quietLogger, requestExit); + + await Promise.resolve(); + expect(requestExit).not.toHaveBeenCalled(); + await new Promise((resolve) => setImmediate(resolve)); + expect(requestExit).toHaveBeenCalledOnce(); + expect(requestExit).toHaveBeenCalledWith(testCase.expectedExitCode); + }); + + it('closes the hook exactly once before publishing its process exit code', async () => { + const events: string[] = []; + const requestExit = createHookExitRequester( + { + close: () => events.push('stop-accepting'), + closeAllConnections: () => events.push('close-connections'), + }, + { + async stop(): Promise { + events.push('stop-runner'); + }, + }, + quietLogger, + (exitCode) => events.push(`exit:${exitCode}`), + ); + + requestExit(0); + requestExit(1); + await new Promise((resolve) => setImmediate(resolve)); + + expect(events).toEqual(['stop-accepting', 'stop-runner', 'close-connections', 'exit:0']); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/server.ts b/lambdas/services/microvm-lifecycle-hooks/src/server.ts new file mode 100644 index 0000000000..c37fd39886 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/server.ts @@ -0,0 +1,275 @@ +import http, { type IncomingMessage, type ServerResponse } from 'node:http'; + +import type { Logger } from './contracts'; +import { consoleLogger } from './contracts'; +import { RunnerLifecycle } from './lifecycle'; +import { HookRequestError, MAX_REQUEST_BYTES } from './payload'; +import { GitHubRunnerLauncher } from './processes'; +import { StorageJitConfigSource } from './storage'; + +export const HOOK_PREFIX = '/aws/lambda-microvms/runtime/v1'; + +const MAX_TIMER_SECONDS = 2_147_483; + +export interface HookLifecycle { + start(body: string): Promise; + stop(): Promise; + resume(): Promise; +} + +export interface HookServerOptions { + headersTimeoutMs?: number; + keepAliveTimeoutMs?: number; + requestTimeoutMs?: number; +} + +export function parsePositiveInteger( + value: string | undefined, + fallback: number, + maximum = Number.MAX_SAFE_INTEGER, +): number { + if (value === undefined || !/^\d+$/.test(value)) { + return fallback; + } + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= maximum ? parsed : fallback; +} + +function timeoutMilliseconds(variable: string, fallbackSeconds: number, maximumSeconds = 60): number { + return ( + parsePositiveInteger(process.env[variable], fallbackSeconds, Math.min(maximumSeconds, MAX_TIMER_SECONDS)) * 1_000 + ); +} + +function respond(response: ServerResponse, status: number, payload: object): void { + const body = Buffer.from(JSON.stringify(payload)); + response.writeHead(status, { + 'Cache-Control': 'no-store', + 'Content-Length': body.length, + 'Content-Type': 'application/json', + }); + response.end(body); +} + +function readBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const contentLength = request.headers['content-length']; + let declaredLength: number | undefined; + if (contentLength !== undefined) { + declaredLength = Number(contentLength); + if (!Number.isInteger(declaredLength) || declaredLength < 0) { + reject(new HookRequestError('Content-Length is invalid')); + request.resume(); + return; + } + if (declaredLength > MAX_REQUEST_BYTES) { + reject(new HookRequestError('request body is too large')); + request.resume(); + return; + } + } + + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + + const fail = (error: Error): void => { + if (settled) { + return; + } + settled = true; + reject(error); + }; + request.on('data', (chunk: Buffer) => { + if (settled) { + return; + } + size += chunk.length; + if (size > MAX_REQUEST_BYTES) { + fail(new HookRequestError('request body is too large')); + request.destroy(); + return; + } + chunks.push(chunk); + }); + request.once('end', () => { + if (settled) { + return; + } + if (declaredLength !== undefined && declaredLength !== size) { + fail(new HookRequestError('Content-Length does not match the request body')); + return; + } + settled = true; + resolve(Buffer.concat(chunks).toString('utf8')); + }); + request.once('aborted', () => fail(new HookRequestError('request body was interrupted'))); + request.once('error', (error) => fail(error)); + }); +} + +export function createHookServer( + lifecycle: HookLifecycle, + logger: Logger = consoleLogger, + options: HookServerOptions = {}, +): http.Server { + const requestTimeout = options.requestTimeoutMs ?? timeoutMilliseconds('HOOK_REQUEST_TIMEOUT_SECONDS', 10); + const headersTimeout = Math.min( + options.headersTimeoutMs ?? timeoutMilliseconds('HOOK_HEADERS_TIMEOUT_SECONDS', 5), + requestTimeout, + ); + const keepAliveTimeout = options.keepAliveTimeoutMs ?? timeoutMilliseconds('HOOK_KEEP_ALIVE_TIMEOUT_SECONDS', 5); + + const server = http.createServer( + { + headersTimeout, + keepAliveTimeout, + maxHeaderSize: 16 * 1024, + requestTimeout, + }, + async (request, response) => { + const path = request.url ?? ''; + if (request.method !== 'POST') { + request.resume(); + respond(response, 405, { error: 'method not allowed' }); + return; + } + + try { + // Consume every POST body so all lifecycle endpoints share the same bounded request handling. + const body = await readBody(request); + if (path === `${HOOK_PREFIX}/ready`) { + respond(response, 200, { status: 'ready' }); + return; + } + if (path === `${HOOK_PREFIX}/validate`) { + respond(response, 200, { status: 'validated' }); + return; + } + if (path === `${HOOK_PREFIX}/run`) { + const started = await lifecycle.start(body); + respond(response, 200, { status: started ? 'started' : 'already-started' }); + return; + } + if (path === `${HOOK_PREFIX}/terminate`) { + await lifecycle.stop(); + respond(response, 200, { status: 'stopped' }); + return; + } + if (path === `${HOOK_PREFIX}/resume`) { + const ready = await lifecycle.resume(); + respond(response, ready ? 200 : 503, { status: ready ? 'ready' : 'not-ready' }); + return; + } + if (path === `${HOOK_PREFIX}/suspend`) { + respond(response, 200, { status: 'ok' }); + return; + } + respond(response, 404, { error: 'unknown lifecycle hook' }); + } catch (error) { + if (error instanceof HookRequestError) { + logger.warn('Rejected invalid lifecycle hook request'); + respond(response, 400, { error: error.message }); + return; + } + // Parse and provider errors can contain credentials in both message and name. + logger.error('Lifecycle hook failed'); + respond(response, 500, { error: 'lifecycle hook failed' }); + } + }, + ); + server.maxConnections = 128; + server.maxHeadersCount = 64; + server.maxRequestsPerSocket = 100; + return server; +} + +export function createDefaultLifecycle(logger: Logger = consoleLogger): RunnerLifecycle { + return new RunnerLifecycle(new StorageJitConfigSource(), new GitHubRunnerLauncher(), logger); +} + +interface ClosableServer { + close(): unknown; + closeAllConnections(): void; +} + +interface StoppableLifecycle { + stop(): Promise; +} + +export async function shutdownHookServer(server: ClosableServer, lifecycle: StoppableLifecycle): Promise { + server.close(); + try { + await lifecycle.stop(); + } finally { + server.closeAllConnections(); + } +} + +function hookExitCode(runnerExitCode: number | null): number { + return runnerExitCode === 0 ? 0 : 1; +} + +export function watchRunnerCompletion( + lifecycle: Pick, + logger: Logger, + requestExit: (exitCode: number) => void, +): void { + void lifecycle.completion.then((runnerExitCode) => { + const exitCode = hookExitCode(runnerExitCode); + if (exitCode === 0) { + logger.info('GitHub Actions runner exited with status %s', runnerExitCode); + } else { + logger.error('GitHub Actions runner exited unexpectedly with status %s', runnerExitCode ?? 'signal'); + } + // Let the /run handler flush its acknowledgement if the runner exits immediately after handoff. + setImmediate(() => requestExit(exitCode)); + }); +} + +export function createHookExitRequester( + server: ClosableServer, + lifecycle: StoppableLifecycle, + logger: Logger, + setExitCode: (exitCode: number) => void = (exitCode) => { + // Let Node exit naturally after lifecycle cleanup and log streams have drained. + process.exitCode = exitCode; + }, +): (exitCode: number) => void { + let exiting = false; + return (exitCode: number): void => { + if (exiting) { + return; + } + exiting = true; + void shutdownHookServer(server, lifecycle).then( + () => setExitCode(exitCode), + () => { + logger.error('Lifecycle hook shutdown failed'); + setExitCode(1); + }, + ); + }; +} + +export async function main(): Promise { + const logger = consoleLogger; + const lifecycle = createDefaultLifecycle(logger); + const server = createHookServer(lifecycle, logger); + const port = parsePositiveInteger(process.env.HOOK_PORT, 8080, 65_535); + + const requestExit = createHookExitRequester(server, lifecycle, logger); + process.once('SIGINT', () => requestExit(0)); + process.once('SIGTERM', () => requestExit(0)); + watchRunnerCompletion(lifecycle, logger, requestExit); + + await new Promise((resolve, reject) => { + const onError = (): void => reject(new Error('lifecycle hook server could not listen')); + server.once('error', onError); + server.listen(port, '0.0.0.0', () => { + server.off('error', onError); + logger.info('Lambda MicroVM lifecycle hooks listening on port %d', port); + resolve(); + }); + }); +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/storage.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/storage.test.ts new file mode 100644 index 0000000000..d1f2ad3400 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/storage.test.ts @@ -0,0 +1,87 @@ +import type { + RunnerConfigConsumer, + RunnerConfigStorageContext, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; + +import { StorageJitConfigSource } from './storage'; + +const SSM_STORAGE: RunnerConfigStorageContext = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', +}; + +describe('StorageJitConfigSource', () => { + it('exports the allowlisted context once before resolving and consuming from the environment', async () => { + const events: string[] = []; + const environment: NodeJS.ProcessEnv = {}; + const consumer: RunnerConfigConsumer = { + consume: vi.fn(async () => { + events.push('consume'); + return 'encoded-jit'; + }), + }; + const exportEnvironment = vi.fn((context: RunnerConfigStorageContext) => { + events.push('export'); + return context; + }); + const createConsumer = vi.fn((target: NodeJS.ProcessEnv) => { + events.push('create'); + expect(target).toBe(environment); + expect(target).toMatchObject(SSM_STORAGE); + return consumer; + }); + const source = new StorageJitConfigSource({ createConsumer, environment, exportEnvironment }); + const signal = new AbortController().signal; + + await expect( + source.consume({ microvmId: 'microvm-1234', storage: SSM_STORAGE }, { deadlineMs: 123_456, signal }), + ).resolves.toEqual({ jitConfig: 'encoded-jit' }); + await expect( + source.consume( + { + microvmId: 'microvm-1234', + storage: { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/tenant/token', + }, + }, + { deadlineMs: 123_457, signal }, + ), + ).resolves.toEqual({ jitConfig: 'encoded-jit' }); + + expect(events).toEqual(['export', 'create', 'consume', 'create', 'consume']); + expect(exportEnvironment).toHaveBeenCalledOnce(); + expect(exportEnvironment).toHaveBeenCalledWith(SSM_STORAGE); + expect(createConsumer).toHaveBeenCalledTimes(2); + expect(consumer.consume).toHaveBeenNthCalledWith(1, 'microvm-1234', { + deadlineMs: 123_456, + signal, + }); + }); + + it('rejects storage context changes after the one-time environment export', async () => { + const environment: NodeJS.ProcessEnv = {}; + const consumer: RunnerConfigConsumer = { consume: vi.fn().mockResolvedValue('encoded-jit') }; + const exportEnvironment = vi.fn((context: RunnerConfigStorageContext) => context); + const createConsumer = vi.fn().mockReturnValue(consumer); + const source = new StorageJitConfigSource({ createConsumer, environment, exportEnvironment }); + const options = { deadlineMs: 123_456, signal: new AbortController().signal }; + + await source.consume({ microvmId: 'microvm-1234', storage: SSM_STORAGE }, options); + await expect( + source.consume( + { + microvmId: 'microvm-1234', + storage: { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/github-action-runners/other/token', + }, + }, + options, + ), + ).rejects.toThrow('storage context cannot change'); + + expect(exportEnvironment).toHaveBeenCalledOnce(); + expect(createConsumer).toHaveBeenCalledOnce(); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/storage.ts b/lambdas/services/microvm-lifecycle-hooks/src/storage.ts new file mode 100644 index 0000000000..528683f59e --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/storage.ts @@ -0,0 +1,49 @@ +import { + createRunnerConfigConsumerFromEnvironment, + exportRunnerConfigStorageEnvironment, + type RunnerConfigConsumer, + type RunnerConfigStorageContext, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; + +import type { ConsumeOptions, JitConfigSource, RunContext, RunnerBootstrap } from './contracts'; + +type RunnerConfigConsumerFactory = typeof createRunnerConfigConsumerFromEnvironment; +type RunnerConfigStorageExporter = typeof exportRunnerConfigStorageEnvironment; + +export interface StorageJitConfigSourceOptions { + createConsumer?: RunnerConfigConsumerFactory; + environment?: NodeJS.ProcessEnv; + exportEnvironment?: RunnerConfigStorageExporter; +} + +function storageContextFingerprint(context: RunnerConfigStorageContext): string { + return JSON.stringify(Object.entries(context).sort(([left], [right]) => left.localeCompare(right))); +} + +/** Adapts the shared provider registry to the lifecycle's one-time bootstrap contract. */ +export class StorageJitConfigSource implements JitConfigSource { + private readonly createConsumer: RunnerConfigConsumerFactory; + private readonly environment: NodeJS.ProcessEnv; + private readonly exportEnvironment: RunnerConfigStorageExporter; + private exportedStorageFingerprint?: string; + + public constructor(options: StorageJitConfigSourceOptions = {}) { + this.createConsumer = options.createConsumer ?? createRunnerConfigConsumerFromEnvironment; + this.environment = options.environment ?? process.env; + this.exportEnvironment = options.exportEnvironment ?? exportRunnerConfigStorageEnvironment; + } + + public async consume(context: RunContext, options: ConsumeOptions): Promise { + const fingerprint = storageContextFingerprint(context.storage); + if (this.exportedStorageFingerprint === undefined) { + Object.assign(this.environment, this.exportEnvironment(context.storage)); + this.exportedStorageFingerprint = fingerprint; + } else if (this.exportedStorageFingerprint !== fingerprint) { + throw new Error('runner configuration storage context cannot change after initialization'); + } + + const consumer: RunnerConfigConsumer = this.createConsumer(this.environment); + const jitConfig = await consumer.consume(context.microvmId, options); + return { jitConfig }; + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/src/timing.test.ts b/lambdas/services/microvm-lifecycle-hooks/src/timing.test.ts new file mode 100644 index 0000000000..b62d8af038 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/timing.test.ts @@ -0,0 +1,32 @@ +import { beforeDeadlineOrAbort, delay } from './timing'; + +describe('timing helpers', () => { + it('removes the delay abort listener after resolving', async () => { + const signal = new AbortController().signal; + const remove = vi.spyOn(signal, 'removeEventListener'); + + await delay(1, signal); + + expect(remove).toHaveBeenCalledOnce(); + }); + + it('removes the delay abort listener after cancellation', async () => { + const controller = new AbortController(); + const remove = vi.spyOn(controller.signal, 'removeEventListener'); + const pending = delay(1_000, controller.signal); + + controller.abort(); + + await expect(pending).rejects.toThrow('operation was cancelled'); + expect(remove).toHaveBeenCalledOnce(); + }); + + it('rejects immediately when an operation is already aborted', async () => { + const controller = new AbortController(); + controller.abort(); + + await expect( + beforeDeadlineOrAbort(Promise.resolve('unused'), Date.now() + 1_000, controller.signal), + ).rejects.toThrow('runner start was cancelled'); + }); +}); diff --git a/lambdas/services/microvm-lifecycle-hooks/src/timing.ts b/lambdas/services/microvm-lifecycle-hooks/src/timing.ts new file mode 100644 index 0000000000..fc791d2e29 --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/src/timing.ts @@ -0,0 +1,66 @@ +export function delay(milliseconds: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = (): void => signal?.removeEventListener('abort', cancel); + const finish = (): void => { + if (settled) { + return; + } + settled = true; + cleanup(); + resolve(); + }; + const timer = setTimeout(finish, milliseconds); + const cancel = (): void => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + cleanup(); + reject(new Error('operation was cancelled')); + }; + signal?.addEventListener('abort', cancel, { once: true }); + if (signal?.aborted) { + cancel(); + } + }); +} + +export async function beforeDeadline(promise: Promise, deadlineMs: number): Promise { + const remaining = deadlineMs - Date.now(); + if (remaining <= 0) { + throw new Error('run-hook deadline elapsed'); + } + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('run-hook deadline elapsed')), remaining); + }); + try { + return await Promise.race([promise, deadline]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } +} + +export async function beforeDeadlineOrAbort( + promise: Promise, + deadlineMs: number, + signal: AbortSignal, +): Promise { + if (signal.aborted) { + throw new Error('runner start was cancelled'); + } + let cancel = (): void => undefined; + const cancelled = new Promise((_resolve, reject) => { + cancel = (): void => reject(new Error('runner start was cancelled')); + signal.addEventListener('abort', cancel, { once: true }); + }); + try { + return await beforeDeadline(Promise.race([promise, cancelled]), deadlineMs); + } finally { + signal.removeEventListener('abort', cancel); + } +} diff --git a/lambdas/services/microvm-lifecycle-hooks/tsconfig.json b/lambdas/services/microvm-lifecycle-hooks/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/services/microvm-lifecycle-hooks/vitest.config.ts b/lambdas/services/microvm-lifecycle-hooks/vitest.config.ts new file mode 100644 index 0000000000..e3c59146ee --- /dev/null +++ b/lambdas/services/microvm-lifecycle-hooks/vitest.config.ts @@ -0,0 +1,12 @@ +import { mergeConfig } from 'vitest/config'; + +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts'], + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index bd96098e13..a4aecfd69b 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -211,6 +211,16 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/microvm-lifecycle-hooks@workspace:services/microvm-lifecycle-hooks": + version: 0.0.0-use.local + resolution: "@aws-github-runner/microvm-lifecycle-hooks@workspace:services/microvm-lifecycle-hooks" + dependencies: + "@aws-github-runner/storage-providers": "npm:*" + "@types/node": "npm:^22.19.3" + esbuild: "npm:^0.27.0" + languageName: unknown + linkType: soft + "@aws-github-runner/scale-set-service@workspace:services/scale-set": version: 0.0.0-use.local resolution: "@aws-github-runner/scale-set-service@workspace:services/scale-set" diff --git a/mkdocs.yaml b/mkdocs.yaml index b8158c5c5d..a8182ec874 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -60,6 +60,7 @@ nav: - Multi-runner v1 to v2 migration: multi-runner-v1-v2-migration.md - Getting started: getting-started.md - Security: security.md + - Lambda MicroVM runners (experimental): microvm-runners.md - Architecture decisions: - MiniStack for integration tests: adr/0001-use-ministack-for-terraform-integration-tests.md - Runner orchestration provider boundary: adr/002-runner-orchestration-provider-boundary.md @@ -82,8 +83,7 @@ nav: - Overview: examples/index.md - Default: examples/default.md - Multi Runner: examples/multi-runner.md - - Multi Runner v2: examples/multi-runner-v2.md - - Multi Runner scale-set: examples/multi-runner-scale-set.md + - Multi-runner orchestration: examples/multi-runner-orchestration.md - Ephemeral: examples/ephemeral.md - External managed secrets: examples/external-managed-ssm-secrets.md - Custom AMI: examples/prebuilt.md diff --git a/tests/ministack/README.md b/tests/ministack/README.md index fe0ad72b9d..8a834cbe53 100644 --- a/tests/ministack/README.md +++ b/tests/ministack/README.md @@ -1,19 +1,19 @@ # MiniStack example tests -The MiniStack workflow runs the `base`, `prebuilt`, `default`, `ephemeral`, -`multi-runner`, `multi-runner-v2`, `multi-runner-scale-set`, and -`termination-watcher` examples directly -with Terraform 1.5.6 and the latest Terraform release, and with OpenTofu 1.11 -and the latest OpenTofu release. +The MiniStack workflow applies the `base`, `prebuilt`, `default`, `ephemeral`, +`multi-runner`, and `termination-watcher` examples with Terraform and OpenTofu. +The combined webhook and scale-set example is exercised by the Python smoke +workflow, which applies it once and tests webhook EC2, webhook MicroVM, and +scale-set EC2. + The examples with input variables get their inputs from their own tfvars files in this directory. The `microvm-foundation` example uses the reusable `base` example module to create its VPC and private subnets, then wires those outputs -into the MicroVM Network Connector. The `termination-watcher` example has no input variables -and uses the configuration checked into the example itself. No override files, -setup module, or Terraform fixture configuration is checked in. The helper -creates and removes a temporary AMI override for `default` and -`ephemeral`, temporary SSM parameters for `multi-runner`, and temporary AMI -fixtures for `multi-runner-v2` and `multi-runner-scale-set`. The migration test +into the MicroVM Network Connector. The `termination-watcher` example has no +input variables and uses the configuration checked into the example itself. No +override files, setup module, or Terraform fixture configuration is checked in. +The helper creates and removes a temporary AMI override for `default` and +`ephemeral`, and temporary SSM parameters for `multi-runner`. The migration test uses its dedicated `run-migration-test.sh` lifecycle script. Start MiniStack, set the AWS endpoint and test credentials, then run: @@ -29,10 +29,6 @@ tests/ministack/run-example.sh apply ephemeral # or tests/ministack/run-example.sh apply multi-runner # or -tests/ministack/run-example.sh apply multi-runner-v2 -# or -tests/ministack/run-example.sh apply multi-runner-scale-set -# or tests/ministack/run-example.sh apply termination-watcher ``` @@ -41,59 +37,146 @@ ZIP fixtures in the paths expected by the modules when they are absent, and removes only the files it created. For `prebuilt`, it seeds AMI metadata through MiniStack's AWS-compatible EC2 API, then removes only the resources it created during cleanup. MiniStack v1.5.11 provides the EC2 image behavior needed by the -`default`, `ephemeral`, `multi-runner`, and `multi-runner-scale-set` examples, -so they are included in the same lifecycle matrix. +`default`, `ephemeral`, and `multi-runner` examples, so they are included in the +same lifecycle matrix. ## Webhook and runner lifecycle smoke test The smoke test covers two independent lifecycle chains. The webhook chain sends signed `workflow_job` webhooks through the API Gateway endpoint and verifies the asynchronous path through EventBridge, the dispatcher Lambda, SQS, -and the scale-up Lambda. It runs scale-up once without a dynamic label and once -with `ghr-ec2-instance-type:m5.large`, checking that the first launch uses a -configured default instance type and the second launch uses exactly `m5.large`. -The scale-up Lambda calls a pinned `mockserver/mockserver` container initialized -from `github-api-expectations.json`; the test uses MockServer's verification API -to confirm the expected GitHub API calls for both jobs. It also checks the +and the scale-up Lambda. Each provider runs scale-up once without a dynamic +label and once with a provider-specific dynamic label, checking the provider's +resolved resource configuration. +The smoke runner connects to an already-running MockServer initialized from +`github-api-expectations.json`; it uses MockServer's verification API to confirm +the expected GitHub API calls for both jobs. It also checks the webhook, dispatcher, and scale-up Lambda log groups for each smoke job ID, then -confirms that both MiniStack EC2 runner instances are removed and terminated. +confirms that each provider resource is removed and terminated. The second, pool chain then invokes the pool Lambda with a pool size of one and verifies every expected GitHub API route for pool reconciliation, including the installation, token, runner-list, and registration-token calls, before confirming that it -creates a second EC2 runner. Installation lookup is mocked for configurations +creates a second provider runner. Installation lookup is mocked for configurations that do not provide a stored installation ID, but is conditional and is not a required assertion. The test also verifies the `ghr:Application`, `ghr:created_by`, `ghr:Type`, and `ghr:Owner` tags used to discover managed -instances. MiniStack v1.5.10 propagates the Terraform launch-template tags to +instances. MiniStack v1.5.15 propagates the Terraform launch-template tags to instances, allowing the scale-down Lambda to discover and remove each runner. The smoke test invokes scale-down for the webhook and pool-created runners and verifies the GitHub API calls and EC2 termination. The pool schedule is configured for a far-future date because the test invokes the Lambda directly. -Build the two real Lambda distributions, start MiniStack, and run: +The smoke deployment uses the `multi-runner-orchestration` example, which creates +both EC2 and MicroVM lanes behind one webhook endpoint. For each provider, the +shared lifecycle runs scale-up without a dynamic label, scale-up with a dynamic +label, one pool scale-up, and scale-down for all three resources. The provider +implementation supplies the event labels, resource discovery, provider-specific +route checks, and compute-resource assertions. The shared example accepts the +built runner-control and webhook Lambda ZIP files as `runners_lambda_zip` and +`webhook_lambda_zip`. + +To exercise the MicroVM image's lifecycle hook after each MicroVM scale-up, start +the image locally and provide its hook URL. The hook container must use the same +MiniStack endpoint as the smoke test. The scale-up Lambda creates the JIT config +in SSM; the smoke test sends the same `runHookPayload` to the hook, which consumes +that SSM value. + +```sh +python3 tests/ministack/run-ministack-smoke.py --webhook-provider microvm +``` + +The test sends the outer JSON request with `runHookPayload` encoded as a JSON +string, then waits for +`/github-action-runners/multi-runner-webhook/microvm/runners/tokens/` +to disappear. This proves that the lifecycle hook consumed the one-time SSM +value before the MicroVM is scaled down. + +### Start MiniStack and MockServer with Docker + +The smoke expects MiniStack on port `4566` and MockServer on port `1080`. Start +both containers before running the smoke. MiniStack needs the Docker socket +mounted so its ECS integration can launch the scale-set controller container. +It also needs a host-gateway entry so that the controller can reach MockServer +at `host.docker.internal:1080`. + +```sh +docker run --detach \ + --name ministack \ + --publish 4566:4566 \ + --add-host=host.docker.internal:host-gateway \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --env MINISTACK_ACCOUNT_ID=000000000000 \ + --env MINISTACK_REGION=eu-west-1 \ + ghcr.io/ministackorg/ministack:latest + +docker run --detach \ + --name ministack-mockserver \ + --publish 1080:1080 \ + mockserver/mockserver:7.6.0 +``` + +Wait for MockServer to become ready: ```sh -(cd lambdas && yarn install --frozen-lockfile) -(cd lambdas && yarn workspace @aws-github-runner/webhook dist) -(cd lambdas && yarn workspace @aws-github-runner/control-plane dist) -sh tests/ministack/run-smoke.sh +until curl --silent --show-error --fail --request PUT \ + http://localhost:1080/mockserver/status; do + sleep 2 +done ``` +The smoke process uses `http://localhost:1080`; the controller container uses +`https://host.docker.internal:1080`. The smoke runner loads its GitHub API +expectations into MockServer when it starts. To stop and remove the containers +after the run: + +```sh +docker rm --force ministack ministack-mockserver +``` + +Build the smoke Lambda archives and run: + +```sh +./.ci/build.sh +# This runs webhook EC2/MicroVM and scale-set EC2 in one deployment. +python3 tests/ministack/run-ministack-smoke.py +# Preserve the deployment and temporary tfvars file for debugging: +python3 tests/ministack/run-ministack-smoke.py --keep-deployment +# Run one provider explicitly when debugging: +python3 tests/ministack/run-ministack-smoke.py --webhook-provider ec2 +python3 tests/ministack/run-ministack-smoke.py --webhook-provider microvm +# Select the scale-set compute provider explicitly: +python3 tests/ministack/run-ministack-smoke.py --scale-set-provider ec2 +``` + +The runner writes detailed command output to `ministack-smoke.log`. + +The Python `smoke/webhook_scenario.py` module owns the provider-neutral scenarios, and +`smoke/webhook_provider.py` defines the provider interface. Shared webhook delivery, log +polling, MockServer route verification, GitHub runner-state fixtures, and Lambda +invocation helpers live in `smoke/common.py`. To add a provider, implement the +interface under `smoke/`, register the provider in +`run-ministack-smoke.py`, and add its provider-specific assertions. Scale-set +MicroVM coverage remains WIP; the combined run currently covers scale-set EC2. + The smoke script generates a temporary RSA key and Terraform variables file, -starts the MockServer container on a temporary port, and removes all temporary -state during cleanup. In CI, the pinned MockServer setup action starts the -server and waits for readiness; the expectations are loaded after checkout. -MiniStack must be able to reach -`host.docker.internal`; -override the hostname with `MINISTACK_GITHUB_MOCK_HOST` when using a different -container runtime. When MiniStack is exposed on a non-default local port, use a -host address reachable from its container for `AWS_ENDPOINT_URL`, for example -`AWS_ENDPOINT_URL=http://:14568`, instead of `127.0.0.1`. - -The workflow also runs `run-scale-set-integration.sh`. It applies the -`multi-runner-scale-set` example and verifies the managed ECS controller, -Fargate task hardening, scale-set environment contract, and reconciler SSM -parameter through MiniStack's AWS-compatible APIs. It does not send webhook -events or exercise webhook scale-up, scale-down, or pool handlers. +expects an already-running MockServer on localhost:1080, loads the +expectations into it, and destroys the Terraform deployment during cleanup. +Pass `--keep-deployment` (or set `MINISTACK_SMOKE_KEEP_DEPLOYMENT=1`) to retain +the deployment for debugging; it prints the generated tfvars path so the +deployment can be destroyed separately with `tests/ministack/run-example.sh destroy`. +MockServer is an external test dependency; the Python smoke runner does not +start or stop it. In CI, the setup action starts it on localhost:1080 and waits +for readiness. The smoke process connects to `http://localhost:1080`; the +MiniStack controller connects to `https://host.docker.internal:1080`. MiniStack +must resolve `host.docker.internal` to the host gateway. When MiniStack is +exposed on a non-default local port, set `AWS_ENDPOINT_URL` to a host address +reachable from its container, such as `http://:14568`, instead of +`localhost`. + +The combined smoke writes one command log for webhook EC2/MicroVM and +scale-set EC2. It applies the `multi-runner-orchestration` example once. MiniStack +must have its Docker engine socket mounted at `/var/run/docker.sock` so ECS can +start the controller container; without it the ECS API may report tasks +without creating Docker containers. diff --git a/tests/ministack/multi-runner-orchestration.tfvars b/tests/ministack/multi-runner-orchestration.tfvars new file mode 100644 index 0000000000..cf33a68551 --- /dev/null +++ b/tests/ministack/multi-runner-orchestration.tfvars @@ -0,0 +1,49 @@ +aws_region = "eu-west-1" +environment = "multi-runner-webhook" + +runners_lambda_zip = "../../lambdas/functions/control-plane/runners.zip" +webhook_lambda_zip = "../../lambdas/functions/webhook/webhook.zip" + +github = { + url = "https://host.docker.internal:1080" + ssl_verify = false + runner_owner = "example" + registration_level = "organization" +} + +scale_set = { + name = "linux-scale-set" + runner_group_name = "experimental-euw1-sl-cicd-forge-emu" + min_runners = 1 + container = { + image = "MINISTACK_SCALE_SET_IMAGE" + } +} + +github_app = { + id = "123" + installation_id = "123" + key_base64 = "ministack-invalid-key" + webhook_secret = "ministack-webhook-secret" +} + +compute_provider = { + aws = { + ec2 = { + instance_types = ["m7a.large", "m5.large"] + ami = { + filter = { + name = ["ministack-webhook-linux-x64"] + state = ["available"] + } + owners = ["self"] + } + } + microvm = { + image_arn = "arn:aws:lambda:eu-west-1:000000000000:microvm-image:ministack" + image_version = "3.0" + egress_network_connectors = ["arn:aws:lambda:eu-west-1:000000000000:network-connector:ministack"] + ingress_network_connectors = [] + } + } +} diff --git a/tests/ministack/multi-runner-scale-set.tfvars b/tests/ministack/multi-runner-scale-set.tfvars deleted file mode 100644 index 7dd1a0ad39..0000000000 --- a/tests/ministack/multi-runner-scale-set.tfvars +++ /dev/null @@ -1,57 +0,0 @@ -environment = "ministack-scale-set" -aws_region = "eu-west-1" - -github = { - url = "https://mockserver:1080" - ssl_verify = false - runner_owner = "example" - registration_level = "organization" -} - -github_app = { - id = "123" - key_base64 = "ministack-invalid-key" - installation_id = "456" -} - -runner_binaries_enabled = false - -ami = { - "linux-arm64" = { - filter = { - name = ["ministack-scale-set-linux-arm64"] - state = ["available"] - } - owners = ["self"] - } - "linux-x64" = { - filter = { - name = ["ministack-scale-set-linux-x64"] - state = ["available"] - } - owners = ["self"] - } - "linux-scale-set" = { - filter = { - name = ["ministack-scale-set-linux-x64"] - state = ["available"] - } - owners = ["self"] - } - "windows-x64" = { - filter = { - name = ["ministack-scale-set-windows-x64"] - state = ["available"] - } - owners = ["self"] - } -} - -scale_set = { - name = "medium" - runner_group_name = "experimental-euw1-sl-cicd-forge-emu" - min_runners = 1 - container = { - image = "localhost:4566/scale-set-controller:smoke" - } -} diff --git a/tests/ministack/multi-runner-webhook.tfvars b/tests/ministack/multi-runner-webhook.tfvars deleted file mode 100644 index aaab6d1da1..0000000000 --- a/tests/ministack/multi-runner-webhook.tfvars +++ /dev/null @@ -1,32 +0,0 @@ -aws_region = "eu-west-1" -environment = "multi-runner-webhook" - -runners_lambda_zip = "../../lambda_output/runners.zip" -webhook_lambda_zip = "../../lambda_output/webhook.zip" - -github_app = { - id = "123" - key_base64 = "ministack-invalid-key" - webhook_secret = "ministack-webhook-secret" -} - -compute_provider = { - aws = { - ec2 = { - instance_types = ["m7a.large", "m5.large"] - ami = { - filter = { - name = ["ministack-webhook-linux-x64"] - state = ["available"] - } - owners = ["self"] - } - } - microvm = { - image_arn = "arn:aws:lambda:eu-west-1:000000000000:microvm-image:ministack" - image_version = "3.0" - egress_network_connectors = ["arn:aws:lambda:eu-west-1:000000000000:network-connector:ministack"] - ingress_network_connectors = [] - } - } -} diff --git a/tests/ministack/run-example.sh b/tests/ministack/run-example.sh index a7707aeec8..5c55bcc7ff 100755 --- a/tests/ministack/run-example.sh +++ b/tests/ministack/run-example.sh @@ -22,7 +22,7 @@ case "$iac_binary" in ;; esac case "$example" in - base | prebuilt | default | ephemeral | multi-runner | multi-runner-webhook | microvm-foundation | multi-runner-scale-set) + base | prebuilt | default | ephemeral | multi-runner | multi-runner-orchestration | microvm-foundation) use_tfvars=true ;; migration-test) @@ -32,15 +32,14 @@ case "$example" in use_tfvars=false ;; *) - echo "Supported examples for the runner are: base, prebuilt, default, ephemeral, multi-runner, multi-runner-webhook, microvm-foundation, multi-runner-scale-set, migration-test, termination-watcher" >&2 - exit 64 - ;; + echo "Supported examples for the runner are: base, prebuilt, default, ephemeral, multi-runner, multi-runner-orchestration, microvm-foundation, migration-test, termination-watcher" >&2 + exit 64 + ;; esac case "$action" in init | plan | apply | destroy | output) ;; *) - echo "Usage: $0 {init|plan|apply|destroy|output} {base|prebuilt|default|ephemeral|multi-runner|multi-runner-webhook|microvm-foundation|multi-runner-scale-set|migration-test|termination-watcher} [TFVARS_FILE]" >&2 exit 64 ;; esac @@ -366,14 +365,9 @@ $lambda_zip" "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-6.1-arm64" \ "ami-0abcdef1234567890" ;; - multi-runner-webhook) + multi-runner-orchestration) create_ami_fixture "ministack-webhook-linux-x64" x86_64 >/dev/null ;; - multi-runner-scale-set) - create_ami_fixture "ministack-scale-set-linux-x64" x86_64 >/dev/null - create_ami_fixture "ministack-scale-set-linux-arm64" arm64 >/dev/null - create_ami_fixture "ministack-scale-set-windows-x64" x86_64 >/dev/null - ;; esac } diff --git a/tests/ministack/run-ministack-smoke.py b/tests/ministack/run-ministack-smoke.py new file mode 100644 index 0000000000..4f6e2607fe --- /dev/null +++ b/tests/ministack/run-ministack-smoke.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Run webhook EC2/MicroVM and scale-set EC2 smoke tests on one deployment.""" + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from smoke import webhook_ec2, webhook_microvm # noqa: E402 +from smoke.common import SmokeContext # noqa: E402 +from smoke.scale_set_ec2 import provider as scale_set_ec2_provider # noqa: E402 +from smoke.scale_set_scenario import prepare as prepare_scale_set # noqa: E402 +from smoke.scale_set_scenario import run as run_scale_set # noqa: E402 +from smoke.scale_set_provider import ScaleSetProvider # noqa: E402 +from smoke.webhook_provider import SmokeProvider # noqa: E402 +from smoke.webhook_scenario import run as run_webhook # noqa: E402 + +SCALE_SET_PROVIDERS = (scale_set_ec2_provider,) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run webhook EC2/MicroVM and scale-set EC2 smoke tests using one MiniStack deployment." + ) + parser.add_argument( + "--webhook-provider", + choices=("all", "ec2", "microvm"), + default="all", + help="Webhook compute-provider scenarios to run", + ) + parser.add_argument( + "--scale-set-provider", + choices=("all", *(provider.slug for provider in SCALE_SET_PROVIDERS)), + default="all", + help="Scale-set compute-provider scenarios to run", + ) + parser.add_argument( + "--keep-deployment", + action="store_true", + help="Keep the shared Terraform deployment and generated tfvars for debugging", + ) + return parser.parse_args() + + +def _select_providers(selection: str) -> tuple[SmokeProvider, ...]: + providers = (webhook_ec2.provider, webhook_microvm.provider) + if selection == "all": + return providers + return tuple(provider for provider in providers if provider.slug == selection) + + +def _select_scale_set_providers(selection: str) -> tuple[ScaleSetProvider, ...]: + if selection == "all": + return SCALE_SET_PROVIDERS + return tuple(provider for provider in SCALE_SET_PROVIDERS if provider.slug == selection) + + +def _run_smoke(args: argparse.Namespace) -> int: + selected = _select_providers(args.webhook_provider) + selected_scale_set = _select_scale_set_providers(args.scale_set_provider) + context = SmokeContext( + Path(__file__).parent, + microvm_enabled=any( + provider.slug == "microvm" for provider in (*selected, *selected_scale_set) + ), + keep_deployment=args.keep_deployment, + ) + try: + with context.step("Prepare combined MiniStack smoke deployment"): + context.configure_mockserver() + with context.step("Build and publish scale-set controller image"): + scale_set_image = prepare_scale_set(context, selected_scale_set[0]) + context.prepare(scale_set_image=scale_set_image) + # Replace MiniStack's initial ECS task with a revision carrying test-only AWS credentials. + with context.step("Scale-set orchestration testing"): + for provider in selected_scale_set: + with context.step(provider.display_name): + run_scale_set(context, provider, scale_set_image) + with context.step("Webhook orchestration testing"): + for provider in selected: + with context.step(provider.display_name): + run_webhook(context, provider) + tested = "EC2 and MicroVM" if args.webhook_provider == "all" else selected[0].display_name + scale_set_tested = " and ".join(provider.display_name for provider in selected_scale_set) + context.progress(f"MiniStack combined smoke passed: webhook {tested}, scale-set {scale_set_tested}.") + finally: + context.cleanup() + return 0 + + +def main() -> int: + return _run_smoke(_parse_args()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/ministack/run-scale-set-integration.sh b/tests/ministack/run-scale-set-integration.sh deleted file mode 100755 index 08e0d156de..0000000000 --- a/tests/ministack/run-scale-set-integration.sh +++ /dev/null @@ -1,600 +0,0 @@ -#!/bin/sh - -set -eu - -export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" -export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" -export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" -export AWS_REGION="${AWS_REGION:-eu-west-1}" -export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" -export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) -example="multi-runner-scale-set" -base_tfvars="$script_dir/$example.tfvars" -tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set.XXXXXX") -app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-key.XXXXXX") -log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-logs.XXXXXX") -controller_log_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-controller-logs.XXXXXX") -config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-config.XXXXXX") -task_definition_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-task-definition.XXXXXX") -instance_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-instances.XXXXXX") -scale_down_config_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-scale-down-config.XXXXXX") -initializer_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-scale-set-mockserver.XXXXXX") -repository_name="scale-set-controller" -image_reference="localhost:4566/${repository_name}:smoke" -mockserver_host="${MINISTACK_GITHUB_MOCK_HOST:-127.0.0.1}" -mockserver_port="${MINISTACK_GITHUB_MOCK_PORT:-1080}" -mockserver_url="${MINISTACK_GITHUB_MOCK_URL:-http://127.0.0.1:${mockserver_port}}" -controller_mock_url="https://${mockserver_host}:${mockserver_port}" -terraform_state_exists=false - -cleanup() { - cleanup_status=$? - set +e - - if [ "$terraform_state_exists" = true ]; then - "$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" >/dev/null 2>&1 - fi - - rm -f "$tfvars_file" "$app_key_file" "$log_file" "$controller_log_file" "$config_file" "$task_definition_file" "$instance_file" "$scale_down_config_file" "$initializer_file" - exit "$cleanup_status" -} -trap cleanup EXIT INT TERM - -require_command() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "$1 is required to run the scale-set MiniStack smoke test." >&2 - exit 69 - fi -} - -for command in aws curl docker openssl python3 terraform; do - require_command "$command" -done - -ministack_aws() { - aws --endpoint-url "$AWS_ENDPOINT_URL" --region "$AWS_DEFAULT_REGION" "$@" -} - -wait_for_http() { - url="$1" - attempts=90 - while ! curl -fsS --max-time 2 "$url" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $url." >&2 - exit 70 - fi - sleep 1 - done -} - -wait_for_scale_set_runner() { - attempts=90 - while :; do - ministack_aws ec2 describe-instances --output json > "$instance_file" - instance_id=$(python3 - "$instance_file" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as instance_file: - response = json.load(instance_file) - -required_tags = { - "ghr:Application": "github-action-runner", - "ghr:created_by": "scale-set-service", - "ghr:environment": "ministack-scale-set-linux-scale-set", - "ghr:Type": "Org", - "ghr:Owner": "example", - "ghr:scale_set_state": "config-published", - "ghr:github_runner_id": "321", -} -active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} -matches = [] -for reservation in response.get("Reservations", []): - for instance in reservation.get("Instances", []): - if instance.get("State", {}).get("Name") not in active_states: - continue - tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} - if all(tags.get(key) == value for key, value in required_tags.items()) and tags.get("ghr:runner_name", "").startswith("scale-set-"): - matches.append(instance["InstanceId"]) - -if len(matches) == 1: - print(matches[0]) -PY -) - if [ -n "$instance_id" ]; then - printf ' [PASS] MiniStack created and registered scale-set EC2 runner %s\n' "$instance_id" - return 0 - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for the scale-set EC2 runner to reach config-published state." >&2 - cat "$instance_file" >&2 || true - exit 1 - fi - sleep 2 - done -} - -wait_for_no_scale_set_runners() { - attempts=90 - while :; do - ministack_aws ec2 describe-instances --output json > "$instance_file" - active_count=$(python3 - "$instance_file" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as instance_file: - response = json.load(instance_file) - -required_tags = { - "ghr:Application": "github-action-runner", - "ghr:created_by": "scale-set-service", - "ghr:environment": "ministack-scale-set-linux-scale-set", - "ghr:Type": "Org", - "ghr:Owner": "example", -} -active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} -count = 0 -for reservation in response.get("Reservations", []): - for instance in reservation.get("Instances", []): - if instance.get("State", {}).get("Name") not in active_states: - continue - tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} - if all(tags.get(key) == value for key, value in required_tags.items()): - count += 1 -print(count) -PY -) - if [ "$active_count" = "0" ]; then - printf '%s\n' ' [PASS] scale-set EC2 runner was terminated after the minimum changed to zero' - return 0 - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for the scale-set EC2 runner to terminate; active count is $active_count." >&2 - cat "$instance_file" >&2 || true - exit 1 - fi - sleep 2 - done -} - -wait_for_http "$AWS_ENDPOINT_URL/_ministack/health" -attempts=90 -while ! curl -fsS --max-time 2 -X PUT "$mockserver_url/mockserver/status" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $mockserver_url/mockserver/status." >&2 - exit 70 - fi - sleep 1 -done -curl -fsS -X PUT "$mockserver_url/mockserver/reset" >/dev/null -CONTROLLER_MOCK_URL="$controller_mock_url" python3 - "$source_root/tests/ministack/initializerJson.json" "$initializer_file" <<'PY' -import os -import sys - -source, destination = sys.argv[1:] -with open(source, encoding="utf-8") as source_file: - fixture = source_file.read() -fixture = fixture.replace("https://mockserver:1080", os.environ["CONTROLLER_MOCK_URL"]) -with open(destination, "w", encoding="utf-8") as destination_file: - destination_file.write(fixture) -PY -curl -fsS -X PUT \ - "$mockserver_url/mockserver/expectation" \ - -H 'Content-Type: application/json' \ - --data-binary "@$initializer_file" \ - >/dev/null - -repository_policy=$(python3 - <<'PY' -import json - -print(json.dumps({ - "Version": "2012-10-17", - "Statement": [{ - "Sid": "AllowAccountPull", - "Effect": "Allow", - "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, - "Action": [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer", - ], - }], -})) -PY -) - -ministack_aws ecr create-repository \ - --repository-name "$repository_name" \ - --image-tag-mutability IMMUTABLE \ - --image-scanning-configuration scanOnPush=false \ - >/dev/null -ministack_aws ecr set-repository-policy \ - --repository-name "$repository_name" \ - --policy-text "$repository_policy" \ - >/dev/null - -docker build \ - --target runtime \ - --file "$source_root/lambdas/services/scale-set/Dockerfile" \ - --tag "$image_reference" \ - "$source_root" - -ministack_aws ecr get-login-password | docker login \ - --username AWS \ - --password-stdin localhost:4566 >/dev/null -docker push "$image_reference" - -ministack_aws ecr describe-images \ - --repository-name "$repository_name" \ - --image-ids imageTag=smoke \ - >/dev/null - -openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null -app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') -APP_KEY_BASE64="$app_key_base64" GITHUB_CONFIG_URL="$controller_mock_url" python3 - "$base_tfvars" "$tfvars_file" <<'PY' -import os -import sys - -source, destination = sys.argv[1:] -replacement = os.environ["APP_KEY_BASE64"] -github_config_url = os.environ["GITHUB_CONFIG_URL"] -with open(source, encoding="utf-8") as source_file: - lines = source_file.readlines() -with open(destination, "w", encoding="utf-8") as destination_file: - for line in lines: - if line.lstrip().startswith("key_base64"): - destination_file.write(f' key_base64 = "{replacement}"\n') - elif line.lstrip().startswith("config_url"): - destination_file.write(f' config_url = "{github_config_url}"\n') - else: - destination_file.write(line) -PY -unset app_key_base64 APP_KEY_BASE64 - -terraform_state_exists=true -"$source_root/tests/ministack/run-example.sh" apply "$example" "$tfvars_file" - -config_path="/ministack-scale-set/scale-set-controller/linux-scale-set/linux-scale-set" -ministack_aws ssm get-parameter \ - --name "$config_path" \ - --query 'Parameter.Value' \ - --output text > "$config_file" -EXPECTED_GITHUB_CONFIG_URL="$controller_mock_url/example" python3 - "$config_file" <<'PY' -import json -import os -import sys - -with open(sys.argv[1], encoding="utf-8") as config_file: - config = json.load(config_file) - -assert config["githubConfigUrl"] == os.environ["EXPECTED_GITHUB_CONFIG_URL"] -assert config["forceGhes"] is True -assert config["sslVerify"] is False -assert config["minRunners"] == 1 -assert config["githubApp"]["appIdParameterName"] -assert config["githubApp"]["installationIdParameterName"] -assert config["githubApp"]["privateKeyParameterName"] -print(" [PASS] SSM manifest has the expected MockServer and GitHub App settings") -PY - -task_definition=$(ministack_aws ecs list-task-definitions \ - --family-prefix ministack-scale-set-ss-linux-scale-se- \ - --sort DESC \ - --query 'taskDefinitionArns[0]' \ - --output text) -if [ -z "$task_definition" ] || [ "$task_definition" = "None" ]; then - echo "The scale-set task definition was not registered." >&2 - exit 1 -fi - -actual_image=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].image | [0]' \ - --output text) -if [ "$actual_image" != "$image_reference" ]; then - echo "Expected the ECS task to use $image_reference, got $actual_image." >&2 - exit 1 -fi -printf '%s\n' ' [PASS] ECS task definition uses the image pushed to MiniStack ECR' - -log_driver=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.containerDefinitions[?name==`scale-set-controller`].logConfiguration.logDriver | [0]' \ - --output text) -if [ "$log_driver" != "awslogs" ]; then - echo "Expected the ECS task to request the awslogs driver, got $log_driver." >&2 - exit 1 -fi -printf '%s\n' ' [PASS] ECS task definition requests the awslogs driver' - -log_group=$(ministack_aws logs describe-log-groups \ - --log-group-name-prefix "/aws/ecs/ministack-scale-set-ss-linux-scale-se-" \ - --query 'logGroups[0].logGroupName' \ - --output text) -if [ -z "$log_group" ] || [ "$log_group" = "None" ]; then - echo "The scale-set CloudWatch log group was not created." >&2 - exit 1 -fi -printf '%s\n' ' [PASS] CloudWatch log group was created for the ECS controller' - -task_family=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.family' \ - --output text) - -# MiniStack exposes ECS credentials through the gateway's container IP. The -# Node.js AWS SDK intentionally rejects that address in -# AWS_CONTAINER_CREDENTIALS_FULL_URI because non-HTTPS full URIs are limited -# to loopback and the real ECS metadata address. Use only synthetic, -# test-scoped credentials in a temporary task-definition revision so the -# smoke test still exercises the pushed image, ECS service, and controller -# lifecycle without changing the production task definition or application. -ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition' \ - --output json > "$task_definition_file" -TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' -import json -import os - -path = os.environ["TASK_DEFINITION_FILE"] -with open(path, encoding="utf-8") as task_definition_file: - task_definition = json.load(task_definition_file) - -for field in ( - "taskDefinitionArn", - "revision", - "status", - "requiresAttributes", - "compatibilities", - "registeredAt", - "registeredBy", -): - task_definition.pop(field, None) - -for container in task_definition["containerDefinitions"]: - if container["name"] != "scale-set-controller": - continue - environment = container.setdefault("environment", []) - environment.extend([ - {"name": "AWS_ACCESS_KEY_ID", "value": "000000000000"}, - {"name": "AWS_SECRET_ACCESS_KEY", "value": "test-only"}, - ]) - -with open(path, "w", encoding="utf-8") as task_definition_file: - json.dump(task_definition, task_definition_file) -PY - -environment_name=$(sed -n 's/^environment[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$base_tfvars") -cluster_name="${environment_name}-scale-set" -task_definition=$(ministack_aws ecs register-task-definition \ - --cli-input-json "file://$task_definition_file" \ - --query 'taskDefinition.taskDefinitionArn' \ - --output text) -ministack_aws ecs update-service \ - --cluster "$cluster_name" \ - --service "$task_family" \ - --task-definition "$task_definition" \ - >/dev/null - -task_revision=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.revision' \ - --output text) -controller_container="" -attempts=90 -while [ -z "$controller_container" ]; do - controller_container=$(docker ps -a \ - --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ - --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ - --filter 'name=scale-set-controller' \ - --format '{{.ID}}' | sed -n '1p') - if [ -n "$controller_container" ]; then - break - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for the MiniStack ECS controller container." >&2 - docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true - exit 1 - fi - sleep 2 -done -printf ' [PASS] MiniStack started ECS controller container %s\n' "$controller_container" - -wait_for_controller_log_event() { - marker="$1" - required_text="${2:-}" - attempts=90 - while :; do - docker logs "$controller_container" > "$controller_log_file" 2>&1 || true - if python3 - "$controller_log_file" "$marker" "$required_text" <<'PY' -import sys - -marker, required = sys.argv[2:] -with open(sys.argv[1], encoding="utf-8") as log_file: - messages = log_file.read().splitlines() -if any(marker in message and (not required or required in message) for message in messages): - raise SystemExit(0) -raise SystemExit(1) -PY - then - printf ' [PASS] CloudWatch logs contain %s\n' "$marker" - return 0 - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for controller log marker '$marker'." >&2 - cat "$controller_log_file" >&2 || true - echo "Controller AWS/ECS metadata environment:" >&2 - docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" \ - | sed -E 's/^(AWS_CONTAINER_AUTHORIZATION_TOKEN|AWS_CONTAINER_CREDENTIALS_FULL_URI)=.*/\1=/' \ - | grep -E '^(AWS_|ECS_)' >&2 || true - echo "Controller network attachments:" >&2 - docker inspect --format '{{json .NetworkSettings.Networks}}' "$controller_container" >&2 || true - echo "Controller credential endpoint probe:" >&2 - docker exec "$controller_container" node -e \ - 'fetch(process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI, {headers: {Authorization: process.env.AWS_CONTAINER_AUTHORIZATION_TOKEN}}).then((response) => { console.error(`status=${response.status}`); process.exit(response.ok ? 0 : 1); }).catch((error) => { console.error(`${error.name}:${error.message}`); process.exit(1); })' \ - >&2 || true - exit 1 - fi - sleep 2 - done -} - -printf '%s\n' ' [INFO] MiniStack 1.5.12 does not emit ECS awslogs streams; validating controller runtime logs instead' -wait_for_controller_log_event 'scale_set_controller_started' -wait_for_controller_log_event 'scale_set_session_created' -wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":1' -wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' -wait_for_scale_set_runner - -wait_for_mock_route() { - method="$1" - route="$2" - body=$(REQUEST_METHOD="$method" REQUEST_PATH="$route" python3 - <<'PY' -import json -import os - -print(json.dumps({ - "httpRequest": { - "method": os.environ["REQUEST_METHOD"], - "path": os.environ["REQUEST_PATH"], - }, - "times": {"atLeast": 1}, -})) -PY - ) - attempts=45 - while ! curl -fsS --max-time 5 -X PUT \ - http://127.0.0.1:1080/mockserver/verify \ - -H 'Content-Type: application/json' \ - --data "$body" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for MockServer route: $method $route" >&2 - curl -sS --max-time 5 \ - 'http://127.0.0.1:1080/mockserver/retrieve?type=REQUESTS&format=JSON' >&2 || true - exit 1 - fi - sleep 2 - done - printf ' [PASS] MockServer received %s %s\n' "$method" "$route" -} - -wait_for_mock_route POST '/api/v3/app/installations/456/access_tokens' -wait_for_mock_route POST '/api/v3/orgs/example/actions/runners/registration-token' -wait_for_mock_route POST '/api/v3/actions/runner-registration' -wait_for_mock_route GET '/tenant/123/_apis/runtime/runnergroups/' -wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets' -wait_for_mock_route GET '/tenant/123/_apis/runtime/runnerscalesets/223' -wait_for_mock_route PATCH '/tenant/123/_apis/runtime/runnerscalesets/223' -wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig' -wait_for_mock_route POST '/tenant/123/_apis/runtime/runnerscalesets/223/sessions' -wait_for_mock_route GET '/messages' - -python3 - "$config_file" "$scale_down_config_file" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as config_file: - reconciler = json.load(config_file) - -assert reconciler["minRunners"] == 1 -reconciler["minRunners"] = 0 -with open(sys.argv[2], "w", encoding="utf-8") as config_file: - json.dump(reconciler, config_file) -PY -ministack_aws ssm put-parameter \ - --name "$config_path" \ - --type String \ - --value "$(cat "$scale_down_config_file")" \ - --overwrite \ - >/dev/null -printf '%s\n' ' [PASS] SSM manifest minimum changed from one runner to zero' - -SCALE_DOWN_CONFIG_FILE="$scale_down_config_file" TASK_DEFINITION_FILE="$task_definition_file" python3 - <<'PY' -import json -import os - -with open(os.environ["SCALE_DOWN_CONFIG_FILE"], encoding="utf-8") as config_file: - reconciler = json.load(config_file) -with open(os.environ["TASK_DEFINITION_FILE"], encoding="utf-8") as task_definition_file: - task_definition = json.load(task_definition_file) - -for container in task_definition["containerDefinitions"]: - if container["name"] != "scale-set-controller": - continue - for environment in container.get("environment", []): - if environment["name"] == "SCALE_SET_CONTROLLER_MANIFEST": - manifest = json.loads(environment["value"]) - assert len(manifest.get("reconcilers", [])) == 1 - manifest["reconcilers"][0]["minRunners"] = reconciler["minRunners"] - environment["value"] = json.dumps(manifest, separators=(",", ":")) - break - else: - raise RuntimeError("scale-set controller task definition has no inline manifest") - -with open(os.environ["TASK_DEFINITION_FILE"], "w", encoding="utf-8") as task_definition_file: - json.dump(task_definition, task_definition_file) -PY -task_definition=$(ministack_aws ecs register-task-definition \ - --cli-input-json "file://$task_definition_file" \ - --query 'taskDefinition.taskDefinitionArn' \ - --output text) -task_revision=$(ministack_aws ecs describe-task-definition \ - --task-definition "$task_definition" \ - --query 'taskDefinition.revision' \ - --output text) -ministack_aws ecs update-service \ - --cluster "$cluster_name" \ - --service "$task_family" \ - --task-definition "$task_definition" \ - --force-new-deployment \ - >/dev/null - -old_controller_container="$controller_container" -new_controller_container="" -attempts=90 -while [ -z "$new_controller_container" ]; do - for candidate in $(docker ps -a \ - --filter "label=com.amazonaws.ecs.task-definition-family=$task_family" \ - --filter "label=com.amazonaws.ecs.task-definition-version=$task_revision" \ - --filter 'name=scale-set-controller' \ - --format '{{.ID}}'); do - if [ "$candidate" != "$old_controller_container" ]; then - new_controller_container="$candidate" - break - fi - done - if [ -n "$new_controller_container" ]; then - break - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo 'Timed out waiting for the ECS service to deploy the scale-down task.' >&2 - docker ps -a --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' >&2 || true - exit 1 - fi - sleep 2 -done -controller_container="$new_controller_container" -printf ' [PASS] ECS service deployed a fresh controller container %s for scale-down\n' "$controller_container" -wait_for_controller_log_event 'scale_set_controller_started' -wait_for_controller_log_event 'scale_set_session_created' -wait_for_controller_log_event 'scale_set_reconciled' '"desiredRunners":0' -wait_for_controller_log_event 'scale_set_reconciled' '"status":"converged"' -wait_for_no_scale_set_runners - -"$source_root/tests/ministack/run-example.sh" destroy "$example" "$tfvars_file" -terraform_state_exists=false -wait_for_mock_route DELETE '/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111' - -echo 'Scale-set MiniStack ECS/MockServer smoke test passed.' diff --git a/tests/ministack/run-smoke.sh b/tests/ministack/run-smoke.sh deleted file mode 100644 index d4d9c7ab3e..0000000000 --- a/tests/ministack/run-smoke.sh +++ /dev/null @@ -1,721 +0,0 @@ -#!/bin/sh - -set -eu - -export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" -export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" -export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" -export AWS_REGION="${AWS_REGION:-eu-west-1}" -export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" -export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) -example_root="$source_root/examples/default" -mock_expectations="$script_dir/github-api-expectations.json" -fixture="$script_dir/workflow_job_event.json" -dynamic_fixture=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-dynamic-workflow-job.XXXXXX") -mock_host="${MINISTACK_GITHUB_MOCK_HOST:-host.docker.internal}" -mock_port="${MINISTACK_GITHUB_MOCK_PORT:-}" -mock_service_url="${MINISTACK_GITHUB_MOCK_URL:-}" -mock_image="${MINISTACK_GITHUB_MOCK_IMAGE:-mockserver/mockserver:7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290}" -mock_container="" -tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-smoke.XXXXXX") -app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-github-app.XXXXXX") -response_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-smoke-response.XXXXXX") -lambda_response_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-lambda-response.XXXXXX") -override_file="$example_root/zz_ministack_smoke_override.tf" -terraform_initialized=false -discovered_instance_ids="" - -cleanup() { - set +e - for instance_id in $discovered_instance_ids; do - aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 terminate-instances \ - --instance-ids "$instance_id" >/dev/null 2>&1 - done - if [ "$terraform_initialized" = true ]; then - "$source_root/tests/ministack/run-example.sh" destroy default "$tfvars_file" >/dev/null 2>&1 - fi - if [ -n "$mock_container" ]; then - docker rm -f "$mock_container" >/dev/null 2>&1 - fi - rm -f "$override_file" "$tfvars_file" "$app_key_file" "$response_file" "$lambda_response_file" "$dynamic_fixture" -} -trap cleanup EXIT INT TERM - -require_command() { - if ! command -v "$1" >/dev/null 2>&1; then - echo "$1 is required to run the MiniStack smoke test." >&2 - exit 69 - fi -} - -for command in aws curl openssl python3 terraform; do - require_command "$command" -done -if [ -z "$mock_service_url" ]; then - require_command docker -fi - -for lambda_zip in \ - "$source_root/lambdas/functions/webhook/webhook.zip" \ - "$source_root/lambdas/functions/control-plane/runners.zip"; do - if [ ! -f "$lambda_zip" ]; then - echo "Missing $lambda_zip. Build the webhook and control-plane distributions first." >&2 - exit 66 - fi -done - -if [ -z "$mock_port" ]; then - if [ -n "$mock_service_url" ]; then - mock_port=1080 - else - mock_port=$(python3 -c 'import socket; s = socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') - fi -fi - -if [ -z "$mock_service_url" ]; then - mock_container="terraform-aws-github-runner-github-api-mock-$$" - mock_service_url="http://127.0.0.1:${mock_port}" -fi - -openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null -app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') -APP_KEY_BASE64="$app_key_base64" python3 - "$script_dir/default.tfvars" "$tfvars_file" <<'PY' -import os -import sys - -source, destination = sys.argv[1:] -replacement = os.environ["APP_KEY_BASE64"] -with open(source, encoding="utf-8") as source_file: - lines = source_file.readlines() -with open(destination, "w", encoding="utf-8") as destination_file: - for line in lines: - if line.lstrip().startswith("key_base64 ="): - destination_file.write(f' key_base64 = "{replacement}"\n') - elif line.lstrip().startswith('id') and '=' in line: - destination_file.write(' id = "123"\n') - else: - destination_file.write(line) -PY -unset app_key_base64 APP_KEY_BASE64 - -printf '%s\n' \ - 'module "runners" {' \ - " ghes_url = \"http://${mock_host}:${mock_port}\"" \ - ' ghes_ssl_verify = false' \ - ' eventbridge = {' \ - ' enable = true' \ - ' accept_events = ["workflow_job"]' \ - ' }' \ - ' delay_webhook_event = 0' \ - ' runners_maximum_count = 1' \ - ' instance_types = ["m7a.large"]' \ - ' enable_dynamic_labels = true' \ - ' minimum_running_time_in_minutes = 0' \ - ' pool_runner_owner = "test-owner"' \ - ' pool_config = [{ schedule_expression = "cron(0 0 1 1 ? 2099)", size = 1 }]' \ - ' scale_down_schedule_expression = "cron(0 0 1 1 ? 2099)"' \ - ' enable_job_queued_check = true' \ - ' enable_jit_config = false' \ - ' enable_runner_binaries_syncer = false' \ - ' log_level = "debug"' \ - '}' \ - '' \ - 'module "webhook_github_app" {' \ - ' count = 0' \ - '}' > "$override_file" - -if [ -n "$mock_container" ]; then - docker run --detach --name "$mock_container" --publish "${mock_port}:1080" \ - --volume "$mock_expectations:/config/github-api-expectations.json:ro" \ - --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/github-api-expectations.json \ - "$mock_image" >/dev/null -fi - -attempts=30 -while ! curl -fsS --max-time 2 -X PUT "${mock_service_url}/mockserver/status" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "MockServer did not become ready." >&2 - if [ -n "$mock_container" ]; then - docker logs "$mock_container" >&2 - fi - exit 70 - fi - sleep 1 -done - -if [ -z "$mock_container" ]; then - MOCKSERVER_URL="$mock_service_url" python3 - "$mock_expectations" <<'PY' -import json -import os -import sys -import urllib.request - -with open(sys.argv[1], encoding="utf-8") as expectations_file: - expectations = json.load(expectations_file) - -for expectation in expectations: - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}/mockserver/expectation', - data=json.dumps(expectation).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="PUT", - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201): - raise RuntimeError(f"MockServer expectation rejected with HTTP {response.status}") -PY -fi - -python3 - "$fixture" "$dynamic_fixture" <<'PY' -import json -import sys - -source, destination = sys.argv[1:] -with open(source, encoding="utf-8") as source_file: - event = json.load(source_file) - -job = event["workflow_job"] -job["id"] = 123457 -job["run_id"] = 654322 -job["run_url"] = job["run_url"].replace("654321", "654322") -job["url"] = job["url"].replace("123456", "123457") -job["html_url"] = job["html_url"].replace("123456", "123457") -job["name"] = "ministack-smoke-dynamic" -job["labels"].append("ghr-ec2-instance-type:m5.large") - -with open(destination, "w", encoding="utf-8") as destination_file: - json.dump(event, destination_file) -PY - -terraform_initialized=true -"$source_root/tests/ministack/run-example.sh" apply default "$tfvars_file" - -printf '%s\n' \ - 'MiniStack smoke chain evidence checklist:' \ - ' [ ] API Gateway accepted the signed workflow_job webhook (HTTP 201)' \ - ' [ ] Webhook Lambda log contains workflow job 123456' \ - ' [ ] EventBridge invoked the dispatcher Lambda (dispatcher log contains 123456)' \ - ' [ ] Dispatcher delivered the job through SQS (scale-up log contains 123456)' \ - ' [ ] Scale-up without a dynamic label called each expected GitHub API route in MockServer' \ - ' [ ] MiniStack EC2 API reports a standard scale-up instance with default EC2 configuration' \ - ' [ ] Scale-up with ghr-ec2-instance-type:m5.large called each expected GitHub API route in MockServer' \ - ' [ ] Dynamic label selected EC2 instance type m5.large' \ - ' [ ] Scale-up EC2 instance has the expected runner discovery tags' \ - ' [ ] Pool called every expected GitHub API route in MockServer' \ - ' [ ] Pool Lambda created a runner instance' \ - ' [ ] Pool EC2 instance has the expected runner discovery tags' \ - ' [ ] Scale-down removed each runner from GitHub and terminated its EC2 instance' - -webhook_endpoint=$(terraform -chdir="$example_root" output -raw webhook_endpoint) -endpoint_host_port=${AWS_ENDPOINT_URL#*://} -endpoint_port=${endpoint_host_port##*:} -api_host_port=${webhook_endpoint#*://} -api_host_port=${api_host_port%%/*} -api_host=${api_host_port%:*} -webhook_secret=$(terraform -chdir="$example_root" output -raw webhook_secret) - -send_webhook() { - fixture_file="$1" - delivery_id="$2" - signature=$(openssl dgst -sha256 -hmac "$webhook_secret" "$fixture_file" | awk '{print $NF}') - status_code=$(curl -sS --max-time 15 -o "$response_file" -w '%{http_code}' \ - --connect-to "${api_host}:4566:127.0.0.1:${endpoint_port}" \ - -X POST "$webhook_endpoint" \ - -H 'Content-Type: application/json' \ - -H 'X-GitHub-Event: workflow_job' \ - -H "X-GitHub-Delivery: ${delivery_id}" \ - -H 'X-GitHub-Hook-Installation-Target-ID: 123' \ - -H "X-Hub-Signature-256: sha256=${signature}" \ - --data-binary "@${fixture_file}") - - if [ "$status_code" != 201 ]; then - echo "Webhook smoke request failed with HTTP $status_code." >&2 - sed -n '1,80p' "$response_file" >&2 - exit 1 - fi - echo " [PASS] API Gateway accepted the signed workflow_job webhook ${delivery_id} (HTTP 201)" -} - -send_webhook "$fixture" "ministack-smoke-123456" - -wait_for_log_event() { - log_group="$1" - marker="$2" - description="$3" - attempts=60 - while ! aws --endpoint-url "$AWS_ENDPOINT_URL" logs filter-log-events \ - --log-group-name "$log_group" --filter-pattern "$marker" --limit 1 --output text 2>/dev/null | grep -Fq "$marker"; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for MiniStack log marker '$marker' in $log_group." >&2 - exit 1 - fi - sleep 2 - done - printf ' [PASS] %s (log group %s contains %s)\n' "$description" "$log_group" "$marker" -} - -wait_for_optional_log_event() { - log_group="$1" - marker="$2" - description="$3" - attempts=60 - while ! aws --endpoint-url "$AWS_ENDPOINT_URL" logs filter-log-events \ - --log-group-name "$log_group" --filter-pattern "$marker" --limit 1 --output text 2>/dev/null | grep -Fq "$marker"; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - printf ' [WARN] %s (log marker %s was not observed in %s)\n' \ - "$description" "$marker" "$log_group" - return 0 - fi - sleep 2 - done - printf ' [PASS] %s (log group %s contains %s)\n' "$description" "$log_group" "$marker" -} - -wait_for_log_event "/aws/lambda/ministack-default-webhook" "123456" \ - "Webhook Lambda received workflow job 123456" -wait_for_log_event "/aws/lambda/ministack-default-dispatch-to-runner" "123456" \ - "EventBridge invoked the dispatcher Lambda" -wait_for_log_event "/aws/lambda/ministack-default-scale-up" "123456" \ - "Dispatcher delivered workflow job 123456 through SQS to scale-up" - -wait_for_mock_route() { - method="$1" - route="$2" - description="$3" - verification_body=$(printf '{"httpRequest":{"method":"%s","path":"%s"},"times":{"atLeast":1}}' "$method" "$route") - attempts=60 - while ! curl -fsS --max-time 5 -X PUT "${mock_service_url}/mockserver/verify" \ - -H 'Content-Type: application/json' \ - --data-binary "$verification_body" >/dev/null 2>&1; do - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for MockServer route: $method $route" >&2 - curl -sS --max-time 5 -X PUT \ - "${mock_service_url}/mockserver/retrieve?type=REQUEST_RESPONSES&format=JSON" >&2 || true - exit 1 - fi - sleep 2 - done - printf ' [PASS] %s (MockServer verified %s %s)\n' "$description" "$method" "$route" -} - -clear_mock_request_log() { - if ! curl -fsS --max-time 5 -X PUT \ - "${mock_service_url}/mockserver/clear?type=log" >/dev/null 2>&1; then - echo "Failed to clear MockServer request history before the next lifecycle phase." >&2 - exit 1 - fi -} - -assert_scale_down_github_routes() { - wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ - "Scale-down requested a GitHub App installation token" - wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners" \ - "Scale-down listed organization runners" - wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners/${1}" \ - "Scale-down checked the runner busy state" - wait_for_mock_route DELETE "/api/v3/orgs/test-owner/actions/runners/${1}" \ - "Scale-down deleted the runner from GitHub" -} - -assert_pool_github_routes() { - wait_for_mock_route GET "/api/v3/orgs/test-owner/installation" \ - "Pool looked up the GitHub App installation" - wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ - "Pool requested a GitHub App installation token" - wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners" \ - "Pool listed organization runners" - wait_for_mock_route POST "/api/v3/orgs/test-owner/actions/runners/registration-token" \ - "Pool requested a GitHub runner registration token" -} - -assert_scale_up_github_routes() { - job_id="$1" - wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ - "Scale-up requested a GitHub App installation token for job ${job_id}" - wait_for_mock_route GET "/api/v3/repos/test-owner/test-repo/actions/jobs/${job_id}" \ - "Scale-up checked the queued GitHub job ${job_id}" - wait_for_mock_route POST "/api/v3/orgs/test-owner/actions/runners/registration-token" \ - "Scale-up requested a GitHub runner registration token for job ${job_id}" -} - -assert_scale_up_github_routes 123456 - -wait_for_ec2_instance() { - source="$1" - description="$2" - attempts=60 - while :; do - found_instance_id=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --filters \ - "Name=instance-state-name,Values=running,pending" \ - "Name=tag:ghr:Application,Values=github-action-runner" \ - "Name=tag:ghr:created_by,Values=$source" \ - --query 'Reservations[].Instances[].InstanceId | [0]' \ - --output text 2>/dev/null || true) - if [ -n "$found_instance_id" ] && [ "$found_instance_id" != "None" ]; then - case " $discovered_instance_ids " in - *" $found_instance_id "*) ;; - *) discovered_instance_ids="$discovered_instance_ids $found_instance_id" ;; - esac - printf ' [PASS] MiniStack EC2 API reports %s: %s\n' "$description" "$found_instance_id" - return - fi - - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $description in the MiniStack EC2 API." >&2 - aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --filters \ - "Name=instance-state-name,Values=running,pending" \ - "Name=tag:ghr:Application,Values=github-action-runner" \ - "Name=tag:ghr:created_by,Values=$source" \ - --output json >&2 || true - exit 1 - fi - sleep 2 - done -} - -wait_for_ec2_instance "scale-up-lambda" "a scale-up instance" -scale_up_instance_id="$found_instance_id" - -assert_ec2_tag() { - instance_id="$1" - key="$2" - expected_value="$3" - description="$4" - actual_value=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query "Reservations[].Instances[].Tags[?Key=='${key}'].Value | [0]" \ - --output text 2>/dev/null || true) - if [ "$actual_value" != "$expected_value" ]; then - echo "Expected $description tag $key=$expected_value on $instance_id, got $actual_value." >&2 - exit 1 - fi -} - -assert_ec2_runner_tags() { - instance_id="$1" - source="$2" - description="$3" - assert_ec2_tag "$instance_id" "ghr:Application" "github-action-runner" "$description" - assert_ec2_tag "$instance_id" "ghr:created_by" "$source" "$description" - assert_ec2_tag "$instance_id" "ghr:Type" "Org" "$description" - assert_ec2_tag "$instance_id" "ghr:Owner" "test-owner" "$description" - printf ' [PASS] MiniStack EC2 API reports correct runner tags on %s\n' "$instance_id" -} - -assert_ec2_runner_tags "$scale_up_instance_id" "scale-up-lambda" "the scale-up runner" - -assert_ec2_default_instance_type() { - instance_id="$1" - actual_type=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].InstanceType' \ - --output text 2>/dev/null || true) - if [ "$actual_type" != "m7a.large" ]; then - echo "Expected standard scale-up to use the configured default m7a.large, got $actual_type." >&2 - exit 1 - fi - printf ' [PASS] Standard scale-up used the configured default EC2 instance type: %s\n' "$actual_type" -} - -assert_ec2_instance_type() { - instance_id="$1" - expected_type="$2" - actual_type=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query 'Reservations[0].Instances[0].InstanceType' \ - --output text 2>/dev/null || true) - if [ "$actual_type" != "$expected_type" ]; then - echo "Expected $instance_id to use EC2 instance type $expected_type, got $actual_type." >&2 - exit 1 - fi - printf ' [PASS] EC2 dynamic label selected instance type %s on %s\n' "$expected_type" "$instance_id" -} - -assert_ec2_default_instance_type "$scale_up_instance_id" - -configure_mock_runner_state() { - instance_id="$1" - runner_id="$2" - MOCKSERVER_URL="$mock_service_url" python3 - "$instance_id" "$runner_id" <<'PY' -import json -import os -import sys -import urllib.request - -instance_id, runner_id = sys.argv[1:] -runner_id = int(runner_id) -base = "/api/v3/orgs/test-owner/actions/runners" - -def control(path, method, payload): - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}{path}', - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method=method, - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201, 202): - raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') - -def clear(method, path): - control("/mockserver/clear", "PUT", {"httpRequest": {"method": method, "path": path}}) - -def expect(method, path, status, body=None): - response = {"statusCode": status} - if body is not None: - response["headers"] = {"Content-Type": ["application/json"]} - response["body"] = json.dumps(body) - control( - "/mockserver/expectation", - "PUT", - {"httpRequest": {"method": method, "path": path}, "httpResponse": response}, - ) - -state_path = f"{base}/{runner_id}" -clear("GET", base) -clear("GET", state_path) -clear("DELETE", state_path) -expect( - "GET", - base, - 200, - { - "total_count": 1, - "runners": [ - { - "id": runner_id, - "name": f"ministack-smoke-{instance_id}", - "os": "linux", - "status": "offline", - "busy": False, - "labels": [], - } - ], - }, -) -expect( - "GET", - state_path, - 200, - { - "id": runner_id, - "name": f"ministack-smoke-{instance_id}", - "os": "linux", - "status": "offline", - "busy": False, - "labels": [], - }, -) -expect("DELETE", state_path, 204) -PY -} - -configure_mock_runner_removed() { - runner_id="$1" - MOCKSERVER_URL="$mock_service_url" python3 - "$runner_id" <<'PY' -import json -import os -import sys -import urllib.request - -runner_id = sys.argv[1] -path = f"/api/v3/orgs/test-owner/actions/runners/{runner_id}" - -def control(path, method, payload): - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}{path}', - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method=method, - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201, 202): - raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') - -control("/mockserver/clear", "PUT", {"httpRequest": {"method": "GET", "path": path}}) -control( - "/mockserver/expectation", - "PUT", - { - "httpRequest": {"method": "GET", "path": path}, - "httpResponse": { - "statusCode": 404, - "headers": {"Content-Type": ["application/json"]}, - "body": '{"message":"Not Found"}', - }, - }, -) -PY -} - -configure_empty_mock_runner_list() { - MOCKSERVER_URL="$mock_service_url" python3 - <<'PY' -import json -import os -import urllib.request - -path = "/api/v3/orgs/test-owner/actions/runners" - -def control(path, method, payload): - request = urllib.request.Request( - f'{os.environ["MOCKSERVER_URL"]}{path}', - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method=method, - ) - with urllib.request.urlopen(request, timeout=10) as response: - if response.status not in (200, 201, 202): - raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') - -control("/mockserver/clear", "PUT", {"httpRequest": {"method": "GET", "path": path}}) -control( - "/mockserver/expectation", - "PUT", - { - "httpRequest": {"method": "GET", "path": path}, - "httpResponse": { - "statusCode": 200, - "headers": {"Content-Type": ["application/json"]}, - "body": '{"total_count":0,"runners":[]}', - }, - }, -) -PY -} - -assert_mock_runner_removed() { - runner_id="$1" - status_code=$(curl -sS --max-time 5 -o "$response_file" -w '%{http_code}' \ - "${mock_service_url}/api/v3/orgs/test-owner/actions/runners/${runner_id}") - if [ "$status_code" != 404 ]; then - echo "Expected GitHub API mock to return 404 for removed runner $runner_id, got HTTP $status_code." >&2 - sed -n '1,80p' "$response_file" >&2 - exit 1 - fi - printf ' [PASS] GitHub API mock reports runner %s removed (HTTP 404)\n' "$runner_id" -} - -wait_for_ec2_termination() { - instance_id="$1" - description="$2" - attempts=60 - while :; do - if state=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ - --instance-ids "$instance_id" \ - --query 'Reservations[].Instances[].State.Name | [0]' \ - --output text 2>/dev/null); then - if [ -z "$state" ] || [ "$state" = "None" ] || [ "$state" = "terminated" ]; then - printf ' [PASS] MiniStack EC2 API reports %s terminated\n' "$description" - return - fi - else - state="describe-instances failed" - fi - attempts=$((attempts - 1)) - if [ "$attempts" -le 0 ]; then - echo "Timed out waiting for $description to terminate; current state: $state." >&2 - exit 1 - fi - sleep 2 - done -} - -invoke_lambda() { - function_name="$1" - payload="$2" - description="$3" - invocation_result=$(aws --endpoint-url "$AWS_ENDPOINT_URL" lambda invoke \ - --cli-binary-format raw-in-base64-out \ - --invocation-type RequestResponse \ - --function-name "$function_name" \ - --payload "$payload" \ - "$lambda_response_file" --output json) - if printf '%s' "$invocation_result" | grep -Fq '"FunctionError"'; then - echo "Lambda invocation returned FunctionError for $function_name." >&2 - exit 1 - fi - printf ' [PASS] %s (Lambda API accepted the request)\n' "$description" -} - -scale_up_runner_id=987654321 -configure_mock_runner_state "$scale_up_instance_id" "$scale_up_runner_id" -clear_mock_request_log -invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-scale-up-scale-down"}' \ - "Scale-down Lambda invoked for the scale-up runner" -wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-scale-up-scale-down" \ - "Scale-down Lambda started processing the scale-up runner" -assert_scale_down_github_routes "$scale_up_runner_id" -configure_mock_runner_removed "$scale_up_runner_id" -assert_mock_runner_removed "$scale_up_runner_id" -wait_for_ec2_termination "$scale_up_instance_id" "the scale-up instance" -wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$scale_up_instance_id" \ - "Scale-down log recorded termination of the scale-up EC2 runner" - -clear_mock_request_log -send_webhook "$dynamic_fixture" "ministack-smoke-123457" -wait_for_log_event "/aws/lambda/ministack-default-webhook" "123457" \ - "Webhook Lambda received dynamic-label workflow job 123457" -wait_for_log_event "/aws/lambda/ministack-default-dispatch-to-runner" "123457" \ - "EventBridge invoked the dispatcher for dynamic-label workflow job 123457" -wait_for_log_event "/aws/lambda/ministack-default-scale-up" "123457" \ - "Dispatcher delivered dynamic-label workflow job 123457 through SQS to scale-up" -assert_scale_up_github_routes 123457 -wait_for_ec2_instance "scale-up-lambda" "a dynamic-label scale-up instance" -dynamic_scale_up_instance_id="$found_instance_id" -assert_ec2_runner_tags "$dynamic_scale_up_instance_id" "scale-up-lambda" \ - "the dynamic-label scale-up runner" -assert_ec2_instance_type "$dynamic_scale_up_instance_id" "m5.large" - -dynamic_scale_up_runner_id=987654323 -configure_mock_runner_state "$dynamic_scale_up_instance_id" "$dynamic_scale_up_runner_id" -clear_mock_request_log -invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-dynamic-scale-up-scale-down"}' \ - "Scale-down Lambda invoked for the dynamic-label scale-up runner" -wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-dynamic-scale-up-scale-down" \ - "Scale-down Lambda started processing the dynamic-label scale-up runner" -assert_scale_down_github_routes "$dynamic_scale_up_runner_id" -configure_mock_runner_removed "$dynamic_scale_up_runner_id" -assert_mock_runner_removed "$dynamic_scale_up_runner_id" -wait_for_ec2_termination "$dynamic_scale_up_instance_id" "the dynamic-label scale-up instance" -wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$dynamic_scale_up_instance_id" \ - "Scale-down log recorded termination of the dynamic-label scale-up EC2 runner" - -echo "MiniStack smoke chain 1 passed: API Gateway -> webhook -> EventBridge -> dispatcher -> SQS -> scale-up without and with EC2 dynamic label -> GitHub API mock." - -configure_empty_mock_runner_list -clear_mock_request_log -invoke_lambda "ministack-default-pool" '{"poolSize":1,"type":"ec2"}' \ - "Pool Lambda invoked to maintain one runner" -assert_pool_github_routes -wait_for_log_event "/aws/lambda/ministack-default-pool" "topped up with 1 runners" \ - "Pool Lambda requested one runner" -wait_for_ec2_instance "pool-lambda" "a pool instance" -pool_instance_id="$found_instance_id" -assert_ec2_runner_tags "$pool_instance_id" "pool-lambda" "the pool runner" - -pool_runner_id=987654322 -configure_mock_runner_state "$pool_instance_id" "$pool_runner_id" -clear_mock_request_log -invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-pool-scale-down"}' \ - "Scale-down Lambda invoked for the pool runner" -wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-pool-scale-down" \ - "Scale-down Lambda started processing the pool runner" -assert_scale_down_github_routes "$pool_runner_id" -configure_mock_runner_removed "$pool_runner_id" -assert_mock_runner_removed "$pool_runner_id" -wait_for_ec2_termination "$pool_instance_id" "the pool instance" -wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$pool_instance_id" \ - "Scale-down log recorded termination of the pool EC2 runner" - -echo "MiniStack smoke chain 2 passed: pool -> GitHub API mock -> EC2 runner creation -> scale-down -> GitHub API mock -> EC2 termination." -echo "MiniStack smoke tests passed: both lifecycle chains completed." diff --git a/tests/ministack/smoke/README.md b/tests/ministack/smoke/README.md new file mode 100644 index 0000000000..e990f71ecb --- /dev/null +++ b/tests/ministack/smoke/README.md @@ -0,0 +1,300 @@ +# MiniStack combined smoke test + +This directory contains the provider-neutral webhook harness for the +`multi-runner-orchestration` example. The combined entry point also runs the ECS +scale-set controller against that same Terraform deployment. Tests use +MiniStack and a GitHub API MockServer without calling GitHub. + +The combined smoke runs webhook EC2, webhook MicroVM, and scale-set EC2 by +default. Scale-set MicroVM coverage is still WIP. + +## Entry point + +Run the harness from the repository root: + +```sh +python3 tests/ministack/run-ministack-smoke.py [--webhook-provider all|ec2|microvm] +``` + +The default is `all`. Use `--keep-deployment` to retain the temporary Terraform +variables and deployed resources after a failure: + +```sh +python3 tests/ministack/run-ministack-smoke.py --keep-deployment +``` + +The ordinary smoke test requires MiniStack on `AWS_ENDPOINT_URL` and an +already-running MockServer listening on localhost:1080. + +Step progress is written to the test step. Detailed subprocess output is +tee'd to `MINISTACK_SMOKE_LOG_FILE` (default: `ministack-smoke.log`) and is +printed to the test step only when a command fails. The smoke runner writes command output to `ministack-smoke.log`, including +Terraform, Packer, Docker, and AWS CLI output. + +## Complete execution flow + +### 1. Select providers + +`run-ministack-smoke.py` creates one `SmokeContext`, selects the requested +webhook and scale-set providers, publishes the scale-set controller image, and +applies `multi-runner-orchestration` once before running each selected +scenario. + +### 2. Configure MockServer + +`SmokeContext.prepare()` first checks the required local commands and waits for +MockServer. It loads the static expectations from: + +```text +tests/ministack/smoke/fixtures/github-api-expectations.json +``` + +Those expectations cover the GitHub job lookup, installation-token exchange, +runner-group lookup, JIT configuration generation, and registration-token +fallback routes. + +Provider-specific expectations are added later when each provider is +configured. + +### 3. Create temporary Terraform input + +The harness generates a temporary RSA key, replaces the invalid fixture +GitHub App key, and adds the local GitHub Enterprise Server configuration. The +temporary variables also point Terraform at the real Lambda ZIPs: + +- `lambdas/functions/control-plane/runners.zip`; +- `lambdas/functions/webhook/webhook.zip`. + +The key and temporary variables are removed during cleanup unless +`--keep-deployment` is used. + +### 4. Apply the `multi-runner-orchestration` example + +The harness invokes: + +```sh +tests/ministack/run-example.sh apply multi-runner-orchestration +``` + +This deploys the webhook, dispatcher, scale-up, scale-down, pool, EC2, and +MicroVM configuration. After apply, the harness reads the Terraform outputs +for the webhook endpoint and webhook secret. + +The webhook endpoint is normally an API Gateway-style hostname such as +`b3855cb6.execute-api.localhost:4566`. Requests are sent to +`localhost:4566` while preserving that hostname in the HTTP `Host` header so +MiniStack routes the request correctly. + +### 5. Configure provider expectations + +Each provider adds the runner-group and JIT configuration expectations required +by its scale-up Lambda. The MicroVM provider keeps this logic in +`webhook_microvm.py`; EC2 keeps its equivalent provider setup in `webhook_ec2.py`. + +The JIT value is an internal MockServer fixture value. It is returned by the +mock GitHub API, written by the scale-up Lambda to MiniStack SSM, and consumed +from SSM by the lifecycle hook. No external JIT configuration variable is +required. + +### 6. Run the standard scale-up scenario + +For job `123456`, the harness: + +1. Deletes the provider's cached runner-group parameter. +2. Clears MockServer request history. +3. Creates and signs a `workflow_job` webhook using the configured secret. +4. Sends the webhook to the deployed endpoint. +5. Waits for the webhook, dispatcher, and provider scale-up Lambda logs. +6. Verifies the GitHub token and queued-job API routes. +7. Discovers the created compute resource. +8. Verifies provider-specific resource state and ownership metadata. + +For EC2 this resource is an instance. For MicroVM it is a MicroVM plus SSM +metadata under the configured MicroVM paths. + +### 7. Exercise the MicroVM lifecycle hook + +When the `microvm` provider is selected, the harness automatically builds and +starts the lifecycle-hook container on `127.0.0.1:8080`: + +```sh +python3 tests/ministack/run-ministack-smoke.py --webhook-provider microvm +``` + +When enabled, `MicrovmProvider.configure()` builds and starts the local image +once before the lifecycle scenarios: + +1. Reads the `microvm` Terraform output. +2. Logs in to MiniStack ECR. +3. Pulls, tags, and pushes the ARM64 Ubuntu base image. +4. Exports the MicroVM foundation outputs to the Packer environment. +5. Runs `packer build .` from `images/microvm-ubuntu`. +6. Copies the lifecycle-hook ZIP already produced by CI into the Docker build + context. +7. Builds the `microvm-lifecycle-hook` ARM64 Docker image. +8. Starts the `microvm-lifecycle-hook` container with Docker's default bridge + network, publishes `8080:8080`, and waits for its readiness endpoint. The + `host.docker.internal` host-gateway mapping lets it reach MiniStack on the + host-published port. + +After the MicroVM scale-up creates the resource, the test waits for the SSM +JIT parameter created by that scale-up. It then sends the same version-1 +`runHookPayload` shape used by the MicroVM control-plane code when it calls +`RunMicrovm`: + +```json +{ + "version": 1, + "imageArn": "", + "imageVersion": "", + "runnerConfigSsmPath": "/github-action-runners/multi-runner-webhook/microvm/runners/config", + "runnerTokenSsmPath": "/github-action-runners/multi-runner-webhook/microvm/runners/tokens" +} +``` + +The outer request contains the MicroVM identifier and the payload encoded as a +JSON string. The hook uses the MicroVM identifier and token path to consume the +JIT value written by scale-up. The test waits until the token parameter is +gone, proving that the one-time SSM value was consumed. + +The image build is guarded by the provider instance and happens only once per +smoke-test run, not once per scale-up. + +The lifecycle ZIP must already exist at: + +```text +lambdas/services/microvm-lifecycle-hooks/microvm-lifecycle-hooks.zip +``` + +The CI pipeline produces this artifact before running the MicroVM image build. + +### 8. Run the standard scale-down scenario + +The provider-specific scale-down implementation: + +1. Adds MockServer runner-list, runner-detail, and delete expectations. +2. Keeps all active smoke resources visible and marks only the selected runner + as removable. +3. Invokes the provider scale-down Lambda directly. +4. Waits for the scale-down log marker. +5. Verifies the installation-token, runner-list, runner-detail, and runner + deletion routes. +6. Changes the selected runner lookup to HTTP 404. +7. Verifies that the compute resource is terminated. + +For MicroVM, the hook termination endpoint is also called after the resource +termination check. + +### 9. Run the dynamic-label scenario + +The same scale-up and scale-down sequence is repeated for job `123457`, but +the event includes the provider-specific dynamic label: + +- EC2: `ghr-ec2-instance-type:m5.large`; +- MicroVM: `ghr-microvm-image-version:3.0`. + +The test verifies that the resource uses the requested dynamic configuration. + +### 10. Run the pool scenario + +The pool path is invoked directly rather than through a webhook: + +```json +{"poolSize": 1, "type": "ec2|microvm"} +``` + +The harness verifies the pool GitHub routes, discovers the created resource, +checks its provider-specific state, and then performs the same scale-down +assertions. + +### 11. Cleanup + +On success or failure, the harness: + +- removes the local MicroVM Docker container when used; +- terminates discovered EC2 instances; +- terminates discovered MicroVMs; +- destroys the `multi-runner-orchestration` Terraform deployment; +- removes temporary variables and response files. + +Use `--keep-deployment` or `MINISTACK_SMOKE_KEEP_DEPLOYMENT=1` when the +Terraform deployment and temporary variables are needed for investigation. + +## Provider responsibilities + +| File | Responsibility | +| --- | --- | +| `webhook_scenario.py` | Shared standard, dynamic, pool, and scale-down scenarios | +| `webhook_provider.py` | Provider interface and resource abstraction | +| `webhook_ec2.py` | EC2 discovery, tag assertions, and termination | +| `webhook_microvm.py` | MicroVM discovery, metadata assertions, image build, hook handoff, and termination | +| `common.py` | Terraform, AWS CLI, HTTP, MockServer, and cleanup plumbing | +| `fixtures/` | GitHub API, workflow-job, and scale-set controller fixtures | +| `scale_set_scenario.py` | ECS scale-set image deployment and controller protocol scenario | +| `scale_set_provider.py` | Provider interface for scale-set runner lifecycle assertions | +| `scale_set_ec2.py` | EC2 runner discovery, ownership checks, and scale-down polling | + +## Troubleshooting + +For a retained deployment, inspect Terraform state and outputs from: + +```sh +terraform -chdir=examples/multi-runner-orchestration output +``` + +If a route assertion times out, check the relevant Lambda log group and the +MockServer request history. If the MicroVM hook is enabled, confirm that the +container is ready at: + +```sh +curl --fail --request POST \ + http://127.0.0.1:8080/aws/lambda-microvms/runtime/v1/ready +``` + +## ECS scale-set integration smoke + +The combined entry point runs the ECS controller protocol after the selected +webhook scenarios. It builds and publishes the controller image, configures +the same `multi-runner-webhook` deployment with the `ec2_scalet_set` lane, and +applies Terraform once: + +```sh +python3 tests/ministack/run-ministack-smoke.py +``` + +It checks the SSM reconciler manifest, ECS task definition and log group, +controller runtime log markers, GitHub API protocol routes, and the created +scale-set EC2 runner. For scale-down it changes the reconciler minimum to zero, +deploys a fresh controller task revision, and waits for the runner to terminate +and the controller session DELETE request to reach MockServer. Scale-set +MicroVM coverage is WIP. + +MiniStack must have its Docker engine socket mounted at `/var/run/docker.sock`. +ECS task metadata can report a task as running without this socket, but MiniStack +cannot start the controller's Docker container for the smoke to inspect. +The controller reaches MockServer at `https://host.docker.internal:1080` to +satisfy the GitHub Enterprise URL contract; MockServer supports HTTP and HTTPS +on the same port. The smoke process loads and verifies expectations at +`http://localhost:1080`. Both addresses and port 1080 are fixed in the runner. + +The combined entry point writes `ministack-smoke.log` once for the full run. +Progress is printed to stdout; detailed subprocess output goes to the log and +is printed only when a command fails. Polling commands are kept out of the +detailed log; controller and ECS/Docker state is recorded once when startup +times out. Set `MINISTACK_SMOKE_LOG_FILE` to choose another log path. Use +`--keep-deployment` (or +`MINISTACK_SMOKE_KEEP_DEPLOYMENT=1`) to retain the Terraform inputs and +deployment for debugging; the temporary GitHub App private key is still +removed during cleanup. + +Both scenario modules use the same provider-adapter pattern. The webhook +scenario exposes `run(context, provider)`; the scale-set scenario exposes +`prepare(context, provider)` and `run(context, provider, image_reference)` +because it must publish its controller image and load MockServer fixtures +before applying the shared Terraform deployment. The `ScaleSetProvider` +interface in `scale_set_provider.py` isolates runner discovery and lifecycle +checks; `scale_set_ec2.py` is the current adapter. Add future scale-set compute +providers as adapters implementing this interface. The shared `SmokeContext` +owns commands, logging, and deployment cleanup. + +`run-ministack-smoke.py` owns the shared deployment lifecycle and logging. diff --git a/tests/ministack/smoke/__init__.py b/tests/ministack/smoke/__init__.py new file mode 100644 index 0000000000..e3cfc2fae6 --- /dev/null +++ b/tests/ministack/smoke/__init__.py @@ -0,0 +1 @@ +"""Provider-specific MiniStack smoke checks.""" diff --git a/tests/ministack/smoke/common.py b/tests/ministack/smoke/common.py new file mode 100644 index 0000000000..f99dfb4a9d --- /dev/null +++ b/tests/ministack/smoke/common.py @@ -0,0 +1,567 @@ +"""Shared MiniStack smoke-test plumbing.""" + +from __future__ import annotations + +import base64 +from contextlib import contextmanager +import hashlib +import hmac +import json +import os +import re +import shlex +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +import zipfile +from pathlib import Path +from typing import Any, Iterator +from urllib.parse import urlsplit, urlunsplit + +class SmokeContext: + def __init__(self, script_dir: Path, *, microvm_enabled: bool, keep_deployment: bool = False) -> None: + self.script_dir = script_dir + self.fixture_dir = Path(__file__).parent / "fixtures" + self.source_root = script_dir.parent.parent + self.example_root = self.source_root / "examples" / "multi-runner-orchestration" + self.aws_endpoint = os.environ.get("AWS_ENDPOINT_URL", "http://localhost:4566") + self.region = os.environ.get("AWS_DEFAULT_REGION", "eu-west-1") + self.environment = os.environ.copy() + self.environment.setdefault("AWS_ACCESS_KEY_ID", "000000000000") + self.environment.setdefault("AWS_SECRET_ACCESS_KEY", "test-only") + self.environment.setdefault("AWS_DEFAULT_REGION", self.region) + self.environment.setdefault("AWS_REGION", self.region) + self.environment.setdefault("AWS_ENDPOINT_URL", self.aws_endpoint) + self.environment.setdefault("AWS_EC2_METADATA_DISABLED", "true") + self.mock_host = "host.docker.internal" + self.mock_port = 1080 + self.mock_url = "http://localhost:1080" + self.tfvars_path: Path | None = None + self.webhook_endpoint = "" + self.webhook_secret = "" + self.discovered_instance_ids: list[str] = [] + self.discovered_microvm_ids: list[str] = [] + self.before_microvm_ids: set[str] = set() + self.response_path = Path(tempfile.mkstemp(prefix="ministack-smoke-response.")[1]) + self.microvm_enabled = microvm_enabled + self.log_path = Path(os.environ.get("MINISTACK_SMOKE_LOG_FILE", "ministack-smoke.log")) + self.keep_deployment = keep_deployment or os.environ.get("MINISTACK_SMOKE_KEEP_DEPLOYMENT") == "1" + self.step_depth = 0 + self.progress(f"Writing smoke command output to {self.log_path}") + + def _append_log(self, value: str) -> None: + if not value: + return + self.log_path.parent.mkdir(parents=True, exist_ok=True) + with self.log_path.open("a", encoding="utf-8") as log_file: + log_file.write(value) + + def progress(self, message: str) -> None: + line = f"{' ' * self.step_depth}{message}" + print(line, flush=True) + self._append_log(f"{line}\n") + + @contextmanager + def step(self, name: str) -> Iterator[None]: + self.progress(name) + self.step_depth += 1 + try: + yield + finally: + self.step_depth -= 1 + + def _log_command_output( + self, + command: list[str], + stdout: str | None, + stderr: str | None, + *, + include_command: bool = True, + ) -> None: + output = "" + if include_command: + output += f"\n$ {shlex.join(command)}\n" + if stdout: + output += stdout + if stderr: + output += stderr + self._append_log(output) + + def command(self, name: str) -> None: + if not shutil_which(name): + raise RuntimeError(f"{name} is required to run the MiniStack smoke test") + + def run( + self, + command: list[str], + *, + check: bool = True, + stream: bool = False, + cwd: Path | None = None, + log_output: bool = True, + input_text: str | None = None, + log_command: bool = True, + ) -> subprocess.CompletedProcess[str]: + if not stream: + try: + result = subprocess.run( + command, + check=check, + input=input_text, + text=True, + capture_output=True, + env=self.environment, + cwd=cwd, + ) + except subprocess.CalledProcessError as error: + if log_output: + self._log_command_output(command, error.stdout, error.stderr, include_command=log_command) + raise + if log_output: + self._log_command_output(command, result.stdout, result.stderr, include_command=log_command) + return result + + log_file = self.log_path.open("a", encoding="utf-8") if log_output else None + try: + process = subprocess.Popen( + command, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + env=self.environment, + cwd=cwd, + ) + except BaseException: + if log_file is not None: + log_file.close() + raise + + output: list[str] = [] + try: + if log_file is not None and log_command: + log_file.write(f"\n$ {shlex.join(command)}\n") + assert process.stdout is not None + for line in process.stdout: + output.append(line) + if log_file is not None: + log_file.write(line) + log_file.flush() + return_code = process.wait() + finally: + if log_file is not None: + log_file.close() + + result = subprocess.CompletedProcess(command, return_code, "".join(output), None) + if check and return_code != 0: + self.progress(f"Command failed: {shlex.join(command)}") + sys.stdout.write(result.stdout) + sys.stdout.flush() + raise subprocess.CalledProcessError(return_code, command, output=result.stdout) + return result + + def aws(self, *args: str, check: bool = True) -> Any: + result = self.run( + ["aws", "--endpoint-url", self.aws_endpoint, "--region", self.region, *args, "--output", "json"], + check=check, + ) + if not result.stdout.strip(): + return None + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + return result.stdout.strip() + + def terraform(self, *args: str, check: bool = True, log_output: bool = True) -> str: + result = self.run( + ["terraform", f"-chdir={self.example_root}", *args], + check=check, + log_output=log_output, + ) + return result.stdout.strip() + + def http(self, method: str, url: str, body: Any = None) -> tuple[int, str]: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request( + url, + data=data, + headers={"Content-Type": "application/json"} if data else {}, + method=method, + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, response.read().decode() + except urllib.error.HTTPError as error: + return error.code, error.read().decode() + except (TimeoutError, urllib.error.URLError) as error: + return 0, str(error) + + def wait_for(self, predicate, description: str, attempts: int = 60, interval: int = 2) -> Any: + for attempt in range(1, attempts + 1): + result = predicate() + if result: + return result + if attempt == 1 or attempt % 10 == 0: + self.progress(f"Still waiting for {description} ({attempt}/{attempts})") + time.sleep(interval) + raise RuntimeError(f"Timed out waiting for {description}") + + def configure_mockserver(self) -> None: + self.progress(f"Using MockServer at {self.mock_url}") + self.wait_for(lambda: self.http("PUT", f"{self.mock_url}/mockserver/status")[0] < 300, "MockServer") + expectations = json.loads((self.fixture_dir / "github-api-expectations.json").read_text()) + for expectation in expectations: + status, body = self.http("PUT", f"{self.mock_url}/mockserver/expectation", expectation) + if status >= 300: + raise RuntimeError(f"MockServer rejected an expectation: {status} {body}") + + def add_expectation(self, method: str, path: str, status: int, body: Any = None) -> None: + response: dict[str, Any] = {"statusCode": status} + if body is not None: + response.update(headers={"Content-Type": ["application/json"]}, body=json.dumps(body)) + code, text = self.http( + "PUT", + f"{self.mock_url}/mockserver/expectation", + {"httpRequest": {"method": method, "path": path}, "httpResponse": response}, + ) + if code >= 300: + raise RuntimeError(f"MockServer expectation failed: {code} {text}") + + def clear_expectation(self, method: str, path: str) -> None: + code, text = self.http( + "PUT", + f"{self.mock_url}/mockserver/clear", + {"httpRequest": {"method": method, "path": path}}, + ) + if code >= 300: + raise RuntimeError(f"MockServer expectation clear failed: {code} {text}") + + def clear_runner_group_cache(self, provider: str) -> None: + parameter_name = ( + f"/github-action-runners/multi-runner-webhook/{provider}/runners/config/runner-group/Default" + ) + self.progress(f"Clearing {provider} runner-group cache") + self.aws("ssm", "delete-parameter", "--name", parameter_name, check=False) + + def clear_requests(self) -> None: + self.progress("Clearing MockServer request history") + code, _ = self.http("PUT", f"{self.mock_url}/mockserver/clear?type=log") + if code >= 300: + raise RuntimeError("Failed to clear MockServer request history") + self.progress("MockServer request history cleared") + + def verify_route(self, method: str, path: str, description: str) -> None: + def verify() -> bool: + code, _ = self.http( + "PUT", + f"{self.mock_url}/mockserver/verify", + {"httpRequest": {"method": method, "path": path}, "times": {"atLeast": 1}}, + ) + return code < 300 + + try: + self.wait_for(verify, description) + except RuntimeError as error: + raise RuntimeError( + f"{error}. Recent messages in the provider Lambda logs may be available in {self.log_path}" + ) from error + + def send_webhook(self, event: dict[str, Any], delivery_id: str) -> None: + payload = json.dumps(event).encode() + signature = hmac.new(self.webhook_secret.encode(), payload, hashlib.sha256).hexdigest() + parsed_endpoint = urlsplit(self.webhook_endpoint) + endpoint = urlunsplit((parsed_endpoint.scheme, f"localhost:{parsed_endpoint.port or 4566}", parsed_endpoint.path, parsed_endpoint.query, parsed_endpoint.fragment)) + self.progress(f"Sending webhook {delivery_id} to {endpoint} (Host: {parsed_endpoint.netloc})") + request = urllib.request.Request( + endpoint, + data=payload, + headers={ + "Content-Type": "application/json", + "Host": parsed_endpoint.netloc, + "X-GitHub-Event": "workflow_job", + "X-GitHub-Delivery": delivery_id, + "X-GitHub-Hook-Installation-Target-ID": "123", + "X-Hub-Signature-256": f"sha256={signature}", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + status = response.status + except urllib.error.HTTPError as error: + status = error.code + body = error.read().decode() + raise RuntimeError( + f"Webhook smoke request failed with HTTP {status} at {endpoint} " + f"(Host: {parsed_endpoint.netloc}): {body}" + ) from error + if status not in (200, 201): + raise RuntimeError(f"Webhook smoke request failed with HTTP {status}") + self.progress(f"Webhook {delivery_id} accepted with HTTP {status}") + + def wait_for_log(self, group: str, marker: str, description: str) -> None: + self.progress(f"Waiting for {description} ({group})") + def found() -> bool: + events = self.aws("logs", "filter-log-events", "--log-group-name", group, "--filter-pattern", marker, "--limit", "1", check=False) + return bool(events and events.get("events")) + + try: + self.wait_for(found, description) + except RuntimeError as error: + groups = self.aws( + "logs", "describe-log-groups", + "--log-group-name-prefix", "/aws/lambda/multi-runner-webhook", + check=False, + ) or {} + available = [item.get("logGroupName") for item in groups.get("logGroups", [])] + recent = self.aws( + "logs", "filter-log-events", "--log-group-name", group, "--limit", "10", check=False, + ) or {} + messages = [item.get("message", "") for item in recent.get("events", [])] + raise RuntimeError( + f"{error}. Available smoke log groups: {available}. " + f"Recent messages in {group}: {messages}" + ) from error + self.progress(f"Found {description}") + + def recent_log_messages(self, group: str, limit: int = 50) -> list[str]: + events = self.aws( + "logs", + "filter-log-events", + "--log-group-name", + group, + "--limit", + str(limit), + check=False, + ) or {} + return [item.get("message", "") for item in events.get("events", [])] + + def invoke(self, function_name: str, payload: dict[str, Any], description: str) -> None: + payload_path = Path(tempfile.mkstemp(prefix="ministack-smoke-payload.")[1]) + output_path = Path(tempfile.mkstemp(prefix="ministack-smoke-lambda.")[1]) + try: + payload_path.write_text(json.dumps(payload)) + result = self.run([ + "aws", "--endpoint-url", self.aws_endpoint, "--region", self.region, + "lambda", "invoke", "--function-name", function_name, + "--payload", f"fileb://{payload_path}", str(output_path), "--output", "json", + ], check=False) + if result.returncode != 0: + raise RuntimeError( + f"{description} failed with exit code {result.returncode}: " + f"stdout={result.stdout.strip()} stderr={result.stderr.strip()}" + ) + metadata = json.loads(result.stdout) + if metadata.get("FunctionError"): + raise RuntimeError(output_path.read_text()) + finally: + payload_path.unlink(missing_ok=True) + output_path.unlink(missing_ok=True) + + def configure_runner_fixtures( + self, + provider: str, + runners: list[tuple[int, str]], + target_runner_id: int, + ) -> None: + """Expose all active GitHub runners while selecting one for removal. + + The scale-down Lambda evaluates every active provider resource. If the + GitHub list only contains the target runner, the remaining provider + resources are incorrectly marked as orphans and later invocations skip + the normal list-runners path. Keep the other runners visible and busy + so the fixture models a real multi-runner environment and only the + selected runner is eligible for termination. + """ + base = "/api/v3/orgs/test-owner/actions/runners" + if not any(runner_id == target_runner_id for runner_id, _ in runners): + raise RuntimeError(f"Target GitHub runner {target_runner_id} is not in the active runner fixtures") + + github_runners = [ + { + "id": runner_id, + "name": f"{provider}-{resource_id}", + "os": "linux", + "status": "offline", + "busy": runner_id != target_runner_id, + "labels": [], + } + for runner_id, resource_id in runners + ] + paths = [("GET", base)] + paths.extend(("GET", f"{base}/{runner_id}") for runner_id, _ in runners) + paths.append(("DELETE", f"{base}/{target_runner_id}")) + for method, path in paths: + self.http("PUT", f"{self.mock_url}/mockserver/clear", {"httpRequest": {"method": method, "path": path}}) + self.add_expectation("GET", base, 200, {"total_count": len(github_runners), "runners": github_runners}) + for runner in github_runners: + self.add_expectation("GET", f"{base}/{runner['id']}", 200, runner) + self.add_expectation("DELETE", f"{base}/{target_runner_id}", 204) + + def configure_runner_removed(self, runner_id: int) -> None: + path = f"/api/v3/orgs/test-owner/actions/runners/{runner_id}" + self.http("PUT", f"{self.mock_url}/mockserver/clear", {"httpRequest": {"method": "GET", "path": path}}) + self.add_expectation("GET", path, 404, {"message": "Not Found"}) + + def configure_empty_runner_list(self) -> None: + path = "/api/v3/orgs/test-owner/actions/runners" + self.http("PUT", f"{self.mock_url}/mockserver/clear", {"httpRequest": {"method": "GET", "path": path}}) + self.add_expectation("GET", path, 200, {"total_count": 0, "runners": []}) + + def assert_runner_removed(self, runner_id: int) -> None: + code, _ = self.http("GET", f"{self.mock_url}/api/v3/orgs/test-owner/actions/runners/{runner_id}") + if code != 404: + raise RuntimeError(f"Expected runner {runner_id} to be removed, got HTTP {code}") + + def scale_down_routes(self, runner_id: int, log_group: str | None = None) -> None: + try: + self.verify_route("POST", "/api/v3/app/installations/123/access_tokens", "Scale-down requested a GitHub App token") + self.verify_route("GET", "/api/v3/orgs/test-owner/actions/runners", "Scale-down listed organization runners") + self.verify_route("GET", f"/api/v3/orgs/test-owner/actions/runners/{runner_id}", "Scale-down checked runner state") + self.verify_route("DELETE", f"/api/v3/orgs/test-owner/actions/runners/{runner_id}", "Scale-down deleted the GitHub runner") + except RuntimeError as error: + if log_group is None: + raise + messages = self.recent_log_messages(log_group) + raise RuntimeError(f"{error}. Recent messages in {log_group}: {messages}") from error + + def scale_up_routes(self, job_id: int, provider: str) -> None: + self.verify_route("POST", "/api/v3/app/installations/123/access_tokens", f"{provider} scale-up requested a GitHub token for {job_id}") + self.verify_route("GET", f"/api/v3/repos/test-owner/test-repo/actions/jobs/{job_id}", f"{provider} scale-up checked queued job {job_id}") + + def pool_routes(self, provider: str) -> None: + self.verify_route("GET", "/api/v3/orgs/test-owner/installation", f"{provider} pool looked up the GitHub App installation") + self.verify_route("POST", "/api/v3/app/installations/123/access_tokens", f"{provider} pool requested a GitHub token") + self.verify_route("GET", "/api/v3/orgs/test-owner/actions/runners", f"{provider} pool listed organization runners") + + def prepare(self, *, scale_set_image: str) -> None: + self.progress("Preparing multi-runner-orchestration smoke deployment") + commands = ("aws", "openssl", "terraform") + for command in commands: + self.command(command) + key_path = Path(tempfile.mkstemp(prefix="ministack-smoke-key.")[1]) + try: + self.run(["openssl", "genpkey", "-algorithm", "RSA", "-pkeyopt", "rsa_keygen_bits:2048", "-out", str(key_path)], check=True) + key = base64.b64encode(key_path.read_bytes()).decode() + finally: + key_path.unlink(missing_ok=True) + source = self.script_dir / "multi-runner-orchestration.tfvars" + text = source.read_text() + text, replacements = re.subn( + r'(?m)^([ \t]*key_base64[ \t]*=[ \t]*)"[^"]*"', + lambda match: f'{match.group(1)}"{key}"', + text, + count=1, + ) + if replacements != 1: + raise RuntimeError("Could not find github_app.key_base64 in the MiniStack tfvars fixture") + if "MINISTACK_SCALE_SET_IMAGE" not in text: + raise RuntimeError("Scale-set image placeholder is missing from the MiniStack tfvars fixture") + text = text.replace("MINISTACK_SCALE_SET_IMAGE", scale_set_image) + lambda_archives = ( + ("runners_lambda_zip", "runners.zip", "lambdas/functions/control-plane/runners.zip"), + ("webhook_lambda_zip", "webhook.zip", "lambdas/functions/webhook/webhook.zip"), + ) + for variable, filename, function_path in lambda_archives: + candidates = ( + self.source_root / "lambda_output" / filename, + self.source_root / function_path, + ) + archive = next((candidate for candidate in candidates if candidate.is_file()), None) + if archive is None: + raise RuntimeError( + f"Missing {filename}; run .ci/build.sh before the MiniStack smoke test" + ) + if not zipfile.is_zipfile(archive): + raise RuntimeError( + f"Invalid or empty {filename} at {archive}; run .ci/build.sh before the MiniStack smoke test" + ) + with zipfile.ZipFile(archive) as archive_file: + if not archive_file.namelist(): + raise RuntimeError( + f"Empty {filename} at {archive}; run .ci/build.sh before the MiniStack smoke test" + ) + assignment = f'{variable} = "{archive}"' + text, replacements = re.subn( + rf"(?m)^{re.escape(variable)}\s*=.*$", + lambda _: assignment, + text, + count=1, + ) + if not replacements: + text += f"\n{assignment}\n" + self.tfvars_path = Path(tempfile.mkstemp(prefix="terraform-aws-github-runner-smoke.")[1]) + self.tfvars_path.write_text(text + "\n") + self.progress("Applying multi-runner-orchestration Terraform example") + self.run( + [str(self.source_root / "tests/ministack/run-example.sh"), "apply", "multi-runner-orchestration", str(self.tfvars_path)], + stream=True, + ) + self.webhook_endpoint = self.terraform("output", "-raw", "webhook_endpoint") + self.webhook_secret = self.terraform("output", "-raw", "webhook_secret", log_output=False) + self.wait_for_webhook_route() + self.progress("Deployment ready") + + def wait_for_webhook_route(self) -> None: + hostname = urlsplit(self.webhook_endpoint).hostname + if not hostname: + raise RuntimeError(f"Invalid webhook endpoint: {self.webhook_endpoint}") + api_id = hostname.split(".", 1)[0] + + def route_ready() -> bool: + routes = self.aws( + "apigatewayv2", + "get-routes", + "--api-id", + api_id, + check=False, + ) or {} + return any(route.get("RouteKey") == "POST /webhook" for route in routes.get("Items", [])) + + self.wait_for(route_ready, "API Gateway POST /webhook route", attempts=30) + + def cleanup(self) -> None: + if self.microvm_enabled: + self.progress("MicroVM lifecycle hook container logs (last 200 lines):") + self.run( + ["docker", "logs", "--timestamps", "--tail", "200", "microvm-lifecycle-hook"], + check=False, + stream=True, + ) + self.run(["docker", "rm", "--force", "microvm-lifecycle-hook"], check=False) + for instance_id in self.discovered_instance_ids: + self.aws("ec2", "terminate-instances", "--instance-ids", instance_id, check=False) + for microvm_id in self.discovered_microvm_ids: + self.aws( + "lambda-microvms", + "terminate-microvm", + "--microvm-identifier", + microvm_id, + check=False, + ) + if self.tfvars_path and self.keep_deployment: + self.progress(f"Terraform deployment retained; tfvars file: {self.tfvars_path}") + elif self.tfvars_path: + self.progress("Destroying multi-runner-orchestration Terraform deployment") + self.run( + [ + str(self.source_root / "tests/ministack/run-example.sh"), + "destroy", + "multi-runner-orchestration", + str(self.tfvars_path), + ], + stream=True, + ) + self.tfvars_path.unlink(missing_ok=True) + self.response_path.unlink(missing_ok=True) + + +def shutil_which(name: str) -> str | None: + for directory in os.environ.get("PATH", "").split(os.pathsep): + candidate = Path(directory) / name + if candidate.is_file() and os.access(candidate, os.X_OK): + return str(candidate) + return None diff --git a/tests/ministack/github-api-expectations.json b/tests/ministack/smoke/fixtures/github-api-expectations.json similarity index 100% rename from tests/ministack/github-api-expectations.json rename to tests/ministack/smoke/fixtures/github-api-expectations.json diff --git a/tests/ministack/initializerJson.json b/tests/ministack/smoke/fixtures/scale-set-initializer.json similarity index 75% rename from tests/ministack/initializerJson.json rename to tests/ministack/smoke/fixtures/scale-set-initializer.json index 443ddb6201..652b4db334 100644 --- a/tests/ministack/initializerJson.json +++ b/tests/ministack/smoke/fixtures/scale-set-initializer.json @@ -2,7 +2,7 @@ { "httpRequest": { "method": "POST", - "path": "/api/v3/app/installations/456/access_tokens" + "path": "/api/v3/app/installations/456/access_tokens" }, "httpResponse": { "statusCode": 201, @@ -51,7 +51,7 @@ "httpResponse": { "statusCode": 200, "headers": { "Content-Type": ["application/json"] }, - "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}]}" + "body": "{\"count\":1,\"value\":[{\"id\":223,\"name\":\"linux-scale-set\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"linux-scale-set\",\"type\":\"system\"}],\"runnerSetting\":{}}]}" } }, { @@ -62,7 +62,7 @@ "httpResponse": { "statusCode": 200, "headers": { "Content-Type": ["application/json"] }, - "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"}],\"runnerSetting\":{}}" + "body": "{\"id\":223,\"name\":\"linux-scale-set\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"linux-scale-set\",\"type\":\"system\"}],\"runnerSetting\":{}}" } }, { @@ -73,7 +73,7 @@ "httpResponse": { "statusCode": 200, "headers": { "Content-Type": ["application/json"] }, - "body": "{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"medium\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}" + "body": "{\"id\":223,\"name\":\"linux-scale-set\",\"runnerGroupId\":48,\"labels\":[{\"name\":\"linux-scale-set\",\"type\":\"system\"},{\"name\":\"linux\",\"type\":\"user\"},{\"name\":\"scale-set\",\"type\":\"user\"},{\"name\":\"self-hosted\",\"type\":\"user\"},{\"name\":\"x64\",\"type\":\"user\"}],\"runnerSetting\":{}}" } }, { @@ -94,7 +94,7 @@ "httpResponse": { "statusCode": 200, "headers": { "Content-Type": ["application/json"] }, - "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.medium\",\"runnerScaleSet\":{\"id\":223,\"name\":\"medium\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}" + "body": "{\"sessionId\":\"11111111-1111-1111-1111-111111111111\",\"ownerName\":\"local.linux-scale-set\",\"runnerScaleSet\":{\"id\":223,\"name\":\"linux-scale-set\",\"runnerGroupId\":48},\"messageQueueUrl\":\"https://mockserver:1080/messages?sessionId=11111111-1111-1111-1111-111111111111&api-version=6.0-preview\",\"messageQueueAccessToken\":\"fake-queue-token\",\"statistics\":{\"totalAvailableJobs\":0,\"totalAcquiredJobs\":0,\"totalAssignedJobs\":0,\"totalRunningJobs\":0,\"totalRegisteredRunners\":0,\"totalBusyRunners\":0,\"totalIdleRunners\":0}}" } }, { diff --git a/tests/ministack/workflow_job_event.json b/tests/ministack/smoke/fixtures/workflow_job_event.json similarity index 100% rename from tests/ministack/workflow_job_event.json rename to tests/ministack/smoke/fixtures/workflow_job_event.json diff --git a/tests/ministack/smoke/scale_set_ec2.py b/tests/ministack/smoke/scale_set_ec2.py new file mode 100644 index 0000000000..9b3016a6c6 --- /dev/null +++ b/tests/ministack/smoke/scale_set_ec2.py @@ -0,0 +1,80 @@ +"""EC2 adapter for scale-set runner lifecycle assertions.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from .scale_set_provider import ScaleSetRunner + +if TYPE_CHECKING: + from .scale_set_scenario import ScaleSetScenario + + +class Ec2ScaleSetProvider: + slug = "ec2" + display_name = "EC2" + group_name = "ec2_scalet_set" + runner_name = "ec2_scalet_set" + + def _instances(self, smoke: ScaleSetScenario, *, runner_only: bool) -> list[dict[str, Any]]: + response = smoke.aws("ec2", "describe-instances") or {} + expected = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": f"{smoke.environment_name}-{smoke.group_name}", + "ghr:Type": "Org", + "ghr:Owner": "example", + } + if runner_only: + expected.update({"ghr:scale_set_state": "config-published", "ghr:github_runner_id": smoke.runner_id}) + active_states = {"pending", "running", "stopping", "stopped", "shutting-down"} + matches = [] + for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + if instance.get("State", {}).get("Name") not in active_states: + continue + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + if all(tags.get(key) == value for key, value in expected.items()): + runner_prefix = "ec2_scalet_set-" + if not runner_only or tags.get("ghr:runner_name", "").startswith(runner_prefix): + matches.append(instance) + return matches + + def wait_for_runner(self, smoke: ScaleSetScenario) -> ScaleSetRunner: + def find_runner() -> ScaleSetRunner | None: + matches = self._instances(smoke, runner_only=True) + if len(matches) == 1 and matches[0].get("InstanceId"): + return ScaleSetRunner(matches[0]["InstanceId"]) + return None + + runner = smoke.wait_for(find_runner, "one config-published scale-set EC2 runner") + smoke.progress(f"MiniStack created and registered scale-set EC2 runner {runner.identifier}") + return runner + + def verify_runner(self, smoke: ScaleSetScenario, runner: ScaleSetRunner) -> None: + response = smoke.aws("ec2", "describe-instances", "--instance-ids", runner.identifier) or {} + try: + instance = response["Reservations"][0]["Instances"][0] + except (IndexError, KeyError, TypeError) as error: + raise RuntimeError(f"Could not inspect scale-set EC2 runner {runner.identifier}") from error + tags = {tag.get("Key"): tag.get("Value") for tag in instance.get("Tags", [])} + expected = { + "ghr:Application": "github-action-runner", + "ghr:created_by": "scale-set-service", + "ghr:environment": f"{smoke.environment_name}-{smoke.group_name}", + "ghr:Type": "Org", + "ghr:Owner": "example", + "ghr:scale_set_state": "config-published", + "ghr:github_runner_id": smoke.runner_id, + } + for key, value in expected.items(): + if tags.get(key) != value: + raise RuntimeError(f"Unexpected EC2 runner tag {key}: expected {value}, got {tags.get(key)}") + + def wait_for_scale_down(self, smoke: ScaleSetScenario) -> None: + smoke.wait_for(lambda: not self._instances(smoke, runner_only=False), + "all scale-set EC2 runners to terminate") + smoke.progress("Scale-set EC2 runner was terminated after the minimum changed to zero") + + +provider = Ec2ScaleSetProvider() diff --git a/tests/ministack/smoke/scale_set_provider.py b/tests/ministack/smoke/scale_set_provider.py new file mode 100644 index 0000000000..023bec8e8d --- /dev/null +++ b/tests/ministack/smoke/scale_set_provider.py @@ -0,0 +1,34 @@ +"""Provider contract for scale-set lifecycle assertions.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol + +if TYPE_CHECKING: + from .scale_set_scenario import ScaleSetScenario + + +@dataclass(frozen=True) +class ScaleSetRunner: + """A runner resource discovered by a compute-provider adapter.""" + + identifier: str + + +class ScaleSetProvider(Protocol): + """Provider-specific configuration and runner lifecycle assertions.""" + + slug: str + display_name: str + group_name: str + runner_name: str + + def wait_for_runner(self, smoke: ScaleSetScenario) -> ScaleSetRunner: + """Wait until the controller has created one provider runner.""" + + def verify_runner(self, smoke: ScaleSetScenario, runner: ScaleSetRunner) -> None: + """Check provider-specific runner metadata after scale-up.""" + + def wait_for_scale_down(self, smoke: ScaleSetScenario) -> None: + """Wait until no active runners owned by this scale set remain.""" diff --git a/tests/ministack/smoke/scale_set_scenario.py b/tests/ministack/smoke/scale_set_scenario.py new file mode 100644 index 0000000000..6f85954d20 --- /dev/null +++ b/tests/ministack/smoke/scale_set_scenario.py @@ -0,0 +1,498 @@ +"""Scale-set controller scenario for the combined MiniStack smoke.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import uuid + +from .common import SmokeContext +from .scale_set_provider import ScaleSetProvider + + +class ScaleSetScenario: + """Run scale-set assertions through the shared MiniStack smoke context.""" + + repository_name = "scale-set-controller" + runner_id = "321" + routes = ( + ("POST", "/api/v3/app/installations/456/access_tokens"), + ("POST", "/api/v3/orgs/example/actions/runners/registration-token"), + ("POST", "/api/v3/actions/runner-registration"), + ("GET", "/tenant/123/_apis/runtime/runnergroups/"), + ("GET", "/tenant/123/_apis/runtime/runnerscalesets"), + ("GET", "/tenant/123/_apis/runtime/runnerscalesets/223"), + ("PATCH", "/tenant/123/_apis/runtime/runnerscalesets/223"), + ("POST", "/tenant/123/_apis/runtime/runnerscalesets/223/generatejitconfig"), + ("POST", "/tenant/123/_apis/runtime/runnerscalesets/223/sessions"), + ("GET", "/messages"), + ) + + def __init__( + self, + context: SmokeContext, + provider: ScaleSetProvider, + *, + image_reference: str | None = None, + ) -> None: + self.context = context + self.provider = provider + self.source_root = context.source_root + self.group_name = provider.group_name + self.runner_name = provider.runner_name + self.environment = context.environment + self.region = context.region + self.aws_endpoint = context.aws_endpoint + self.mock_host = context.mock_host + self.mock_port = context.mock_port + self.mock_url = context.mock_url.rstrip("/") + self.controller_mock_url = f"https://{self.mock_host}:{self.mock_port}" + self.temp_dir = Path(tempfile.mkdtemp(prefix="ministack-scale-set-smoke.")) + self.image_tag = f"smoke-{uuid.uuid4().hex}" + self.image_reference = image_reference or f"localhost:4566/{self.repository_name}:{self.image_tag}" + self.task_definition_path = self.temp_dir / "task-definition.json" + self.log_path = context.log_path + self.environment_name = "multi-runner-webhook" + self.config_path = f"/{self.environment_name}/scale-set-controller/{self.group_name}/{self.runner_name}" + self.cluster_name = f"{self.environment_name}-scale-set" + safe_group = re.sub(r"[^a-z0-9_-]", "-", self.group_name.lower())[:14] + suffix = hashlib.sha256(self.group_name.encode()).hexdigest()[:8] + self.service_name = f"{self.environment_name}-ss-{safe_group}-{suffix}" + + def cleanup(self) -> None: + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def progress(self, message: str) -> None: + self.context.progress(message) + + def _log(self, value: str) -> None: + self.context._append_log(value) + + def step(self, name: str): + return self.context.step(name) + + def run( + self, + command: list[str], + *, + check: bool = True, + stream: bool = False, + cwd: Path | None = None, + input_text: str | None = None, + log_output: bool = True, + log_command: bool = True, + ) -> subprocess.CompletedProcess[str]: + return self.context.run( + command, check=check, stream=stream, cwd=cwd, + input_text=input_text, log_output=log_output, log_command=log_command, + ) + + def require_commands(self) -> None: + for command in ("aws", "docker", "openssl", "python3", "terraform"): + self.context.command(command) + + def aws(self, *args: str, check: bool = True): + return self.context.aws(*args, check=check) + + def terraform(self, *args: str, check: bool = True) -> str: + return self.context.terraform(*args, check=check) + + def http(self, method: str, url: str, body=None) -> tuple[int, str]: + return self.context.http(method, url, body) + + def wait_for(self, predicate, description: str, *, attempts: int = 90, interval: int = 2): + return self.context.wait_for(predicate, description, attempts=attempts, interval=interval) + + def wait_http(self, url: str, method: str = "GET") -> None: + self.wait_for(lambda: self.http(method, url)[0] in range(200, 300), url, attempts=90, interval=1) + + def prepare_shared_image(self) -> None: + """Publish the controller image before the shared webhook example is applied.""" + self.require_commands() + self.wait_http(f"{self.aws_endpoint.rstrip('/')}/_ministack/health") + policy = { + "Version": "2012-10-17", + "Statement": [{ + "Sid": "AllowAccountPull", "Effect": "Allow", + "Principal": {"AWS": "arn:aws:iam::000000000000:root"}, + "Action": ["ecr:BatchCheckLayerAvailability", "ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer"], + }], + } + create_repository = self.run([ + "aws", "--endpoint-url", self.aws_endpoint, "--region", self.region, + "ecr", "create-repository", "--repository-name", self.repository_name, + "--image-tag-mutability", "IMMUTABLE", "--image-scanning-configuration", "scanOnPush=false", + "--output", "json", + ], check=False) + if create_repository.returncode and "RepositoryAlreadyExistsException" not in create_repository.stderr: + raise subprocess.CalledProcessError( + create_repository.returncode, create_repository.args, + create_repository.stdout, create_repository.stderr, + ) + if create_repository.returncode: + self.progress(f"Reusing existing ECR repository {self.repository_name}") + self.aws("ecr", "set-repository-policy", "--repository-name", self.repository_name, + "--policy-text", json.dumps(policy, separators=(",", ":"))) + self.run([ + "docker", "build", "--target", "runtime", "--file", + str(self.source_root / "lambdas/services/scale-set/Dockerfile"), + "--tag", self.image_reference, str(self.source_root), + ], stream=True) + password = self.run([ + "aws", "--endpoint-url", self.aws_endpoint, "--region", self.region, + "ecr", "get-login-password", + ], log_output=False).stdout + self.run(["docker", "login", "--username", "AWS", "--password-stdin", "localhost:4566"], input_text=password) + self.run(["docker", "push", self.image_reference], stream=True) + self.aws("ecr", "describe-images", "--repository-name", self.repository_name, + "--image-ids", f"imageTag={self.image_tag}") + + def add_shared_mockserver_expectations(self) -> None: + """Replace scale-set API fixtures while preserving shared webhook routes.""" + self.wait_http(f"{self.mock_url}/mockserver/status", "PUT") + fixture_path = Path(__file__).resolve().parent / "fixtures" / "scale-set-initializer.json" + expectations = json.loads(fixture_path.read_text(encoding="utf-8")) + for expectation in expectations: + request = expectation.get("httpRequest", {}) + method = request.get("method") + path = request.get("path") + if isinstance(path, str): + request["path"] = path.replace("/installations/456/", "/installations/123/") + if path == "/messages" or path.startswith("/tenant/123/"): + code, body = self.http( + "PUT", + f"{self.mock_url}/mockserver/clear", + {"httpRequest": {"method": method, "path": path}}, + ) + if code >= 300: + raise RuntimeError(f"Could not replace scale-set MockServer expectation {method} {path}: HTTP {code}: {body}") + self._replace_fixture_urls(expectations) + code, body = self.http("PUT", f"{self.mock_url}/mockserver/expectation", expectations) + if code not in (200, 201, 202): + raise RuntimeError(f"Could not initialize scale-set MockServer expectations: HTTP {code}: {body}") + + def run_shared(self) -> None: + """Run the scale-set lifecycle against the already-applied shared example.""" + with self.step("Scale-up"): + definition = self.verify_ecs() + container = self.scale_up(definition) + with self.step("Scale-down"): + self.scale_down(definition, container) + with self.step("Cleanup"): + self.wait_for( + lambda: self.http("PUT", f"{self.mock_url}/mockserver/verify", { + "httpRequest": { + "method": "DELETE", + "path": "/tenant/123/_apis/runtime/runnerscalesets/223/sessions/11111111-1111-1111-1111-111111111111", + }, + "times": {"atLeast": 1}, + })[0] in range(200, 300), + "MockServer controller session DELETE", attempts=45, + ) + self.context.progress("Scale-set lifecycle completed") + + def _replace_fixture_urls(self, expectations: list[dict]) -> None: + for expectation in expectations: + response = expectation.get("httpResponse", {}) + if isinstance(response.get("body"), str): + response["body"] = response["body"].replace("https://mockserver:1080", self.controller_mock_url) + template = expectation.get("httpResponseTemplate", {}) + if isinstance(template.get("template"), str): + template["template"] = template["template"].replace("https://mockserver:1080", self.controller_mock_url) + + def verify_configuration(self) -> None: + response = self.aws("ssm", "get-parameter", "--name", self.config_path) + try: + config = json.loads(response["Parameter"]["Value"]) + except (KeyError, TypeError, json.JSONDecodeError) as error: + raise RuntimeError("Scale-set SSM reconciler parameter is missing or invalid") from error + expected = { + "githubConfigUrl": f"{self.controller_mock_url}/example", + "forceGhes": True, + "sslVerify": False, + "minRunners": 1, + } + for key, value in expected.items(): + if config.get(key) != value: + raise RuntimeError(f"SSM reconciler setting {key}: expected {value!r}, got {config.get(key)!r}") + app = config.get("githubApp", {}) + for key in ("appIdParameterName", "installationIdParameterName", "privateKeyParameterName"): + if not app.get(key): + raise RuntimeError(f"SSM GitHub App configuration is missing {key}") + self.progress("SSM manifest has the expected MockServer and GitHub App settings") + + def task_definition(self) -> dict: + definitions = self.aws("ecs", "list-task-definitions", "--family-prefix", self.service_name, "--sort", "DESC") + arns = definitions.get("taskDefinitionArns", []) if isinstance(definitions, dict) else [] + if not arns: + # Preserve the raw discovery evidence when MiniStack does not honor the prefix filter. + definitions = self.aws("ecs", "list-task-definitions", "--sort", "DESC") + arns = definitions.get("taskDefinitionArns", []) if isinstance(definitions, dict) else [] + for arn in arns: + result = self.aws("ecs", "describe-task-definition", "--task-definition", arn) + definition = result.get("taskDefinition", {}) if isinstance(result, dict) else {} + if definition.get("family") == self.service_name: + return definition + raise RuntimeError(f"No ECS task definition found for scale-set service {self.service_name}") + + def verify_ecs(self) -> dict: + definition = self.task_definition() + containers = {item.get("name"): item for item in definition.get("containerDefinitions", [])} + controller = containers.get("scale-set-controller") + if controller is None: + raise RuntimeError("ECS task definition has no scale-set-controller container") + if controller.get("image") != self.image_reference: + raise RuntimeError(f"Expected ECS image {self.image_reference}, got {controller.get('image')}") + if controller.get("logConfiguration", {}).get("logDriver") != "awslogs": + raise RuntimeError("ECS scale-set controller task does not request the awslogs driver") + logs = self.aws("logs", "describe-log-groups", "--log-group-name-prefix", f"/aws/ecs/{self.service_name}") + if not logs.get("logGroups"): + raise RuntimeError("No CloudWatch log group was created for the ECS controller") + self.progress("ECS task definition uses the published image and awslogs driver") + return definition + + def _temporary_task_definition(self, definition: dict, *, minimum: int) -> str: + definition = json.loads(json.dumps(definition)) + for key in ("taskDefinitionArn", "revision", "status", "requiresAttributes", "compatibilities", "registeredAt", "registeredBy"): + definition.pop(key, None) + controller = next((item for item in definition.get("containerDefinitions", []) if item.get("name") == "scale-set-controller"), None) + if controller is None: + raise RuntimeError("Scale-set controller container is missing") + environment = controller.setdefault("environment", []) + values = {item.get("name"): item for item in environment} + values["AWS_ACCESS_KEY_ID"] = {"name": "AWS_ACCESS_KEY_ID", "value": "000000000000"} + values["AWS_SECRET_ACCESS_KEY"] = {"name": "AWS_SECRET_ACCESS_KEY", "value": "test-only"} + controller["environment"] = list(values.values()) + for item in controller["environment"]: + if item.get("name") == "SCALE_SET_CONTROLLER_MANIFEST": + manifest = json.loads(item["value"]) + reconcilers = manifest.get("reconcilers", []) + if len(reconcilers) != 1: + raise RuntimeError("Expected exactly one reconciler in the controller manifest") + reconcilers[0]["minRunners"] = minimum + item["value"] = json.dumps(manifest, separators=(",", ":")) + break + else: + raise RuntimeError("Scale-set controller task definition has no inline manifest") + self.task_definition_path.write_text(json.dumps(definition), encoding="utf-8") + registered = self.aws("ecs", "register-task-definition", "--cli-input-json", f"file://{self.task_definition_path}") + arn = registered.get("taskDefinition", {}).get("taskDefinitionArn") + if not arn: + raise RuntimeError("MiniStack did not return an ARN for the temporary controller task revision") + return arn + + def _deploy_task_revision(self, arn: str, *, old_container: str | None = None) -> tuple[str, str]: + self.aws("ecs", "update-service", "--cluster", self.cluster_name, "--service", self.service_name, + "--task-definition", arn, "--force-new-deployment") + described = self.aws("ecs", "describe-task-definition", "--task-definition", arn)["taskDefinition"] + revision = str(described["revision"]) + + def container_ids(*filters: str) -> list[str]: + result = self.run( + ["docker", "ps", "-a", *filters, "--format", "{{.ID}}"], + check=False, + log_output=False, + log_command=False, + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + def find_container(): + family_filter = f"label=com.amazonaws.ecs.task-definition-family={self.service_name}" + name_filter = "name=scale-set-controller" + exact = container_ids( + "--filter", family_filter, + "--filter", f"label=com.amazonaws.ecs.task-definition-version={revision}", + "--filter", name_filter, + ) + exact_fresh = next((item for item in exact if item != old_container), None) + if exact_fresh: + return exact_fresh + + # Some MiniStack ECS versions omit or lag the revision label. Keep + # the family/name constraints, then verify any reported revision. + family_matches = container_ids( + "--filter", family_filter, + "--filter", name_filter, + ) + for candidate in family_matches: + if candidate == old_container: + continue + inspect = self.run([ + "docker", "inspect", "--format", + '{{index .Config.Labels "com.amazonaws.ecs.task-definition-version"}}', + candidate, + ], check=False, log_output=False, log_command=False) + if inspect.returncode: + continue + candidate_revision = inspect.stdout.strip() + if not candidate_revision or candidate_revision == revision: + return candidate + return None + + try: + container = self.wait_for(find_container, "a fresh MiniStack ECS controller container") + except RuntimeError as error: + self._log_controller_startup_diagnostics(revision) + raise RuntimeError( + f"Timed out waiting for controller container for task revision {revision}; " + f"ECS and Docker diagnostics were written to {self.log_path}" + ) from error + return container, revision + + def _log_controller_startup_diagnostics(self, revision: str) -> None: + service = self.aws( + "ecs", "describe-services", "--cluster", self.cluster_name, + "--services", self.service_name, check=False, + ) + tasks = self.aws( + "ecs", "list-tasks", "--cluster", self.cluster_name, + "--service-name", self.service_name, check=False, + ) + task_arns = tasks.get("taskArns", []) if isinstance(tasks, dict) else [] + task_details = self.aws( + "ecs", "describe-tasks", "--cluster", self.cluster_name, + "--tasks", *task_arns, check=False, + ) if task_arns else {} + task_summaries = [ + { + "taskArn": task.get("taskArn"), + "taskDefinitionArn": task.get("taskDefinitionArn"), + "desiredStatus": task.get("desiredStatus"), + "lastStatus": task.get("lastStatus"), + "stoppedReason": task.get("stoppedReason"), + "containers": [ + { + key: container.get(key) + for key in ("name", "image", "runtimeId", "lastStatus", "exitCode", "reason") + } + for container in task.get("containers", []) + ], + } + for task in task_details.get("tasks", []) + ] if isinstance(task_details, dict) else task_details + containers = self.run([ + "docker", "ps", "-a", "--filter", "name=scale-set-controller", + "--format", "{{.ID}} {{.Image}} {{.Status}} {{.Names}}", + ], check=False) + self._log(f"\nController task revision expected: {revision}\n") + self._log("ECS service state:\n" + json.dumps(service, indent=2, default=str) + "\n") + self._log("ECS service tasks:\n" + json.dumps(tasks, indent=2, default=str) + "\n") + self._log("ECS task details:\n" + json.dumps(task_summaries, indent=2, default=str) + "\n") + self._log("Scale-set Docker containers:\n" + (containers.stdout or "\n")) + if task_arns and not containers.stdout.strip(): + self._log( + "No controller Docker container was visible. Verify MiniStack has the Docker engine socket " + "mounted at /var/run/docker.sock.\n" + ) + self.progress(f"Controller startup diagnostics written to {self.log_path}") + + def wait_controller_event(self, container: str, marker: str, required: str = "") -> list[str]: + last_lines: list[str] = [] + + def contains_event() -> bool: + nonlocal last_lines + result = self.run( + ["docker", "logs", container], check=False, + log_output=False, log_command=False, + ) + last_lines = (result.stdout + (result.stderr or "")).splitlines() + return any(marker in line and (not required or required in line) for line in last_lines) + + try: + self.wait_for(contains_event, f"controller log marker {marker}") + except RuntimeError as error: + code, request_body = self.http("PUT", f"{self.mock_url}/mockserver/retrieve?type=REQUESTS", {}) + if code in range(200, 300): + try: + requests = json.loads(request_body) + except json.JSONDecodeError: + requests = [] + if isinstance(requests, list): + self.progress("MockServer request paths at timeout:") + for request in requests[-40:]: + if isinstance(request, dict): + method = request.get("method", "?") + path = request.get("path", "?") + self.progress(f"{method} {path}") + logs = self.run( + ["docker", "logs", container], check=False, + log_output=False, log_command=False, + ) + self.progress("Controller logs at timeout:") + for line in (logs.stdout + (logs.stderr or "")).splitlines(): + self.progress(line) + inspect = self.run(["docker", "inspect", "--format", "{{json .NetworkSettings.Networks}}", container], check=False) + self.progress(f"Controller network attachments: {inspect.stdout.strip()}") + raise error + self._log(f"\n$ docker logs {container}\n" + "\n".join(last_lines) + "\n") + self.progress(f"Controller logs contain {marker}") + return last_lines + + def verify_routes(self) -> None: + for method, path in self.routes: + path = path.replace("/installations/456/", "/installations/123/") + body = {"httpRequest": {"method": method, "path": path}, "times": {"atLeast": 1}} + self.wait_for( + lambda: self.http("PUT", f"{self.mock_url}/mockserver/verify", body)[0] in range(200, 300), + f"MockServer request {method} {path}", attempts=45, + ) + self.progress(f"MockServer received {method} {path}") + + def scale_up(self, task_definition: dict) -> str: + self.verify_configuration() + self.verify_ecs() + arn = self._temporary_task_definition(task_definition, minimum=1) + container, _revision = self._deploy_task_revision(arn) + self.progress(f"MiniStack started ECS controller container {container}") + self.progress("MiniStack does not emit ECS awslogs streams; checking controller runtime logs instead") + self.wait_controller_event(container, "scale_set_controller_started") + self.wait_controller_event(container, "scale_set_session_created") + self.wait_controller_event(container, "scale_set_reconciled", '"desiredRunners":1') + self.wait_controller_event(container, "scale_set_reconciled", '"status":"converged"') + runner = self.provider.wait_for_runner(self) + self.provider.verify_runner(self, runner) + self.verify_routes() + return container + + def scale_down(self, task_definition: dict, old_container: str) -> None: + response = self.aws("ssm", "get-parameter", "--name", self.config_path) + reconciler = json.loads(response["Parameter"]["Value"]) + if reconciler.get("minRunners") != 1: + raise RuntimeError("Expected the SSM minimum to be one before scale-down") + reconciler["minRunners"] = 0 + self.aws("ssm", "put-parameter", "--name", self.config_path, "--type", "String", + "--value", json.dumps(reconciler, separators=(",", ":")), "--overwrite") + self.progress("SSM manifest minimum changed from one runner to zero") + arn = self._temporary_task_definition(task_definition, minimum=0) + container, _revision = self._deploy_task_revision(arn, old_container=old_container) + self.progress(f"ECS service deployed a fresh controller container {container} for scale-down") + self.wait_controller_event(container, "scale_set_controller_started") + self.wait_controller_event(container, "scale_set_session_created") + self.wait_controller_event(container, "scale_set_reconciled", '"desiredRunners":0') + self.wait_controller_event(container, "scale_set_reconciled", '"status":"converged"') + self.provider.wait_for_scale_down(self) + + +def prepare(context: SmokeContext, provider: ScaleSetProvider) -> str: + """Prepare shared scale-set fixtures and return the controller image reference.""" + scenario = ScaleSetScenario(context, provider) + try: + scenario.prepare_shared_image() + scenario.add_shared_mockserver_expectations() + return scenario.image_reference + finally: + scenario.cleanup() + + +def run(context: SmokeContext, provider: ScaleSetProvider, image_reference: str) -> None: + """Run the scale-set lifecycle with the selected compute provider.""" + scenario = ScaleSetScenario(context, provider, image_reference=image_reference) + try: + scenario.run_shared() + finally: + scenario.cleanup() diff --git a/tests/ministack/smoke/webhook_ec2.py b/tests/ministack/smoke/webhook_ec2.py new file mode 100644 index 0000000000..6561959f2e --- /dev/null +++ b/tests/ministack/smoke/webhook_ec2.py @@ -0,0 +1,160 @@ +"""EC2 implementation of the provider smoke-test interface.""" + +import base64 +import json +from typing import Any + +from .common import SmokeContext +from .webhook_provider import RunnerResource + + +class Ec2Provider: + slug = "ec2" + display_name = "EC2" + + def configure(self, context: SmokeContext) -> None: + self._configure_jit_expectations(context) + + def _configure_jit_expectations(self, context: SmokeContext) -> None: + context.add_expectation( + "GET", + "/api/v3/orgs/test-owner/actions/runner-groups", + 200, + [{"id": 1, "name": "Default"}], + ) + context.add_expectation( + "POST", + "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", + 200, + { + "runner": {"id": 987654321, "labels": [{"name": "self-hosted"}, {"name": "linux"}]}, + # EC2 does not launch the runner in this smoke; keep the fixture + # Base64-shaped so it cannot mask a JIT handoff failure. + "encoded_jit_config": base64.b64encode(b"{}").decode(), + }, + ) + + def event(self, context: SmokeContext, job_id: int, dynamic: bool) -> dict[str, Any]: + value = json.loads((context.fixture_dir / "workflow_job_event.json").read_text()) + job = value["workflow_job"] + job["id"] = job_id + job["name"] = f"multi-runner-webhook-ec2-{job_id}" + job["labels"] = ["self-hosted", "linux", "x64", "ec2"] + if dynamic: + job["labels"].append("ghr-ec2-instance-type:m5.large") + return value + + def verify_scale_up_routes(self, context: SmokeContext, job_id: int) -> None: + context.verify_route( + "GET", + "/api/v3/orgs/test-owner/actions/runner-groups", + f"EC2 scale-up resolved the runner group for {job_id}", + ) + context.verify_route( + "POST", + "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", + f"EC2 scale-up generated JIT configuration for {job_id}", + ) + + def verify_pool_routes(self, context: SmokeContext) -> None: + context.verify_route( + "GET", + "/api/v3/orgs/test-owner/actions/runner-groups", + "EC2 pool resolved the runner group", + ) + context.verify_route( + "POST", + "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", + "EC2 pool generated JIT configuration", + ) + + def _wait_for_instance(self, context: SmokeContext, source: str, description: str) -> RunnerResource: + def find() -> str | None: + result = context.aws( + "ec2", "describe-instances", "--filters", + "Name=instance-state-name,Values=running,pending", + "Name=tag:ghr:Application,Values=github-action-runner", + f"Name=tag:ghr:created_by,Values={source}", check=False, + ) or {} + for reservation in result.get("Reservations", []): + for instance in reservation.get("Instances", []): + instance_id = instance.get("InstanceId") + if instance_id and instance_id not in context.discovered_instance_ids: + context.discovered_instance_ids.append(instance_id) + return instance_id + return None + + return RunnerResource(context.wait_for(find, description)) + + def _instance(self, context: SmokeContext, resource: RunnerResource) -> dict[str, Any]: + result = context.aws("ec2", "describe-instances", "--instance-ids", resource.identifier) + return result["Reservations"][0]["Instances"][0] + + def _assert_tags(self, context: SmokeContext, resource: RunnerResource, source: str) -> None: + tags = {tag["Key"]: tag["Value"] for tag in self._instance(context, resource).get("Tags", [])} + expected = { + "ghr:Application": "github-action-runner", + "ghr:created_by": source, + "ghr:Type": "Org", + "ghr:Owner": "test-owner", + } + for key, value in expected.items(): + if tags.get(key) != value: + raise RuntimeError(f"Unexpected EC2 runner tag {key}: expected {value}, got {tags.get(key)}") + + def wait_for_scale_up(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_instance(context, source, "an EC2 scale-up instance") + + def assert_scale_up(self, context: SmokeContext, resource: RunnerResource, dynamic: bool) -> None: + expected_type = "m5.large" if dynamic else "m7a.large" + actual_type = self._instance(context, resource).get("InstanceType") + if actual_type != expected_type: + raise RuntimeError(f"EC2 scale-up used {actual_type}, expected {expected_type}") + self._assert_tags(context, resource, "scale-up-lambda") + + def start_scale_up_runner(self, context: SmokeContext, resource: RunnerResource) -> bool: + return False + + def wait_for_pool(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_instance(context, source, "an EC2 pool instance") + + def assert_pool(self, context: SmokeContext, resource: RunnerResource) -> None: + self._assert_tags(context, resource, "pool-lambda") + + def _wait_for_termination(self, context: SmokeContext, resource: RunnerResource) -> None: + def terminated() -> bool: + result = context.aws("ec2", "describe-instances", "--instance-ids", resource.identifier, check=False) + if not result: + return True + state = result.get("Reservations", [{}])[0].get("Instances", [{}])[0].get("State", {}).get("Name") + return state in (None, "terminated") + + context.wait_for(terminated, f"EC2 instance {resource.identifier} termination") + + def scale_down( + self, + context: SmokeContext, + resource: RunnerResource, + runner_id: int, + marker: str, + active_runners: list[tuple[int, RunnerResource]], + ) -> None: + context.configure_runner_fixtures( + self.slug, + [(active_runner_id, active_resource.identifier) for active_runner_id, active_resource in active_runners], + runner_id, + ) + context.clear_requests() + context.invoke( + "multi-runner-webhook-ec2-scale-down", + {"smokeMarker": marker, "type": "ec2"}, + "EC2 scale-down Lambda invoked", + ) + context.wait_for_log("/aws/lambda/multi-runner-webhook-ec2-scale-down", marker, "EC2 scale-down Lambda started") + context.scale_down_routes(runner_id, "/aws/lambda/multi-runner-webhook-ec2-scale-down") + context.configure_runner_removed(runner_id) + context.assert_runner_removed(runner_id) + self._wait_for_termination(context, resource) + + +provider = Ec2Provider() diff --git a/tests/ministack/smoke/webhook_microvm.py b/tests/ministack/smoke/webhook_microvm.py new file mode 100644 index 0000000000..4610b32a97 --- /dev/null +++ b/tests/ministack/smoke/webhook_microvm.py @@ -0,0 +1,486 @@ +"""MicroVM implementation of the provider smoke-test interface.""" + +import base64 +import json +import shutil +from typing import Any + +from .common import SmokeContext +from .webhook_provider import RunnerResource + +MICROVM_HOOK_CONTAINER = "microvm-lifecycle-hook" +MICROVM_HOOK_PORT = 8080 +MICROVM_HOOK_URL = f"http://127.0.0.1:{MICROVM_HOOK_PORT}" + + +def _base64_json(value: dict[str, Any]) -> str: + return base64.b64encode(json.dumps(value, separators=(",", ":")).encode()).decode() + + +def _smoke_jit_config(context: SmokeContext) -> str: + """Return a synthetic but runner-compatible JIT configuration for MockServer.""" + # The smoke validates JIT handoff, not the runner service protocol. Keep the + # launched runner away from MockServer, whose REST expectations are for the + # control plane and GitHub API only. + runner_server_url = "http://127.0.0.1:65535" + files = { + ".runner": _base64_json( + { + "AgentId": 987654321, + "AgentName": "ministack-microvm", + "DisableUpdate": True, + "Ephemeral": True, + "PoolId": 1, + "PoolName": "Default", + "ServerUrl": runner_server_url, + "WorkFolder": "_work", + } + ), + ".credentials": _base64_json( + { + "scheme": "OAuth", + "data": { + "clientId": "00000000-0000-0000-0000-000000000000", + "authorizationUrl": f"{runner_server_url}/_apis/oauth2/token", + }, + } + ), + # This is a throwaway RSA key used only to let the runner pass its local + # JIT bootstrap. It does not authenticate against a real GitHub service. + ".credentials_rsaparams": _base64_json( + { + "d": "BnkRwk8qg/fMob7o5QboXqqTJsPX2mO7uw7QQAZdFw8FY0P7GmpaiGRPsyu6hhRHH5n6vkMw3gRWvIcP+0rBQ3S32U+tKf4+CaARP9iongbice0xUDdKZKXrTlqSZ9AUND5ZIGIuFNDFn5qXS2J6SyrvF/LopAUu13lWxDrduyEpWRtJkR4RPNKEHi7Lk8NsaZ6N8AIz3+/y0dkWI3pPmelzi+rAssDnz7soK4o6CG9RjIXzeBQzJ4BXefD6zEeXM++mDZylnVJNOoHJWNLvPN+aL5vCfBmgYkk6KZgYzMCFFsxkwtvw+6el3PumxHayginTZ9kd8QJBTypCLtWHWQ==", + "dp": "lkubjli9JdkRuXqnHXHlSDy38RaNGKy+qEa1v4yQTg6h8ni3ZBDMfDhsNhUtlgBHF1AhYod2qwkCZKRNRWAVg00G1Pxswmt57b+4jfW4J0LQ1AytrxhTSrthlyQR5ikUK3d/kEMQs+yP0f2SapkYqXzdaHiU1RA6IhT35bFhHfU=", + "dq": "r7ZtgewAZCZt21o3fBkhAB08Ct+QV0KkzCJlcDr0QbmbjLg/0Nuy6zeiA2QC609LZPgGv6BnPhHMG11bT401WsSkgu/h56L77fK8GwVKCpeZ8SSn3fwyCSpjRHQx3duTerLgi3paVuJTVgL3FdFKSko7wkdSDI1eu9BSHGYnu18=", + "exponent": "AQAB", + "inverseQ": "FSukzwvdLNKkglKWjmYcs1ZCqcgAxecU3bczzVi1TCIYmLN7bLavs15ezr2Xe7MnUJCVz9Lk61sCDVxAA1XK/Bx88iuZvC9GFuM9wZEflvibycx6KI4dvmfSgM0Gff8BnoLs5WinopSuz/fvCzpB26aNfsuv4eCnBgAn8F8bDGU=", + "modulus": "mEkM5pFWZbsCIhVBw2PHC3OfcgP6UtrabLxkAHw6NfxNxdyfRErU6BeI2e6Sh9bRNlo3GbHtq4CizVhcmwJo6CKc1/r1Zrgbb1xQ/FiiHJDA8J6b7cxY894N7rY2r0PqOAxBruGfyAUgG3eFSC5ZSxJfiJe/sd6gwtetrh1ncoCXfeI3IGzZa/dQtIZkefFoqgv5h45gy5KwcAODZ5G0M0aYksFQyUHPLEqSGESsz8LWUOMm1Fgauj2poy8ZHC4xGvfKISPoONRAbHOQDQ7IP09v/w1iAKt02qy9Xdr4vHzZK34a6/Ug/YJpYuIE4fxyTgi096FVIrOl5v+QZg8h4w==", + "p": "zEqTwQJRTY7JmI+zamHZJ/GZ+Hv6n2RSBwFX7BdBS3rxUzAxrijax7fgmhyd4WUgTnMliWZWW3B61Ez/pzXT39ZtSrelOMa6TCMZBbAfq964X5nlWwAdEfHN0SAffKLjXlVBcF6Ov0nhjB8Ci081kjebO1hEgH8ri5awtJLw2S0=", + "q": "vtSnRA6EtRsr+o2gq0E0RA44hBpMe7NMyMIIAxcM4vyMRsKhGc2+vaHIyXLejiaomPlTYWLjCBGoNx8wFN/K0ZoshEJAPAYBVgX9hX9eywigdoGWufkaJqHG1a5YmIqTRqO9dQs8rItpGGPeJmToPLwPaPz1rZH1/BoBO7pksU8=", + } + ), + } + return _base64_json(files) + + +class MicrovmProvider: + slug = "microvm" + display_name = "MicroVM" + image_arn = "arn:aws:lambda:eu-west-1:000000000000:microvm-image:ministack" + image_version = "3.0" + hook_url = MICROVM_HOOK_URL + runner_config_path = "/github-action-runners/multi-runner-webhook/microvm/runners/config" + runner_token_path = "/github-action-runners/multi-runner-webhook/microvm/runners/tokens" + metadata_path = "/github-action-runners/multi-runner-webhook/microvm/runners/config/microvm-metadata" + + def __init__(self) -> None: + self._runner_image_built = False + self._hook_needs_restart = False + + def configure(self, context: SmokeContext) -> None: + self.build_runner_image(context) + self._configure_jit_expectations(context) + context.before_microvm_ids = set(self._metadata_by_path(context)) + + def _configure_jit_expectations(self, context: SmokeContext) -> None: + runner_group_path = "/api/v3/orgs/test-owner/actions/runner-groups" + jit_config_path = "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig" + context.clear_expectation("GET", runner_group_path) + context.clear_expectation("POST", jit_config_path) + context.add_expectation( + "GET", + runner_group_path, + 200, + [{"id": 1, "name": "Default"}], + ) + context.add_expectation( + "POST", + jit_config_path, + 200, + { + "runner": {"id": 987654321, "labels": [{"name": "self-hosted"}, {"name": "linux"}]}, + "encoded_jit_config": _smoke_jit_config(context), + }, + ) + + def build_runner_image(self, context: SmokeContext) -> None: + """Build the local ARM64 runner image once before lifecycle checks.""" + if self._runner_image_built: + return + + with context.step("Test Packer"): + output = json.loads(context.terraform("output", "-json", "microvm")) + foundation = output["microvm_foundation"] + ecr_repository_uri = output["ecr_repo"] + repository_name = ecr_repository_uri.rsplit("/", 1)[-1] + docker_registry = "localhost:4566" + image_tag = "latest" + docker_base_image = f"{docker_registry}/{repository_name}:{image_tag}" + ubuntu_image = ( + ecr_repository_uri + if ":" in ecr_repository_uri.rsplit("/", 1)[-1] + else f"{ecr_repository_uri}:{image_tag}" + ) + image_root = context.source_root / "images" / "microvm-ubuntu" + image_context = image_root / "packer" / "scripts" / "microvm" / "image" + lifecycle_hook_zip = ( + context.source_root + / "lambda_output/" + / "microvm-lifecycle-hooks.zip" + ) + + with context.step("Build base image"): + context.run( + [ + "bash", + "-o", + "pipefail", + "-c", + "aws ecr get-login-password | " + f"docker login --username AWS --password-stdin {docker_registry}", + ], + stream=True, + ) + context.run(["docker", "pull", "--platform", "linux/arm64", "ubuntu:24.04"], stream=True) + context.run(["docker", "tag", "ubuntu:24.04", docker_base_image], stream=True) + context.run(["docker", "push", docker_base_image], stream=True) + + context.environment.update( + { + "MICROVM_ARTIFACT_BUCKET": foundation["artifact_bucket_name"], + "MICROVM_BUILD_ROLE_ARN": foundation["build_role_arn"], + "MICROVM_EGRESS_NETWORK_CONNECTOR_ARN": foundation["connector_arns"]["ministack"], + "MICROVM_IMAGE_NAME": "micro-ubuntu24", + "MICROVM_MEMORY_MIB": "8192", + "MICROVM_IDEMPOTENCY_NONCE": context.environment.get( + "MICROVM_IDEMPOTENCY_NONCE", "ministack-smoke" + ), + "MICROVM_LOG_GROUP": output.get("log_group", "/aws/lambda/microvms/ubuntu24"), + "MICROVM_UBUNTU_IMAGE": ubuntu_image, + "MICROVM_LIFECYCLE_HOOK_ZIP": str(lifecycle_hook_zip), + } + ) + + with context.step("Build MicroVM image"): + context.run(["packer", "build", "."], cwd=image_root, stream=True) + + with context.step("Build lifecycle-hook image"): + shutil.copy2(lifecycle_hook_zip, image_context / "microvm-lifecycle-hooks.zip") + context.run( + [ + "docker", + "build", + "--platform", + "linux/arm64", + "-f", + str(image_context / "ubuntu24.arm64.Dockerfile"), + "--build-arg", + f"UBUNTU_IMAGE={docker_base_image}", + "--tag", + MICROVM_HOOK_CONTAINER, + str(image_context), + ], + stream=True, + ) + + self._start_microvm_hook(context) + self._runner_image_built = True + + def _start_microvm_hook(self, context: SmokeContext) -> None: + with context.step("Start lifecycle-hook container"): + context.run(["docker", "rm", "--force", MICROVM_HOOK_CONTAINER], check=False) + context.run( + [ + "docker", + "run", + "--detach", + "--rm", + "--platform", + "linux/arm64", + "--name", + MICROVM_HOOK_CONTAINER, + "--add-host=host.docker.internal:host-gateway", + "--publish", + f"{MICROVM_HOOK_PORT}:8080", + "--env", + "AWS_ENDPOINT_URL=http://host.docker.internal:4566", + "--env", + "AWS_REGION=eu-west-1", + "--env", + "AWS_DEFAULT_REGION=eu-west-1", + "--env", + "AWS_ACCESS_KEY_ID=000000000000", + "--env", + "AWS_SECRET_ACCESS_KEY=test", + "--env", + "MICROVM_ID=ministack-microvm", + "--env", + f"RUNNER_CONFIG_SSM_PATH={self.runner_config_path}", + MICROVM_HOOK_CONTAINER, + ], + stream=True, + ) + with context.step("Wait for lifecycle-hook readiness"): + context.wait_for( + lambda: context.run( + [ + "curl", + "--fail", + "--silent", + "--show-error", + "--request", + "POST", + f"http://127.0.0.1:{MICROVM_HOOK_PORT}/aws/lambda-microvms/runtime/v1/ready", + ], + check=False, + ).returncode + == 0, + "MicroVM lifecycle hook container readiness", + attempts=30, + ) + + def event(self, context: SmokeContext, job_id: int, dynamic: bool) -> dict[str, Any]: + value = json.loads((context.fixture_dir / "workflow_job_event.json").read_text()) + job = value["workflow_job"] + job["id"] = job_id + job["name"] = f"multi-runner-webhook-microvm-{job_id}" + job["labels"] = ["self-hosted", "linux", "arm64", "microvm"] + if dynamic: + job["labels"].append(f"ghr-microvm-image-version:{self.image_version}") + return value + + def verify_scale_up_routes(self, context: SmokeContext, job_id: int) -> None: + context.verify_route("GET", "/api/v3/orgs/test-owner/actions/runner-groups", "MicroVM scale-up resolved the runner group") + context.verify_route("POST", "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", "MicroVM scale-up generated JIT configuration") + + def verify_pool_routes(self, context: SmokeContext) -> None: + context.verify_route("GET", "/api/v3/orgs/test-owner/actions/runner-groups", "MicroVM pool resolved the runner group") + context.verify_route("POST", "/api/v3/orgs/test-owner/actions/runners/generate-jitconfig", "MicroVM pool generated JIT configuration") + + def _metadata(self, context: SmokeContext, microvm_id: str) -> dict[str, Any]: + value = context.aws( + "ssm", "get-parameter", + "--name", f"{self.metadata_path}/{microvm_id}", + check=False, + ) + if not value or value.get("Parameter", {}).get("Value") in (None, "None"): + raise RuntimeError(f"Missing MicroVM ownership metadata for {microvm_id}") + return json.loads(value["Parameter"]["Value"]) + + def _metadata_by_path(self, context: SmokeContext) -> dict[str, dict[str, Any]]: + result = context.aws( + "ssm", + "get-parameters-by-path", + "--path", + self.metadata_path, + check=False, + ) or {} + prefix = f"{self.metadata_path}/" + metadata: dict[str, dict[str, Any]] = {} + for parameter in result.get("Parameters", []): + name = parameter.get("Name", "") + if not name.startswith(prefix): + continue + microvm_id = name[len(prefix):] + if "." in microvm_id: + continue + value = parameter.get("Value") + if value in (None, "None"): + continue + metadata[microvm_id] = json.loads(value) + return metadata + + def _wait_for_microvm(self, context: SmokeContext, source: str, description: str) -> RunnerResource: + def find() -> str | None: + for microvm_id, metadata in self._metadata_by_path(context).items(): + if microvm_id in context.before_microvm_ids or microvm_id in context.discovered_microvm_ids: + continue + if metadata.get("source") != source: + continue + details = self._details(context, RunnerResource(microvm_id)) + if details.get("state") not in ("PENDING", "RUNNING", "SUSPENDING", "SUSPENDED"): + continue + context.discovered_microvm_ids.append(microvm_id) + return microvm_id + return None + + return RunnerResource(context.wait_for(find, description)) + + def _details(self, context: SmokeContext, resource: RunnerResource) -> dict[str, Any]: + return context.aws( + "lambda-microvms", + "get-microvm", + "--microvm-identifier", + resource.identifier, + check=False, + ) or {} + + def _assert_resource(self, context: SmokeContext, resource: RunnerResource, source: str, dynamic: bool) -> None: + details = self._details(context, resource) + if details.get("state") not in ("PENDING", "RUNNING", "SUSPENDING", "SUSPENDED"): + raise RuntimeError(f"MicroVM {resource.identifier} is not active: {details}") + if details.get("imageArn") != self.image_arn: + raise RuntimeError(f"MicroVM {resource.identifier} used {details.get('imageArn')}, expected {self.image_arn}") + if dynamic and details.get("imageVersion") != self.image_version: + raise RuntimeError(f"MicroVM {resource.identifier} used image version {details.get('imageVersion')}, expected {self.image_version}") + metadata = self._metadata(context, resource.identifier) + expected = { + "environment": "multi-runner-webhook-microvm", + "source": source, + "runnerOwner": "test-owner", + "runnerType": "Org", + } + if any(metadata.get(key) != value for key, value in expected.items()): + raise RuntimeError(f"Unexpected MicroVM metadata: {metadata}") + + def wait_for_scale_up(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_microvm(context, source, "a MicroVM scale-up resource") + + def assert_scale_up(self, context: SmokeContext, resource: RunnerResource, dynamic: bool) -> None: + self._assert_resource(context, resource, "scale-up-lambda", dynamic) + + def start_scale_up_runner(self, context: SmokeContext, resource: RunnerResource) -> bool: + if self._hook_needs_restart: + with context.step("Restart lifecycle-hook container"): + self._start_microvm_hook(context) + self._hook_needs_restart = False + + details = self._details(context, resource) + image_arn = details.get("imageArn") + image_version = details.get("imageVersion") + if not isinstance(image_arn, str) or not isinstance(image_version, str): + raise RuntimeError(f"MicroVM {resource.identifier} has incomplete image details: {details}") + + parameter_name = f"{self.runner_token_path.rstrip('/')}/{resource.identifier}" + context.wait_for( + lambda: context.aws("ssm", "get-parameter", "--name", parameter_name, check=False), + f"MicroVM scale-up to create {parameter_name}", + ) + + run_hook_payload = json.dumps( + { + "version": 1, + "imageArn": image_arn, + "imageVersion": image_version, + "runnerConfigSsmPath": self.runner_config_path, + "runnerTokenSsmPath": self.runner_token_path, + }, + separators=(",", ":"), + ) + request_body = json.dumps( + {"microvmId": resource.identifier, "runHookPayload": run_hook_payload}, + separators=(",", ":"), + ) + context.progress(f"Curling MicroVM runner hook for {resource.identifier}") + result = context.run( + [ + "curl", + "--fail-with-body", + "--silent", + "--show-error", + "--max-time", + "10", + "--request", + "POST", + f"{self.hook_url}/aws/lambda-microvms/runtime/v1/run", + "--header", + "Content-Type: application/json", + "--data-raw", + request_body, + ], + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + "MicroVM runner hook curl failed with " + f"exit code {result.returncode}: {result.stderr.strip() or result.stdout.strip()}" + ) + + context.wait_for( + lambda: not context.aws("ssm", "get-parameter", "--name", parameter_name, check=False), + f"MicroVM runner hook to consume {parameter_name}", + ) + github_runner_id_parameter = f"{self.metadata_path}/{resource.identifier}.github-runner-id" + context.wait_for( + lambda: bool( + context.aws( + "ssm", + "get-parameter", + "--name", + github_runner_id_parameter, + check=False, + ) + ), + f"MicroVM scale-up to persist {github_runner_id_parameter}", + ) + return True + + def wait_for_pool(self, context: SmokeContext, source: str) -> RunnerResource: + return self._wait_for_microvm(context, source, "a MicroVM pool resource") + + def assert_pool(self, context: SmokeContext, resource: RunnerResource) -> None: + self._assert_resource(context, resource, "pool-lambda", False) + + def _wait_for_termination(self, context: SmokeContext, resource: RunnerResource) -> None: + def terminated() -> bool: + details = context.aws( + "lambda-microvms", + "get-microvm", + "--microvm-identifier", + resource.identifier, + check=False, + ) + return not details or details.get("state") == "TERMINATED" + + context.wait_for(terminated, f"MicroVM {resource.identifier} termination") + + def _wait_for_listed_microvm(self, context: SmokeContext, resource: RunnerResource) -> None: + def listed() -> bool: + result = context.aws("lambda-microvms", "list-microvms", check=False) or {} + return any( + item.get("microvmId") == resource.identifier + and item.get("state") in ("PENDING", "RUNNING", "SUSPENDING", "SUSPENDED") + for item in result.get("items", []) + ) + + context.wait_for(listed, f"MicroVM {resource.identifier} to appear in ListMicrovms") + + def scale_down( + self, + context: SmokeContext, + resource: RunnerResource, + runner_id: int, + marker: str, + active_runners: list[tuple[int, RunnerResource]], + ) -> None: + self._wait_for_listed_microvm(context, resource) + context.configure_runner_fixtures( + self.slug, + [(active_runner_id, active_resource.identifier) for active_runner_id, active_resource in active_runners], + runner_id, + ) + context.clear_requests() + context.invoke( + "multi-runner-webhook-microvm-scale-down", + {"smokeMarker": marker, "type": "microvm"}, + "MicroVM scale-down Lambda invoked", + ) + context.wait_for_log("/aws/lambda/multi-runner-webhook-microvm-scale-down", marker, "MicroVM scale-down Lambda started") + context.scale_down_routes(runner_id, "/aws/lambda/multi-runner-webhook-microvm-scale-down") + context.configure_runner_removed(runner_id) + context.assert_runner_removed(runner_id) + self._wait_for_termination(context, resource) + self.stop_microvm_hook(context) + + def stop_microvm_hook(self, context: SmokeContext) -> None: + status, body = context.http( + "POST", + f"{self.hook_url}/aws/lambda-microvms/runtime/v1/terminate", + {}, + ) + if status not in (0, 200, 404): + raise RuntimeError(f"MicroVM lifecycle hook termination failed with HTTP {status}: {body}") + self._hook_needs_restart = True + + +provider = MicrovmProvider() diff --git a/tests/ministack/smoke/webhook_provider.py b/tests/ministack/smoke/webhook_provider.py new file mode 100644 index 0000000000..c154e03883 --- /dev/null +++ b/tests/ministack/smoke/webhook_provider.py @@ -0,0 +1,57 @@ +"""Interface implemented by each compute provider smoke test.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + +from .common import SmokeContext + + +@dataclass(frozen=True) +class RunnerResource: + """Provider resource created by one smoke scenario.""" + + identifier: str + + +class SmokeProvider(Protocol): + slug: str + display_name: str + + def configure(self, context: SmokeContext) -> None: + """Add provider-specific MockServer expectations.""" + + def event(self, context: SmokeContext, job_id: int, dynamic: bool) -> dict[str, Any]: + """Build a workflow_job event that selects this provider.""" + + def verify_scale_up_routes(self, context: SmokeContext, job_id: int) -> None: + """Verify provider-specific GitHub API calls made during scale-up.""" + + def verify_pool_routes(self, context: SmokeContext) -> None: + """Verify provider-specific GitHub API calls made during pool scale-up.""" + + def wait_for_scale_up(self, context: SmokeContext, source: str) -> RunnerResource: + """Find the resource created by a scale-up Lambda.""" + + def assert_scale_up(self, context: SmokeContext, resource: RunnerResource, dynamic: bool) -> None: + """Check provider-specific scale-up state and ownership.""" + + def start_scale_up_runner(self, context: SmokeContext, resource: RunnerResource) -> bool: + """Start the provider-specific runner lifecycle after scale-up.""" + + def wait_for_pool(self, context: SmokeContext, source: str) -> RunnerResource: + """Find the resource created by a pool Lambda.""" + + def assert_pool(self, context: SmokeContext, resource: RunnerResource) -> None: + """Check provider-specific pool state and ownership.""" + + def scale_down( + self, + context: SmokeContext, + resource: RunnerResource, + runner_id: int, + marker: str, + active_runners: list[tuple[int, RunnerResource]], + ) -> None: + """Run provider-specific scale-down checks for one resource.""" diff --git a/tests/ministack/smoke/webhook_scenario.py b/tests/ministack/smoke/webhook_scenario.py new file mode 100644 index 0000000000..19d7cb1d88 --- /dev/null +++ b/tests/ministack/smoke/webhook_scenario.py @@ -0,0 +1,125 @@ +"""Provider-neutral webhook and runner lifecycle scenarios.""" + +from __future__ import annotations + +from uuid import uuid4 + +from .common import SmokeContext +from .webhook_provider import RunnerResource, SmokeProvider + +MOCK_JIT_RUNNER_ID = 987654321 + + +def _log_group(provider: SmokeProvider, stage: str) -> str: + # The webhook and EventBridge dispatcher are shared by all compute + # providers. Only the scale-up Lambda is provider-specific. + if stage in ("webhook", "dispatch-to-runner"): + return f"/aws/lambda/multi-runner-webhook-{stage}" + return f"/aws/lambda/multi-runner-webhook-{provider.slug}-{stage}" + + +def _wait_for_webhook_chain(context: SmokeContext, provider: SmokeProvider, job_id: int) -> None: + context.wait_for_log(_log_group(provider, "webhook"), str(job_id), f"{provider.display_name} webhook received job {job_id}") + context.wait_for_log(_log_group(provider, "dispatch-to-runner"), str(job_id), f"{provider.display_name} dispatcher received job {job_id}") + context.wait_for_log(_log_group(provider, "scale-up"), str(job_id), f"{provider.display_name} scale-up received job {job_id}") + + +def _scale_up( + context: SmokeContext, + provider: SmokeProvider, + job_id: int, + dynamic: bool, + source: str, +) -> RunnerResource: + label_mode = "with dynamic label" if dynamic else "without dynamic label" + with context.step(f"Scale-up {label_mode}"): + with context.step("Prepare scale-up fixtures"): + context.clear_runner_group_cache(provider.slug) + context.clear_requests() + with context.step("Send webhook"): + context.send_webhook( + provider.event(context, job_id, dynamic), + f"multi-runner-webhook-{provider.slug}-{job_id}", + ) + with context.step("Wait for webhook chain"): + _wait_for_webhook_chain(context, provider, job_id) + with context.step("Verify shared scale-up routes"): + context.scale_up_routes(job_id, provider.display_name) + with context.step("Verify provider scale-up routes"): + provider.verify_scale_up_routes(context, job_id) + with context.step("Wait for compute resource"): + resource = provider.wait_for_scale_up(context, source) + with context.step("Validate compute resource"): + provider.assert_scale_up(context, resource, dynamic) + with context.step("Start runner"): + provider.start_scale_up_runner(context, resource) + return resource + + +def _pool(context: SmokeContext, provider: SmokeProvider, pool_size: int) -> RunnerResource: + with context.step("Pool"): + with context.step("Prepare pool fixtures"): + context.configure_empty_runner_list() + context.clear_runner_group_cache(provider.slug) + context.clear_requests() + with context.step("Invoke pool Lambda"): + context.invoke( + f"multi-runner-webhook-{provider.slug}-pool", + {"poolSize": pool_size, "type": provider.slug}, + f"{provider.display_name} pool Lambda invoked", + ) + with context.step("Verify shared pool routes"): + context.pool_routes(provider.display_name) + with context.step("Verify provider pool routes"): + provider.verify_pool_routes(context) + with context.step("Wait for compute resource"): + resource = provider.wait_for_pool(context, "pool-lambda") + with context.step("Validate compute resource"): + context.progress(f"Pool created resource {resource.identifier}") + provider.assert_pool(context, resource) + return resource + + +def _scale_down( + context: SmokeContext, + provider: SmokeProvider, + resource: RunnerResource, + runner_id: int, + marker: str, +) -> None: + with context.step("Scale-down"): + context.progress(f"Scaling down resource {resource.identifier}") + with context.step("Run provider scale-down checks"): + provider.scale_down(context, resource, runner_id, marker, [(runner_id, resource)]) + + +def run(context: SmokeContext, provider: SmokeProvider) -> None: + with context.step("Configure"): + provider.configure(context) + + scale_up = _scale_up(context, provider, 123456, False, "scale-up-lambda") + _scale_down( + context, + provider, + scale_up, + MOCK_JIT_RUNNER_ID, + f"multi-runner-webhook-{provider.slug}-scale-up-scale-down-{uuid4().hex}", + ) + + dynamic_scale_up = _scale_up(context, provider, 123457, True, "scale-up-lambda") + _scale_down( + context, + provider, + dynamic_scale_up, + MOCK_JIT_RUNNER_ID, + f"multi-runner-webhook-{provider.slug}-dynamic-scale-down-{uuid4().hex}", + ) + + pool = _pool(context, provider, pool_size=1) + _scale_down( + context, + provider, + pool, + MOCK_JIT_RUNNER_ID, + f"multi-runner-webhook-{provider.slug}-pool-scale-down-{uuid4().hex}", + )