ci: reuse finished builds from R2 and cache compiles with sccache #16
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build V8 matrix | |
| # Runs on demand and on every push to the branch so the matrix is verified | |
| # before anything is tagged. Pushing a tag additionally cuts a release. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| platforms: | |
| description: Which platforms to build | |
| type: choice | |
| options: [all, android, ios] | |
| default: all | |
| push: | |
| branches: [main] | |
| tags: ['v8-*'] | |
| pull_request: | |
| paths: | |
| - 'config.env' | |
| - 'patches/**' | |
| - 'scripts/matrix/**' | |
| - '.github/workflows/build-matrix.yml' | |
| permissions: | |
| contents: write | |
| env: | |
| SCCACHE_VERSION: 0.10.0 | |
| # Nine jobs a run, several hours each -- superseded runs are not worth finishing. | |
| # Tags are excluded from cancellation so a release is never interrupted. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }} | |
| jobs: | |
| config: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| v8_version: ${{ steps.cfg.outputs.v8_version }} | |
| ndk_release: ${{ steps.cfg.outputs.ndk_release }} | |
| inputs_hash: ${{ steps.cfg.outputs.inputs_hash }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - id: cfg | |
| run: | | |
| set -a; . ./config.env; set +a | |
| echo "v8_version=$V8_VERSION" >> "$GITHUB_OUTPUT" | |
| echo "ndk_release=$ANDROID_NDK_RELEASE" >> "$GITHUB_OUTPUT" | |
| # Everything the artifacts are a function of, hashed by blob id. | |
| # Commits with an unchanged hash produce identical artifacts, so a | |
| # finished build in R2 can be reused instead of repeated. | |
| echo "inputs_hash=$(git ls-files -s config.env patches scripts/matrix .github/workflows/build-matrix.yml \ | |
| | git hash-object --stdin)" >> "$GITHUB_OUTPUT" | |
| android: | |
| needs: config | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.platforms == 'all' || inputs.platforms == 'android' }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 300 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| abi: [arm64-v8a, x86_64, armeabi-v7a, x86] | |
| # The R2 bucket has two prefixes with separate lifecycle rules: | |
| # artifacts/<inputs-hash>/ holds finished tarballs (expired quickly; a miss | |
| # just means building, as before) and sccache/ holds the compiler cache | |
| # (kept long). Every R2 step is skipped when the secrets are absent, e.g. | |
| # on a fork PR, and the build then runs exactly as it would without R2. | |
| env: | |
| R2_BUCKET: ${{ vars.R2_BUCKET }} | |
| R2_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: auto | |
| # aws-cli >= 2.23 defaults to CRC checksums R2 rejects. | |
| AWS_REQUEST_CHECKSUM_CALCULATION: when_required | |
| AWS_RESPONSE_CHECKSUM_VALIDATION: when_required | |
| SCCACHE_BUCKET: ${{ vars.R2_BUCKET }} | |
| SCCACHE_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com | |
| SCCACHE_REGION: auto | |
| SCCACHE_S3_KEY_PREFIX: sccache | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Check R2 for prebuilt artifacts | |
| id: cache | |
| if: env.AWS_ACCESS_KEY_ID != '' | |
| run: | | |
| keys="v8-${{ needs.config.outputs.v8_version }}-android-${{ matrix.abi }}.tar.gz" | |
| if [ "${{ matrix.abi }}" = "arm64-v8a" ]; then | |
| keys="$keys v8-${{ needs.config.outputs.v8_version }}-src-headers.tar.gz" | |
| fi | |
| prefix="artifacts/${{ needs.config.outputs.inputs_hash }}" | |
| hit=true | |
| for k in $keys; do | |
| aws s3api head-object --endpoint-url "$R2_ENDPOINT" \ | |
| --bucket "$R2_BUCKET" --key "$prefix/$k" >/dev/null 2>&1 || hit=false | |
| done | |
| if [ "$hit" = true ]; then | |
| for k in $keys; do | |
| aws s3 cp --endpoint-url "$R2_ENDPOINT" "s3://$R2_BUCKET/$prefix/$k" "$k" | |
| done | |
| fi | |
| echo "hit=$hit" >> "$GITHUB_OUTPUT" | |
| # A 32-bit target builds mksnapshot and the bytecode-builtins generator | |
| # as 32-bit x86 host binaries. Without the i386 runtime they link and | |
| # then fail to execute, which surfaces only as a failed | |
| # generate_bytecode_builtins_list action. | |
| - name: Enable i386 multiarch | |
| if: steps.cache.outputs.hit != 'true' | |
| run: | | |
| sudo dpkg --add-architecture i386 | |
| sudo apt-get update -qq | |
| sudo apt-get install -y -qq --no-install-recommends \ | |
| libc6:i386 libstdc++6:i386 libatomic1:i386 zlib1g:i386 rsync | |
| - name: Install the Android NDK ${{ needs.config.outputs.ndk_release }} | |
| if: steps.cache.outputs.hit != 'true' | |
| run: | | |
| curl -fsSL -o /tmp/ndk.zip \ | |
| "https://dl.google.com/android/repository/android-ndk-${{ needs.config.outputs.ndk_release }}-linux.zip" | |
| unzip -q /tmp/ndk.zip -d "$HOME" | |
| echo "ANDROID_NDK_ROOT=$HOME/android-ndk-${{ needs.config.outputs.ndk_release }}" >> "$GITHUB_ENV" | |
| - name: Install depot_tools | |
| if: steps.cache.outputs.hit != 'true' | |
| run: | | |
| git clone -q --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "$HOME/depot_tools" | |
| echo "$HOME/depot_tools" >> "$GITHUB_PATH" | |
| - name: Install sccache | |
| if: env.AWS_ACCESS_KEY_ID != '' && steps.cache.outputs.hit != 'true' | |
| run: | | |
| curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ | |
| | tar -xz -C /tmp | |
| mkdir -p "$HOME/.local/bin" | |
| install "/tmp/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" "$HOME/.local/bin/" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Fetch and patch V8 | |
| if: steps.cache.outputs.hit != 'true' | |
| run: scripts/matrix/fetch.sh --platform android --no-history | |
| - name: Build ${{ matrix.abi }} | |
| if: steps.cache.outputs.hit != 'true' | |
| run: > | |
| scripts/matrix/build-android.sh --abi ${{ matrix.abi }} | |
| ${{ env.AWS_ACCESS_KEY_ID != '' && '--cc-wrapper sccache' || '' }} | |
| - name: sccache stats | |
| if: env.AWS_ACCESS_KEY_ID != '' && steps.cache.outputs.hit != 'true' | |
| run: sccache --show-stats | |
| - name: Package | |
| if: steps.cache.outputs.hit != 'true' | |
| run: | | |
| tar -czf "v8-${{ needs.config.outputs.v8_version }}-android-${{ matrix.abi }}.tar.gz" \ | |
| -C dist "android-${{ matrix.abi }}" | |
| # The internal headers are architecture-independent, so one job produces | |
| # them. It piggybacks on a build rather than getting its own job because | |
| # src/inspector/protocol/*.h are generated into gen/ and only exist once | |
| # something has been built. | |
| - name: Package the internal headers | |
| if: matrix.abi == 'arm64-v8a' && steps.cache.outputs.hit != 'true' | |
| run: | | |
| scripts/matrix/package-src-headers.sh \ | |
| --gen-dir .v8/v8/out.gn/android-arm64-release/gen | |
| tar -czf "v8-${{ needs.config.outputs.v8_version }}-src-headers.tar.gz" \ | |
| -C dist src-headers | |
| - name: Upload the artifacts to R2 | |
| if: env.AWS_ACCESS_KEY_ID != '' && steps.cache.outputs.hit != 'true' | |
| run: | | |
| prefix="artifacts/${{ needs.config.outputs.inputs_hash }}" | |
| for f in v8-*.tar.gz; do | |
| aws s3 cp --endpoint-url "$R2_ENDPOINT" "$f" "s3://$R2_BUCKET/$prefix/$f" | |
| done | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: v8-android-${{ matrix.abi }} | |
| path: v8-*-android-${{ matrix.abi }}.tar.gz | |
| retention-days: 7 | |
| - uses: actions/upload-artifact@v7 | |
| if: matrix.abi == 'arm64-v8a' | |
| with: | |
| name: v8-src-headers | |
| path: v8-*-src-headers.tar.gz | |
| retention-days: 7 | |
| ios: | |
| needs: config | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.platforms == 'all' || inputs.platforms == 'ios' }} | |
| runs-on: macos-15 | |
| timeout-minutes: 300 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| variant: [arm64-device, arm64-simulator, x64-simulator, arm64-catalyst, x64-catalyst] | |
| # See the android job for the R2 bucket layout and the missing-secrets | |
| # fallback; this job mirrors it. | |
| env: | |
| R2_BUCKET: ${{ vars.R2_BUCKET }} | |
| R2_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: auto | |
| AWS_REQUEST_CHECKSUM_CALCULATION: when_required | |
| AWS_RESPONSE_CHECKSUM_VALIDATION: when_required | |
| SCCACHE_BUCKET: ${{ vars.R2_BUCKET }} | |
| SCCACHE_ENDPOINT: https://${{ secrets.R2_ACCOUNT_ID }}.r2.cloudflarestorage.com | |
| SCCACHE_REGION: auto | |
| SCCACHE_S3_KEY_PREFIX: sccache | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Check R2 for a prebuilt artifact | |
| id: cache | |
| if: env.AWS_ACCESS_KEY_ID != '' | |
| run: | | |
| key="artifacts/${{ needs.config.outputs.inputs_hash }}/v8-${{ needs.config.outputs.v8_version }}-ios-${{ matrix.variant }}.tar.gz" | |
| if aws s3api head-object --endpoint-url "$R2_ENDPOINT" \ | |
| --bucket "$R2_BUCKET" --key "$key" >/dev/null 2>&1; then | |
| aws s3 cp --endpoint-url "$R2_ENDPOINT" "s3://$R2_BUCKET/$key" . | |
| echo "hit=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "hit=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Install depot_tools | |
| if: steps.cache.outputs.hit != 'true' | |
| run: | | |
| git clone -q --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "$HOME/depot_tools" | |
| echo "$HOME/depot_tools" >> "$GITHUB_PATH" | |
| - name: Install sccache | |
| if: env.AWS_ACCESS_KEY_ID != '' && steps.cache.outputs.hit != 'true' | |
| run: | | |
| curl -fsSL "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-aarch64-apple-darwin.tar.gz" \ | |
| | tar -xz -C /tmp | |
| mkdir -p "$HOME/.local/bin" | |
| install "/tmp/sccache-v${SCCACHE_VERSION}-aarch64-apple-darwin/sccache" "$HOME/.local/bin/" | |
| echo "$HOME/.local/bin" >> "$GITHUB_PATH" | |
| - name: Fetch and patch V8 | |
| if: steps.cache.outputs.hit != 'true' | |
| run: scripts/matrix/fetch.sh --platform ios --no-history | |
| - name: Build ${{ matrix.variant }} | |
| if: steps.cache.outputs.hit != 'true' | |
| run: > | |
| scripts/matrix/build-ios.sh --variant ${{ matrix.variant }} | |
| ${{ env.AWS_ACCESS_KEY_ID != '' && '--cc-wrapper sccache' || '' }} | |
| - name: sccache stats | |
| if: env.AWS_ACCESS_KEY_ID != '' && steps.cache.outputs.hit != 'true' | |
| run: sccache --show-stats | |
| - name: Package | |
| if: steps.cache.outputs.hit != 'true' | |
| run: | | |
| tar -czf "v8-${{ needs.config.outputs.v8_version }}-ios-${{ matrix.variant }}.tar.gz" \ | |
| -C dist "ios-${{ matrix.variant }}" | |
| - name: Upload the artifact to R2 | |
| if: env.AWS_ACCESS_KEY_ID != '' && steps.cache.outputs.hit != 'true' | |
| run: | | |
| f="v8-${{ needs.config.outputs.v8_version }}-ios-${{ matrix.variant }}.tar.gz" | |
| aws s3 cp --endpoint-url "$R2_ENDPOINT" "$f" \ | |
| "s3://$R2_BUCKET/artifacts/${{ needs.config.outputs.inputs_hash }}/$f" | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: v8-ios-${{ matrix.variant }} | |
| path: v8-*-ios-${{ matrix.variant }}.tar.gz | |
| retention-days: 7 | |
| release: | |
| needs: [config, android, ios] | |
| if: startsWith(github.ref, 'refs/tags/v8-') | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: actions/download-artifact@v8 | |
| with: | |
| path: artifacts | |
| merge-multiple: true | |
| # The checksums are the point of shipping this way: consumers pin a tag | |
| # and verify the archive rather than trusting whatever the URL returns. | |
| - name: Generate checksums | |
| working-directory: artifacts | |
| run: | | |
| sha256sum *.tar.gz | tee SHA256SUMS | |
| ls -la | |
| - name: Create the release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| { | |
| echo "V8 \`${{ needs.config.outputs.v8_version }}\` built from \`${GITHUB_SHA}\`." | |
| echo | |
| echo "Built against Android NDK \`${{ needs.config.outputs.ndk_release }}\`; it must match the" | |
| echo "NDK the Android runtime is built with, because libc++ is only ABI-compatible" | |
| echo "with itself across a static link." | |
| echo | |
| echo "Patches applied: \`v8_resurrecting_finalizers.patch\` (restores" | |
| echo "\`WeakCallbackType::kFinalizer\`), \`android_build.patch\` (API 21 floor," | |
| echo "selectable \`android_ndk_root\`, macOS host support)." | |
| echo | |
| echo "Verify with \`sha256sum -c SHA256SUMS\`. The gn args are in \`scripts/matrix/build-*.sh\`." | |
| } > notes.md | |
| gh release create "${GITHUB_REF_NAME}" \ | |
| --title "${GITHUB_REF_NAME}" \ | |
| --notes-file notes.md \ | |
| artifacts/*.tar.gz artifacts/SHA256SUMS |