diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index ad22ceef1..19f968d55 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -1,7 +1,6 @@ -# NOTE!!!: This does not work with forks right now, because this needs repository secrets. -# It would be unsafe to allow them in this workflow. -# Bencher has a documented 2-step workflow on how to accomplish this. I'll follow up with that in another PR. - +# Handles running Bencher perf tests to detect regressions. +# Runs only when PRs are made from an internal branch (NOT A FORK) to main. +# Forks are processed via 2 steps [bencher_fork_run.yml, bencher_fork_track.yml] # TODO: I have some big plans for this file. Please contact @jkaczman before planning to modify with new tests. name: bencher @@ -41,9 +40,6 @@ jobs: bench: needs: image - permissions: - checks: write - pull-requests: write runs-on: blacksmith-2vcpu-ubuntu-2404 strategy: fail-fast: false diff --git a/.github/workflows/bencher_fork_run.yml b/.github/workflows/bencher_fork_run.yml new file mode 100644 index 000000000..e4ad3ff0b --- /dev/null +++ b/.github/workflows/bencher_fork_run.yml @@ -0,0 +1,46 @@ +# Fork version of Bencher (Step 1 of 2) +# +# This file is more-or-less a repeat of what we do in the main workflow for building the image +# except, now, the image gets saved as an artifact, and ran in a separate workflow. +# +# Why the separation? This allows for us to build directly on the fork, WITHOUT USING GITHUB SECRETS, +# and then have a separate workflow, that runs after this one, with privileged secrets, +# that can use the image built, and run on the main branch (mitigating any security concerns) +# +# This is the documented 2-step workflow that Bencher mentions in their docs. +name: bencher-fork-run +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + image: + if: github.event.pull_request.head.repo.full_name != github.repository + # This initial runner is to assemble the docker image for this PR for the Bencher runner. It doesn't run any tests itself. + # Bencher's runners do not have outside network access; requires us to have everything ready-to-go. + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v6 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: build-v1 + - name: Build + run: cargo build --release --bin pgdog + - name: Build image + run: | + docker build --build-context bin=target/release -f benches/perf/Dockerfile -t pgdog-bench . + docker save pgdog-bench | gzip > image.tar.gz + + # Deviation from main workflow (as seen in header of this file) + - uses: actions/upload-artifact@v4 + with: + name: image + path: image.tar.gz + + # Save which PR requested the run, so the workflow running on main knows what PR to comment on + - uses: actions/upload-artifact@v4 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/bencher_fork_track.yml b/.github/workflows/bencher_fork_track.yml new file mode 100644 index 000000000..e7bea8db8 --- /dev/null +++ b/.github/workflows/bencher_fork_track.yml @@ -0,0 +1,100 @@ +# Fork version of Bencher (Step 2 of 2) +# +# We built the image in bencher_fork_run.yml on the fork PR. +# Now, we can use it (ON THE MAIN BRANCH, where it's okay to use secrets), +# and basically do what we do in the main Bencher workflow, with a pre-built image this time. + +name: bencher-fork-track +on: + workflow_run: + workflows: [bencher-fork-run] + types: [completed] + +env: + IMAGE: registry.bencher.dev/pgdog:${{ github.event.workflow_run.head_sha }} + +jobs: + image: + # Download image from previous workflow (2-step). Push image to bencher. + if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name != github.repository + permissions: + actions: read + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/download-artifact@v4 + with: + name: image + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + docker load < image.tar.gz + docker tag pgdog-bench $IMAGE + echo '${{ secrets.BENCHER_API_KEY }}' | docker login registry.bencher.dev -u pgdog --password-stdin + docker push $IMAGE + + bench: + needs: image + permissions: + actions: read + runs-on: blacksmith-2vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + suite: [pooler, lb, sharding] + steps: + - uses: actions/download-artifact@v4 + with: + name: event.json + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + # Since we're running this on main branch, we have to download the artifact + # from the PR that requested this run to figure out which one it was, + # so that we can safely run this on main and have all the required info. + - name: Export PR event data + run: | + echo "PR_HEAD=$(jq -r .pull_request.head.ref event.json)" >> "$GITHUB_ENV" + echo "PR_HEAD_SHA=$(jq -r .pull_request.head.sha event.json)" >> "$GITHUB_ENV" + echo "PR_BASE=$(jq -r .pull_request.base.ref event.json)" >> "$GITHUB_ENV" + echo "PR_BASE_SHA=$(jq -r .pull_request.base.sha event.json)" >> "$GITHUB_ENV" + echo "PR_NUMBER=$(jq -r .number event.json)" >> "$GITHUB_ENV" + - uses: bencherdev/bencher@80847436d7157936b40525db4c2a1064124f04a + - name: Run ${{ matrix.suite }} on Bencher bare metal + # This will run one test per job (all in parallel). + # It'll error if a test's throughput is more than 3% BELOW the mean of the baseline's previous runs (main) + # + # After testing 5x back-to-back, variance was <1%. This should be enough + # of a difference, unless something goes on with their servers in the future. + # + # TODO: we can re-evaluate a proper variance later + # + # Why --github-actions? This allows it to comment on the PR with the result. + # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts + # Why --detach & --ci-callback-token? This allows us to exit early, and defer posting a comment until repository_dispatch is fired on + # a separate workflow (bencher_followup.yml) + run: | + bencher run \ + --project pgdog \ + --key '${{ secrets.BENCHER_API_KEY }}' \ + --branch "$PR_HEAD" \ + --hash "$PR_HEAD_SHA" \ + --start-point "$PR_BASE" \ + --start-point-hash "$PR_BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --spec intel-v1 \ + --threshold-measure throughput \ + --threshold-test percentage \ + --threshold-lower-boundary 0.03 \ + --thresholds-reset \ + --error-on-alert \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-number "$PR_NUMBER" \ + --ci-id ${{ matrix.suite }} \ + --ci-only-on-alert \ + --adapter json \ + --image $IMAGE \ + --detach \ + --ci-callback-token '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ + "bash benches/perf/bencher.sh ${{ matrix.suite }}"