From 6a357c47c6bbe98b83a8c26afb1e94aaa2319f5d Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 20:15:26 -0400 Subject: [PATCH 1/5] Try using PAT instead of GITHUB_TOKEN --- .github/workflows/bencher.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index ad22ceef1..aa76a7f9e 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -41,9 +41,6 @@ jobs: bench: needs: image - permissions: - checks: write - pull-requests: write runs-on: blacksmith-2vcpu-ubuntu-2404 strategy: fail-fast: false @@ -79,7 +76,7 @@ jobs: --threshold-lower-boundary 0.03 \ --thresholds-reset \ --error-on-alert \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --github-actions '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ --ci-id ${{ matrix.suite }} \ --ci-only-on-alert \ --adapter json \ From fb65e4070f4b5863afc13113690eb7672e10e980 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Wed, 23 Sep 2026 08:24:11 -0400 Subject: [PATCH 2/5] ci: also run Bencher perf tests on fork PRs --- .github/workflows/bencher.yml | 7 +- .github/workflows/bencher_fork_run.yml | 46 ++++++++++++ .github/workflows/bencher_fork_track.yml | 95 ++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/bencher_fork_run.yml create mode 100644 .github/workflows/bencher_fork_track.yml diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index aa76a7f9e..ce8b176e4 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -1,7 +1,6 @@ -# NOTE!!!: This does not work with forks right now, because this needs repository secrets. -# It would be unsafe to allow them in this workflow. -# Bencher has a documented 2-step workflow on how to accomplish this. I'll follow up with that in another PR. - +# Handles running Bencher perf tests to detect regressions. +# Runs only when PRs are made from an internal branch (NOT A FORK) to main. +# Forks are processed via 2 steps [bencher_fork_run.yml, bencher_fork_track.yml] # TODO: I have some big plans for this file. Please contact @jkaczman before planning to modify with new tests. name: bencher diff --git a/.github/workflows/bencher_fork_run.yml b/.github/workflows/bencher_fork_run.yml new file mode 100644 index 000000000..e4ad3ff0b --- /dev/null +++ b/.github/workflows/bencher_fork_run.yml @@ -0,0 +1,46 @@ +# Fork version of Bencher (Step 1 of 2) +# +# This file is more-or-less a repeat of what we do in the main workflow for building the image +# except, now, the image gets saved as an artifact, and ran in a separate workflow. +# +# Why the separation? This allows for us to build directly on the fork, WITHOUT USING GITHUB SECRETS, +# and then have a separate workflow, that runs after this one, with privileged secrets, +# that can use the image built, and run on the main branch (mitigating any security concerns) +# +# This is the documented 2-step workflow that Bencher mentions in their docs. +name: bencher-fork-run +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + image: + if: github.event.pull_request.head.repo.full_name != github.repository + # This initial runner is to assemble the docker image for this PR for the Bencher runner. It doesn't run any tests itself. + # Bencher's runners do not have outside network access; requires us to have everything ready-to-go. + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v6 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: build-v1 + - name: Build + run: cargo build --release --bin pgdog + - name: Build image + run: | + docker build --build-context bin=target/release -f benches/perf/Dockerfile -t pgdog-bench . + docker save pgdog-bench | gzip > image.tar.gz + + # Deviation from main workflow (as seen in header of this file) + - uses: actions/upload-artifact@v4 + with: + name: image + path: image.tar.gz + + # Save which PR requested the run, so the workflow running on main knows what PR to comment on + - uses: actions/upload-artifact@v4 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/bencher_fork_track.yml b/.github/workflows/bencher_fork_track.yml new file mode 100644 index 000000000..8e2aad2f6 --- /dev/null +++ b/.github/workflows/bencher_fork_track.yml @@ -0,0 +1,95 @@ +# Fork version of Bencher (Step 2 of 2) +# +# We built the image in bencher_fork_run.yml on the fork PR. +# Now, we can use it (ON THE MAIN BRANCH, where it's okay to use secrets), +# and basically do what we do in the main Bencher workflow, with a pre-built image this time. + +name: bencher-fork-track +on: + workflow_run: + workflows: [bencher-fork-run] + types: [completed] + +env: + IMAGE: registry.bencher.dev/pgdog:${{ github.event.workflow_run.head_sha }} + +jobs: + image: + # Download image from previous workflow (2-step). Push image to bencher. + if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name != github.repository + permissions: + actions: read + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/download-artifact@v4 + with: + name: image + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + docker load < image.tar.gz + docker tag pgdog-bench $IMAGE + echo '${{ secrets.BENCHER_API_KEY }}' | docker login registry.bencher.dev -u pgdog --password-stdin + docker push $IMAGE + + bench: + needs: image + permissions: + actions: read + runs-on: blacksmith-2vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + suite: [pooler, lb, sharding] + steps: + - uses: actions/download-artifact@v4 + with: + name: event.json + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + # Since we're running this on main branch, we have to download the artifact + # from the PR that requested this run to figure out which one it was, + # so that we can safely run this on main and have all the required info. + - name: Export PR event data + run: | + echo "PR_HEAD=$(jq -r .pull_request.head.ref event.json)" >> "$GITHUB_ENV" + echo "PR_HEAD_SHA=$(jq -r .pull_request.head.sha event.json)" >> "$GITHUB_ENV" + echo "PR_BASE=$(jq -r .pull_request.base.ref event.json)" >> "$GITHUB_ENV" + echo "PR_BASE_SHA=$(jq -r .pull_request.base.sha event.json)" >> "$GITHUB_ENV" + echo "PR_NUMBER=$(jq -r .number event.json)" >> "$GITHUB_ENV" + - uses: bencherdev/bencher@main + - name: Run ${{ matrix.suite }} on Bencher bare metal + # This will run one test per job (all in parallel). + # It'll error if a test's throughput is more than 3% BELOW the mean of the baseline's previous runs (main) + # + # After testing 5x back-to-back, variance was <1%. This should be enough + # of a difference, unless something goes on with their servers in the future. + # + # TODO: we can re-evaluate a proper variance later + # + # Why --github-actions? This allows it to comment on the PR with the result. + # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts + run: | + bencher run \ + --project pgdog \ + --key '${{ secrets.BENCHER_API_KEY }}' \ + --branch "$PR_HEAD" \ + --hash "$PR_HEAD_SHA" \ + --start-point "$PR_BASE" \ + --start-point-hash "$PR_BASE_SHA" \ + --start-point-clone-thresholds \ + --spec intel-v1 \ + --threshold-measure throughput \ + --threshold-test percentage \ + --threshold-lower-boundary 0.03 \ + --thresholds-reset \ + --error-on-alert \ + --github-actions '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ + --ci-number "$PR_NUMBER" \ + --ci-id ${{ matrix.suite }} \ + --ci-only-on-alert \ + --adapter json \ + --image $IMAGE \ + "bash benches/perf/bencher.sh ${{ matrix.suite }}" From 29d939106e52b997c595498f692b26189289adf1 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Wed, 23 Sep 2026 18:48:51 -0400 Subject: [PATCH 3/5] BENCHER_GITHUB_TOKEN (PAT) -> GITHUB_TOKEN regular --- .github/workflows/bencher.yml | 2 +- .github/workflows/bencher_fork_track.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index ce8b176e4..19f968d55 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -75,7 +75,7 @@ jobs: --threshold-lower-boundary 0.03 \ --thresholds-reset \ --error-on-alert \ - --github-actions '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ --ci-id ${{ matrix.suite }} \ --ci-only-on-alert \ --adapter json \ diff --git a/.github/workflows/bencher_fork_track.yml b/.github/workflows/bencher_fork_track.yml index 8e2aad2f6..f2db705bb 100644 --- a/.github/workflows/bencher_fork_track.yml +++ b/.github/workflows/bencher_fork_track.yml @@ -86,7 +86,7 @@ jobs: --threshold-lower-boundary 0.03 \ --thresholds-reset \ --error-on-alert \ - --github-actions '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ --ci-number "$PR_NUMBER" \ --ci-id ${{ matrix.suite }} \ --ci-only-on-alert \ From d8aeba2e3bdc18240bea75930af0363076600190 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Mon, 28 Sep 2026 13:53:29 -0400 Subject: [PATCH 4/5] Use --detach and --ci-callback-token for forks as well --- .github/workflows/bencher_fork_track.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bencher_fork_track.yml b/.github/workflows/bencher_fork_track.yml index f2db705bb..bdbf06404 100644 --- a/.github/workflows/bencher_fork_track.yml +++ b/.github/workflows/bencher_fork_track.yml @@ -59,7 +59,7 @@ jobs: echo "PR_BASE=$(jq -r .pull_request.base.ref event.json)" >> "$GITHUB_ENV" echo "PR_BASE_SHA=$(jq -r .pull_request.base.sha event.json)" >> "$GITHUB_ENV" echo "PR_NUMBER=$(jq -r .number event.json)" >> "$GITHUB_ENV" - - uses: bencherdev/bencher@main + - uses: bencherdev/bencher@80847436d7157936b40525db4c2a1064124f04a - name: Run ${{ matrix.suite }} on Bencher bare metal # This will run one test per job (all in parallel). # It'll error if a test's throughput is more than 3% BELOW the mean of the baseline's previous runs (main) @@ -71,6 +71,8 @@ jobs: # # Why --github-actions? This allows it to comment on the PR with the result. # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts + # Why --detach & --ci-callback-token? This allows us to exit early, and defer posting a comment until repository_dispatch is fired on + # a separate workflow (bencher_followup.yml) run: | bencher run \ --project pgdog \ @@ -92,4 +94,6 @@ jobs: --ci-only-on-alert \ --adapter json \ --image $IMAGE \ + --detach \ + --ci-callback-token '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ "bash benches/perf/bencher.sh ${{ matrix.suite }}" From 1866550600c7c0efb780902d2497108b81beac4f Mon Sep 17 00:00:00 2001 From: jkaczman Date: Mon, 28 Sep 2026 17:43:58 -0400 Subject: [PATCH 5/5] --start-point-reset --- .github/workflows/bencher_fork_track.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/bencher_fork_track.yml b/.github/workflows/bencher_fork_track.yml index bdbf06404..e7bea8db8 100644 --- a/.github/workflows/bencher_fork_track.yml +++ b/.github/workflows/bencher_fork_track.yml @@ -82,6 +82,7 @@ jobs: --start-point "$PR_BASE" \ --start-point-hash "$PR_BASE_SHA" \ --start-point-clone-thresholds \ + --start-point-reset \ --spec intel-v1 \ --threshold-measure throughput \ --threshold-test percentage \