ci: also run Bencher perf tests on fork PRs

.github/workflows/bencher.yml

@@ -1,7 +1,6 @@
-# NOTE!!!: This does not work with forks right now, because this needs repository secrets.
-# It would be unsafe to allow them in this workflow.
-# Bencher has a documented 2-step workflow on how to accomplish this. I'll follow up with that in another PR.
-
+# Handles running Bencher perf tests to detect regressions.
+# Runs only when PRs are made from an internal branch (NOT A FORK) to main.
+# Forks are processed via 2 steps [bencher_fork_run.yml, bencher_fork_track.yml]
 # TODO: I have some big plans for this file. Please contact @jkaczman before planning to modify with new tests.
 
 name: bencher
@@ -41,9 +40,6 @@ jobs:
 
   bench:
     needs: image
-    permissions:
-      checks: write
-      pull-requests: write
     runs-on: blacksmith-2vcpu-ubuntu-2404
     strategy:
       fail-fast: false

.github/workflows/bencher_fork_run.yml

@@ -0,0 +1,46 @@
+# Fork version of Bencher (Step 1 of 2)
+# 
+# This file is more-or-less a repeat of what we do in the main workflow for building the image
+# except, now, the image gets saved as an artifact, and ran in a separate workflow.
+#
+# Why the separation? This allows for us to build directly on the fork, WITHOUT USING GITHUB SECRETS,
+# and then have a separate workflow, that runs after this one, with privileged secrets,
+# that can use the image built, and run on the main branch (mitigating any security concerns)
+#
+# This is the documented 2-step workflow that Bencher mentions in their docs.
+name: bencher-fork-run
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  image:
+    if: github.event.pull_request.head.repo.full_name != github.repository
+    # This initial runner is to assemble the docker image for this PR for the Bencher runner. It doesn't run any tests itself.
+    # Bencher's runners do not have outside network access; requires us to have everything ready-to-go.
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install CI deps
+        run: bash integration/ci/install-deps.sh
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: build-v1
+      - name: Build
+        run: cargo build --release --bin pgdog
+      - name: Build image
+        run: |
+          docker build --build-context bin=target/release -f benches/perf/Dockerfile -t pgdog-bench .
+          docker save pgdog-bench | gzip > image.tar.gz
+
+      # Deviation from main workflow (as seen in header of this file)    
+      - uses: actions/upload-artifact@v4
+        with:
+          name: image
+          path: image.tar.gz
+
+      # Save which PR requested the run, so the workflow running on main knows what PR to comment on
+      - uses: actions/upload-artifact@v4
+        with:
+          name: event.json
+          path: ${{ github.event_path }}

.github/workflows/bencher_fork_track.yml

@@ -0,0 +1,100 @@
+# Fork version of Bencher (Step 2 of 2)
+# 
+# We built the image in bencher_fork_run.yml on the fork PR.
+# Now, we can use it (ON THE MAIN BRANCH, where it's okay to use secrets),
+# and basically do what we do in the main Bencher workflow, with a pre-built image this time.
+
+name: bencher-fork-track
+on:
+  workflow_run:
+    workflows: [bencher-fork-run]
+    types: [completed]
+
+env:
+  IMAGE: registry.bencher.dev/pgdog:${{ github.event.workflow_run.head_sha }}
+
+jobs:
+  image:
+    # Download image from previous workflow (2-step). Push image to bencher.
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name != github.repository
+    permissions:
+      actions: read
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: image
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Push image
+        run: |
+          docker load < image.tar.gz
+          docker tag pgdog-bench $IMAGE
+          echo '${{ secrets.BENCHER_API_KEY }}' | docker login registry.bencher.dev -u pgdog --password-stdin
+          docker push $IMAGE
+
+  bench:
+    needs: image
+    permissions:
+      actions: read
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: [pooler, lb, sharding]
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: event.json
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Since we're running this on main branch, we have to download the artifact
+      # from the PR that requested this run to figure out which one it was,
+      # so that we can safely run this on main and have all the required info.
+      - name: Export PR event data
+        run: |
+          echo "PR_HEAD=$(jq -r .pull_request.head.ref event.json)" >> "$GITHUB_ENV"
+          echo "PR_HEAD_SHA=$(jq -r .pull_request.head.sha event.json)" >> "$GITHUB_ENV"
+          echo "PR_BASE=$(jq -r .pull_request.base.ref event.json)" >> "$GITHUB_ENV"
+          echo "PR_BASE_SHA=$(jq -r .pull_request.base.sha event.json)" >> "$GITHUB_ENV"
+          echo "PR_NUMBER=$(jq -r .number event.json)" >> "$GITHUB_ENV"
+      - uses: bencherdev/bencher@80847436d7157936b40525db4c2a1064124f04a
+      - name: Run ${{ matrix.suite }} on Bencher bare metal
+        # This will run one test per job (all in parallel).
+        # It'll error if a test's throughput is more than 3% BELOW the mean of the baseline's previous runs (main)
+        #
+        # After testing 5x back-to-back, variance was <1%. This should be enough
+        # of a difference, unless something goes on with their servers in the future.
+        #
+        # TODO: we can re-evaluate a proper variance later
+        #
+        # Why --github-actions? This allows it to comment on the PR with the result.
+        # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts
+        # Why --detach & --ci-callback-token? This allows us to exit early, and defer posting a comment until repository_dispatch is fired on
+        # a separate workflow (bencher_followup.yml)
+        run: |
+          bencher run \
+            --project pgdog \
+            --key '${{ secrets.BENCHER_API_KEY }}' \
+            --branch "$PR_HEAD" \
+            --hash "$PR_HEAD_SHA" \
+            --start-point "$PR_BASE" \
+            --start-point-hash "$PR_BASE_SHA" \
+            --start-point-clone-thresholds \
+            --start-point-reset \
+            --spec intel-v1 \
+            --threshold-measure throughput \
+            --threshold-test percentage \
+            --threshold-lower-boundary 0.03 \
+            --thresholds-reset \
+            --error-on-alert \
+            --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+            --ci-number "$PR_NUMBER" \
+            --ci-id ${{ matrix.suite }} \
+            --ci-only-on-alert \
+            --adapter json \
+            --image $IMAGE \
+            --detach \
+            --ci-callback-token '${{ secrets.BENCHER_GITHUB_TOKEN }}' \
+            "bash benches/perf/bencher.sh ${{ matrix.suite }}"