ci: bencher v1, benches/perf/{lb,sharding,pooler}, detect perf regressions

.github/workflows/bencher.yml

@@ -0,0 +1,89 @@
+# NOTE!!!: This does not work with forks right now, because this needs repository secrets.
+# It would be unsafe to allow them in this workflow.
+# Bencher has a documented 2-step workflow on how to accomplish this. I'll follow up with that in another PR.
+
+# TODO: I have some big plans for this file. Please contact @jkaczman before planning to modify with new tests.
+
+name: bencher
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+
+env:
+  IMAGE: registry.bencher.dev/pgdog:${{ github.sha }}
+
+jobs:
+  image:
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
+    # This initial runner is to assemble the docker image for the Bencher runner. It doesn't run any tests itself.
+    # Bencher's runners do not have outside network access; requires us to have everything ready-to-go.
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install CI deps
+        run: bash integration/ci/install-deps.sh
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: build-v1
+      - name: Build
+        run: cargo build --release --bin pgdog
+
+      - name: Build image
+        run: docker build --build-context bin=target/release -f benches/perf/Dockerfile -t $IMAGE .
+      - name: Push image
+        run: |
+          echo '${{ secrets.BENCHER_API_KEY }}' | docker login registry.bencher.dev -u pgdog --password-stdin
+          docker push $IMAGE
+
+  bench:
+    needs: image
+    permissions:
+      checks: write
+      pull-requests: write
+    runs-on: blacksmith-2vcpu-ubuntu-2404
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: [pooler, lb, sharding]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a
+      - name: Run ${{ matrix.suite }} on Bencher bare metal
+        env:
+          START_POINT: ${{ github.event_name == 'pull_request' && format('--start-point {0} --start-point-hash {1} --start-point-clone-thresholds --start-point-reset', github.base_ref, github.event.pull_request.base.sha) || '' }}
+        # This will run one test per job (all in parallel).
+        # It'll error if a test's throughput is more than 3% BELOW the mean of the baseline's previous runs (main)
+        #
+        # After testing 5x back-to-back, variance was <1%. This should be enough
+        # of a difference, unless something goes on with their servers in the future.
+        # 
+        # TODO: we can re-evaluate a proper variance later
+        #
+        # Why --github-actions? This allows it to comment on the PR with the result.
+        # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts
+        # Why --detach & --ci-callback-token? This allows us to exit early, and defer posting a comment until repository_dispatch is fired on
+        # a separate workflow (bencher_followup.yml)
+        run: |
+          bencher run \
+            --project pgdog \
+            --key '${{ secrets.BENCHER_API_KEY }}' \
+            --branch '${{ github.head_ref || github.ref_name }}' \
+            $START_POINT \
+            --spec intel-v1 \
+            --threshold-measure throughput \
+            --threshold-test percentage \
+            --threshold-lower-boundary 0.03 \
+            --thresholds-reset \
+            --error-on-alert \
+            --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+            --ci-id ${{ matrix.suite }} \
+            --ci-only-on-alert \
+            --adapter json \
+            --image $IMAGE \
+            --detach \
+            --ci-callback-token '${{ secrets.BENCHER_GITHUB_TOKEN }}' \
+            "bash benches/perf/bencher.sh ${{ matrix.suite }}"

.github/workflows/bencher_followup.yml

@@ -0,0 +1,26 @@
+# This is just the workflow Bencher has in its docs; haven't customized it yet
+# <https://bencher.dev/docs/how-to/github-actions/#pull-requests-on-bare-metal>
+on:
+  repository_dispatch:
+    types: [bencher_run]
+
+jobs:
+  attach_pr_benchmarks:
+    name: Post PR Benchmark Results from Bencher Bare Metal
+    permissions:
+      checks: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a
+      - name: Post PR Benchmark Results with Bencher
+        env:
+          JOB_UUID: ${{ github.event.client_payload.bencher.job }}
+        run: |
+          bencher run \
+          --project pgdog \
+          --key '${{ secrets.BENCHER_API_KEY }}' \
+          --job "$JOB_UUID" \
+          --error-on-alert \
+          --ci-only-on-alert \
+          --github-actions '${{ secrets.GITHUB_TOKEN }}'

benches/perf/Dockerfile

@@ -0,0 +1,18 @@
+# Re-uses the base runtime image; injects this branch's built pgdog binary and
+# the perf scripts we're gonna run in Bencher.
+
+FROM ghcr.io/pgdogdev/pgdog-base-runtime:latest
+
+RUN apt update && apt install -y --no-install-recommends postgresql-18 sudo iproute2
+
+WORKDIR /pgdog
+COPY --from=bin pgdog /usr/local/bin/pgdog
+COPY integration/common.sh integration/setup.sh integration/
+COPY integration/ci/setup.sh integration/ci/
+COPY pgdog/src/backend/schema/setup.sql pgdog/src/backend/schema/
+COPY benches/perf benches/perf
+
+# This happens here because we don't have network access in Bencher.
+RUN bash integration/ci/setup.sh && pg_ctlcluster 18 main stop
+
+CMD ["bash", "benches/perf/bencher.sh", "pooler", "lb", "sharding"]

benches/perf/bencher.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+export PGDOG_BIN=/usr/local/bin/pgdog
+
+# We have to make some changes to account for this being ran in a Firecracker microVM, not a Docker container (as
+# this pgdog-base-runtime image was based on that)
+printf '127.0.0.1 localhost\n::1 localhost\n' > /etc/hosts
+ip link set lo up
+mkdir -p /dev/shm && mount -t tmpfs tmpfs /dev/shm
+chown -R postgres:postgres /var/lib/postgresql /etc/postgresql /var/log/postgresql
+chgrp -R ssl-cert /etc/ssl/private
+pg_ctlcluster 18 main start
+
+# This assembles into "Bencher Metric Format"; required because we don't use something like criterion here,
+# we have our own custom runtime scripts
+json=""
+for name in "$@"; do
+    bash benches/perf/run.sh $name > $name.txt 2>&1
+    cat $name.txt >&2
+    tps=$(awk '/^tps/ {print $3}' $name.txt)
+    json+="\"$name\": {\"throughput\": {\"value\": $tps}},"
+done
+
+echo "{${json%,}}"

benches/perf/lb/run.sh

@@ -8,4 +8,4 @@ export PGHOST=127.0.0.1
 export PGPORT=6432
 
 pgbench -i
-pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/../select_1.sql -P 1
+pgbench -c 10 -j 2 -T 180 -f ${SCRIPT_DIR}/../select_1.sql -P 1

benches/perf/pooler/run.sh

@@ -8,4 +8,4 @@ export PGHOST=127.0.0.1
 export PGPORT=6432
 
 pgbench -i
-pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/../select_1.sql -P 1
+pgbench -c 10 -j 2 -T 180 -f ${SCRIPT_DIR}/../select_1.sql -P 1

benches/perf/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source ${SCRIPT_DIR}/../../integration/common.sh
+
+run_pgdog benches/perf/$1
+wait_for_pgdog
+
+bash ${SCRIPT_DIR}/$1/run.sh
+
+stop_pgdog

benches/perf/sharding/run.sh

@@ -7,4 +7,4 @@ export PGPORT=6432
 
 psql -f ${SCRIPT_DIR}/setup.sql
 pgbench -i
-pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/script.sql -P 1 --protocol extended
+pgbench -c 10 -j 2 -T 180 -f ${SCRIPT_DIR}/script.sql -P 1 --protocol extended