diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml new file mode 100644 index 000000000..ad22ceef1 --- /dev/null +++ b/.github/workflows/bencher.yml @@ -0,0 +1,89 @@ +# NOTE!!!: This does not work with forks right now, because this needs repository secrets. +# It would be unsafe to allow them in this workflow. +# Bencher has a documented 2-step workflow on how to accomplish this. I'll follow up with that in another PR. + +# TODO: I have some big plans for this file. Please contact @jkaczman before planning to modify with new tests. + +name: bencher +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +env: + IMAGE: registry.bencher.dev/pgdog:${{ github.sha }} + +jobs: + image: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + # This initial runner is to assemble the docker image for the Bencher runner. It doesn't run any tests itself. + # Bencher's runners do not have outside network access; requires us to have everything ready-to-go. + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v6 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: build-v1 + - name: Build + run: cargo build --release --bin pgdog + + - name: Build image + run: docker build --build-context bin=target/release -f benches/perf/Dockerfile -t $IMAGE . + - name: Push image + run: | + echo '${{ secrets.BENCHER_API_KEY }}' | docker login registry.bencher.dev -u pgdog --password-stdin + docker push $IMAGE + + bench: + needs: image + permissions: + checks: write + pull-requests: write + runs-on: blacksmith-2vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + suite: [pooler, lb, sharding] + steps: + - uses: actions/checkout@v6 + - uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a + - name: Run ${{ matrix.suite }} on Bencher bare metal + env: + START_POINT: ${{ github.event_name == 'pull_request' && format('--start-point {0} --start-point-hash {1} --start-point-clone-thresholds --start-point-reset', github.base_ref, github.event.pull_request.base.sha) || '' }} + # This will run one test per job (all in parallel). + # It'll error if a test's throughput is more than 3% BELOW the mean of the baseline's previous runs (main) + # + # After testing 5x back-to-back, variance was <1%. This should be enough + # of a difference, unless something goes on with their servers in the future. + # + # TODO: we can re-evaluate a proper variance later + # + # Why --github-actions? This allows it to comment on the PR with the result. + # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts + # Why --detach & --ci-callback-token? This allows us to exit early, and defer posting a comment until repository_dispatch is fired on + # a separate workflow (bencher_followup.yml) + run: | + bencher run \ + --project pgdog \ + --key '${{ secrets.BENCHER_API_KEY }}' \ + --branch '${{ github.head_ref || github.ref_name }}' \ + $START_POINT \ + --spec intel-v1 \ + --threshold-measure throughput \ + --threshold-test percentage \ + --threshold-lower-boundary 0.03 \ + --thresholds-reset \ + --error-on-alert \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-id ${{ matrix.suite }} \ + --ci-only-on-alert \ + --adapter json \ + --image $IMAGE \ + --detach \ + --ci-callback-token '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ + "bash benches/perf/bencher.sh ${{ matrix.suite }}" diff --git a/.github/workflows/bencher_followup.yml b/.github/workflows/bencher_followup.yml new file mode 100644 index 000000000..c87088f8e --- /dev/null +++ b/.github/workflows/bencher_followup.yml @@ -0,0 +1,26 @@ +# This is just the workflow Bencher has in its docs; haven't customized it yet +# +on: + repository_dispatch: + types: [bencher_run] + +jobs: + attach_pr_benchmarks: + name: Post PR Benchmark Results from Bencher Bare Metal + permissions: + checks: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a + - name: Post PR Benchmark Results with Bencher + env: + JOB_UUID: ${{ github.event.client_payload.bencher.job }} + run: | + bencher run \ + --project pgdog \ + --key '${{ secrets.BENCHER_API_KEY }}' \ + --job "$JOB_UUID" \ + --error-on-alert \ + --ci-only-on-alert \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' diff --git a/benches/perf/Dockerfile b/benches/perf/Dockerfile new file mode 100644 index 000000000..fdc51c0ff --- /dev/null +++ b/benches/perf/Dockerfile @@ -0,0 +1,18 @@ +# Re-uses the base runtime image; injects this branch's built pgdog binary and +# the perf scripts we're gonna run in Bencher. + +FROM ghcr.io/pgdogdev/pgdog-base-runtime:latest + +RUN apt update && apt install -y --no-install-recommends postgresql-18 sudo iproute2 + +WORKDIR /pgdog +COPY --from=bin pgdog /usr/local/bin/pgdog +COPY integration/common.sh integration/setup.sh integration/ +COPY integration/ci/setup.sh integration/ci/ +COPY pgdog/src/backend/schema/setup.sql pgdog/src/backend/schema/ +COPY benches/perf benches/perf + +# This happens here because we don't have network access in Bencher. +RUN bash integration/ci/setup.sh && pg_ctlcluster 18 main stop + +CMD ["bash", "benches/perf/bencher.sh", "pooler", "lb", "sharding"] diff --git a/benches/perf/bencher.sh b/benches/perf/bencher.sh new file mode 100644 index 000000000..17a764fc9 --- /dev/null +++ b/benches/perf/bencher.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e +export PGDOG_BIN=/usr/local/bin/pgdog + +# We have to make some changes to account for this being ran in a Firecracker microVM, not a Docker container (as +# this pgdog-base-runtime image was based on that) +printf '127.0.0.1 localhost\n::1 localhost\n' > /etc/hosts +ip link set lo up +mkdir -p /dev/shm && mount -t tmpfs tmpfs /dev/shm +chown -R postgres:postgres /var/lib/postgresql /etc/postgresql /var/log/postgresql +chgrp -R ssl-cert /etc/ssl/private +pg_ctlcluster 18 main start + +# This assembles into "Bencher Metric Format"; required because we don't use something like criterion here, +# we have our own custom runtime scripts +json="" +for name in "$@"; do + bash benches/perf/run.sh $name > $name.txt 2>&1 + cat $name.txt >&2 + tps=$(awk '/^tps/ {print $3}' $name.txt) + json+="\"$name\": {\"throughput\": {\"value\": $tps}}," +done + +echo "{${json%,}}" diff --git a/benches/perf/lb/run.sh b/benches/perf/lb/run.sh index 32af9f744..9b1ac007d 100644 --- a/benches/perf/lb/run.sh +++ b/benches/perf/lb/run.sh @@ -8,4 +8,4 @@ export PGHOST=127.0.0.1 export PGPORT=6432 pgbench -i -pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/../select_1.sql -P 1 +pgbench -c 10 -j 2 -T 180 -f ${SCRIPT_DIR}/../select_1.sql -P 1 diff --git a/benches/perf/pooler/run.sh b/benches/perf/pooler/run.sh index 32af9f744..9b1ac007d 100644 --- a/benches/perf/pooler/run.sh +++ b/benches/perf/pooler/run.sh @@ -8,4 +8,4 @@ export PGHOST=127.0.0.1 export PGPORT=6432 pgbench -i -pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/../select_1.sql -P 1 +pgbench -c 10 -j 2 -T 180 -f ${SCRIPT_DIR}/../select_1.sql -P 1 diff --git a/benches/perf/run.sh b/benches/perf/run.sh new file mode 100644 index 000000000..9d5add3b6 --- /dev/null +++ b/benches/perf/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source ${SCRIPT_DIR}/../../integration/common.sh + +run_pgdog benches/perf/$1 +wait_for_pgdog + +bash ${SCRIPT_DIR}/$1/run.sh + +stop_pgdog diff --git a/benches/perf/sharding/run.sh b/benches/perf/sharding/run.sh index 065852d2e..c24e40e71 100644 --- a/benches/perf/sharding/run.sh +++ b/benches/perf/sharding/run.sh @@ -7,4 +7,4 @@ export PGPORT=6432 psql -f ${SCRIPT_DIR}/setup.sql pgbench -i -pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/script.sql -P 1 --protocol extended +pgbench -c 10 -j 2 -T 180 -f ${SCRIPT_DIR}/script.sql -P 1 --protocol extended