From 819a425f4c1f0f6fb4a5124351b772af9feb7b6a Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 13:56:12 -0400 Subject: [PATCH 01/13] testing to see blacksmith run variance --- .github/workflows/benchmarks.yml | 30 ++++++++++++++++++++++++++++++ benches/perf/lb/run.sh | 2 +- benches/perf/pooler/run.sh | 2 +- benches/perf/run.sh | 11 +++++++++++ benches/perf/sharding/run.sh | 2 +- 5 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 benches/perf/run.sh mode change 100644 => 100755 benches/perf/sharding/run.sh diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 000000000..6a0561f78 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,30 @@ +name: benchmarks +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + bench: + runs-on: blacksmith-32vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v6 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - uses: Swatinem/rust-cache@v2 + with: + prefix-key: build-v1 + - name: Build + run: cargo build --release --bin pgdog + - name: Setup PostgreSQL + run: bash integration/ci/setup.sh + - name: Run benchmarks + env: + PGDOG_BIN: ${{ github.workspace }}/target/release/pgdog + shell: bash + run: | + for name in pooler lb sharding; do + bash benches/perf/run.sh $name | tee ${{ runner.temp }}/$name.txt + done + - name: TPS + run: grep '^tps' ${{ runner.temp }}/{pooler,lb,sharding}.txt diff --git a/benches/perf/lb/run.sh b/benches/perf/lb/run.sh index 32af9f744..c166f4173 100644 --- a/benches/perf/lb/run.sh +++ b/benches/perf/lb/run.sh @@ -8,4 +8,4 @@ export PGHOST=127.0.0.1 export PGPORT=6432 pgbench -i -pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/../select_1.sql -P 1 +pgbench -c 10 -j 2 -T 30 -f ${SCRIPT_DIR}/../select_1.sql -P 1 diff --git a/benches/perf/pooler/run.sh b/benches/perf/pooler/run.sh index 32af9f744..c166f4173 100644 --- a/benches/perf/pooler/run.sh +++ b/benches/perf/pooler/run.sh @@ -8,4 +8,4 @@ export PGHOST=127.0.0.1 export PGPORT=6432 pgbench -i -pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/../select_1.sql -P 1 +pgbench -c 10 -j 2 -T 30 -f ${SCRIPT_DIR}/../select_1.sql -P 1 diff --git a/benches/perf/run.sh b/benches/perf/run.sh new file mode 100644 index 000000000..9d5add3b6 --- /dev/null +++ b/benches/perf/run.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source ${SCRIPT_DIR}/../../integration/common.sh + +run_pgdog benches/perf/$1 +wait_for_pgdog + +bash ${SCRIPT_DIR}/$1/run.sh + +stop_pgdog diff --git a/benches/perf/sharding/run.sh b/benches/perf/sharding/run.sh old mode 100644 new mode 100755 index 065852d2e..179188feb --- a/benches/perf/sharding/run.sh +++ b/benches/perf/sharding/run.sh @@ -7,4 +7,4 @@ export PGPORT=6432 psql -f ${SCRIPT_DIR}/setup.sql pgbench -i -pgbench -c 10 -j 2 -t 10000000 -f ${SCRIPT_DIR}/script.sql -P 1 --protocol extended +pgbench -c 10 -j 2 -T 30 -f ${SCRIPT_DIR}/script.sql -P 1 --protocol extended From 186b80d17c91c0b3777d4f59980248615316abc1 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:06:08 -0400 Subject: [PATCH 02/13] Try out another run From 49d72281a25b61004b406663576b1ac982c33f80 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:06:41 -0400 Subject: [PATCH 03/13] Try out another run x2 From abb6809fb261384820fa418ad90d49d29e794397 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:06:44 -0400 Subject: [PATCH 04/13] Try out another run x3 From ee5c5aceb4fa87712888d5703db585629360053a Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:06:59 -0400 Subject: [PATCH 05/13] Try out another run x4 From 66b0a1e97348074f36a613302f057fe9bfbfff1a Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:18:35 -0400 Subject: [PATCH 06/13] Lets see what happens with macOS runners! --- .github/workflows/benchmarks.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6a0561f78..1b87c76aa 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -6,18 +6,23 @@ on: jobs: bench: - runs-on: blacksmith-32vcpu-ubuntu-2404 + runs-on: blacksmith-12vcpu-macos-latest steps: - uses: actions/checkout@v6 - - name: Install CI deps - run: bash integration/ci/install-deps.sh - uses: Swatinem/rust-cache@v2 with: prefix-key: build-v1 - name: Build run: cargo build --release --bin pgdog - name: Setup PostgreSQL - run: bash integration/ci/setup.sh + run: | + brew install postgresql@18 + echo "$(brew --prefix postgresql@18)/bin" >> "$GITHUB_PATH" + export PATH="$(brew --prefix postgresql@18)/bin:$PATH" + brew services start postgresql@18 + until pg_isready -q; do sleep 1; done + createdb + bash integration/setup.sh || (brew services restart postgresql@18 && until pg_isready -q; do sleep 1; done && bash integration/setup.sh) - name: Run benchmarks env: PGDOG_BIN: ${{ github.workspace }}/target/release/pgdog From af960b93a2695c8d77a2cd99479c38846517ede6 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:24:20 -0400 Subject: [PATCH 07/13] Try out another run x5 From a3bd36420438c9da3a95ac2fcec5dab68fec717f Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:24:25 -0400 Subject: [PATCH 08/13] Try out another run x6 From 72843bb7ff68d5d2b5c4f5af9aa66ecdd7fb0705 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:24:32 -0400 Subject: [PATCH 09/13] Try out another run x7 From b27c5b47bf89d400814558ad591785ae7e4094ba Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:44:24 -0400 Subject: [PATCH 10/13] Run main AND current PR changes in the workflow; prevent cross-runner variance --- .github/workflows/benchmarks.yml | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 1b87c76aa..3ed715b34 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -6,30 +6,34 @@ on: jobs: bench: - runs-on: blacksmith-12vcpu-macos-latest + runs-on: blacksmith-32vcpu-ubuntu-2404 steps: - uses: actions/checkout@v6 + - name: Install CI deps + run: bash integration/ci/install-deps.sh - uses: Swatinem/rust-cache@v2 with: prefix-key: build-v1 - - name: Build - run: cargo build --release --bin pgdog - - name: Setup PostgreSQL + - name: Build base + run: | + git fetch --depth=1 origin ${{ github.event.pull_request.base.sha || 'main' }} + git checkout FETCH_HEAD + cargo build --release --bin pgdog + cp target/release/pgdog ${{ runner.temp }}/pgdog-base + git checkout ${{ github.sha }} + - name: Build head run: | - brew install postgresql@18 - echo "$(brew --prefix postgresql@18)/bin" >> "$GITHUB_PATH" - export PATH="$(brew --prefix postgresql@18)/bin:$PATH" - brew services start postgresql@18 - until pg_isready -q; do sleep 1; done - createdb - bash integration/setup.sh || (brew services restart postgresql@18 && until pg_isready -q; do sleep 1; done && bash integration/setup.sh) + cargo build --release --bin pgdog + cp target/release/pgdog ${{ runner.temp }}/pgdog-head + - name: Setup PostgreSQL + run: bash integration/ci/setup.sh - name: Run benchmarks - env: - PGDOG_BIN: ${{ github.workspace }}/target/release/pgdog shell: bash run: | for name in pooler lb sharding; do - bash benches/perf/run.sh $name | tee ${{ runner.temp }}/$name.txt + for rev in base head; do + PGDOG_BIN=${{ runner.temp }}/pgdog-$rev bash benches/perf/run.sh $name | tee ${{ runner.temp }}/$name-$rev.txt + done done - name: TPS - run: grep '^tps' ${{ runner.temp }}/{pooler,lb,sharding}.txt + run: grep '^tps' ${{ runner.temp }}/{pooler,lb,sharding}-{base,head}.txt From 907c00f5fa48d7afabf9b36a87dd2c4ee292777f Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:49:29 -0400 Subject: [PATCH 11/13] Try out another run x1000 From 8ac9c4b618346d9544a7369c9d9e81f62234f757 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:49:34 -0400 Subject: [PATCH 12/13] Try out another run x1001 From ee71ec6d378258fb539f7c4d5ddbed3b83cb8043 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Tue, 22 Sep 2026 14:49:40 -0400 Subject: [PATCH 13/13] Try out another run x1002