Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
304 changes: 304 additions & 0 deletions docs/benchmarks/2026-10-12/baseline-refresh/README.md

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

32 changes: 32 additions & 0 deletions docs/benchmarks/2026-10-12/baseline-refresh/quick-run-1.log
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# run 1 (judge against stored baseline), 2026-10-12 15:19:24 +0200
## pmset before
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
## uptime
15:19 up 13 days, 4:25, 4 users, load averages: 2.74 4.29 5.58
SLOWER: tile t=1 total 0.70 -> 0.78 s (+10.9 %, band 5.0 %)
SLOWER: tile t=1 refine 0.47 -> 0.52 s (+11.2 %, band 5.0 %)
SLOWER: numedalslagen total 8.83 -> 9.48 s (+7.3 %, band 5.0 %)
SLOWER: numedalslagen decode 0.92 -> 1.00 s (+8.5 %, band 5.0 %)
SLOWER: numedalslagen features clip 6.00 -> 6.38 s (+6.3 %, band 5.0 %)
SLOWER: numedalslagen features clip: clean-up 5.94 -> 6.31 s (+6.3 %, band 5.0 %)
SLOWER: lagan total 14.54 -> 15.49 s (+6.5 %, band 5.0 %)
SLOWER: lagan features read 0.92 -> 1.10 s (+19.8 %, band 5.5 %)
SLOWER: lagan features clip 6.34 -> 6.69 s (+5.5 %, band 5.0 %)
SLOWER: lagan features clip: clean-up 5.39 -> 5.68 s (+5.4 %, band 5.0 %)
SLOWER: romsdal-slope total 3.58 -> 3.92 s (+9.5 %, band 5.0 %)
SLOWER: romsdal-slope refine 2.01 -> 2.24 s (+11.3 %, band 5.0 %)
SLOWER: romsdal-slope refine: scan (parallel) 0.56 -> 0.63 s (+11.5 %, band 5.0 %)
SLOWER: romsdal-slope refine: split + flip (serial) 1.18 -> 1.33 s (+12.5 %, band 5.0 %)
HOTSPOT: tile refine 44 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine 68 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine: scan (parallel) 50 % of the run (not in hotspots.toml)
HOTSPOT: quarter refine 46 % of the run (not in hotspots.toml)
HOTSPOT: numedalslagen features clip: clean-up 67 % of the run (not in hotspots.toml)
HOTSPOT: geilo-al-ramp decode 43 % of the run (not in hotspots.toml)
HOTSPOT: romsdal-slope refine 57 % of the run (not in hotspots.toml)
SLOWER
exit 1
## pmset after
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
37 changes: 37 additions & 0 deletions docs/benchmarks/2026-10-12/baseline-refresh/quick-run-2-save.log
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# run 2 (--save-baseline), 2026-10-12 15:30:37 +0200
## pmset before
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
## uptime
15:30 up 13 days, 4:36, 4 users, load averages: 3.02 3.43 4.32
SLOWER: tile t=1 total 0.70 -> 0.79 s (+12.8 %, band 5.0 %)
SLOWER: tile t=1 refine 0.47 -> 0.54 s (+15.5 %, band 5.0 %)
SLOWER: tile t=1 refine: scan (parallel) 0.35 -> 0.40 s (+14.3 %, band 5.0 %)
SLOWER: numedalslagen total 8.83 -> 9.34 s (+5.8 %, band 5.0 %)
SLOWER: numedalslagen decode 0.92 -> 0.99 s (+7.1 %, band 5.0 %)
SLOWER: numedalslagen features clip 6.00 -> 6.34 s (+5.7 %, band 5.0 %)
SLOWER: numedalslagen features clip: clean-up 5.94 -> 6.28 s (+5.7 %, band 5.0 %)
SLOWER: numedalslagen refine 0.66 -> 0.72 s (+9.1 %, band 5.0 %)
SLOWER: lagan total 14.54 -> 15.48 s (+6.5 %, band 5.0 %)
SLOWER: lagan features read 0.92 -> 1.09 s (+18.7 %, band 5.5 %)
SLOWER: lagan features clip 6.34 -> 6.74 s (+6.2 %, band 5.0 %)
SLOWER: lagan features clip: clean-up 5.39 -> 5.73 s (+6.2 %, band 5.0 %)
SLOWER: romsdal-slope total 3.58 -> 3.88 s (+8.3 %, band 5.0 %)
SLOWER: romsdal-slope refine 2.01 -> 2.24 s (+11.4 %, band 5.0 %)
SLOWER: romsdal-slope refine: scan (parallel) 0.56 -> 0.63 s (+10.9 %, band 5.0 %)
SLOWER: romsdal-slope refine: split + flip (serial) 1.18 -> 1.34 s (+12.9 %, band 5.0 %)
HOTSPOT: tile refine 44 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine 69 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine: scan (parallel) 51 % of the run (not in hotspots.toml)
HOTSPOT: quarter refine 47 % of the run (not in hotspots.toml)
HOTSPOT: numedalslagen features clip: clean-up 67 % of the run (not in hotspots.toml)
HOTSPOT: geilo-al-ramp decode 44 % of the run (not in hotspots.toml)
HOTSPOT: romsdal-slope refine 58 % of the run (not in hotspots.toml)
baseline saved: /Users/skavhaug/projects/rasputin/.claude/worktrees/baseline-1012/docs/benchmarks/quick/baseline-ac.json
SLOWER
exit 1
## pmset after
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
## uptime after
15:32 up 13 days, 4:39, 4 users, load averages: 4.51 3.69 4.27
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# run 3 (wait for load < 1.5, then --save-baseline), started 2026-10-12 15:34:34 +0200
15:34:34 load1 2.58
15:35:04 load1 2.38
15:35:34 load1 3.10
15:36:04 load1 2.91
15:36:34 load1 2.30
15:37:04 load1 2.74
15:37:34 load1 2.21
15:38:04 load1 2.49
15:38:46 load1 2.45
15:40:01 load1 2.51
16:00:17 load1 3.89
16:00:47 load1 3.72
16:01:17 load1 3.61
16:01:47 load1 2.92
16:02:17 load1 3.51
16:02:47 load1 3.50
16:03:17 load1 2.76
16:03:48 load1 2.37
16:04:18 load1 2.28
16:04:48 load1 3.02
16:05:18 load1 2.93
16:05:48 load1 2.38
16:06:18 load1 1.90
16:06:48 load1 1.87
16:07:18 load1 1.94
16:07:48 load1 1.87
16:08:18 load1 1.71
16:08:48 load1 2.13
16:09:18 load1 1.89
16:09:48 load1 2.10
16:10:18 load1 1.81
16:10:48 load1 1.50
16:11:18 load1 2.44
16:11:49 load1 2.29
16:12:19 load1 2.05
16:12:49 load1 1.86
16:13:19 load1 1.76
16:13:49 load1 1.84
16:14:19 load1 1.97
16:14:49 load1 1.90
16:15:19 load1 2.26
16:15:49 load1 3.75
16:16:19 load1 3.75
16:16:49 load1 3.64
16:17:19 load1 3.58
16:17:49 load1 2.68
16:18:19 load1 3.60
16:18:49 load1 4.61
16:19:19 load1 5.11
GAVE UP: the load never fell under 1.5 on two samples before 16:19:49
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# run 4 (wait for load < 2.0 twice, then --save-baseline), started 2026-10-12 20:15:03 +0200
20:15:03 load1 2.06
20:15:34 load1 2.09
20:16:04 load1 2.94
20:16:34 load1 3.32
20:17:04 load1 3.31
20:17:34 load1 2.81
20:18:04 load1 2.97
20:18:34 load1 3.03
20:19:04 load1 2.47
20:19:34 load1 2.03
20:20:04 load1 2.74
20:20:34 load1 2.34
20:21:04 load1 2.04
20:21:34 load1 2.09
20:22:04 load1 2.56
20:22:34 load1 2.50
20:23:04 load1 2.50
20:23:34 load1 2.73
20:24:04 load1 3.04
20:24:34 load1 2.63
20:25:04 load1 2.25
20:25:34 load1 2.10
20:26:04 load1 2.43
20:26:34 load1 2.65
20:27:05 load1 3.37
20:27:35 load1 3.04
20:28:05 load1 2.91
20:28:35 load1 2.51
20:29:05 load1 1.96
20:29:35 load1 1.76
## pmset before
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
## uptime
20:29 up 13 days, 9:35, 5 users, load averages: 1.76 2.49 5.13
## top CPU users
75.2 /Applications/iTerm.app/Contents/XPCServices/pidinfo.xpc/Contents/MacOS/pidinfo
16.2 /Applications/iTerm.app/Contents/MacOS/iTerm2
5.1 /System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.WebContent.xpc/Contents/MacOS/com.apple.WebKit.WebContent
3.8 /System/Volumes/Preboot/Cryptexes/App/System/Applications/Safari.app/Contents/MacOS/Safari
3.7 /Applications/Xerox/Xerox Workplace Cloud Client.app/Contents/MacOS/Xerox Workplace Cloud Client
1.7 /System/Library/CoreServices/Finder.app/Contents/MacOS/Finder
HOTSPOT: tile refine 45 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine 66 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine: scan (parallel) 50 % of the run (not in hotspots.toml)
HOTSPOT: quarter refine 47 % of the run (not in hotspots.toml)
HOTSPOT: numedalslagen features clip: clean-up 67 % of the run (not in hotspots.toml)
HOTSPOT: geilo-al-ramp decode 43 % of the run (not in hotspots.toml)
HOTSPOT: romsdal-slope refine 55 % of the run (not in hotspots.toml)
baseline saved: /Users/skavhaug/projects/rasputin/.claude/worktrees/baseline-1012/docs/benchmarks/quick/baseline-ac.json
NO BASELINE: input $RASPUTIN_DATA/sweden_glo30_cache changed
exit 2
## pmset after
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
## uptime after
20:31 up 13 days, 9:37, 5 users, load averages: 4.98 3.19 5.02
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# run 5 (increment 38, --tree /Users/skavhaug/projects/rasputin/.claude/worktrees/threads, judged against the baseline run 4 saved), started 2026-10-12 20:31:45 +0200
## tree 38 head
8f9289c49f69b95e6e9207a3b2f80ab2467d3351
## pmset before
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
## uptime
20:31 up 13 days, 9:37, 5 users, load averages: 4.98 3.19 5.02
## top CPU users
75.6 /Applications/iTerm.app/Contents/XPCServices/pidinfo.xpc/Contents/MacOS/pidinfo
11.0 /Applications/iTerm.app/Contents/MacOS/iTerm2
3.5 /usr/libexec/airportd
3.4 /Applications/Xerox/Xerox Workplace Cloud Client.app/Contents/MacOS/Xerox Workplace Cloud Client
3.0 /System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.WebContent.xpc/Contents/MacOS/com.apple.WebKit.WebContent
2.8 /System/Volumes/Preboot/Cryptexes/App/System/Applications/Safari.app/Contents/MacOS/Safari
HOTSPOT: tile refine 45 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine 67 % of the run (not in hotspots.toml)
HOTSPOT: tile t=1 refine: scan (parallel) 50 % of the run (not in hotspots.toml)
HOTSPOT: quarter refine 47 % of the run (not in hotspots.toml)
HOTSPOT: numedalslagen features clip: clean-up 67 % of the run (not in hotspots.toml)
HOTSPOT: geilo-al-ramp decode 43 % of the run (not in hotspots.toml)
HOTSPOT: romsdal-slope refine 56 % of the run (not in hotspots.toml)
NO CHANGE
exit 0
## pmset after
Now drawing from 'AC Power'
-InternalBattery-0 (id=24117347) 100%; charged; 0:00 remaining present: true
## uptime after
20:34 up 13 days, 9:40, 5 users, load averages: 3.57 3.24 4.73
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Stored baseline `357808b1` (2026-10-11) vs the refreshed baseline `969c49e2` (run 4, 2026-10-12 20:29, idle, AC)

Every phase at 5 % or more of its case's total, as `bench_quick.verdict`
selects them; band = max(5 %, the old record's (max - min) / median). The
tool itself printed `NO BASELINE: input $RASPUTIN_DATA/sweden_glo30_cache
changed` (the Lagan GLO-30 cache gained tiles), so this table was made by
calling `verdict` on the two records with the old inputs substituted into
the new one (`.venv/bin/python`, `tools/bench_quick.py` at `969c49e2`); the
judge lines it gives are:

SLOWER: lagan features read 0.92 -> 1.03 s (+12.1 %, band 5.5 %)

No `MESH CHANGED`, no `BROKEN`; every case's mesh SHA-256 and `max_error`
equal the stored ones.

| line | old s | new s | change | band |
|---|---:|---:|---:|---:|
| tile total | 0.416 | 0.430 | +3.3 % | 5.0 % |
| tile decode | 0.060 | 0.070 | +15.8 % | 5.5 % |
| tile start mesh: triangulate | 0.029 | 0.029 | -0.1 % | 5.0 % |
| tile refine | 0.190 | 0.192 | +1.2 % | 5.0 % |
| tile refine: scan (parallel) | 0.056 | 0.057 | +1.5 % | 5.0 % |
| tile refine: split + flip (serial) | 0.106 | 0.108 | +1.3 % | 6.2 % |
| tile refine: setup + output | 0.025 | 0.026 | +4.0 % | 5.0 % |
| tile t=1 total | 0.699 | 0.711 | +1.7 % | 5.0 % |
| tile t=1 decode | 0.060 | 0.068 | +13.3 % | 5.8 % |
| tile t=1 refine | 0.472 | 0.473 | +0.2 % | 5.0 % |
| tile t=1 refine: scan (parallel) | 0.352 | 0.352 | +0.1 % | 5.0 % |
| tile t=1 refine: split + flip (serial) | 0.094 | 0.094 | -0.1 % | 5.0 % |
| quarter total | 0.355 | 0.366 | +3.1 % | 5.0 % |
| quarter decode | 0.044 | 0.051 | +15.9 % | 5.7 % |
| quarter refine | 0.172 | 0.171 | -0.4 % | 5.6 % |
| quarter refine: scan (parallel) | 0.045 | 0.046 | +1.2 % | 5.0 % |
| quarter refine: split + flip (serial) | 0.107 | 0.106 | -1.4 % | 7.8 % |
| numedalslagen total | 8.829 | 8.894 | +0.7 % | 5.0 % |
| numedalslagen decode | 0.921 | 0.950 | +3.2 % | 5.0 % |
| numedalslagen features clip | 6.000 | 6.022 | +0.4 % | 5.0 % |
| numedalslagen features clip: clean-up | 5.937 | 5.956 | +0.3 % | 5.0 % |
| numedalslagen refine | 0.659 | 0.661 | +0.4 % | 5.0 % |
| lagan total | 14.542 | 14.736 | +1.3 % | 5.0 % |
| lagan decode | 4.578 | 4.615 | +0.8 % | 5.0 % |
| lagan features read | 0.922 | 1.033 | +12.1 % | 5.5 % |
| lagan features clip | 6.342 | 6.353 | +0.2 % | 5.0 % |
| lagan features clip: clean-up | 5.392 | 5.397 | +0.1 % | 5.0 % |
| geilo-al-ramp total | 0.329 | 0.344 | +4.7 % | 5.0 % |
| geilo-al-ramp decode | 0.138 | 0.148 | +7.2 % | 5.0 % |
| geilo-al-ramp refine | 0.099 | 0.101 | +1.5 % | 5.0 % |
| geilo-al-ramp refine: scan (parallel) | 0.083 | 0.083 | +1.0 % | 5.0 % |
| romsdal-slope total | 3.578 | 3.624 | +1.3 % | 5.0 % |
| romsdal-slope refine | 2.009 | 2.010 | +0.1 % | 5.0 % |
| romsdal-slope refine: scan (parallel) | 0.564 | 0.567 | +0.6 % | 5.0 % |
| romsdal-slope refine: split + flip (serial) | 1.184 | 1.183 | -0.1 % | 5.0 % |
| romsdal-slope refine: setup + output | 0.257 | 0.262 | +2.1 % | 5.0 % |

The small `decode` phases (tile, tile t=1, quarter, geilo-al-ramp: +7 to
+16 %) are over their percentage band but under the tool's 0.05 s floor
(0.007 to 0.010 s each), so the tool does not flag them; `lagan decode`,
the one large decode, is +0.8 %.
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
stored baseline commit 357808b1 vs run 2 (15:30, loaded machine) commit c317c7e4; medians of the timed runs, seconds

| case | phase | stored | run 2 | change | mesh same |
|---|---|---:|---:|---:|---|
| tile | total | 0.416 | 0.453 | +8.9 % | yes |
| tile | decode | 0.060 | 0.067 | +12.1 % | |
| tile | start mesh: triangulate | 0.029 | 0.030 | +2.5 % | |
| tile | refine | 0.190 | 0.201 | +5.9 % | |
| tile | refine: scan (parallel) | 0.056 | 0.061 | +8.0 % | |
| tile | refine: split + flip (serial) | 0.106 | 0.113 | +6.3 % | |
| tile | refine: setup + output | 0.025 | 0.027 | +6.4 % | |
| tile t=1 | total | 0.699 | 0.789 | +12.8 % | yes |
| tile t=1 | decode | 0.060 | 0.068 | +13.3 % | |
| tile t=1 | refine | 0.472 | 0.545 | +15.5 % | |
| tile t=1 | refine: scan (parallel) | 0.352 | 0.402 | +14.3 % | |
| tile t=1 | refine: split + flip (serial) | 0.094 | 0.113 | +20.5 % | |
| quarter | total | 0.355 | 0.388 | +9.3 % | yes |
| quarter | decode | 0.044 | 0.053 | +20.1 % | |
| quarter | refine | 0.172 | 0.181 | +5.4 % | |
| quarter | refine: scan (parallel) | 0.045 | 0.050 | +9.9 % | |
| quarter | refine: split + flip (serial) | 0.107 | 0.112 | +4.1 % | |
| numedalslagen | total | 8.829 | 9.343 | +5.8 % | yes |
| numedalslagen | decode | 0.921 | 0.986 | +7.1 % | |
| numedalslagen | features clip | 6.000 | 6.342 | +5.7 % | |
| numedalslagen | features clip: clean-up | 5.937 | 6.275 | +5.7 % | |
| numedalslagen | refine | 0.659 | 0.719 | +9.1 % | |
| lagan | total | 14.542 | 15.483 | +6.5 % | yes |
| lagan | decode | 4.578 | 4.783 | +4.5 % | |
| lagan | features read | 0.922 | 1.094 | +18.7 % | |
| lagan | features clip | 6.342 | 6.737 | +6.2 % | |
| lagan | features clip: clean-up | 5.392 | 5.726 | +6.2 % | |
| geilo-al-ramp | total | 0.329 | 0.364 | +10.9 % | yes |
| geilo-al-ramp | decode | 0.138 | 0.159 | +15.0 % | |
| geilo-al-ramp | refine | 0.099 | 0.107 | +7.9 % | |
| geilo-al-ramp | refine: scan (parallel) | 0.083 | 0.089 | +7.3 % | |
| romsdal-slope | total | 3.578 | 3.876 | +8.3 % | yes |
| romsdal-slope | refine | 2.009 | 2.238 | +11.4 % | |
| romsdal-slope | refine: scan (parallel) | 0.564 | 0.625 | +10.9 % | |
| romsdal-slope | refine: split + flip (serial) | 1.184 | 1.337 | +12.9 % | |
| romsdal-slope | refine: setup + output | 0.257 | 0.276 | +7.5 % | |
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/zsh
# Evening, 2026-10-12. Wait until the 1-minute load average is under 2.0 on
# two samples 30 s apart (at most 30 minutes), then:
# run 4: the quick check on this tree (master f09b9863 merged in) with
# --save-baseline, judged against the stored 357808b1 baseline;
# run 5: increment 38's tree (worktrees/threads) judged against the
# baseline run 4 just saved. Nothing else of ours runs beside them.
set -u
TREE=/Users/skavhaug/projects/rasputin/.claude/worktrees/baseline-1012
TREE38=/Users/skavhaug/projects/rasputin/.claude/worktrees/threads
DIR=$TREE/docs/benchmarks/2026-10-12/baseline-refresh
LOG4=$DIR/quick-run-4-idle-save.log
LOG5=$DIR/quick-run-5-increment-38.log
cd "$TREE" || exit 3
deadline=$(( $(date +%s) + 30 * 60 ))
low=0
echo "# run 4 (wait for load < 2.0 twice, then --save-baseline), started $(date '+%Y-%m-%d %H:%M:%S %z')" > "$LOG4"
while [ "$(date +%s)" -lt "$deadline" ]; do
load=$(sysctl -n vm.loadavg | awk '{print $2}')
echo "$(date +%H:%M:%S) load1 $load" >> "$LOG4"
if [ "$(echo "$load < 2.0" | bc)" = 1 ]; then low=$((low + 1)); else low=0; fi
if [ "$low" -ge 2 ]; then
{ echo "## pmset before"; pmset -g batt; echo "## uptime"; uptime;
echo "## top CPU users"; ps -eo pcpu,comm | sort -rn | head -6; } >> "$LOG4"
RASPUTIN_DATA=/Users/skavhaug/projects/rasputin_data .venv/bin/python tools/bench_quick.py run --save-baseline >> "$LOG4" 2>&1
echo "exit $?" >> "$LOG4"
{ echo "## pmset after"; pmset -g batt; echo "## uptime after"; uptime; } >> "$LOG4"
echo "# run 5 (increment 38, --tree $TREE38, judged against the baseline run 4 saved), started $(date '+%Y-%m-%d %H:%M:%S %z')" > "$LOG5"
{ echo "## tree 38 head"; git -C "$TREE38" rev-parse HEAD; echo "## pmset before"; pmset -g batt; echo "## uptime"; uptime;
echo "## top CPU users"; ps -eo pcpu,comm | sort -rn | head -6; } >> "$LOG5"
RASPUTIN_DATA=/Users/skavhaug/projects/rasputin_data .venv/bin/python tools/bench_quick.py run --tree "$TREE38" >> "$LOG5" 2>&1
echo "exit $?" >> "$LOG5"
{ echo "## pmset after"; pmset -g batt; echo "## uptime after"; uptime; } >> "$LOG5"
exit 0
fi
sleep 30
done
echo "GAVE UP: the load never fell under 2.0 on two samples before $(date '+%H:%M:%S')" >> "$LOG4"
exit 4
Loading
Loading