diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 7d884b49..7cbe6395 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -2,17 +2,24 @@ # with their binaries uploaded as artifacts. # This is done on GitHub infrastructure, to achieve maximum parallelization. # -# The benchmark itself runs on a self-hosted runner which we completely control; -# a machine with stable performance characteristics is what makes the results comparable at all. +# The benchmark itself runs on a Github-hosted Arm runner, a fresh 4-core VM for every job, so +# nothing about the machine carries over between runs and nothing else shares it. +# The two sides are measured alternately, one fork at a time, and never one after the other, so +# both meet the same machine and anything it does divides out of their ratio. That is worth about a +# third of the variance here; the rest is a speed offset drawn once per JVM, which no schedule can +# cancel and only more forks can average away. The fork count is what buys precision. # # A benchmark subjob produces no report of its own - -# it only uploads its raw JMH results as an artifact. +# it uploads one raw JMH result for each fork, which a merge step joins into the single-entry file +# the report expects. That merge, not JMH, computes the statistics: a fork keeps whatever shape it +# compiled for its whole life, so the fork is the unit, not the iteration. # After every subjob finishes, one summarizing job downloads all of them and aggregates them into a single table, # which is the only report this workflow produces. # -# Each benchmark gives a 99.9 % confidence interval. -# The confidence intervals are compared to determine if the branch under test is a regression or an improvement. -# The error threshold is expected to be below +/- 2.0 %. +# A row is judged on its delta against a fixed +/- 3 % band, and carries its resolution: +# the 99.9 % confidence interval on that delta, computed by pairing fork N of one side with fork N +# of the other. A row whose resolution is no better than the band could not have caught a +# regression worth failing on, and is marked as such. # # [1] Unless the baseline is a release tag, in which case its binaries are downloaded from a repository. # @@ -263,7 +270,8 @@ jobs: group: perf-score_director-${{ matrix.example }} cancel-in-progress: true needs: [ decisions, build_baseline, build_sut ] - runs-on: self-hosted # We need a stable machine to actually run the benchmarks. + runs-on: ubuntu-24.04-arm # Github-hosted Arm runner; free and GA for this public repo. + timeout-minutes: 300 strategy: fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue. matrix: @@ -291,6 +299,10 @@ jobs: with: repository: TimefoldAI/timefold-solver-benchmarks path: ./timefold-solver-benchmarks + # The harness and the binaries have to come from one commit, for the same reason + # build_baseline gives: pinned to a branch, a harness fix on any other branch + # was silently never exercised by this workflow. + ref: ${{ github.sha }} - name: Configure the benchmark working-directory: ./timefold-solver-benchmarks @@ -298,14 +310,45 @@ jobs: env: EXAMPLE: ${{ matrix.example }} run: | - echo "forks=20" > scoredirector-benchmark.properties - echo "warmup_iterations=5" >> scoredirector-benchmark.properties - echo "measurement_iterations=5" >> scoredirector-benchmark.properties - echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties + # - One fork for each invocation, on purpose. + # The two sides are measured alternately, one fork at a time, so that anything the + # machine does lands on both of them equally. Baseline fork N and SUT fork N run about a + # minute apart and their speeds correlate at +0.39, so the pairing is worth about a + # third of the variance. + # The fork count therefore lives in FORK_COUNT below, not here. + # - One measurement iteration for each fork, on purpose. + # A JVM keeps the shape it compiled for its whole life, so several iterations inside one + # fork are repeated looks at a single sample. JMH divides their spread by the square root + # of their count, which understates the error. + # With one iteration for each fork, every sample is its own JVM. + # - Three warmup iterations. + # What is left after the pairing is a speed offset drawn once per JVM: forks of one side + # do identical work and burn identical CPU, yet finish different numbers of operations, + # and the offset is already fully present in the first second of the measurement window + # and never moves. A longer measurement cannot average that away; only more forks can. + # So warmup is held to what the JIT needs and the rest of the clock goes to forks. + # Whether three is enough cannot be seen in these runs - at one measurement iteration + # the warmup curve is invisible. Checking it needs a one-off dispatch with several + # measurement iterations. + echo "forks=1" > scoredirector-benchmark.properties + echo "warmup_iterations=3" >> scoredirector-benchmark.properties + echo "measurement_iterations=1" >> scoredirector-benchmark.properties + # Only a console warning from Main, about one side's own spread. It is not what decides + # whether a row can be trusted - that is the report's resolution column, which compares + # the two sides fork by fork instead of looking at either one alone. + # Set loosely enough that it speaks up only when a side is genuinely wild. + echo "relative_score_error_threshold=0.05" >> scoredirector-benchmark.properties echo "score_director_type=cs" >> scoredirector-benchmark.properties echo "example=$EXAMPLE" >> scoredirector-benchmark.properties cat scoredirector-benchmark.properties chmod +x run-scoredirector.sh + # How many forks each side gets, spent one at a time by the loop below. + # Sized to the clock: a fork costs about 10 s for each warmup and measurement iteration + # plus startup, and the job holds two sides, so 40 forks land it near 65 minutes. + # Resolution improves only as the square root of this, so doubling precision costs four + # times the job. At 40 every example resolves to better than the +/- 3 % band except + # patient_admission_scheduling, which sits just over it and carries the marker. + echo "FORK_COUNT=40" >> "$GITHUB_ENV" - name: Download the baseline binary uses: actions/download-artifact@v8 @@ -318,31 +361,17 @@ jobs: name: benchmarks-sut path: ./timefold-solver-benchmarks + # Both JDKs are installed before the benchmark, because the loop below switches between them + # for every fork and a step-level action cannot run inside a loop. - name: (Baseline) Setup JDK and Maven uses: actions/setup-java@v6.0.1 with: java-version: ${{ github.event.inputs.jdk_baseline }} distribution: 'temurin' check-latest: true - - - name: (Baseline) Run the benchmark - working-directory: ./timefold-solver-benchmarks + - name: (Baseline) Record the JDK shell: bash - env: - RUN_ID: ${{ env.BASELINE_DIR }} - TIMEFOLD_LICENSE: ${{ secrets.TIMEFOLD_SOLVER_CI_PROD_LICENSE }} - run: | - mkdir target - cp benchmarks-baseline.jar target/benchmarks.jar - ./run-scoredirector.sh - cp results/scoredirector/$RUN_ID/results.json baseline-results.json - # JMH does not fail the process when one benchmark trial errors out - it just omits it - # from the results, so a crashed run can still exit 0. One example must be present. - COUNT=$(jq 'length' baseline-results.json) - if [ "$COUNT" -ne 1 ]; then - echo "::error::Baseline run produced $COUNT/1 benchmark results - it did not produce metrics." - exit 1 - fi + run: echo "JAVA_HOME_BASELINE=$JAVA_HOME" >> "$GITHUB_ENV" - name: (SUT) Setup JDK and Maven uses: actions/setup-java@v6.0.1 @@ -350,23 +379,173 @@ jobs: java-version: ${{ github.event.inputs.jdk_branch }} distribution: 'temurin' check-latest: true + - name: (SUT) Record the JDK + shell: bash + run: echo "JAVA_HOME_SUT=$JAVA_HOME" >> "$GITHUB_ENV" + + - name: Verify the JDKs + shell: bash + env: + JDK_BASELINE: ${{ github.event.inputs.jdk_baseline }} + JDK_BRANCH: ${{ github.event.inputs.jdk_branch }} + run: | + # Fail now rather than benchmark the wrong JVM for an hour. + check_jdk() { # label, java home, requested major version + if [ -z "$2" ] || [ ! -x "$2/bin/java" ]; then + echo "::error::The $1 JDK is missing at '$2'." + exit 1 + fi + # java.specification.version is a bare major version; the -version banner is not. + ACTUAL=$("$2/bin/java" -XshowSettings:properties -version 2>&1 \ + | sed -n 's/.*java\.specification\.version = //p' | tr -d '[:space:]') + if [ "$ACTUAL" != "$3" ]; then + echo "::error::The $1 JDK is $ACTUAL, but $3 was requested." + exit 1 + fi + echo "$1 JDK $ACTUAL at $2" + } + check_jdk baseline "$JAVA_HOME_BASELINE" "$JDK_BASELINE" + check_jdk sut "$JAVA_HOME_SUT" "$JDK_BRANCH" + # setup-java hands back the same installation for the same version, + # so the two may only differ when they were asked to. + if [ "$JDK_BASELINE" != "$JDK_BRANCH" ] && [ "$JAVA_HOME_BASELINE" = "$JAVA_HOME_SUT" ]; then + echo "::error::Different JDKs were requested, but both sides resolved to $JAVA_HOME_BASELINE." + exit 1 + fi + + - name: Prepare the machine for profiling + shell: bash + run: | + # The async profiler needs perf_event_paranoid at 1 or less to open perf events, and + # kptr_restrict at 0 to resolve kernel symbols. Done once, here, rather than once for + # every fork inside run-scoredirector.sh. + # Both currently apply: the recordings carry engine="perf_events" and + # kernelSymbols="true", which is the check to repeat after any runner change - + # jfr print --events ActiveSetting + # The runner image is Github's, not ours, so these have to be set here on every job and + # can only be asked for, never assumed. "sudo -n" fails immediately rather than waiting + # on a password prompt no job can answer, and the warning below carries the reason it + # gave, because a silent fallback to the ctimer engine is easy to miss. + ensure_sysctl() { # key, wanted value, comparison + # Try to set it, then report what the kernel actually holds - the attempt succeeding + # is not the same as the value being right. Carry sudo's own complaint into the + # warning, flattened to one line, because that names the reason. + REASON=$(sudo -n sysctl -q "$1=$2" 2>&1 | tr '\n' ' ') || true + ACTUAL=$(cat "/proc/sys/${1//.//}" 2>/dev/null || echo unknown) + if [ "$ACTUAL" != unknown ] && [ "$ACTUAL" -"$3" "$2" ] 2>/dev/null; then + echo "$1 = $ACTUAL" + else + echo "::warning::$1 is $ACTUAL, not $2; the profiler falls back to its ctimer engine and records no kernel symbols. $REASON" + fi + } + ensure_sysctl kernel.perf_event_paranoid 1 le + ensure_sysctl kernel.kptr_restrict 0 eq - - name: (SUT) Run the benchmark + - name: Run the benchmark working-directory: ./timefold-solver-benchmarks shell: bash env: - RUN_ID: ${{ env.SUT_DIR }} TIMEFOLD_LICENSE: ${{ secrets.TIMEFOLD_SOLVER_CI_PROD_LICENSE }} run: | - rm target/benchmarks.jar - cp benchmarks-sut.jar target/benchmarks.jar - ./run-scoredirector.sh - cp results/scoredirector/$RUN_ID/results.json sut-results.json - # JMH does not fail the process when one benchmark trial errors out - it just omits it - # from the results, so a crashed run can still exit 0. One example must be present. - COUNT=$(jq 'length' sut-results.json) - if [ "$COUNT" -ne 1 ]; then - echo "::error::SUT run produced $COUNT/1 benchmark results - it did not produce metrics." + # RUN_ID may hold a slash, so each fork writes its own results.json and its own JFR + # instead of overwriting the one before it. + mkdir -p logs + run_fork() { # side, fork id + case $1 in + baseline) HOME_DIR=$JAVA_HOME_BASELINE; RESULT_DIR=$BASELINE_DIR ;; + sut) HOME_DIR=$JAVA_HOME_SUT; RESULT_DIR=$SUT_DIR ;; + esac + RESULT_FILE="results/scoredirector/$RESULT_DIR/fork-$2/results.json" + LOG="logs/$1-fork-$2.log" + for ATTEMPT in 1 2; do + STARTED=$SECONDS + # Every one of these invocations is a single fork, which JMH announces as + # "Fork: 1 of 1", so its output cannot say how far along the run is. + # Keep it in a file and print one line for each fork instead. + JAVA_HOME="$HOME_DIR" BENCHMARK_JAR="benchmarks-$1.jar" RUN_ID="$RESULT_DIR/fork-$2" \ + ./run-scoredirector.sh --skip-flamegraphs > "$LOG" 2>&1 || true + # JMH does not fail the process when a benchmark trial errors out - it just omits it + # from the results, so a crashed run can still exit 0. + if [ -s "$RESULT_FILE" ]; then + printf '%10s fork %3s / %3s: %8.0f ops/s in %3s s\n' "$1" "$2" "$FORK_COUNT" \ + "$(jq -r '.[0].primaryMetric.score' "$RESULT_FILE")" "$((SECONDS - STARTED))" + return 0 + fi + # A retried fork runs later than its partner on the other side, which weakens the + # pairing that the alternation exists to create. Say so loudly, with the evidence: + # this is the only place that log is ever read. + echo "::warning::The $1 fork-$2 produced no results on attempt $ATTEMPT." + tail -n 30 "$LOG" + done + echo "::error::The $1 fork-$2 produced no results, even after a retry." + exit 1 + } + for FORK in $(seq -w 1 "$FORK_COUNT"); do + run_fork baseline "$FORK" + run_fork sut "$FORK" + done + + - name: Merge the results + working-directory: ./timefold-solver-benchmarks + shell: bash + env: + JDK_BASELINE: ${{ github.event.inputs.jdk_baseline }} + JDK_BRANCH: ${{ github.event.inputs.jdk_branch }} + run: | + merge_side() { # side, result dir, results file, requested major version + COUNT=$(find "results/scoredirector/$2" -name results.json | wc -l) + if [ "$COUNT" -ne "$FORK_COUNT" ]; then + echo "::error::The $1 side produced $COUNT/$FORK_COUNT fork results." + exit 1 + fi + python3 merge-scoredirector-results.py \ + results/scoredirector/"$2"/fork-*/results.json --output "$3" --expect-jdk "$4" + COUNT=$(jq 'length' "$3") + if [ "$COUNT" -ne 1 ]; then + echo "::error::The $1 side merged to $COUNT/1 benchmark results." + exit 1 + fi + # JMH names every fork's recording jfr-cpu.jfr in one directory for each benchmark, + # with no fork index, so a single multi-fork run kept only the last fork's profile. + # Concatenated recordings are one valid multi-chunk file, in which chunk N is fork N + # byte for byte, so nothing is lost by keeping only the merge: + # jfr disassemble --max-chunks 1 --output jfr-cpu.jfr + RECORDINGS=$(find "results/scoredirector/$2" -path '*/fork-*/*' -name 'jfr-cpu.jfr' | sort) + if [ -z "$RECORDINGS" ]; then + # Only happens when the async profiler could not be loaded; it warns and carries on. + echo "::warning::The $1 side produced no JFR recordings." + return 0 + fi + echo "$RECORDINGS" | xargs cat > "results/scoredirector/$2/jfr-cpu.jfr" + echo "$RECORDINGS" | xargs rm + # One fork holds too few samples to draw a useful flame graph, so the graphs are made + # here, once, from the merge. The per-fork recordings are gone, so the conversion + # finds exactly this one file. + JAVA_HOME="$5" BENCHMARK_JAR="benchmarks-$1.jar" RUN_ID="$2" \ + ./run-scoredirector.sh --flamegraphs-only + } + merge_side baseline "$BASELINE_DIR" baseline-results.json "$JDK_BASELINE" "$JAVA_HOME_BASELINE" + merge_side sut "$SUT_DIR" sut-results.json "$JDK_BRANCH" "$JAVA_HOME_SUT" + + - name: Verify what actually ran + working-directory: ./timefold-solver-benchmarks + shell: bash + env: + JDK_BASELINE: ${{ github.event.inputs.jdk_baseline }} + JDK_BRANCH: ${{ github.event.inputs.jdk_branch }} + run: | + # JMH records the JVM that ran the benchmark, which is evidence, unlike the environment + # this job set up for it. + BASELINE_JVM=$(jq -r '.[0].jvm' baseline-results.json) + SUT_JVM=$(jq -r '.[0].jvm' sut-results.json) + echo "baseline ran on $BASELINE_JVM" + echo "sut ran on $SUT_JVM" + if [ "$JDK_BASELINE" = "$JDK_BRANCH" ] && [ "$BASELINE_JVM" != "$SUT_JVM" ]; then + echo "::error::One JDK was requested for both sides, but they ran on different JVMs." + exit 1 + fi + if [ "$JDK_BASELINE" != "$JDK_BRANCH" ] && [ "$BASELINE_JVM" = "$SUT_JVM" ]; then + echo "::error::Different JDKs were requested, but one JVM ran both sides." exit 1 fi @@ -380,11 +559,13 @@ jobs: uses: actions/upload-artifact@v7 with: name: assets-${{ matrix.example }} + # The recording and its flame graphs now sit directly in the side's directory, one file + # for each side; the fork-* subdirectories below it hold only results.json. path: | - ./timefold-solver-benchmarks/results/scoredirector/${{ env.BASELINE_DIR }}/**/*.jfr - ./timefold-solver-benchmarks/results/scoredirector/${{ env.BASELINE_DIR }}/**/*.html - ./timefold-solver-benchmarks/results/scoredirector/${{ env.SUT_DIR }}/**/*.jfr - ./timefold-solver-benchmarks/results/scoredirector/${{ env.SUT_DIR }}/**/*.html + ./timefold-solver-benchmarks/results/scoredirector/${{ env.BASELINE_DIR }}/*.jfr + ./timefold-solver-benchmarks/results/scoredirector/${{ env.BASELINE_DIR }}/*.html + ./timefold-solver-benchmarks/results/scoredirector/${{ env.SUT_DIR }}/*.jfr + ./timefold-solver-benchmarks/results/scoredirector/${{ env.SUT_DIR }}/*.html if-no-files-found: warn # Empty when the step above found no assets (if-no-files-found: warn uploads nothing, so @@ -414,6 +595,9 @@ jobs: uses: actions/checkout@v7 with: path: ./timefold-solver-benchmarks + # Same reason as the benchmark job: the summarizer has to come from the commit + # that dispatched this run, or a change to it is never exercised before it is merged. + ref: ${{ github.sha }} - name: Setup Python uses: actions/setup-python@v7 diff --git a/merge-scoredirector-results.py b/merge-scoredirector-results.py new file mode 100644 index 00000000..984a15dc --- /dev/null +++ b/merge-scoredirector-results.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Merge the single-fork JMH result files of one side into the one file the report expects. + +CI runs the baseline and the SUT one fork at a time, alternating, so that anything the machine +does lands on both sides equally instead of on whichever block it happens to cover. That leaves one +results.json for each fork. This script joins them back into a single entry for each benchmark, identical in +shape to what a single multi-fork JMH run would have written, so summarize-scoredirector.py needs +no knowledge of how the measurements were collected. + +A one-fork JMH file reports "scoreError": NaN, which the summarizer reads as "cannot say which side +is faster". The merge therefore has to happen before the data artifact is built, not in the report. + +Usage: + python3 merge-scoredirector-results.py FORK_RESULTS... -o MERGED [--expect-jdk MAJOR] + python3 merge-scoredirector-results.py --selftest + +Pass the fork files in the order they ran; the merged rawData keeps that order, which is what makes +fork N of the merged JFR line up with fork N here. +""" +import argparse +import json +import math +import statistics +import sys + +# JMH reports the half-width of a 99.9 % confidence interval, so each tail holds 0.05 %. +CONFIDENCE_TAIL = 0.0005 + +# A fork keeps whatever shape the JIT gave it for its whole life, so the iterations inside one fork +# are repeated looks at one sample, not independent ones. Every statistic below is therefore +# computed over fork means. With one measurement iteration for each fork this is exactly what JMH +# would have computed itself; it stays correct if the iteration count is ever raised again. +PERCENTILES = ("0.0", "50.0", "90.0", "95.0", "99.0", "99.9", "99.99", "99.999", "99.9999", "100.0") + +# Fields describing the JVM that ran the benchmark. They must agree across every fork of one side - +# a fork that quietly ran on another JDK makes the comparison meaningless, so it is an error here +# rather than something to notice later in a report. +JVM_FIELDS = ("jvm", "jdkVersion", "vmName", "vmVersion") + + +def _incomplete_beta(a: float, b: float, x: float) -> float: + """Regularised incomplete beta, by the usual continued fraction. Avoids a scipy dependency.""" + if x <= 0: + return 0.0 + if x >= 1: + return 1.0 + + def fraction(a: float, b: float, x: float) -> float: + tiny = 1e-30 + c, d = 1.0, 1 - (a + b) * x / (a + 1) + d = 1 / (d if abs(d) > tiny else tiny) + h = d + for m in range(1, 300): + m2 = 2 * m + for numerator in (m * (b - m) * x / ((a - 1 + m2) * (a + m2)), + -(a + m) * (a + b + m) * x / ((a + m2) * (a + 1 + m2))): + d = 1 + numerator * d + d = 1 / (d if abs(d) > tiny else tiny) + c = 1 + numerator / (c if abs(c) > tiny else tiny) + c = c if abs(c) > tiny else tiny + delta = d * c + h *= delta + if abs(delta - 1) < 3e-16: + break + return h + + log_beta = math.lgamma(a) + math.lgamma(b) - math.lgamma(a + b) + front = math.exp(a * math.log(x) + b * math.log(1 - x) - log_beta) + if x < (a + 1) / (a + b + 2): + return front / a * fraction(a, b, x) + return 1 - math.exp(b * math.log(1 - x) + a * math.log(x) - log_beta) / b * fraction(b, a, 1 - x) + + +def t_upper_tail(t: float, degrees_of_freedom: int) -> float: + return _incomplete_beta(degrees_of_freedom / 2, 0.5, + degrees_of_freedom / (degrees_of_freedom + t * t)) / 2 + + +def t_quantile(tail: float, degrees_of_freedom: int) -> float: + """The t value whose upper tail is `tail`, found by bisection. Monotonic, so this is safe.""" + low, high = 0.0, 1.0 + while t_upper_tail(high, degrees_of_freedom) > tail: + high *= 2 + if high > 1e6: + raise ValueError("The t quantile did not bracket; degrees of freedom (%d) must be positive." + % degrees_of_freedom) + for _ in range(200): + middle = (low + high) / 2 + if t_upper_tail(middle, degrees_of_freedom) > tail: + low = middle + else: + high = middle + return (low + high) / 2 + + +def percentile(sorted_values: list, rank: float) -> float: + """Linear interpolation between the neighbouring values, as JMH's own percentiles do.""" + if not sorted_values: + return math.nan + position = (len(sorted_values) - 1) * rank / 100 + below = math.floor(position) + above = min(below + 1, len(sorted_values) - 1) + return sorted_values[below] + (sorted_values[above] - sorted_values[below]) * (position - below) + + +def entry_key(entry: dict) -> str: + return json.dumps([entry["benchmark"], entry["params"]], sort_keys=True) + + +def merge_entries(entries: list) -> dict: + """Builds one JMH entry out of the same benchmark run once for each fork.""" + first = entries[0] + for other in entries[1:]: + for field in JVM_FIELDS: + if other[field] != first[field]: + raise ValueError("The %s (%s) of one fork does not match the other forks (%s). " + "Maybe the two sides did not run on the same JDK." + % (field, other[field], first[field])) + + raw_data = [fork for entry in entries for fork in entry["primaryMetric"]["rawData"]] + iteration_counts = {len(fork) for fork in raw_data} + if len(iteration_counts) != 1: + raise ValueError("The forks (%s) do not all have the same iteration count." + % sorted(iteration_counts)) + fork_means = [statistics.fmean(fork) for fork in raw_data] + fork_count = len(fork_means) + if fork_count < 2: + raise ValueError("The forkCount (%d) must be at least 2 for an error to be computable." + % fork_count) + + score = statistics.fmean(fork_means) + error = t_quantile(CONFIDENCE_TAIL, fork_count - 1) \ + * statistics.stdev(fork_means) / math.sqrt(fork_count) + ordered = sorted(value for fork in raw_data for value in fork) + + merged = dict(first) + merged["forks"] = fork_count + merged["measurementIterations"] = iteration_counts.pop() + merged["primaryMetric"] = dict(first["primaryMetric"]) + merged["primaryMetric"].update({ + "score": score, + "scoreError": error, + "scoreConfidence": [score - error, score + error], + "scorePercentiles": {rank: percentile(ordered, float(rank)) for rank in PERCENTILES}, + "rawData": raw_data, + }) + return merged + + +def merge_files(paths: list, expected_jdk: "str | None" = None) -> list: + grouped = {} + for path in paths: + with open(path) as file: + for entry in json.load(file): + grouped.setdefault(entry_key(entry), []).append(entry) + if not grouped: + raise ValueError("The fileCount (%d) held no benchmark results." % len(paths)) + + merged = [merge_entries(entries) for entries in grouped.values()] + if expected_jdk is not None: + for entry in merged: + actual = str(entry["jdkVersion"]) + if actual != expected_jdk and not actual.startswith(expected_jdk + "."): + raise ValueError("The jdkVersion (%s) of benchmark (%s) is not the requested " + "major version (%s)." % (actual, entry["benchmark"], expected_jdk)) + return merged + + +def _selftest() -> None: + # JMH itself used 3.392 for 100 iterations; recovering that number validates the quantile. + assert abs(t_quantile(CONFIDENCE_TAIL, 99) - 3.392) < 0.001, t_quantile(CONFIDENCE_TAIL, 99) + for degrees_of_freedom, expected in ((1, 636.619), (10, 4.587), (19, 3.883), (25, 3.725)): + actual = t_quantile(CONFIDENCE_TAIL, degrees_of_freedom) + assert abs(actual - expected) < max(0.001, expected * 1e-5), (degrees_of_freedom, actual) + + assert percentile([1.0, 2.0, 3.0], 0.0) == 1.0 + assert percentile([1.0, 2.0, 3.0], 50.0) == 2.0 + assert percentile([1.0, 2.0, 3.0], 100.0) == 3.0 + + def fork(*values, jvm="/jdk25/bin/java", version="25.0.4.1"): + return {"benchmark": "Bench.run", "params": {"csExample": "CLOUD_BALANCING"}, + "jvm": jvm, "jdkVersion": version, "vmName": "OpenJDK 64-Bit Server VM", + "vmVersion": "25+1", "forks": 1, "measurementIterations": len(values), + "primaryMetric": {"score": statistics.fmean(values), "scoreError": float("nan"), + "scoreConfidence": [float("nan"), float("nan")], + "scorePercentiles": {}, "scoreUnit": "ops/s", + "rawData": [list(values)]}} + + merged = merge_entries([fork(100.0), fork(102.0), fork(98.0), fork(104.0)]) + assert merged["forks"] == 4 and merged["measurementIterations"] == 1 + assert merged["primaryMetric"]["score"] == 101.0 + expected_error = t_quantile(CONFIDENCE_TAIL, 3) * statistics.stdev([100.0, 102.0, 98.0, 104.0]) / 2 + assert abs(merged["primaryMetric"]["scoreError"] - expected_error) < 1e-9 + low, high = merged["primaryMetric"]["scoreConfidence"] + assert abs(high - low - 2 * expected_error) < 1e-9 + assert merged["primaryMetric"]["rawData"] == [[100.0], [102.0], [98.0], [104.0]], "fork order is kept" + assert merged["primaryMetric"]["scorePercentiles"]["0.0"] == 98.0 + assert merged["primaryMetric"]["scoreUnit"] == "ops/s", "unrelated fields survive" + + # Iterations inside a fork are not independent, so they must not inflate the sample count. + spread = merge_entries([fork(100.0, 100.0, 100.0), fork(104.0, 104.0, 104.0)]) + assert spread["measurementIterations"] == 3 and spread["forks"] == 2 + assert spread["primaryMetric"]["score"] == 102.0 + two_fork_error = t_quantile(CONFIDENCE_TAIL, 1) * statistics.stdev([100.0, 104.0]) / math.sqrt(2) + assert abs(spread["primaryMetric"]["scoreError"] - two_fork_error) < 1e-9 + + for broken, reason in ( + ([fork(100.0), fork(102.0, jvm="/jdk21/bin/java")], "a different JVM"), + ([fork(100.0), fork(102.0, version="21.0.1")], "a different JDK version"), + ([fork(100.0), fork(102.0, 103.0)], "a different iteration count"), + ([fork(100.0)], "a single fork")): + try: + merge_entries(broken) + raise AssertionError("merge_entries accepted " + reason) + except ValueError: + pass + + print("merge-scoredirector-results.py: selftest OK") + + +def main() -> None: + if "--selftest" in sys.argv: + _selftest() + return + + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("results", nargs="+", help="Single-fork results.json files, in the order they ran") + parser.add_argument("-o", "--output", required=True, help="Where to write the merged results.json") + parser.add_argument("--expect-jdk", help="Major JDK version every fork must report, e.g. 25") + parser.add_argument("--selftest", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + + try: + merged = merge_files(args.results, args.expect_jdk) + except ValueError as error: + # A traceback tells a CI log reader nothing they can act on. + print("::error::%s" % error, file=sys.stderr) + sys.exit(1) + with open(args.output, "w") as file: + json.dump(merged, file, indent=4) + for entry in merged: + metric = entry["primaryMetric"] + print("%s: %d forks, %.3f %s ± %.2f %%" + % (entry["benchmark"], entry["forks"], metric["score"], metric["scoreUnit"], + abs(metric["scoreError"] / metric["score"]) * 100)) + + +if __name__ == "__main__": + main() diff --git a/run-scoredirector.sh b/run-scoredirector.sh index 16b2676e..fea4e6be 100755 --- a/run-scoredirector.sh +++ b/run-scoredirector.sh @@ -1,4 +1,10 @@ #!/bin/bash -sudo -i sysctl kernel.perf_event_paranoid=1 -sudo -i sysctl kernel.kptr_restrict=0 -java -cp target/benchmarks.jar -Djmh.ignoreLock=true ai.timefold.solver.benchmarks.micro.scoredirector.Main +# The async profiler needs kernel.perf_event_paranoid at 1 or less and kernel.kptr_restrict at 0. +# Setting them belongs to whoever prepares the machine, once - see "Prepare the machine for +# profiling" in performance_score_director.yml - not to this script, which runs once for every fork. +# JAVA_HOME picks the JDK, instead of whichever java happens to come first on the PATH; +# JMH forks its children from the host process, so pinning the host pins every fork. +# BENCHMARK_JAR lets the caller alternate between two binaries without copying a file +# between measurements. Both default to what a plain local run expects. +"${JAVA_HOME:+$JAVA_HOME/bin/}java" -cp "${BENCHMARK_JAR:-target/benchmarks.jar}" \ + ai.timefold.solver.benchmarks.micro.scoredirector.Main "$@" diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java b/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java index 47248eee..2c78e655 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/common/AbstractMain.java @@ -50,6 +50,8 @@ import org.slf4j.LoggerFactory; import one.convert.Arguments; +import one.convert.JfrToFlame; +import one.convert.JfrToHeatmap; import one.profiler.AsyncProfilerLoader; public abstract class AbstractMain { @@ -146,9 +148,9 @@ private void visualizeJfr(Path jfrFilePath, VisualizationType visualizationType, var args = argStream.toArray(String[]::new); try { if (visualizationType == VisualizationType.FLAME_GRAPH) { - one.convert.JfrToFlame.convert(inputPath.toString(), output.toString(), new Arguments(args)); + JfrToFlame.convert(inputPath.toString(), output.toString(), new Arguments(args)); } else if (visualizationType == VisualizationType.HEAT_MAP) { - one.convert.JfrToHeatmap.convert(inputPath.toString(), output.toString(), new Arguments(args)); + JfrToHeatmap.convert(inputPath.toString(), output.toString(), new Arguments(args)); } else { throw new IllegalArgumentException("Unsupported visualization: " + visualizationType); } @@ -188,7 +190,7 @@ public ChainedOptionsBuilder getBaseJmhConfig(C configuration) { // the thread keeps running in the interpreter or C1 while C2 compiles, // so which profile the final code is built from is a race. // Each JVM lost or won it once and kept that shape for its whole life, - // which would split the forks of one benchmark into two speeds 10-30 % apart. + // which split the forks of the move-provider benchmark into two distinct speeds. "-Xbatch") .result(resultsDirectory.resolve("results.json").toAbsolutePath().toString()) .resultFormat(ResultFormatType.JSON) diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java index fcdcfc35..63036152 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Set; import ai.timefold.solver.benchmarks.micro.common.AbstractMain; @@ -42,6 +43,19 @@ public final class Main extends AbstractMain { + /** + * Convert the JFR recordings already in the results directory and do nothing else. + * CI runs one fork for each invocation, so the flame graphs are made once, at the end, + * from the concatenation of every fork's recording. + */ + private static final String FLAME_GRAPHS_ONLY = "--flamegraphs-only"; + /** + * Run the benchmark, but leave the JFR recording alone. + * One fork does not have enough samples to draw a useful flame graph, + * and converting each of them would multiply the size of the archived assets. + */ + private static final String SKIP_FLAME_GRAPHS = "--skip-flamegraphs"; + public Main() { super("scoredirector"); } @@ -61,15 +75,30 @@ public static void main(String[] args) throws RunnerException, IOException { } public void run(String[] args) throws RunnerException, IOException { + var arguments = Set.of(args); + if (arguments.contains(FLAME_GRAPHS_ONLY)) { + convertJfrToFlameGraphs(); + return; + } var configuration = readConfiguration(); - var options = getBaseJmhConfig(configuration); + var options = getBaseJmhConfig(configuration) + // Commit and touch the whole heap at startup, instead of faulting it in during warmup. + // Forks of one side do identical work and burn identical CPU, yet differ in how many + // operations they finish, by a constant that is already fully present in the first second of + // measurement and never changes afterwards - a speed drawn once per JVM. Lazy page backing of + // the heap fits that shape, so this takes one candidate draw away. It has not been shown to + // help; it is kept because it is cheap, and it is a hedge, not a fix. + // Appended here, not in AbstractMain: coldstart measures startup, which is what this costs. + .jvmArgsAppend("-XX:+AlwaysPreTouch"); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED); options = initAsyncProfiler(options); var runner = new Runner(options.build()); var runResults = runner.run(); - convertJfrToFlameGraphs(); + if (!arguments.contains(SKIP_FLAME_GRAPHS)) { + convertJfrToFlameGraphs(); + } var relativeScoreErrorThreshold = configuration.getRelativeScoreErrorThreshold(); var thresholdForPrint = ((int) Math.round(relativeScoreErrorThreshold * 10_000)) / 100.0D; diff --git a/summarize-scoredirector.py b/summarize-scoredirector.py index 000797a7..918f8e5f 100644 --- a/summarize-scoredirector.py +++ b/summarize-scoredirector.py @@ -7,6 +7,10 @@ regresses or improves, and prints one markdown table plus a legend. Its exit code is the CI verdict: 0 only if every expected row is present and either within tolerance or an improvement. +Each row also reports its resolution: how small a difference this particular run could have told +apart from zero. That is the trust signal - a delta only means something when the run that measured +it could resolve something that size. + Usage: python3 summarize-scoredirector.py DATA_DIR --expect JSON_ARRAY --baseline REF --branch REF --owner OWNER @@ -15,25 +19,41 @@ import argparse import collections import glob +import importlib.util import json import math import os import re +import statistics import sys -# The band inside which a delta counts as noise. Two sides run on the same self-hosted machine, one -# after the other, so the noise of their difference is only a little more than one side's own error -# - unlike a shared runner, there is no separate JVM/JIT-shape lottery to absorb. -TOLERANCE_PCT = 4.0 -# A row is marked with HIGH_ERROR when one side's own error is more than the band divided by -# sqrt(2) - that is, when the band above no longer covers this example. -HIGH_ERROR_BAND_FRACTION = math.sqrt(2) +# The band inside which a delta counts as noise. Measured, not guessed: across 36 main-against-main +# comparisons the deltas had an RMS of 0.55 % and a worst case of 1.61 %, so 3 % leaves about a +# factor of two over anything actually seen and lets through roughly one false failure in a +# thousand runs. Narrower is not free: the noise is per-fork and independent, so the only way to +# buy a tighter band is to spend forks. Re-measure this whenever the runners change. +TOLERANCE_PCT = 3.0 -RUNNER_LABEL = "self-hosted" +RUNNER_LABEL = "ubuntu-24.04-arm" -HIGH_ERROR = "⚠️" +UNRESOLVED = "⚠️" ABSENT = "—" +def _load_merge_module(): + """The t quantile lives in the merge script, which needs the same 99.9 % convention JMH uses. + + Both scripts sit side by side in the repository root and are checked out together, but the + file name has hyphens, so it cannot be imported by name. + """ + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "merge-scoredirector-results.py") + spec = importlib.util.spec_from_file_location("merge_scoredirector_results", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_MERGE = _load_merge_module() + Verdict = collections.namedtuple("Verdict", ("emoji", "label")) REGRESSION = Verdict("‼️", "regression") @@ -73,6 +93,8 @@ def load_side(path: str) -> dict: "score_error": _to_float(metric["scoreError"]), "conf_lo": _to_float(conf[0]), "conf_hi": _to_float(conf[1]), + # One value for each fork, in the order they ran, which is what pairs the two sides. + "forks": [_to_float(value) for fork in metric.get("rawData", []) for value in fork], } return result @@ -102,13 +124,33 @@ def relative_error(side: dict) -> float: return abs(side["score_error"] / side["score"]) -def evaluate_row(old: "dict | None", new: "dict | None") -> tuple["float | None", Verdict, bool]: - """Returns (delta_pct, verdict, high_error). delta_pct is None only for MISSING.""" +def resolution(old: dict, new: dict) -> float: + """Half-width of the 99.9 % confidence interval on the delta, in percent. NaN when unknowable. + + Paired, because the workflow alternates: fork k of one side runs about a minute before fork k + of the other, so both met the same machine. Taking the ratio inside each pair cancels whatever + the machine was doing between them. The two forks of a pair correlate at +0.39 here, so the + pairing is worth about a third of the variance; most of the spread is a per-JVM constant that + no schedule can cancel. Pairing is never worse than not pairing, so it is unconditional. + + A side carrying no rawData cannot be paired, and neither can two sides of different lengths, + which is what a retried fork leaves behind; both give NaN. + """ + old_forks, new_forks = old["forks"], new["forks"] + if len(old_forks) != len(new_forks) or len(old_forks) < 2: + return math.nan + if any(value <= 0 for value in old_forks + new_forks): + return math.nan + ratios = [math.log(after / before) for before, after in zip(old_forks, new_forks)] + quantile = _MERGE.t_quantile(_MERGE.CONFIDENCE_TAIL, len(ratios) - 1) + return quantile * statistics.stdev(ratios) / math.sqrt(len(ratios)) * 100 + + +def evaluate_row(old: "dict | None", new: "dict | None") -> tuple["float | None", Verdict, float]: + """Returns (delta_pct, verdict, resolution_pct). delta_pct is None only for MISSING.""" if old is None or new is None: - return None, MISSING, False + return None, MISSING, math.nan delta_pct = (new["score"] / old["score"] - 1) * 100 - error_limit = TOLERANCE_PCT / 100 / HIGH_ERROR_BAND_FRACTION - high_error = relative_error(old) > error_limit or relative_error(new) > error_limit if abs(delta_pct) <= TOLERANCE_PCT: verdict = TOLERANCE elif new["conf_lo"] > old["conf_hi"]: @@ -117,7 +159,12 @@ def evaluate_row(old: "dict | None", new: "dict | None") -> tuple["float | None" verdict = REGRESSION else: verdict = UNDETERMINED - return delta_pct, verdict, high_error + return delta_pct, verdict, resolution(old, new) + + +def is_unresolved(resolution_pct: float) -> bool: + """True when the run could not have told a band-sized change apart from no change at all.""" + return not math.isnan(resolution_pct) and resolution_pct >= TOLERANCE_PCT def format_count(value: float) -> str: @@ -126,18 +173,15 @@ def format_count(value: float) -> str: def format_throughput(old: "dict | None", new: "dict | None", delta_pct: "float | None", - verdict: Verdict, high_error: bool) -> str: - marker = verdict.emoji + (" " + HIGH_ERROR if high_error else "") + verdict: Verdict, resolution_pct: float) -> str: + marker = verdict.emoji + (" " + UNRESOLVED if is_unresolved(resolution_pct) else "") if old is None or new is None: return f"{marker} {ABSENT} → {ABSENT}" return f"{marker} {format_count(old['score'])} → {format_count(new['score'])} ({delta_pct:+.1f} %)" -def format_margin(side: "dict | None") -> str: - if side is None: - return ABSENT - value = relative_error(side) - return ABSENT if math.isnan(value) else f"± {value * 100:.1f} %" +def format_resolution(resolution_pct: float) -> str: + return ABSENT if math.isnan(resolution_pct) else f"± {resolution_pct:.1f} %" def format_example(example: str, url: "str | None") -> str: @@ -158,27 +202,26 @@ def build_rows(expect: list, baseline_data: dict, sut_data: dict, asset_urls: di rows = [] for example in sorted(expect): old, new = baseline_data.get(example), sut_data.get(example) - delta_pct, verdict, high_error = evaluate_row(old, new) - rows.append((example, old, new, delta_pct, verdict, high_error, asset_urls.get(example))) + delta_pct, verdict, resolution_pct = evaluate_row(old, new) + rows.append((example, old, new, delta_pct, verdict, resolution_pct, asset_urls.get(example))) return rows def render_table(rows: list) -> list: - lines = ["| Example | Throughput | Old ± | New ± |", - "|---|---:|---:|---:|"] - for example, old, new, delta_pct, verdict, high_error, url in rows: - lines.append("| {} | {} | {} | {} |".format( + lines = ["| Example | Throughput | Resolution |", + "|---|---:|---:|"] + for example, old, new, delta_pct, verdict, resolution_pct, url in rows: + lines.append("| {} | {} | {} |".format( format_example(example, url), - format_throughput(old, new, delta_pct, verdict, high_error), - format_margin(old), - format_margin(new))) + format_throughput(old, new, delta_pct, verdict, resolution_pct), + format_resolution(resolution_pct))) return lines def render_legend() -> list: return ["", " · ".join(f"{v.emoji} {v.label}" for v in _ALL_VERDICTS) - + f" · {HIGH_ERROR} score error too big for its band", + + f" · {UNRESOLVED} could not resolve a change this size", "", "#### Noise band", "", @@ -186,15 +229,26 @@ def render_legend() -> list: "Positive is faster.", f"- A delta inside its band counts as noise: ± {TOLERANCE_PCT:.0f} %.", "", + "#### Resolution", + "", + "- The smallest difference this run could tell apart from no difference at all: " + "the 99.9 % confidence interval on the delta itself.", + "- It is paired. The two sides alternate fork by fork, so fork N of each ran about a " + "minute apart and met the same machine; comparing them within the pair cancels " + "whatever the machine was doing in between. Most of the remaining spread is a speed " + "offset drawn once per JVM, which only more forks can average away.", + "- It shrinks with the square root of the fork count, so it is also the answer to " + "\"how many forks do we need?\".", + "", "#### What the marks mean", "", f"- {UNDETERMINED.emoji} {UNDETERMINED.label}: the delta is outside its band, " f"but the two confidence intervals overlap, so this run cannot say which side is faster. " f"It fails the build, the same as {REGRESSION.emoji} {REGRESSION.label}. " - f"Read the Old ±/New ± columns, then run it again with more forks.", - f"- {HIGH_ERROR}: one side's own error is more than its band allows for. " - f"The band is a fixed number from an earlier run, " - f"so this says that run no longer describes this example.", + f"Read the resolution, then run it again with more forks.", + f"- {UNRESOLVED}: the resolution is no better than the band itself, so this row could " + f"not have caught a regression worth failing on. Its delta says nothing either way - " + f"the run needs more forks, or the machine was too busy to measure on.", "", "#### Notes", "", @@ -226,10 +280,30 @@ def build_report(data_dir: str, expect: list, baseline_ref: str, branch_ref: str def _selftest() -> None: - fast = {"score": 100.0, "score_error": 1.0, "conf_lo": 99.0, "conf_hi": 101.0} - slow = {"score": 80.0, "score_error": 1.0, "conf_lo": 79.0, "conf_hi": 81.0} - same = {"score": 100.5, "score_error": 1.0, "conf_lo": 99.5, "conf_hi": 101.5} - noisy = {"score": 90.0, "score_error": 0.0, "conf_lo": math.nan, "conf_hi": math.nan} + def side(score, error, lo, hi, forks=()): + return {"score": score, "score_error": error, "conf_lo": lo, "conf_hi": hi, + "forks": list(forks)} + + fast = side(100.0, 1.0, 99.0, 101.0) + slow = side(80.0, 1.0, 79.0, 81.0) + same = side(100.5, 1.0, 99.5, 101.5) + noisy = side(90.0, 0.0, math.nan, math.nan) + + # Resolution is paired: drift shared by both sides cancels, so a pair of sides that move + # together resolves far better than either side's own spread suggests. + drifting_old = side(100.0, 0.0, math.nan, math.nan, [90, 95, 100, 105, 110]) + drifting_new = side(100.0, 0.0, math.nan, math.nan, [90, 95, 100, 105, 110]) + assert resolution(drifting_old, drifting_new) == 0.0, "identical sides resolve perfectly" + scattered_old = side(100.0, 0.0, math.nan, math.nan, [100, 100, 100, 100, 100]) + scattered_new = side(100.0, 0.0, math.nan, math.nan, [90, 95, 100, 105, 110]) + assert resolution(scattered_old, scattered_new) > 10.0, "unpaired scatter is not hidden" + # Anything that cannot be paired resolves to nothing rather than to a wrong number. + assert math.isnan(resolution(fast, slow)), "no rawData, no resolution" + assert math.isnan(resolution(drifting_old, side(100.0, 0.0, 0.0, 0.0, [100, 100]))) + assert math.isnan(resolution(side(1.0, 0.0, 0.0, 0.0, [0.0, 1.0, 2.0]), drifting_old)) + + assert is_unresolved(TOLERANCE_PCT + 0.1) and not is_unresolved(TOLERANCE_PCT - 0.1) + assert not is_unresolved(math.nan), "unknown resolution must not raise the flag" # Regression: new (slow) is strictly below old (fast). delta, verdict, _ = evaluate_row(fast, slow) @@ -250,19 +324,17 @@ def _selftest() -> None: assert verdict == UNDETERMINED, verdict # Missing side never crashes and is reported distinctly. - delta, verdict, high_error = evaluate_row(None, fast) - assert verdict == MISSING and delta is None and high_error is False + delta, verdict, resolution_pct = evaluate_row(None, fast) + assert verdict == MISSING and delta is None and math.isnan(resolution_pct) - # High relative error only annotates; it must not override a tolerance/regression verdict. - high_err_side = {"score": 100.0, "score_error": 5.0, "conf_lo": 90.0, "conf_hi": 110.0} - delta, verdict, high_error = evaluate_row(fast, high_err_side) - assert verdict == TOLERANCE and high_error is True + # A coarse resolution only annotates; it must not override the verdict. + coarse_old = side(100.0, 0.0, math.nan, math.nan, [100, 100, 100, 100, 100]) + coarse_new = side(100.0, 0.0, math.nan, math.nan, [80, 90, 100, 110, 120]) + delta, verdict, resolution_pct = evaluate_row(coarse_old, coarse_new) + assert verdict == TOLERANCE and is_unresolved(resolution_pct), resolution_pct - # A margin on a zero score is no percentage at all; it prints as absent rather than crashing. - zero = {"score": 0.0, "score_error": 1.0, "conf_lo": math.nan, "conf_hi": math.nan} - assert format_margin(zero) == ABSENT - assert format_margin(None) == ABSENT - assert format_margin(fast) == "± 1.0 %" + assert format_resolution(math.nan) == ABSENT + assert format_resolution(1.34) == "± 1.3 %" # Row order: alphabetical, from a deliberately scrambled input. expect = ["vehicle_routing", "cloud_balancing", "examination"] @@ -274,13 +346,13 @@ def _selftest() -> None: report, exit_code = render_report(rows, "v1.0.0", "main", "TimefoldAI") assert exit_code == 1, "a regression must fail the build" - assert "| Example | Throughput | Old ± | New ± |" in report + assert "| Example | Throughput | Resolution |" in report assert report.count("| Example |") == 1 # The legend is sectioned. - for heading in ("#### Noise band", "#### What the marks mean", "#### Notes"): + for heading in ("#### Noise band", "#### Resolution", "#### What the marks mean", "#### Notes"): assert heading in report, heading - for mark in (f"{UNDETERMINED.emoji} {UNDETERMINED.label}:", f"{HIGH_ERROR}:"): + for mark in (f"{UNDETERMINED.emoji} {UNDETERMINED.label}:", f"{UNRESOLVED}:"): assert f"- {mark}" in report, mark # A missing side renders as a distinct row rather than vanishing.