From 37b1be847314d85b55b5a3448ceb4709e0699520 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Fri, 11 Sep 2026 16:03:16 +0200 Subject: [PATCH 1/9] ci: continue stabilization --- .../benchmarks/micro/scoredirector/Main.java | 6 +++- .../scoredirector/jmh/AbstractBenchmark.java | 29 ++++++++++++------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java index f10b87b5..03ba7446 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java @@ -67,7 +67,11 @@ public void run(String[] args) throws RunnerException, IOException { // still land 6-18 % apart here. Unlike coldstart, this benchmark only cares about // steady-state throughput, so skipping tiering removes that remaining lottery: // every fork goes straight to C2. - .jvmArgsAppend("-XX:-TieredCompilation"); + // With a fixed heap (-Xms == -Xmx), pages can still be lazily committed and + // first-touched mid-run, so an allocation can stall on a page fault right when GC + // also runs - JFR showed CPU dips lining up with a subset of GC events. Pre-touching + // commits and zeroes every heap page at startup instead, once, before measurement. + .jvmArgsAppend("-XX:-TieredCompilation", "-XX:+AlwaysPreTouch"); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED); options = initAsyncProfiler(options); diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java index d05f22c5..03ce7716 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java @@ -8,6 +8,7 @@ import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -18,6 +19,14 @@ @BenchmarkMode(Mode.Throughput) public abstract class AbstractBenchmark { + /** + * Batches this many independent draw-execute-score cycles into one JMH invocation, + * by calling {@link Problem}'s per-invocation lifecycle in a plain loop + * instead of letting JMH's {@code Level.Invocation} machinery drive it once per call. + * Without this, JMH's own per-invocation bookkeeping lands on a single cheap operation instead of being amortized. + */ + private static final int BATCH_SIZE = 100; + public Problem problem; abstract protected ScoreDirectorType getScoreDirectorType(); @@ -35,19 +44,17 @@ public void setupIteration() { problem.setupIteration(); } - @Setup(Level.Invocation) - public void setupInvocation() { - problem.setupInvocation(); - } - @Benchmark + @OperationsPerInvocation(BATCH_SIZE) public Object run(Blackhole blackhole) { - return problem.runInvocation(); - } - - @TearDown(Level.Invocation) - public void teardownInvocation() { - problem.tearDownInvocation(); + Object result = null; + for (var i = 0; i < BATCH_SIZE; i++) { + problem.setupInvocation(); + result = problem.runInvocation(); + blackhole.consume(result); + problem.tearDownInvocation(); + } + return result; } @TearDown(Level.Iteration) From f2b770662fff16280bf645615a2bd6bbbe63ef19 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Fri, 11 Sep 2026 21:55:29 +0200 Subject: [PATCH 2/9] experiment --- .../solver/benchmarks/micro/scoredirector/Main.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java index 03ba7446..37f4d23c 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java @@ -36,6 +36,7 @@ import ai.timefold.solver.benchmarks.micro.common.AbstractMain; +import org.openjdk.jmh.profile.GCProfiler; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; @@ -71,7 +72,15 @@ public void run(String[] args) throws RunnerException, IOException { // first-touched mid-run, so an allocation can stall on a page fault right when GC // also runs - JFR showed CPU dips lining up with a subset of GC events. Pre-touching // commits and zeroes every heap page at startup instead, once, before measurement. - .jvmArgsAppend("-XX:-TieredCompilation", "-XX:+AlwaysPreTouch"); + .jvmArgsAppend("-XX:-TieredCompilation", "-XX:+AlwaysPreTouch") + // Diagnostic only: none of forkCount/warmup/GC page-faults/JMH-invocation-overhead/ + // JIT-inlining explains why some examples' forks spread far more than others (see the + // score-director stability investigation). GCProfiler reads GC MXBean deltas around + // each iteration - negligible overhead, no effect on the primary throughput metric - + // and reports per-fork GC pause count/time as secondary metrics in the same + // results.json, to test whether the noisy examples' slow forks are the ones paying + // for more/longer GC pauses. + .addProfiler(GCProfiler.class); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED); options = initAsyncProfiler(options); From e6939fd68387a4273cc2d983adb326b04fef6198 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Sat, 12 Sep 2026 07:48:14 +0200 Subject: [PATCH 3/9] fix CI --- .github/workflows/performance_score_director.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 4153e0e1..89bd752e 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -88,7 +88,11 @@ jobs: with: repository: TimefoldAI/timefold-solver-benchmarks path: ./timefold-solver-benchmarks - ref: main # Assume the ref is compatible with both baseline and SUT + # Same commit for both baseline and SUT builds, so a harness-only change never confounds + # the solver comparison. Pinning this to a fixed branch name meant a harness fix on any + # other branch (incl. this one) was silently never exercised by this workflow; use the + # commit that dispatched this run instead, whichever branch that happens to be. + ref: ${{ github.sha }} - name: Setup JDK and Maven uses: actions/setup-java@v6.0.0 @@ -163,7 +167,11 @@ jobs: with: repository: TimefoldAI/timefold-solver-benchmarks path: ./timefold-solver-benchmarks - ref: main # Assume the ref is compatible with both baseline and SUT + # Same commit for both baseline and SUT builds, so a harness-only change never confounds + # the solver comparison. Pinning this to a fixed branch name meant a harness fix on any + # other branch (incl. this one) was silently never exercised by this workflow; use the + # commit that dispatched this run instead, whichever branch that happens to be. + ref: ${{ github.sha }} - name: Setup JDK and Maven uses: actions/setup-java@v6.0.0 From ccba2c0f27864fd24d7545cd17f81bc898ac7596 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Sat, 12 Sep 2026 10:24:42 +0200 Subject: [PATCH 4/9] fix CI more --- .github/workflows/performance_move_provider.yml | 12 ++++++++++-- .github/workflows/performance_score_director.yml | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/performance_move_provider.yml b/.github/workflows/performance_move_provider.yml index b129873b..65b09b16 100644 --- a/.github/workflows/performance_move_provider.yml +++ b/.github/workflows/performance_move_provider.yml @@ -140,7 +140,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): baseline is often a tag # (e.g. a release version). Falls back to the already-checked-out ref if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi @@ -224,7 +228,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): the branch under test may # also be a tag. Falls back to the already-checked-out ref if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 89bd752e..25da4234 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -137,7 +137,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): baseline is often a tag # (e.g. a release version). Falls back to the already-checked-out default if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi @@ -227,7 +231,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): the branch under test may # also be a tag. Falls back to the already-checked-out default if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi From 401263eb9e3b2fb232a8e95c9517a44817ea5cf6 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Sat, 12 Sep 2026 12:27:38 +0200 Subject: [PATCH 5/9] more stabilization --- .../benchmarks/micro/scoredirector/Main.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java index 37f4d23c..f10b87b5 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java @@ -36,7 +36,6 @@ import ai.timefold.solver.benchmarks.micro.common.AbstractMain; -import org.openjdk.jmh.profile.GCProfiler; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; @@ -68,19 +67,7 @@ public void run(String[] args) throws RunnerException, IOException { // still land 6-18 % apart here. Unlike coldstart, this benchmark only cares about // steady-state throughput, so skipping tiering removes that remaining lottery: // every fork goes straight to C2. - // With a fixed heap (-Xms == -Xmx), pages can still be lazily committed and - // first-touched mid-run, so an allocation can stall on a page fault right when GC - // also runs - JFR showed CPU dips lining up with a subset of GC events. Pre-touching - // commits and zeroes every heap page at startup instead, once, before measurement. - .jvmArgsAppend("-XX:-TieredCompilation", "-XX:+AlwaysPreTouch") - // Diagnostic only: none of forkCount/warmup/GC page-faults/JMH-invocation-overhead/ - // JIT-inlining explains why some examples' forks spread far more than others (see the - // score-director stability investigation). GCProfiler reads GC MXBean deltas around - // each iteration - negligible overhead, no effect on the primary throughput metric - - // and reports per-fork GC pause count/time as secondary metrics in the same - // results.json, to test whether the noisy examples' slow forks are the ones paying - // for more/longer GC pauses. - .addProfiler(GCProfiler.class); + .jvmArgsAppend("-XX:-TieredCompilation"); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED); options = initAsyncProfiler(options); From 8caab3352570da415a9e297c0adad9f6c6758491 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Sat, 12 Sep 2026 12:42:00 +0200 Subject: [PATCH 6/9] fine-tuning --- .../workflows/performance_score_director.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 25da4234..b897bd17 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -290,15 +290,26 @@ jobs: path: ./timefold-solver-benchmarks # Fine-tuned for stability on GHA. + # forks/warmup/measurement below are not arbitrary - + # they're sized against this job's actual "ubuntu-24.04-arm" runner, + # measured from real CI logs: + # - warmup=4/measurement=3 (7 iterations) was chosen because + # warmup throughput plateaus by iteration 2-3 on every example checked (incl. the biggest). + # - forks=27 selected as the max value to fit within the time budget of ~1h: + # Between-fork variance (JIT/scheduling noise that differs per JVM process) + # turned out to be as large as within-fork variance here, + # and only fork count - not iteration count - shrinks that term, + # so this reallocation nets tighter confidence intervals for the same CI time. + # If the runner type/hardware changes, re-measure rather than assume these still hold. - name: Configure the benchmark working-directory: ./timefold-solver-benchmarks shell: bash env: EXAMPLE: ${{ matrix.example }} run: | - echo "forks=20" > scoredirector-benchmark.properties - echo "warmup_iterations=5" >> scoredirector-benchmark.properties - echo "measurement_iterations=5" >> scoredirector-benchmark.properties + echo "forks=27" > scoredirector-benchmark.properties + echo "warmup_iterations=4" >> scoredirector-benchmark.properties + echo "measurement_iterations=3" >> scoredirector-benchmark.properties echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties echo "score_director_type=cs" >> scoredirector-benchmark.properties echo "example=$EXAMPLE" >> scoredirector-benchmark.properties From fd78b831a53a46b190c5a9c72e3e87118c475638 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Sat, 12 Sep 2026 15:33:25 +0200 Subject: [PATCH 7/9] another experiment --- .../solver/benchmarks/micro/scoredirector/Main.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java index f10b87b5..916a17a9 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java @@ -67,7 +67,17 @@ public void run(String[] args) throws RunnerException, IOException { // still land 6-18 % apart here. Unlike coldstart, this benchmark only cares about // steady-state throughput, so skipping tiering removes that remaining lottery: // every fork goes straight to C2. - .jvmArgsAppend("-XX:-TieredCompilation"); + // + // EXPERIMENT, not an established fix: forks here still split into a "normal" and a + // ~10-15 % slower mode, with every iteration inside one fork agreeing - so whatever + // decides it is fixed at JVM startup, not transient. moveprovider, on the same + // runners, spreads only 1.4-2.8 %, so it is not the hardware; the one large + // difference is that this benchmark allocates ~28 KB/op at ~480 MB/s, which makes + // it far more exposed to how the heap is backed by physical pages. Huge pages are + // decided per process, once, and last for its whole life - the exact shape of the + // observed modes. AlwaysPreTouch is required alongside, so that backing is settled + // up front instead of being materialized piecemeal mid-measurement. + .jvmArgsAppend("-XX:-TieredCompilation", "-XX:+AlwaysPreTouch", "-XX:+UseTransparentHugePages"); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED); options = initAsyncProfiler(options); From ba3dfb286086b59e7821f883dc992f63108b47e5 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Sat, 12 Sep 2026 18:34:20 +0200 Subject: [PATCH 8/9] more --- .../workflows/performance_score_director.yml | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index b897bd17..6079c35c 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -293,13 +293,17 @@ jobs: # forks/warmup/measurement below are not arbitrary - # they're sized against this job's actual "ubuntu-24.04-arm" runner, # measured from real CI logs: - # - warmup=4/measurement=3 (7 iterations) was chosen because - # warmup throughput plateaus by iteration 2-3 on every example checked (incl. the biggest). - # - forks=27 selected as the max value to fit within the time budget of ~1h: - # Between-fork variance (JIT/scheduling noise that differs per JVM process) - # turned out to be as large as within-fork variance here, - # and only fork count - not iteration count - shrinks that term, - # so this reallocation nets tighter confidence intervals for the same CI time. + # - warmup=4 because warmup throughput plateaus by iteration 2-3 + # on every example checked (incl. the biggest). + # - measurement=1, matching moveprovider. A fork's measurement iterations are near-duplicates + # of each other here (a slow fork measured slow in every one of its iterations), so they are + # not independent samples. JMH nonetheless pools all fork x iteration samples when computing + # scoreError, which inflates N and reports a confidence interval ~1.4x tighter than the data + # supports. One iteration per fork makes that pooling a no-op: fork x iteration == fork. + # - forks=36 spends everything the shorter forks free up, at ~58 s/fork + # (5 iterations x 10 s + ~8 s JVM startup, both measured on this runner). + # The fork is the real unit of replication, so this is the only lever that buys + # genuine precision - extra iterations per fork mostly buy the appearance of it. # If the runner type/hardware changes, re-measure rather than assume these still hold. - name: Configure the benchmark working-directory: ./timefold-solver-benchmarks @@ -307,9 +311,9 @@ jobs: env: EXAMPLE: ${{ matrix.example }} run: | - echo "forks=27" > scoredirector-benchmark.properties + echo "forks=36" > scoredirector-benchmark.properties echo "warmup_iterations=4" >> scoredirector-benchmark.properties - echo "measurement_iterations=3" >> scoredirector-benchmark.properties + echo "measurement_iterations=1" >> scoredirector-benchmark.properties echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties echo "score_director_type=cs" >> scoredirector-benchmark.properties echo "example=$EXAMPLE" >> scoredirector-benchmark.properties From e754fcbf2f18deab2afcdf67f28575a72558cf31 Mon Sep 17 00:00:00 2001 From: Lukas Petrovicky Date: Tue, 15 Sep 2026 13:06:01 +0200 Subject: [PATCH 9/9] revert most of it --- .../workflows/performance_score_director.yml | 27 +++++-------------- .../benchmarks/micro/scoredirector/Main.java | 17 +----------- 2 files changed, 8 insertions(+), 36 deletions(-) diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 6079c35c..a6cffc8b 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -2,6 +2,9 @@ # with their binaries uploaded as artifacts. # This is done on GitHub infrastructure, to achieve maximum parallelization. # +# The benchmark itself runs on a self-hosted runner which we completely control; +# a machine with stable performance characteristics is what makes the results comparable at all. +# # A benchmark subjob produces no report of its own - # it only uploads its raw JMH results as an artifact. # After every subjob finishes, one summarizing job downloads all of them and aggregates them into a single table, @@ -260,7 +263,7 @@ jobs: group: perf-score_director-${{ matrix.example }} cancel-in-progress: true needs: [ decisions, build_baseline, build_sut ] - runs-on: ubuntu-24.04-arm # Github-hosted Arm runner; free and GA for this public repo. + runs-on: self-hosted # We need a stable machine to actually run the benchmarks. strategy: fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue. matrix: @@ -289,31 +292,15 @@ jobs: repository: TimefoldAI/timefold-solver-benchmarks path: ./timefold-solver-benchmarks - # Fine-tuned for stability on GHA. - # forks/warmup/measurement below are not arbitrary - - # they're sized against this job's actual "ubuntu-24.04-arm" runner, - # measured from real CI logs: - # - warmup=4 because warmup throughput plateaus by iteration 2-3 - # on every example checked (incl. the biggest). - # - measurement=1, matching moveprovider. A fork's measurement iterations are near-duplicates - # of each other here (a slow fork measured slow in every one of its iterations), so they are - # not independent samples. JMH nonetheless pools all fork x iteration samples when computing - # scoreError, which inflates N and reports a confidence interval ~1.4x tighter than the data - # supports. One iteration per fork makes that pooling a no-op: fork x iteration == fork. - # - forks=36 spends everything the shorter forks free up, at ~58 s/fork - # (5 iterations x 10 s + ~8 s JVM startup, both measured on this runner). - # The fork is the real unit of replication, so this is the only lever that buys - # genuine precision - extra iterations per fork mostly buy the appearance of it. - # If the runner type/hardware changes, re-measure rather than assume these still hold. - name: Configure the benchmark working-directory: ./timefold-solver-benchmarks shell: bash env: EXAMPLE: ${{ matrix.example }} run: | - echo "forks=36" > scoredirector-benchmark.properties - echo "warmup_iterations=4" >> scoredirector-benchmark.properties - echo "measurement_iterations=1" >> scoredirector-benchmark.properties + echo "forks=20" > scoredirector-benchmark.properties + echo "warmup_iterations=5" >> scoredirector-benchmark.properties + echo "measurement_iterations=5" >> scoredirector-benchmark.properties echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties echo "score_director_type=cs" >> scoredirector-benchmark.properties echo "example=$EXAMPLE" >> scoredirector-benchmark.properties diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java index 916a17a9..fcdcfc35 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java @@ -62,22 +62,7 @@ public static void main(String[] args) throws RunnerException, IOException { public void run(String[] args) throws RunnerException, IOException { var configuration = readConfiguration(); - var options = getBaseJmhConfig(configuration) - // -Xbatch alone still leaves the C1->C2 promotion point timing-dependent, so forks - // still land 6-18 % apart here. Unlike coldstart, this benchmark only cares about - // steady-state throughput, so skipping tiering removes that remaining lottery: - // every fork goes straight to C2. - // - // EXPERIMENT, not an established fix: forks here still split into a "normal" and a - // ~10-15 % slower mode, with every iteration inside one fork agreeing - so whatever - // decides it is fixed at JVM startup, not transient. moveprovider, on the same - // runners, spreads only 1.4-2.8 %, so it is not the hardware; the one large - // difference is that this benchmark allocates ~28 KB/op at ~480 MB/s, which makes - // it far more exposed to how the heap is backed by physical pages. Huge pages are - // decided per process, once, and last for its whole life - the exact shape of the - // observed modes. AlwaysPreTouch is required alongside, so that backing is settled - // up front instead of being materialized piecemeal mid-measurement. - .jvmArgsAppend("-XX:-TieredCompilation", "-XX:+AlwaysPreTouch", "-XX:+UseTransparentHugePages"); + var options = getBaseJmhConfig(configuration); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED); options = initAsyncProfiler(options);