ci: scoredirector benchmark stabilization

.github/workflows/performance_move_provider.yml

@@ -140,7 +140,11 @@ jobs:
         run: |
           # Match a branch OR a tag named $TARGET_BRANCH (no --heads): baseline is often a tag
           # (e.g. a release version). Falls back to the already-checked-out ref if no match exists.
-          if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
+          # Skip for "main": that's the common default solver ref, not a deliberate companion
+          # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching
+          # on it would silently discard the commit checked out above (the one dispatching this
+          # run) and revert to origin/main, undoing that checkout for every ordinary run.
+          if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
             git fetch --depth=1 origin "$TARGET_BRANCH"
             git checkout -B "$TARGET_BRANCH" FETCH_HEAD
           fi
@@ -224,7 +228,11 @@ jobs:
         run: |
           # Match a branch OR a tag named $TARGET_BRANCH (no --heads): the branch under test may
           # also be a tag. Falls back to the already-checked-out ref if no match exists.
-          if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
+          # Skip for "main": that's the common default solver ref, not a deliberate companion
+          # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching
+          # on it would silently discard the commit checked out above (the one dispatching this
+          # run) and revert to origin/main, undoing that checkout for every ordinary run.
+          if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
             git fetch --depth=1 origin "$TARGET_BRANCH"
             git checkout -B "$TARGET_BRANCH" FETCH_HEAD
           fi

.github/workflows/performance_score_director.yml

@@ -2,6 +2,9 @@
 # with their binaries uploaded as artifacts.
 # This is done on GitHub infrastructure, to achieve maximum parallelization.
 #
+# The benchmark itself runs on a self-hosted runner which we completely control;
+# a machine with stable performance characteristics is what makes the results comparable at all.
+#
 # A benchmark subjob produces no report of its own -
 # it only uploads its raw JMH results as an artifact.
 # After every subjob finishes, one summarizing job downloads all of them and aggregates them into a single table,
@@ -88,7 +91,11 @@ jobs:
         with:
           repository: TimefoldAI/timefold-solver-benchmarks
           path: ./timefold-solver-benchmarks
-          ref: main # Assume the ref is compatible with both baseline and SUT
+          # Same commit for both baseline and SUT builds, so a harness-only change never confounds
+          # the solver comparison. Pinning this to a fixed branch name meant a harness fix on any
+          # other branch (incl. this one) was silently never exercised by this workflow; use the
+          # commit that dispatched this run instead, whichever branch that happens to be.
+          ref: ${{ github.sha }}
 
       - name: Setup JDK and Maven
         uses: actions/setup-java@v6.0.0
@@ -133,7 +140,11 @@ jobs:
         run: |
           # Match a branch OR a tag named $TARGET_BRANCH (no --heads): baseline is often a tag
           # (e.g. a release version). Falls back to the already-checked-out default if no match exists.
-          if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
+          # Skip for "main": that's the common default solver ref, not a deliberate companion
+          # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching
+          # on it would silently discard the commit checked out above (the one dispatching this
+          # run) and revert to origin/main, undoing that checkout for every ordinary run.
+          if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
             git fetch --depth=1 origin "$TARGET_BRANCH"
             git checkout -B "$TARGET_BRANCH" FETCH_HEAD
           fi
@@ -163,7 +174,11 @@ jobs:
         with:
           repository: TimefoldAI/timefold-solver-benchmarks
           path: ./timefold-solver-benchmarks
-          ref: main # Assume the ref is compatible with both baseline and SUT
+          # Same commit for both baseline and SUT builds, so a harness-only change never confounds
+          # the solver comparison. Pinning this to a fixed branch name meant a harness fix on any
+          # other branch (incl. this one) was silently never exercised by this workflow; use the
+          # commit that dispatched this run instead, whichever branch that happens to be.
+          ref: ${{ github.sha }}
 
       - name: Setup JDK and Maven
         uses: actions/setup-java@v6.0.0
@@ -219,7 +234,11 @@ jobs:
         run: |
           # Match a branch OR a tag named $TARGET_BRANCH (no --heads): the branch under test may
           # also be a tag. Falls back to the already-checked-out default if no match exists.
-          if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
+          # Skip for "main": that's the common default solver ref, not a deliberate companion
+          # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching
+          # on it would silently discard the commit checked out above (the one dispatching this
+          # run) and revert to origin/main, undoing that checkout for every ordinary run.
+          if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then
             git fetch --depth=1 origin "$TARGET_BRANCH"
             git checkout -B "$TARGET_BRANCH" FETCH_HEAD
           fi
@@ -244,7 +263,7 @@ jobs:
       group: perf-score_director-${{ matrix.example }}
       cancel-in-progress: true
     needs: [ decisions, build_baseline, build_sut ]
-    runs-on: ubuntu-24.04-arm # Github-hosted Arm runner; free and GA for this public repo.
+    runs-on: self-hosted # We need a stable machine to actually run the benchmarks.
     strategy:
       fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue.
       matrix:
@@ -273,7 +292,6 @@ jobs:
           repository: TimefoldAI/timefold-solver-benchmarks
           path: ./timefold-solver-benchmarks
 
-      # Fine-tuned for stability on GHA.
       - name: Configure the benchmark
         working-directory: ./timefold-solver-benchmarks
         shell: bash

src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java

@@ -62,12 +62,7 @@ public static void main(String[] args) throws RunnerException, IOException {
 
     public void run(String[] args) throws RunnerException, IOException {
         var configuration = readConfiguration();
-        var options = getBaseJmhConfig(configuration)
-                // -Xbatch alone still leaves the C1->C2 promotion point timing-dependent, so forks
-                // still land 6-18 % apart here. Unlike coldstart, this benchmark only cares about
-                // steady-state throughput, so skipping tiering removes that remaining lottery:
-                // every fork goes straight to C2.
-                .jvmArgsAppend("-XX:-TieredCompilation");
+        var options = getBaseJmhConfig(configuration);
         options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS);
         options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED);
         options = initAsyncProfiler(options);

src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java

@@ -8,6 +8,7 @@
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OperationsPerInvocation;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
@@ -18,6 +19,14 @@
 @BenchmarkMode(Mode.Throughput)
 public abstract class AbstractBenchmark {
 
+    /**
+     * Batches this many independent draw-execute-score cycles into one JMH invocation,
+     * by calling {@link Problem}'s per-invocation lifecycle in a plain loop
+     * instead of letting JMH's {@code Level.Invocation} machinery drive it once per call.
+     * Without this, JMH's own per-invocation bookkeeping lands on a single cheap operation instead of being amortized.
+     */
+    private static final int BATCH_SIZE = 100;
+
     public Problem problem;
 
     abstract protected ScoreDirectorType getScoreDirectorType();
@@ -35,19 +44,17 @@ public void setupIteration() {
         problem.setupIteration();
     }
 
-    @Setup(Level.Invocation)
-    public void setupInvocation() {
-        problem.setupInvocation();
-    }
-
     @Benchmark
+    @OperationsPerInvocation(BATCH_SIZE)
     public Object run(Blackhole blackhole) {
-        return problem.runInvocation();
-    }
-
-    @TearDown(Level.Invocation)
-    public void teardownInvocation() {
-        problem.tearDownInvocation();
+        Object result = null;
+        for (var i = 0; i < BATCH_SIZE; i++) {
+            problem.setupInvocation();
+            result = problem.runInvocation();
+            blackhole.consume(result);
+            problem.tearDownInvocation();
+        }
+        return result;
     }
 
     @TearDown(Level.Iteration)