diff --git a/.github/workflows/performance_move_provider.yml b/.github/workflows/performance_move_provider.yml index b129873b..65b09b16 100644 --- a/.github/workflows/performance_move_provider.yml +++ b/.github/workflows/performance_move_provider.yml @@ -140,7 +140,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): baseline is often a tag # (e.g. a release version). Falls back to the already-checked-out ref if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi @@ -224,7 +228,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): the branch under test may # also be a tag. Falls back to the already-checked-out ref if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 4153e0e1..a6cffc8b 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -2,6 +2,9 @@ # with their binaries uploaded as artifacts. # This is done on GitHub infrastructure, to achieve maximum parallelization. # +# The benchmark itself runs on a self-hosted runner which we completely control; +# a machine with stable performance characteristics is what makes the results comparable at all. +# # A benchmark subjob produces no report of its own - # it only uploads its raw JMH results as an artifact. # After every subjob finishes, one summarizing job downloads all of them and aggregates them into a single table, @@ -88,7 +91,11 @@ jobs: with: repository: TimefoldAI/timefold-solver-benchmarks path: ./timefold-solver-benchmarks - ref: main # Assume the ref is compatible with both baseline and SUT + # Same commit for both baseline and SUT builds, so a harness-only change never confounds + # the solver comparison. Pinning this to a fixed branch name meant a harness fix on any + # other branch (incl. this one) was silently never exercised by this workflow; use the + # commit that dispatched this run instead, whichever branch that happens to be. + ref: ${{ github.sha }} - name: Setup JDK and Maven uses: actions/setup-java@v6.0.0 @@ -133,7 +140,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): baseline is often a tag # (e.g. a release version). Falls back to the already-checked-out default if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi @@ -163,7 +174,11 @@ jobs: with: repository: TimefoldAI/timefold-solver-benchmarks path: ./timefold-solver-benchmarks - ref: main # Assume the ref is compatible with both baseline and SUT + # Same commit for both baseline and SUT builds, so a harness-only change never confounds + # the solver comparison. Pinning this to a fixed branch name meant a harness fix on any + # other branch (incl. this one) was silently never exercised by this workflow; use the + # commit that dispatched this run instead, whichever branch that happens to be. + ref: ${{ github.sha }} - name: Setup JDK and Maven uses: actions/setup-java@v6.0.0 @@ -219,7 +234,11 @@ jobs: run: | # Match a branch OR a tag named $TARGET_BRANCH (no --heads): the branch under test may # also be a tag. Falls back to the already-checked-out default if no match exists. - if git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then + # Skip for "main": that's the common default solver ref, not a deliberate companion + # harness branch, and timefold-solver-benchmarks always has its own "main" too - matching + # on it would silently discard the commit checked out above (the one dispatching this + # run) and revert to origin/main, undoing that checkout for every ordinary run. + if [ "$TARGET_BRANCH" != "main" ] && git ls-remote --exit-code origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD fi @@ -244,7 +263,7 @@ jobs: group: perf-score_director-${{ matrix.example }} cancel-in-progress: true needs: [ decisions, build_baseline, build_sut ] - runs-on: ubuntu-24.04-arm # Github-hosted Arm runner; free and GA for this public repo. + runs-on: self-hosted # We need a stable machine to actually run the benchmarks. strategy: fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue. matrix: @@ -273,7 +292,6 @@ jobs: repository: TimefoldAI/timefold-solver-benchmarks path: ./timefold-solver-benchmarks - # Fine-tuned for stability on GHA. - name: Configure the benchmark working-directory: ./timefold-solver-benchmarks shell: bash diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java index f10b87b5..fcdcfc35 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/Main.java @@ -62,12 +62,7 @@ public static void main(String[] args) throws RunnerException, IOException { public void run(String[] args) throws RunnerException, IOException { var configuration = readConfiguration(); - var options = getBaseJmhConfig(configuration) - // -Xbatch alone still leaves the C1->C2 promotion point timing-dependent, so forks - // still land 6-18 % apart here. Unlike coldstart, this benchmark only cares about - // steady-state throughput, so skipping tiering removes that remaining lottery: - // every fork goes straight to C2. - .jvmArgsAppend("-XX:-TieredCompilation"); + var options = getBaseJmhConfig(configuration); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS); options = processBenchmark(options, configuration, ScoreDirectorType.CONSTRAINT_STREAMS_JUSTIFIED); options = initAsyncProfiler(options); diff --git a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java index d05f22c5..03ce7716 100644 --- a/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java +++ b/src/main/java/ai/timefold/solver/benchmarks/micro/scoredirector/jmh/AbstractBenchmark.java @@ -8,6 +8,7 @@ import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -18,6 +19,14 @@ @BenchmarkMode(Mode.Throughput) public abstract class AbstractBenchmark { + /** + * Batches this many independent draw-execute-score cycles into one JMH invocation, + * by calling {@link Problem}'s per-invocation lifecycle in a plain loop + * instead of letting JMH's {@code Level.Invocation} machinery drive it once per call. + * Without this, JMH's own per-invocation bookkeeping lands on a single cheap operation instead of being amortized. + */ + private static final int BATCH_SIZE = 100; + public Problem problem; abstract protected ScoreDirectorType getScoreDirectorType(); @@ -35,19 +44,17 @@ public void setupIteration() { problem.setupIteration(); } - @Setup(Level.Invocation) - public void setupInvocation() { - problem.setupInvocation(); - } - @Benchmark + @OperationsPerInvocation(BATCH_SIZE) public Object run(Blackhole blackhole) { - return problem.runInvocation(); - } - - @TearDown(Level.Invocation) - public void teardownInvocation() { - problem.tearDownInvocation(); + Object result = null; + for (var i = 0; i < BATCH_SIZE; i++) { + problem.setupInvocation(); + result = problem.runInvocation(); + blackhole.consume(result); + problem.tearDownInvocation(); + } + return result; } @TearDown(Level.Iteration)