diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 75757d96c..201a657ad 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -1,10 +1,11 @@ -name: CodSpeed (simulation + memory) +name: CodSpeed (memory + simulation) -# CPU instruction counts (cachegrind) AND heap allocations in ONE upload. +# Heap allocations AND CPU instruction counts (cachegrind) in ONE upload. # They must share a run: CodSpeed records one run per (commit, environment), # and a second upload to the same environment overwrites the first — the other # instrument's benchmarks then show as "skipped". So both are measured serially -# via ``mode: simulation,memory`` (CodSpeedHQ/action >= 4.12). +# via ``mode: memory,simulation`` — memory first so the fast, always-on signal +# lands even if the slow cachegrind pass is interrupted (CodSpeedHQ/action >= 4.12). # # Cadence: master push (establish/update both baselines) + manual dispatch + # opt-in per-PR via the ``trigger:benchmark`` label — which keeps the PR context @@ -25,7 +26,7 @@ concurrency: jobs: codspeed: - name: CodSpeed simulation + memory + name: CodSpeed memory + simulation # Always on master push / dispatch; on PRs only when explicitly labelled. if: >- ${{ github.event_name != 'pull_request' || @@ -52,10 +53,10 @@ jobs: python -m pip install uv uv pip install --system -e ".[dev,benchmarks]" - - name: Run benchmarks under CodSpeed (simulation + memory) + - name: Run benchmarks under CodSpeed (memory + simulation) uses: CodSpeedHQ/action@v4 with: - mode: simulation,memory + mode: memory,simulation run: | pytest benchmarks/ --quick --codspeed diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 17d74bcbc..212b22165 100644 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -1,36 +1,24 @@ """ Ad-hoc benchmarking of arbitrary callables on the *current* linopy tree. -Where the pytest suite measures the fixed registry grid and ``sweep`` -measures across installed linopy versions, ``bench`` is for the -interactive middle: time or memory-profile any callable — a registry -builder, a phase verb applied to a model you built by hand, or a one-off -lambda — get a result object back, and either inspect it as a DataFrame -or drop it into a snapshot the existing ``plot`` / ``compare`` machinery -already understands:: +Where the pytest suite measures the fixed registry grid and ``sweep`` measures +across installed linopy versions, ``bench`` is the interactive middle: time or +memory-profile any callable, get a result object, inspect it as a DataFrame or +drop it into a snapshot the ``plot`` / ``compare`` machinery already reads:: from benchmarks import bench, REGISTRY r = bench.time(REGISTRY["basic"].build, 100) - r # rich repr in a notebook r.to_snapshot("a.json", spec="basic", size=100, phase="build") - bench.compare({"v1": f1, "v2": f2}).to_snapshot("cmp.json") -This plugs into the *output* side of the pipeline (snapshot JSON read by -``snapshot.load_long_df``), not into ``sweep``: a sweep runs pytest inside -per-version venvs as subprocesses, so it can only measure importable -registry models — an in-process callable can't cross that boundary. To -sweep a custom model across versions, promote it to ``benchmarks/models/``. - -**Methodology.** Timing is built on :class:`timeit.Timer`: an -``autorange`` calibration picks the inner iteration count (so timer -resolution doesn't dominate fast callables), then the per-iteration time -is sampled across rounds with the suite's min-of-N convention (the -fastest sample approximates the no-noise floor). It is *not* -pytest-benchmark's calibrated timer, so absolute numbers are not -interchangeable with suite snapshots — compare ``bench`` to ``bench`` and -suite to suite. +It feeds the output side (snapshot JSON), not ``sweep`` — a sweep runs pytest in +per-version venvs, so only importable registry models cross that boundary; +promote a custom model to ``benchmarks/models/`` to sweep it. + +Timing uses :class:`timeit.Timer` (``autorange`` picks the inner iteration count +so resolution doesn't dominate) with the suite's min-of-N convention. It is *not* +pytest-benchmark's timer, so compare ``bench`` to ``bench``, suite to suite. """ from __future__ import annotations diff --git a/benchmarks/memory.py b/benchmarks/memory.py index 83d13897a..7f5c69a01 100644 --- a/benchmarks/memory.py +++ b/benchmarks/memory.py @@ -1,37 +1,22 @@ """ Measure and compare peak memory across the registry × phase grid. -Each measurement uses ``memray.Tracker`` directly so the model construction -(setup) lives *outside* the tracked region and the peak reflects only the -phase work itself:: +Each measurement runs the phase work inside ``memray.Tracker`` with model +construction *outside* it, so the peak reflects only the phase:: m = spec.build(size) # setup, not tracked with memray.Tracker(bin_path): wrapper(m) # tracked - peak = FileReader(bin_path).metadata.peak_memory - -This module exposes ``save(label, ...)`` and ``compare(label_a, label_b)`` as -plain functions; user-facing invocation goes through the typer CLI:: - - python -m benchmarks memory save