docs(bench): plot the results, and fold the tables under them

bench/conftest.py

@@ -70,10 +70,16 @@ def pytest_addoption(parser: pytest.Parser) -> None:
 #: `pytest-benchmem` is one of them: a fix to its isolated pass moves `rss`
 #: without a line of lpspec changing, so a result file that does not name the
 #: version that measured it cannot be compared across such a release.
+#:
+#: `gurobipy` and `scipy` for the same reason one level out: the `gurobi` sink
+#: is measurable now, and a published ratio through a solver has to say which
+#: solver — scipy being what carries the matrix into it.
 TRACKED = (
     'lpspec',
     'linopy',
     'highspy',
+    'gurobipy',
+    'scipy',
     'polars',
     'pandas',
     'numpy',

bench/plot.py

@@ -1,7 +1,12 @@
-"""Refresh the numbers inside ``docs/benchmarks-scaling.html``.
+"""Refresh what the docs plot: the chart page's numbers, and the SVG figures.
 
     uv run python -m bench.plot
 
+Two outputs from one read of the results, so a page and a figure can never
+disagree about the same run. ``docs/charts/*.svg`` are what
+``docs/benchmarks.md`` embeds — see :mod:`bench.svg` for why they are committed
+files rather than something drawn at view time.
+
 The page is a tracked source file, not a build artifact: its markup, prose and
 renderer are edited by hand and reviewed in the diff. Only the measurements go
 stale, so this rewrites exactly one line of it — the ``const DATA = {...};``
@@ -30,6 +35,21 @@
 _DATA = re.compile(r'^const DATA = .*;$', re.MULTILINE)
 
 
+def measurements(name: str) -> Path:
+    """The results file called *name*, in whichever format is committed.
+
+    The harness writes `.json` now; `bench/results/` still holds the `.jsonl`
+    the previous one wrote. Preferring the new and falling back to the old is
+    what lets this run against the tree as it stands rather than only after
+    the next full ladder.
+    """
+    for suffix in ('.json', '.jsonl'):
+        path = Path('bench/results') / f'{name}{suffix}'
+        if path.exists():
+            return path
+    raise SystemExit(f'no bench/results/{name}.json — run the ladder first (bench/README.md)')
+
+
 def best(path: Path, sink: str) -> dict[str, dict[Any, Any]]:
     """``(case, size, arm) -> fastest repeat``. Minimum, because noise only adds."""
     out: dict[str, dict[Any, Any]] = {'wall': {}, 'peak': {}, 'cols': {}}
@@ -56,9 +76,125 @@ def panel(t: dict[str, dict[Any, Any]], case: str, rungs: tuple[str, ...], arms:
     }
 
 
+#: Which sinks the figures face. A sink absent from the results is dropped
+#: rather than drawn empty — a facet with no line reads as a measurement of
+#: zero, and a reader has no way to tell it from one.
+SINKS = ('highs', 'lp', 'gurobi')
+
+
+def figures(results: Path) -> list[tuple[str, Any]]:
+    """The figures `docs/benchmarks.md` embeds, as (name, builder) pairs."""
+    from bench import svg
+
+    stamp = f'generated by bench.plot from {results.name}'
+    per_sink = {s: best(results, s) for s in SINKS}
+    present = [s for s in SINKS if per_sink[s]['wall']]
+
+    def lines(metric: str, unit: str, label: str, caption: str) -> svg.Figure:
+        panels = []
+        for sink in present:
+            table = per_sink[sink]
+            panels.append(
+                svg.Panel(
+                    title=f'{sink} sink',
+                    series=[
+                        svg.Series(
+                            NAME[arm],
+                            [
+                                (_at(table['cols'], ('dispatch', r)), _at(table[metric], ('dispatch', r, arm)))
+                                for r in LADDER
+                            ],
+                            slot=i,
+                        )
+                        for i, arm in enumerate(('lpspec', 'linopy'))
+                    ],
+                )
+            )
+        return svg.Figure(
+            panels=panels,
+            x_label='variables in the model',
+            y_label=label,
+            y_unit=unit,
+            caption=caption,
+            provenance=stamp,
+        )
+
+    def cases(scheme: svg.Scheme) -> str:
+        table = per_sink['highs']
+        names = sorted({c for c, _, _ in table['wall']})
+        panels = [
+            svg.Panel(
+                title=name,
+                series=[
+                    svg.Series(
+                        NAME[arm],
+                        [(_at(table['cols'], (name, r)), _at(table['wall'], (name, r, arm))) for r in LADDER],
+                        slot=i,
+                    )
+                    for i, arm in enumerate(('lpspec', 'linopy'))
+                ],
+            )
+            for name in names
+        ]
+        return svg.render(
+            svg.Figure(
+                panels=panels,
+                x_label='variables in the model',
+                y_label='wall seconds',
+                caption='Every model in the corpus, through the highs sink. Same two lines, same order, six shapes.',
+                provenance=stamp,
+                width=1180,
+                panel_height=190,
+            ),
+            scheme,
+        )
+
+    def sinks(scheme: svg.Scheme) -> str:
+        """The `l` rung, every destination, both arms.
+
+        This replaces a phase breakdown the harness no longer produces: #448
+        made `test_emit` one number for build-and-hand-over, and the only other
+        measurement is a *warm* in-process rebuild, so there is no honest way
+        to split the bar any more. What three sinks make askable instead is
+        where the advantage sits per destination — the LP file, HiGHS, Gurobi —
+        which the ratio tables answer one cell at a time.
+        """
+        rows = [
+            r
+            for r in bench_results.records(results)
+            if r.get('record') == 'timing' and r.get('size') == 'l' and 'error' not in r
+        ]
+        groups = []
+        for case in sorted({r['case'] for r in rows}):
+            bars_ = []
+            for sink in ('lp', 'highs', 'gurobi'):
+                for arm in ('lpspec', 'linopy'):
+                    hits = [
+                        r['wall_seconds'] for r in rows if r['case'] == case and r['sink'] == sink and r['arm'] == arm
+                    ]
+                    if hits:
+                        bars_.append((f'{sink} · {NAME[arm]}', [('wall', min(hits))]))
+            if bars_:
+                groups.append((case, bars_))
+        return svg.bars(
+            groups,
+            ('wall seconds',),
+            scheme,
+            caption='The l rung, every sink. Minimum of the timed rounds; the search is never started.',
+            provenance=stamp,
+        )
+
+    return [
+        ('wall', lines('wall', 's', 'wall seconds', 'Wall time to a loaded solver. Both axes are log.')),
+        ('peak', lines('peak', 'GB', 'peak RSS (GB)', 'Peak resident memory for the same builds. Both axes are log.')),
+        ('cases', cases),
+        ('sinks', sinks),
+    ]
+
+
 def main() -> int:
-    ladder = best(Path('bench/results/latest.json'), 'highs')
-    scaling = best(Path('bench/results/scaling.json'), 'lp')
+    ladder = best(measurements('latest'), 'highs')
+    scaling = best(measurements('scaling'), 'lp')
     cases = sorted({c for c, _, _ in ladder['wall']})
     data = {
         'scaling': panel(scaling, 'dispatch', SCALING, ('lpspec', 'linopy')),
@@ -73,6 +209,11 @@ def main() -> int:
         raise SystemExit(f'{page} has no `const DATA = ...;` line — keep the literal on one line of its own')
     page.write_text(_DATA.sub(lambda _: 'const DATA = ' + json.dumps(data) + ';', text, count=1))
     print(f'{page} refreshed')
+
+    from bench import svg
+
+    written = svg.write(figures(measurements('latest')), 'docs/charts')
+    print(f'{len(written)} figures -> docs/charts/')
     return 0
 
 

bench/report.py

@@ -95,6 +95,32 @@ def sizes_of(case: str, rows: dict[Key, Row], sink: str = 'lp', *, density: bool
     return sorted(seen, key=lambda s: seen[s])
 
 
+#: The figures `bench.plot` writes, in the order the page reads them. Paired
+#: light/dark because mkdocs-material's toggle stamps the *host* page and an
+#: `<img>`-referenced SVG cannot see it; GitHub takes the first of the pair.
+FIGURES = (
+    ('wall', 'Wall time to a loaded solver, by model size'),
+    ('peak', 'Peak resident memory, by model size'),
+    ('cases', 'Every model in the corpus, through the highs sink'),
+    ('sinks', 'The l rung through every sink, both arms'),
+)
+
+
+def figures() -> str:
+    """The figure embeds, as markdown that renders in both places."""
+    out = []
+    for name, alt in FIGURES:
+        out.append(f'![{alt}](charts/{name}-light.svg#only-light)')
+        out.append(f'![{alt}](charts/{name}-dark.svg#only-dark)')
+        out.append('')
+    # one pointer for the four, not four for the four: these are pictures, and
+    # reading a value off one is what the interactive page is for
+    out.append(
+        '*Static, so they render anywhere. The same data with a cursor: [the chart page](benchmarks-scaling.html).*'
+    )
+    return '\n'.join(out)
+
+
 #: How each arm reaches each sink, said once so a table can name its own seam.
 _SEAM = {
     'lp': 'lpspec writes the LP file, linopy through its `lp-polars` writer.',
@@ -124,7 +150,10 @@ def table(case: str, rows: dict[Key, Row], sink: str = 'lp') -> str:
         + ['peak', 'LP']
     )
     lines = [
-        f'### {case} — {sink} sink',
+        # bold, not a heading: these live inside a collapsed <details>, and a
+        # heading in there still lands in the table of contents — twelve rail
+        # entries for tables the page has just decided are the appendix
+        f'**{case} — {sink} sink**',
         '',
         _SEAM[sink],
         '',
@@ -306,12 +335,22 @@ def main(argv: list[str] | None = None) -> int:
         )
     else:
         print('Parity gate: enforced at measurement time — every arm below built the same model.')
+    print()
+    print(figures())
     for case in sorted({c for c, _, _, _ in rows}):
-        for sink in sorted({k for c, _, k, _ in rows if c == case}):
-            if not sizes_of(case, rows, sink):
-                continue
-            print()
+        sinks = [k for k in sorted({k for c, _, k, _ in rows if c == case}) if sizes_of(case, rows, k)]
+        if not sinks:
+            continue
+        print()
+        # Collapsed, and `<details>` rather than mkdocs' `???` because these
+        # pages are read on GitHub too, where only the HTML form folds. The
+        # figures above are the reading; the numbers stay one click away
+        # because a chart nobody can check is decoration.
+        print(f'<details markdown="1">\n<summary><b>{case}</b> — every rung, every sink</summary>\n')
+        for sink in sinks:
             print(table(case, rows, sink))
+            print()
+        print('</details>')
     loop_table = marginal(loop)
     if loop_table:
         print()

bench/results.py

@@ -77,7 +77,20 @@ def _counts(extra: dict[str, Any]) -> dict[str, Any]:
 
 
 def records(path: Path) -> Iterator[dict[str, Any]]:
-    """Every measurement in *path*, in the shape the report and the plot read."""
+    """Every measurement in *path*, in the shape the report and the plot read.
+
+    Two formats, because the harness changed and the committed results did
+    not. ``.jsonl`` is what the runner before #448 wrote — one record per
+    line, already in this shape — and it is still what `bench/results/` holds,
+    so the reader takes it verbatim. `.json` is pytest-benchmark's document,
+    which the rest of this function unpacks.
+    """
+    if path.suffix == '.jsonl':
+        for line in path.read_text().splitlines():
+            if line.strip():
+                yield json.loads(line)
+        return
+
     doc = json.loads(path.read_text())
     machine, commit = doc.get('machine_info', {}), doc.get('commit_info', {})
     yield {