From 3e85ce8415d1ad75e069c6557b7b6f6af6821352 Mon Sep 17 00:00:00 2001 From: Sergei Winitzki Date: Thu, 10 Sep 2026 17:21:42 +0200 Subject: [PATCH 1/3] show benchmark uncertainty and axis controls Co-authored-by: Cursor --- src/Perf/DB/Materialize.hs | 12 ++++++++-- src/Perf/Web/Plot.hs | 49 +++++++++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/Perf/DB/Materialize.hs b/src/Perf/DB/Materialize.hs index a17dca8..eb1ac84 100644 --- a/src/Perf/DB/Materialize.hs +++ b/src/Perf/DB/Materialize.hs @@ -22,7 +22,8 @@ type BenchmarkSeries key metric = (Map key metric))) data DisplayMetric = DisplayMetric - { mean :: Double + { mean :: Double, + stddev :: Double } deriving (Eq, Show) @@ -191,5 +192,12 @@ materializeExternalSnapshots snapshots = metrics = Map.fromList $ flip map test.metrics \metric -> - (Prim.MetricLabel metric.metric, Map.singleton label DisplayMetric {mean = metric.mean}) + ( Prim.MetricLabel metric.metric, + Map.singleton + label + DisplayMetric + { mean = metric.mean, + stddev = metric.stddev + } + ) in (factors, metrics)) diff --git a/src/Perf/Web/Plot.hs b/src/Perf/Web/Plot.hs index 6b0afba..70cc243 100644 --- a/src/Perf/Web/Plot.hs +++ b/src/Perf/Web/Plot.hs @@ -54,6 +54,7 @@ generateCommitPlotsWith masterCtx branchCommits = branchCommits shortCommitLabel (.metricMean) + (.metricStddev) generateExternalPlots :: BenchmarkSeries Text DisplayMetric -> Html () generateExternalPlots benchmarks = @@ -62,6 +63,7 @@ generateExternalPlots benchmarks = (collectKeys benchmarks) id (.mean) + (.stddev) benchmarks generatePlotsWith :: @@ -70,9 +72,10 @@ generatePlotsWith :: [key] -> (key -> Text) -> (metric -> Double) -> + (metric -> Double) -> BenchmarkSeries key metric -> Html () -generatePlotsWith masterCtx branchKeys renderKey metricMean benchmarks = do +generatePlotsWith masterCtx branchKeys renderKey metricMean metricStddev benchmarks = do let masterKeys = case masterCtx of MasterComparisonDisabled -> [] MasterComparisonEnabled cs -> cs @@ -104,12 +107,18 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean benchmarks = do Foldable.for_ (zip [0 :: Int ..] (Set.toList metrics)) \(metricIdx, metricLabel) -> do let dataSets = flip map (Map.toList tests) \(factors, allMetrics) -> - (factors, toSeries orderedKeys (Map.findWithDefault Map.empty metricLabel allMetrics)) + let metricSeries = Map.findWithDefault Map.empty metricLabel allMetrics + in ( factors, + toSeries metricMean orderedKeys metricSeries, + toSeries ((2 *) . metricStddev) orderedKeys metricSeries + ) labels = map renderKey orderedKeys (plotData, layout) = makePlotlyConfig metricLabel labels dataSets chartId = T.pack (show benchmarkIdx) <> "-" <> T.pack (show metricIdx) legendEntries = - zip (cycle plotColors) (map (factorsSmall . fst) dataSets) + zip + (cycle plotColors) + (map (\(factors, _, _) -> factorsSmall factors) dataSets) div_ [class_ "chart-cell"] do chart_ ChartOptions @@ -125,9 +134,9 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean benchmarks = do -- Script must run after plot containers are in the DOM. script_ plotControlsScript where - toSeries keys metricMap = + toSeries accessor keys metricMap = flip map keys \key -> - maybe Null (toJSON . metricMean) $ + maybe Null (toJSON . accessor) $ Map.lookup key metricMap collectKeys :: Ord key => BenchmarkSeries key metric -> [key] @@ -170,6 +179,17 @@ plotControls_ masterEnabled defaultMasterShow options = do <> [makeAttributes "selected" "selected" | n == defaultMasterShow] ) (toHtml (show n)) + label_ + [ for_ "start-y-at-zero", + style_ "display: flex; align-items: center; gap: 0.4rem; white-space: nowrap;" + ] + do + input_ + [ type_ "checkbox", + id_ "start-y-at-zero", + makeAttributes "checked" "checked" + ] + "Start Y axis at zero" -- | Color the first N x-axis ticks green via CSS (survives Plotly resize redraws). masterTickStyles :: Text @@ -191,6 +211,7 @@ plotControlsScript = (function () { const search = document.getElementById('plot-search'); const masterSelect = document.getElementById('master-commits'); + const startYAtZero = document.getElementById('start-y-at-zero'); const config = ${plotlyConfigJson}; const traceVisibility = new Map(); const singleClickTimers = new Map(); @@ -242,6 +263,8 @@ plotControlsScript = document.querySelectorAll('.benchmark-plot').forEach((el) => { const fullData = JSON.parse(el.getAttribute('data-full') || '[]'); const layout = JSON.parse(el.getAttribute('data-layout') || '{}'); + layout.yaxis = layout.yaxis || {}; + layout.yaxis.rangemode = (!startYAtZero || startYAtZero.checked) ? 'tozero' : 'normal'; const masterCount = parseInt(el.getAttribute('data-master-count') || '0', 10) || 0; const data = slicePlotData(fullData, masterCount, show); ensureVisibility(el.id, data.length); @@ -288,6 +311,7 @@ plotControlsScript = }); if (search) search.addEventListener('input', applySearch); if (masterSelect) masterSelect.addEventListener('change', redrawPlots); + if (startYAtZero) startYAtZero.addEventListener('change', redrawPlots); applySearch(); redrawPlots(); })(); @@ -296,7 +320,7 @@ plotControlsScript = makePlotlyConfig :: Prim.MetricLabel -> [Text] -> - [(Set Prim.GeneralFactor, [Value])] -> + [(Set Prim.GeneralFactor, [Value], [Value])] -> (Value, Value) makePlotlyConfig metricName labels dataSets = (toJSON traces, layout) @@ -309,9 +333,15 @@ makePlotlyConfig metricName labels dataSets = "mode" .= ("lines+markers" :: Text), "name" .= factorsSmall factors, "line" .= object ["color" .= color, "width" .= (2 :: Int)], - "marker" .= object ["size" .= (7 :: Int), "color" .= color] + "marker" .= object ["size" .= (7 :: Int), "color" .= color], + "error_y" + .= object + [ "type" .= ("data" :: Text), + "array" .= errors, + "visible" .= True + ] ] - | ((factors, series), color) <- zip dataSets $ cycle plotColors + | ((factors, series, errors), color) <- zip dataSets $ cycle plotColors ] layout = object @@ -323,6 +353,9 @@ makePlotlyConfig metricName labels dataSets = "xaxis" .= object [ "title" .= ("" :: Text), + "type" .= ("category" :: Text), + "categoryorder" .= ("array" :: Text), + "categoryarray" .= labels, "tickangle" .= (-30 :: Int), "automargin" .= True, "tickfont" .= object ["family" .= ("monospace" :: Text), "color" .= ("#111111" :: Text)] From 18256a887aa1901123694daa376add855d6cb2e6 Mon Sep 17 00:00:00 2001 From: Sergei Winitzki Date: Thu, 10 Sep 2026 19:30:03 +0200 Subject: [PATCH 2/3] sort "time" plots first --- src/Perf/Web/Plot.hs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/Perf/Web/Plot.hs b/src/Perf/Web/Plot.hs index 70cc243..bbc7393 100644 --- a/src/Perf/Web/Plot.hs +++ b/src/Perf/Web/Plot.hs @@ -2,7 +2,9 @@ module Perf.Web.Plot where import Data.Aeson import Data.Foldable qualified as Foldable +import Data.List (sortBy) import Data.Map qualified as Map +import Data.Ord (comparing) import Data.Set (Set) import Data.Set qualified as Set import Data.Text (Text) @@ -93,8 +95,11 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean metricStddev benchma unless (Map.null benchmarks) $ plotControls_ masterEnabled defaultMasterShow (masterCommitOptions viewingMaster) Foldable.for_ (zip [0 :: Int ..] (Map.toList benchmarks)) \(benchmarkIdx, (subject, tests)) -> do - let metrics :: Set Prim.MetricLabel = - Set.fromList $ concatMap Map.keys $ Map.elems tests + let metrics = + orderMetrics $ + Set.fromList $ + concatMap Map.keys $ + Map.elems tests div_ [ class_ "benchmark-subject", makeAttributes "data-plot-title" (subjectText subject), @@ -104,7 +109,7 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean metricStddev benchma do h2_ $ toHtml subject div_ [class_ "chart-grid"] do - Foldable.for_ (zip [0 :: Int ..] (Set.toList metrics)) \(metricIdx, metricLabel) -> do + Foldable.for_ (zip [0 :: Int ..] metrics) \(metricIdx, metricLabel) -> do let dataSets = flip map (Map.toList tests) \(factors, allMetrics) -> let metricSeries = Map.findWithDefault Map.empty metricLabel allMetrics @@ -404,6 +409,14 @@ subjectText (Prim.SubjectName t) = t metricText :: Prim.MetricLabel -> Text metricText (Prim.MetricLabel t) = t +-- | Metrics whose names start with "time" come first; the rest stay alphabetical. +orderMetrics :: Set Prim.MetricLabel -> [Prim.MetricLabel] +orderMetrics = + sortBy (comparing isNotTime <> comparing metricText) . Set.toList + where + isNotTime label = + not $ T.isPrefixOf "time" $ T.toLower $ metricText label + shortCommitLabel :: DB.Commit -> Text shortCommitLabel commit = T.take 8 $ From 7669725bcd578c8825d7b863110d7001ee3c1e08 Mon Sep 17 00:00:00 2001 From: Sergei Winitzki Date: Fri, 11 Sep 2026 12:17:36 +0200 Subject: [PATCH 3/3] update max number of commits on master and on branch --- src/Perf/Web/Plot.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Perf/Web/Plot.hs b/src/Perf/Web/Plot.hs index bbc7393..e1584f9 100644 --- a/src/Perf/Web/Plot.hs +++ b/src/Perf/Web/Plot.hs @@ -27,8 +27,8 @@ data MasterPlotContext key -- | Selectable values for the master-commits control. masterCommitOptions :: Bool -> [Int] masterCommitOptions viewingMaster - | viewingMaster = [0, 1, 2, 3, 5, 10, 20, 30, 50, 100] - | otherwise = [0, 1, 2, 3, 5, 10, 20] + | viewingMaster = [0, 1, 2, 3, 5, 10, 20, 30, 50, 100, 150, 200, 250, 300, 400, 500] + | otherwise = [0, 1, 2, 3, 5, 10, 20, 30, 50] -- | How many master commits to load when viewing master. maxMasterCommits :: Int @@ -90,7 +90,7 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean metricStddev benchma viewingMaster = masterEnabled && null branchKeys defaultMasterShow | viewingMaster = 20 - | masterEnabled = 1 + | masterEnabled = 3 | otherwise = 0 unless (Map.null benchmarks) $ plotControls_ masterEnabled defaultMasterShow (masterCommitOptions viewingMaster)