Skip to content

show benchmark uncertainty and axis controls #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
winitzki merged 3 commits into from
Sep 11, 2026
Merged

show benchmark uncertainty and axis controls #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3e85ce8
show benchmark uncertainty and axis controls
Sep 10, 2026
18256a8
sort "time" plots first
Sep 10, 2026
7669725
update max number of commits on master and on branch
Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/Perf/DB/Materialize.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ type BenchmarkSeries key metric =
(Map key metric)))

data DisplayMetric = DisplayMetric
{ mean :: Double
{ mean :: Double,
stddev :: Double
}
deriving (Eq, Show)

Expand Down Expand Up @@ -191,5 +192,12 @@ materializeExternalSnapshots snapshots =
metrics =
Map.fromList $
flip map test.metrics \metric ->
(Prim.MetricLabel metric.metric, Map.singleton label DisplayMetric {mean = metric.mean})
( Prim.MetricLabel metric.metric,
Map.singleton
label
DisplayMetric
{ mean = metric.mean,
stddev = metric.stddev
}
)
in (factors, metrics))
74 changes: 60 additions & 14 deletions src/Perf/Web/Plot.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ module Perf.Web.Plot where

import Data.Aeson
import Data.Foldable qualified as Foldable
import Data.List (sortBy)
import Data.Map qualified as Map
import Data.Ord (comparing)
import Data.Set (Set)
import Data.Set qualified as Set
import Data.Text (Text)
Expand All @@ -25,8 +27,8 @@ data MasterPlotContext key
-- | Selectable values for the master-commits control.
masterCommitOptions :: Bool -> [Int]
masterCommitOptions viewingMaster
| viewingMaster = [0, 1, 2, 3, 5, 10, 20, 30, 50, 100]
| otherwise = [0, 1, 2, 3, 5, 10, 20]
| viewingMaster = [0, 1, 2, 3, 5, 10, 20, 30, 50, 100, 150, 200, 250, 300, 400, 500]
| otherwise = [0, 1, 2, 3, 5, 10, 20, 30, 50]

-- | How many master commits to load when viewing master.
maxMasterCommits :: Int
Expand Down Expand Up @@ -54,6 +56,7 @@ generateCommitPlotsWith masterCtx branchCommits =
branchCommits
shortCommitLabel
(.metricMean)
(.metricStddev)

generateExternalPlots :: BenchmarkSeries Text DisplayMetric -> Html ()
generateExternalPlots benchmarks =
Expand All @@ -62,6 +65,7 @@ generateExternalPlots benchmarks =
(collectKeys benchmarks)
id
(.mean)
(.stddev)
benchmarks

generatePlotsWith ::
Expand All @@ -70,9 +74,10 @@ generatePlotsWith ::
[key] ->
(key -> Text) ->
(metric -> Double) ->
(metric -> Double) ->
BenchmarkSeries key metric ->
Html ()
generatePlotsWith masterCtx branchKeys renderKey metricMean benchmarks = do
generatePlotsWith masterCtx branchKeys renderKey metricMean metricStddev benchmarks = do
let masterKeys = case masterCtx of
MasterComparisonDisabled -> []
MasterComparisonEnabled cs -> cs
Expand All @@ -85,13 +90,16 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean benchmarks = do
viewingMaster = masterEnabled && null branchKeys
defaultMasterShow
| viewingMaster = 20
| masterEnabled = 1
| masterEnabled = 3
| otherwise = 0
unless (Map.null benchmarks) $
plotControls_ masterEnabled defaultMasterShow (masterCommitOptions viewingMaster)
Foldable.for_ (zip [0 :: Int ..] (Map.toList benchmarks)) \(benchmarkIdx, (subject, tests)) -> do
let metrics :: Set Prim.MetricLabel =
Set.fromList $ concatMap Map.keys $ Map.elems tests
let metrics =
orderMetrics $
Set.fromList $
concatMap Map.keys $
Map.elems tests
div_
[ class_ "benchmark-subject",
makeAttributes "data-plot-title" (subjectText subject),
Expand All @@ -101,15 +109,21 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean benchmarks = do
do
h2_ $ toHtml subject
div_ [class_ "chart-grid"] do
Foldable.for_ (zip [0 :: Int ..] (Set.toList metrics)) \(metricIdx, metricLabel) -> do
Foldable.for_ (zip [0 :: Int ..] metrics) \(metricIdx, metricLabel) -> do
let dataSets =
flip map (Map.toList tests) \(factors, allMetrics) ->
(factors, toSeries orderedKeys (Map.findWithDefault Map.empty metricLabel allMetrics))
let metricSeries = Map.findWithDefault Map.empty metricLabel allMetrics
in ( factors,
toSeries metricMean orderedKeys metricSeries,
toSeries ((2 *) . metricStddev) orderedKeys metricSeries
)
labels = map renderKey orderedKeys
(plotData, layout) = makePlotlyConfig metricLabel labels dataSets
chartId = T.pack (show benchmarkIdx) <> "-" <> T.pack (show metricIdx)
legendEntries =
zip (cycle plotColors) (map (factorsSmall . fst) dataSets)
zip
(cycle plotColors)
(map (\(factors, _, _) -> factorsSmall factors) dataSets)
div_ [class_ "chart-cell"] do
chart_
ChartOptions
Expand All @@ -125,9 +139,9 @@ generatePlotsWith masterCtx branchKeys renderKey metricMean benchmarks = do
-- Script must run after plot containers are in the DOM.
script_ plotControlsScript
where
toSeries keys metricMap =
toSeries accessor keys metricMap =
flip map keys \key ->
maybe Null (toJSON . metricMean) $
maybe Null (toJSON . accessor) $
Map.lookup key metricMap

collectKeys :: Ord key => BenchmarkSeries key metric -> [key]
Expand Down Expand Up @@ -170,6 +184,17 @@ plotControls_ masterEnabled defaultMasterShow options = do
<> [makeAttributes "selected" "selected" | n == defaultMasterShow]
)
(toHtml (show n))
label_
[ for_ "start-y-at-zero",
style_ "display: flex; align-items: center; gap: 0.4rem; white-space: nowrap;"
]
do
input_
[ type_ "checkbox",
id_ "start-y-at-zero",
makeAttributes "checked" "checked"
]
"Start Y axis at zero"

-- | Color the first N x-axis ticks green via CSS (survives Plotly resize redraws).
masterTickStyles :: Text
Expand All @@ -191,6 +216,7 @@ plotControlsScript =
(function () {
const search = document.getElementById('plot-search');
const masterSelect = document.getElementById('master-commits');
const startYAtZero = document.getElementById('start-y-at-zero');
const config = ${plotlyConfigJson};
const traceVisibility = new Map();
const singleClickTimers = new Map();
Expand Down Expand Up @@ -242,6 +268,8 @@ plotControlsScript =
document.querySelectorAll('.benchmark-plot').forEach((el) => {
const fullData = JSON.parse(el.getAttribute('data-full') || '[]');
const layout = JSON.parse(el.getAttribute('data-layout') || '{}');
layout.yaxis = layout.yaxis || {};
layout.yaxis.rangemode = (!startYAtZero || startYAtZero.checked) ? 'tozero' : 'normal';
const masterCount = parseInt(el.getAttribute('data-master-count') || '0', 10) || 0;
const data = slicePlotData(fullData, masterCount, show);
ensureVisibility(el.id, data.length);
Expand Down Expand Up @@ -288,6 +316,7 @@ plotControlsScript =
});
if (search) search.addEventListener('input', applySearch);
if (masterSelect) masterSelect.addEventListener('change', redrawPlots);
if (startYAtZero) startYAtZero.addEventListener('change', redrawPlots);
applySearch();
redrawPlots();
})();
Expand All @@ -296,7 +325,7 @@ plotControlsScript =
makePlotlyConfig ::
Prim.MetricLabel ->
[Text] ->
[(Set Prim.GeneralFactor, [Value])] ->
[(Set Prim.GeneralFactor, [Value], [Value])] ->
(Value, Value)
makePlotlyConfig metricName labels dataSets =
(toJSON traces, layout)
Expand All @@ -309,9 +338,15 @@ makePlotlyConfig metricName labels dataSets =
"mode" .= ("lines+markers" :: Text),
"name" .= factorsSmall factors,
"line" .= object ["color" .= color, "width" .= (2 :: Int)],
"marker" .= object ["size" .= (7 :: Int), "color" .= color]
"marker" .= object ["size" .= (7 :: Int), "color" .= color],
"error_y"
.= object
[ "type" .= ("data" :: Text),
"array" .= errors,
"visible" .= True
]
]
| ((factors, series), color) <- zip dataSets $ cycle plotColors
| ((factors, series, errors), color) <- zip dataSets $ cycle plotColors
]
layout =
object
Expand All @@ -323,6 +358,9 @@ makePlotlyConfig metricName labels dataSets =
"xaxis"
.= object
[ "title" .= ("" :: Text),
"type" .= ("category" :: Text),

@winitzki winitzki Sep 11, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should fix the formatting git commit hashes as text and not as auto-detected type.

"categoryorder" .= ("array" :: Text),
"categoryarray" .= labels,
"tickangle" .= (-30 :: Int),
"automargin" .= True,
"tickfont" .= object ["family" .= ("monospace" :: Text), "color" .= ("#111111" :: Text)]
Expand Down Expand Up @@ -371,6 +409,14 @@ subjectText (Prim.SubjectName t) = t
metricText :: Prim.MetricLabel -> Text
metricText (Prim.MetricLabel t) = t

-- | Metrics whose names start with "time" come first; the rest stay alphabetical.
orderMetrics :: Set Prim.MetricLabel -> [Prim.MetricLabel]
orderMetrics =
sortBy (comparing isNotTime <> comparing metricText) . Set.toList
where
isNotTime label =
not $ T.isPrefixOf "time" $ T.toLower $ metricText label

shortCommitLabel :: DB.Commit -> Text
shortCommitLabel commit =
T.take 8 $
Expand Down
Loading