diff --git a/Project.toml b/Project.toml index f811cbe..8785dba 100644 --- a/Project.toml +++ b/Project.toml @@ -46,7 +46,7 @@ DoubleEnded = "0.2.3" ImmutableList = "0.4.2" ListUtil = "0.4.1" MetaModelica = "0.4.2" -OM = "0.7.0" +OM = "0.9" OMBackend = "0.8.0" OMFrontend = "1.7.5" OMParser = "0.2.1" diff --git a/README.md b/README.md index 7c61109..a763dc0 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ OMLibraryTesting.jl/ │ ├── analysis.jl # Error categorization and analysis utilities │ ├── report.jl # HTML report generation │ ├── site.jl # Coverage site: one page per run, index, badges +│ ├── signals.jl # The compared signals as JSON (signals.json) │ └── testsuite.jl # Fetch + run the upstream OpenModelica testsuite (rtest / OM.jl modes) ├── models/ │ ├── models.toml # Per-model overrides (atol, reltol, expected phase, signal mapping), MSL 3.2.3 @@ -103,7 +104,9 @@ that every model is there once) and passes them to the publisher. It also adds t coverage site, served by GitHub Pages from the `gh-pages` branch (`scripts/publish_site.jl`): `runs//` with the analysis and the per-model report, an index over all runs, and the badges (`badge.json`, models that validate; `badges/.json`); MSL 4.1.0 has the same in `msl-4.1.0/`, docs/COVERAGE_4.1.0.md and its own -section above. The same by hand (`OMJL_COVERAGE_MSL`, default `MSL:3.2.3`, selects the version for all three): +section above. A run's `signals.json` holds the signals each validation compared, OM.jl's values and the +reference's at the comparison times, passed or not (`signals_json`); the latest 5 runs of a version keep it. +`generate_report` writes the same next to the HTML report (`.signals.json`). The same by hand (`OMJL_COVERAGE_MSL`, default `MSL:3.2.3`, selects the version for all three): ```bash OMJL_COVERAGE_NOSKIP=1 julia --project=. scripts/run_full_coverage.jl diff --git a/src/OMLibraryTesting.jl b/src/OMLibraryTesting.jl index 8a3b9c1..882d3d6 100644 --- a/src/OMLibraryTesting.jl +++ b/src/OMLibraryTesting.jl @@ -46,6 +46,7 @@ include("discovery.jl") include("runner.jl") include("analysis.jl") include("report.jl") +include("signals.jl") include("site.jl") include("rerun_failed.jl") include("testsuite.jl") @@ -59,6 +60,7 @@ export discover_experiments, record_experiments, merge_overrides!, clear_discove export run_coverage, run_coverage_and_report, run_model, WorkerManager, WorkerPool, ensure_pool! export shard_specs, merge_shards export generate_report, print_summary +export signals_json, write_signals export coverage_funnel, publish_site, site_history export categorize_error, categorize_results, print_error_analysis export models_with_error, error_for_model, run_frontend_coverage diff --git a/src/comparison.jl b/src/comparison.jl index 81c4d0c..6f103ad 100644 --- a/src/comparison.jl +++ b/src/comparison.jl @@ -41,21 +41,6 @@ struct ReferenceData signals::Dict{String, Vector{Float64}} end -""" - SignalComparison - -Result of comparing one signal against its reference. -""" -struct SignalComparison - name::String - passed::Bool - max_abs_err::Float64 - max_rel_err::Float64 - worst_time::Float64 - worst_actual::Float64 - worst_expected::Float64 -end - """ load_reference_csv(path) -> ReferenceData @@ -244,13 +229,16 @@ function compare_signal(sol, ref::ReferenceData, signal_name::String, worst_expected = 0.0 passed = true times = range(0.0, stopTime, length = npoints) + local sampleTimes::Vector{Float64} = collect(times) + local expectedValues::Vector{Float64} = [interpolate_reference(ref.time, ref_values, t) for t in times] + local actualValues::Vector{Float64} = fill(NaN, npoints) #= At a reference discontinuity the sample instant carries both limits (set-valued jump); accept the actual value if it matches either one-sided reference limit within tolerance. =# knot_eps = length(ref.time) > 1 ? 1.5 * (ref.time[end] - ref.time[1]) / (length(ref.time) - 1) : 0.0 - for t in times - expected = interpolate_reference(ref.time, ref_values, t) + for (i, t) in enumerate(times) + expected = expectedValues[i] actual = try sol(t, idxs = resolved_sym) catch @@ -264,12 +252,15 @@ function compare_signal(sol, ref::ReferenceData, signal_name::String, #= Not in the solution: not compared (passed = false, NaN errors), which fails the validation (validate_against_reference). It passed, so a model validated on a subset of its signals; and a sample after a failed one discarded that failure. =# - return SignalComparison(signal_name, false, NaN, NaN, t, NaN, expected) + return SignalComparison(signal_name, false, NaN, NaN, t, NaN, expected, + sampleTimes, expectedValues, actualValues) end end + actual isa Real && (actualValues[i] = Float64(actual)) #= A non-finite value is a failure: NaN compared as never above the threshold. =# if !(actual isa Real) || !isfinite(actual) - return SignalComparison(signal_name, false, Inf, Inf, t, actual isa Real ? Float64(actual) : NaN, expected) + return SignalComparison(signal_name, false, Inf, Inf, t, actual isa Real ? Float64(actual) : NaN, expected, + sampleTimes, expectedValues, actualValues) end abs_err = abs(actual - expected) rel_err = abs(expected) > 1e-15 ? abs_err / abs(expected) : abs_err @@ -302,7 +293,8 @@ function compare_signal(sol, ref::ReferenceData, signal_name::String, end end return SignalComparison(signal_name, passed, max_abs, max_rel, - worst_t, worst_actual, worst_expected) + worst_t, worst_actual, worst_expected, + sampleTimes, expectedValues, actualValues) end """ diff --git a/src/report.jl b/src/report.jl index 2e477a9..00336b6 100644 --- a/src/report.jl +++ b/src/report.jl @@ -119,6 +119,11 @@ function generate_report(results::Vector{ModelResult}; write(filepath, content) @info "Report saved to $filepath" + #= the compared signals next to it, for looking at how a model behaved (signals.jl) =# + if format === :html + local signalsPath = splitext(filepath)[1] * ".signals.json" + write_signals(signalsPath, results) && @info "Signals saved to $signalsPath" + end return filepath end @@ -155,9 +160,10 @@ function _generate_html(results, filepath, ts, total, broken, tested, val_cell = phase_cell(r.phases, VALIDATE) total_model_time = sum(p.time_s for p in r.phases; init = 0.0) time_cell = isempty(r.phases) ? "—" : "$(round(total_model_time, digits=1))s" + local signals_link = isempty(r.signals) ? "" : _signals_link(r) print(rows, """ - $name_short + $name_short$signals_link $fe_cell $be_cell $sim_cell @@ -215,6 +221,7 @@ function _generate_html(results, filepath, ts, total, broken, tested, .broken-row { opacity: 0.5; } .broken-row td { font-style: italic; } .domain-header td { background: #e9ecef; font-size: 0.85em; padding: 6px 10px; } + $(_SIGNALS_CSS) @@ -250,7 +257,7 @@ function _generate_html(results, filepath, ts, total, broken, tested,
- Error detail + Error detail 

       

     

@@ -258,6 +265,8 @@ function _generate_html(results, filepath, ts, total, broken, tested,
       function showError(td) {
         var msg = td.getAttribute('data-error');
         if (!msg) return;
+        document.getElementById('error-modal-title').textContent = 'Error detail';
+        document.getElementById('error-modal-text').className = '';
         document.getElementById('error-modal-text').textContent = msg;
         document.getElementById('error-modal').classList.add('open');
       }
@@ -271,11 +280,223 @@ function _generate_html(results, filepath, ts, total, broken, tested,
         if (e.key === 'Escape') closeError();
       });
     
+    
     
     
     """
 end
 
+#= A model's link to the signals its validation compared (signals.jl): how many passed of how many. =#
+function _signals_link(r::ModelResult)::String
+    local n::Int = length(r.signals)
+    local passed::Int = count(s -> s.passed, r.signals)
+    local cls::String = passed < n ? "sig-link sig-has-fail" : "sig-link"
+    return " signals $passed/$n"
+end
+
+const _SIGNALS_CSS = raw"""
+      .sig-link { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+                  font-size: 0.85em; margin-left: 0.6em; color: #0366d6; text-decoration: none; white-space: nowrap; }
+      .sig-link:hover { text-decoration: underline; }
+      .sig-link.sig-has-fail { color: #b31d28; font-weight: 600; }
+      .sig-row > td { background: #fafbfc; padding: 10px 12px; }
+      .sig-head { font-size: 0.85em; color: #555; margin-bottom: 8px; }
+      .sig-key-ref { color: #555; font-weight: 600; }
+      .sig-key-omjl { color: #0366d6; font-weight: 600; }
+      .sig-grid { display: flex; flex-wrap: wrap; gap: 12px; }
+      .sig-card { border: 1px solid #ddd; border-radius: 4px; background: #fff; padding: 6px 8px; width: 376px; }
+      .sig-card.sig-fail { border-color: #e4a3a8; }
+      .sig-title { font-family: monospace; font-size: 0.85em; word-break: break-all; }
+      .sig-title a { color: #333; text-decoration: none; }
+      .sig-title a:hover { text-decoration: underline; }
+      .sig-mark-ok { color: #155724; }
+      .sig-mark-fail { color: #b31d28; font-weight: 600; }
+      .sig-stats, .sig-readout { font-size: 0.75em; color: #666; font-family: monospace; min-height: 1.3em; }
+      .sig-plot { display: block; max-width: 100%; height: auto; }
+      .sig-plot .frame { fill: none; stroke: #ddd; }
+      .sig-plot .band { fill: #e6e6e6; }
+      .sig-plot .ref { fill: none; stroke: #555; stroke-width: 1.5; }
+      .sig-plot .omjl { fill: none; stroke: #0366d6; stroke-width: 1.5; stroke-dasharray: 5 3; }
+      .sig-plot .worst { fill: none; stroke: #b31d28; stroke-width: 2; }
+      .sig-plot .cursor { stroke: #999; stroke-dasharray: 2 2; }
+      .sig-plot text { font-size: 10px; fill: #666; font-family: monospace; }
+      #error-modal-text.values { color: #333; }
+"""
+
+#= The viewer of a model's compared signals: the run's signals.json (on the site), or the report's
+   .signals.json beside it, read on the first click; a report opened from disk cannot read a
+   file, so it asks for it. =#
+const _SIGNALS_JS = raw"""
+var SIGNALS = null, SIGNALS_LOADING = null, SVGNS = 'http://www.w3.org/2000/svg';
+function signalFiles() {
+  var page = location.pathname.split('/').pop() || 'report.html';
+  return [page.replace(/\.html?$/, '') + '.signals.json', 'signals.json'];
+}
+function loadSignals() {
+  if (SIGNALS) return Promise.resolve(SIGNALS);
+  if (SIGNALS_LOADING) return SIGNALS_LOADING;
+  SIGNALS_LOADING = signalFiles().reduce(function(p, f) {
+    return p.catch(function() {
+      return fetch(f).then(function(r) { if (!r.ok) throw new Error(f + ': ' + r.status); return r.json(); });
+    });
+  }, Promise.reject(new Error('none'))).then(function(d) { SIGNALS = d.models; return SIGNALS; });
+  SIGNALS_LOADING.catch(function() { SIGNALS_LOADING = null; });
+  return SIGNALS_LOADING;
+}
+function toggleSignals(a) {
+  var tr = a.closest('tr'), next = tr.nextElementSibling;
+  if (next && next.classList.contains('sig-row')) { next.remove(); return false; }
+  var row = document.createElement('tr'), td = document.createElement('td');
+  row.className = 'sig-row'; td.colSpan = 6; row.appendChild(td);
+  tr.parentNode.insertBefore(row, tr.nextSibling);
+  td.textContent = 'Loading the signals...';
+  var model = a.getAttribute('data-model');
+  loadSignals().then(function(models) { showModel(td, models, model); },
+                     function() { askForFile(td, model); });
+  return false;
+}
+function showModel(td, models, model) {
+  if (models[model]) renderSignals(td, models[model]);
+  else td.textContent = 'The signals file has no signals of ' + model + '.';
+}
+function askForFile(td, model) {
+  td.textContent = '';
+  var p = document.createElement('p');
+  p.textContent = 'The signals could not be read. The coverage site keeps them (signals.json) for the latest 5 runs ' +
+    'of a version; a report opened from disk cannot read the .signals.json beside it: choose that file here. ';
+  var input = document.createElement('input');
+  input.type = 'file'; input.accept = '.json,application/json';
+  input.onchange = function() {
+    if (!input.files[0]) return;
+    input.files[0].text().then(function(t) { SIGNALS = JSON.parse(t).models; showModel(td, SIGNALS, model); });
+  };
+  p.appendChild(input); td.appendChild(p);
+}
+function fmt(x) {
+  if (x === null || x === undefined) return '—';
+  var a = Math.abs(x);
+  return (a !== 0 && (a >= 1e5 || a < 1e-3)) ? x.toExponential(3) : String(Number(x.toPrecision(6)));
+}
+function fmtAxis(x) {
+  var a = Math.abs(x);
+  return (a !== 0 && (a >= 1e5 || a < 1e-3)) ? x.toExponential(2) : String(Number(x.toPrecision(4)));
+}
+function sampleTimes(s, m) {
+  var T = s.time || m.time, n = T[2], t = [];
+  for (var i = 0; i < n; i++) t.push(n > 1 ? T[0] + i * (T[1] - T[0]) / (n - 1) : T[0]);
+  return t;
+}
+function renderSignals(td, m) {
+  td.textContent = '';
+  var sigs = m.signals.slice().sort(function(x, y) { return x.passed - y.passed; });   // failed first
+  var failed = sigs.filter(function(s) { return !s.passed; }).length;
+  var head = document.createElement('div');
+  head.className = 'sig-head';
+  head.innerHTML = '— reference   - - OM.jl   ' +
+    'grey: the tolerance around the reference (atol ' + fmt(m.atol) + ', reltol ' + fmt(m.reltol) + '); values with 6 significant digits. ';
+  head.appendChild(document.createTextNode(sigs.length + ' signals compared, ' + failed + ' outside the tolerance' +
+    (m.validated ? '.' : '; the model did not validate.') + ' Point at a plot to read its values; click a name for all of them.'));
+  td.appendChild(head);
+  var grid = document.createElement('div');
+  grid.className = 'sig-grid';
+  sigs.forEach(function(s) { grid.appendChild(signalCard(s, m)); });
+  td.appendChild(grid);
+}
+function signalCard(s, m) {
+  var card = document.createElement('div'), title = document.createElement('div'), name = document.createElement('a');
+  var mark = document.createElement('span'), stats = document.createElement('div'), readout = document.createElement('div');
+  card.className = 'sig-card ' + (s.passed ? 'sig-pass' : 'sig-fail');
+  title.className = 'sig-title';
+  name.href = '#'; name.textContent = s.name; name.title = 'All compared values';
+  name.onclick = function() { showValues(s, m); return false; };
+  mark.className = s.passed ? 'sig-mark-ok' : 'sig-mark-fail';
+  mark.textContent = s.passed ? ' ✓' : ' ✗ outside the tolerance';
+  title.appendChild(name); title.appendChild(mark);
+  stats.className = 'sig-stats';
+  stats.textContent = s.max_abs_err === null ? 'not compared: no OM.jl value' :
+    'max |error| ' + fmt(s.max_abs_err) + ' (relative ' + fmt(s.max_rel_err) + ') at t = ' + fmt(s.worst_time);
+  readout.className = 'sig-readout';
+  card.appendChild(title); card.appendChild(stats); card.appendChild(signalPlot(s, m, readout)); card.appendChild(readout);
+  return card;
+}
+function svgEl(name, attrs) {
+  var e = document.createElementNS(SVGNS, name);
+  for (var k in attrs) e.setAttribute(k, attrs[k]);
+  return e;
+}
+function svgText(svg, x, y, s, anchor) {
+  var t = svgEl('text', {x: x, y: y, 'text-anchor': anchor});
+  t.textContent = s; svg.appendChild(t);
+}
+function svgLine(svg, t, v, X, Y, cls) {   // one polyline per run of values (none where null)
+  var pts = [];
+  function flush() { if (pts.length) svg.appendChild(svgEl('polyline', {points: pts.join(' '), 'class': cls})); pts = []; }
+  for (var i = 0; i < t.length; i++) { if (v[i] === null) flush(); else pts.push(X(t[i]) + ',' + Y(v[i])); }
+  flush();
+}
+function signalPlot(s, m, readout) {
+  var W = 360, H = 160, L = 62, R = 8, T = 8, B = 20;
+  var t = sampleTimes(s, m), ref = s.reference, om = s.omjl, n = t.length, i;
+  var tol = ref.map(function(v) { return v === null ? null : m.atol + m.reltol * Math.abs(v); });
+  var ys = [];
+  for (i = 0; i < n; i++) {
+    if (ref[i] !== null) ys.push(ref[i] + tol[i], ref[i] - tol[i]);
+    if (om[i] !== null) ys.push(om[i]);
+  }
+  var y0 = ys.length ? Math.min.apply(null, ys) : 0, y1 = ys.length ? Math.max.apply(null, ys) : 1;
+  if (y1 - y0 <= 1e-12 * Math.max(1, Math.abs(y1))) { var c = y0, d = 1e-6 * Math.max(1, Math.abs(c)); y0 = c - d; y1 = c + d; }
+  var pad = 0.05 * (y1 - y0); y0 -= pad; y1 += pad;
+  var t0 = t[0], t1 = t[n - 1] > t0 ? t[n - 1] : t0 + 1;
+  function X(x) { return L + (x - t0) / (t1 - t0) * (W - L - R); }
+  function Y(y) { return T + (y1 - y) / (y1 - y0) * (H - T - B); }
+  var svg = svgEl('svg', {viewBox: '0 0 ' + W + ' ' + H, width: W, height: H, 'class': 'sig-plot'});
+  svg.appendChild(svgEl('rect', {x: L, y: T, width: W - L - R, height: H - T - B, 'class': 'frame'}));
+  var up = [], lo = [];
+  for (i = 0; i < n; i++) if (ref[i] !== null) { up.push(X(t[i]) + ',' + Y(ref[i] + tol[i])); lo.unshift(X(t[i]) + ',' + Y(ref[i] - tol[i])); }
+  if (up.length) svg.appendChild(svgEl('polygon', {points: up.concat(lo).join(' '), 'class': 'band'}));
+  svgLine(svg, t, ref, X, Y, 'ref');
+  svgLine(svg, t, om, X, Y, 'omjl');
+  if (!s.passed && s.worst_time !== null) {
+    var w = Math.round((s.worst_time - t0) / (t1 - t0) * (n - 1));
+    w = Math.max(0, Math.min(n - 1, w));
+    if (om[w] !== null) svg.appendChild(svgEl('circle', {cx: X(t[w]), cy: Y(om[w]), r: 4, 'class': 'worst'}));
+  }
+  svgText(svg, L - 4, T + 9, fmtAxis(y1 - pad), 'end');
+  svgText(svg, L - 4, H - B, fmtAxis(y0 + pad), 'end');
+  svgText(svg, L, H - 5, fmtAxis(t[0]), 'start');
+  svgText(svg, W - R, H - 5, 't = ' + fmtAxis(t[n - 1]), 'end');
+  var cursor = svgEl('line', {y1: T, y2: H - B, 'class': 'cursor', visibility: 'hidden'});
+  svg.appendChild(cursor);
+  var hit = svgEl('rect', {x: L, y: T, width: W - L - R, height: H - T - B, fill: 'transparent'});
+  hit.addEventListener('mousemove', function(ev) {
+    var box = svg.getBoundingClientRect(), x = (ev.clientX - box.left) * W / box.width;
+    var k = Math.max(0, Math.min(n - 1, Math.round((x - L) / (W - L - R) * (n - 1))));
+    cursor.setAttribute('x1', X(t[k])); cursor.setAttribute('x2', X(t[k])); cursor.setAttribute('visibility', 'visible');
+    readout.textContent = 't = ' + fmt(t[k]) + '  reference ' + fmt(ref[k]) + '  OM.jl ' + fmt(om[k]) +
+      (ref[k] !== null && om[k] !== null ? '  |diff| ' + fmt(Math.abs(om[k] - ref[k])) : '');
+  });
+  hit.addEventListener('mouseleave', function() { cursor.setAttribute('visibility', 'hidden'); readout.textContent = ''; });
+  svg.appendChild(hit);
+  return svg;
+}
+function showValues(s, m) {
+  var t = sampleTimes(s, m), lines = [];
+  function pad(text) { return (text + '                ').slice(0, 16); }
+  lines.push(pad('t') + pad('reference') + pad('OM.jl') + '|diff|');
+  for (var i = 0; i < t.length; i++) {
+    var r = s.reference[i], o = s.omjl[i];
+    lines.push(pad(fmt(t[i])) + pad(fmt(r)) + pad(fmt(o)) + (r !== null && o !== null ? fmt(Math.abs(o - r)) : '—'));
+  }
+  document.getElementById('error-modal-title').textContent = s.name + ': the compared values';
+  var pre = document.getElementById('error-modal-text');
+  pre.className = 'values'; pre.textContent = lines.join('\n');
+  document.getElementById('error-modal').classList.add('open');
+}
+"""
+
 function _generate_markdown(results, filepath, ts, total, broken, tested,
                             frontend_pass, backend_pass, simulate_pass, validate_pass,
                             has_ref, pct, changelog)
diff --git a/src/runner.jl b/src/runner.jl
index e2f2fb0..b3c6066 100644
--- a/src/runner.jl
+++ b/src/runner.jl
@@ -366,12 +366,13 @@ end
 """
     _run_model_phases(model_name, msl_version, stop_time, reference_file,
                       reference, atol, reltol, signal_mapping, ref_dir,
-                      phase_ints) -> (results, highest)
+                      phase_ints) -> (results, highest, signals)
 
 Runs all phases for a single model. Intended to execute on a worker process.
 Returns primitive types to avoid Distributed.jl type identity issues.
 
-Each phase result is `(phase_int, success, time_s, error_or_nothing)`.
+Each phase result is `(phase_int, success, time_s, error_or_nothing)`; `signals` holds the reference
+signals the validation compared, passed or not, as the fields of a `SignalComparison` (`SIGNAL_TUPLE`).
 """
 function _run_model_phases(model_name::String,
                            msl_version::String,
@@ -394,6 +395,7 @@ function _run_model_phases(model_name::String,
     results = Tuple{Int, Bool, Float64, Union{Nothing, String}}[]
     highest = 0
     sol = nothing
+    signals = SIGNAL_TUPLE[]
 
     has_csv_ref = !isempty(reference_file)
     has_inline_ref = !isempty(reference)
@@ -448,6 +450,7 @@ function _run_model_phases(model_name::String,
                         Set{Phase}())
                     (passed, comparisons) = validate_against_reference(sol, temp_spec, ref_dir;
                         terminatedAt = sol.retcode == ReturnCode.Terminated ? Float64(sol.t[end]) : nothing)
+                    signals = [_signal_tuple(c) for c in comparisons]
                     if !passed
                         failed = filter(c -> !c.passed, comparisons)
                         isempty(failed) &&
@@ -497,9 +500,17 @@ function _run_model_phases(model_name::String,
         @warn "    $(model_name): forgetModel failed" exception = e
     end
 
-    return (results, highest)
+    return (results, highest, signals)
 end
 
+#= A SignalComparison's fields, for a worker's result (plain types across processes). =#
+const SIGNAL_TUPLE = Tuple{String, Bool, Float64, Float64, Float64, Float64, Float64,
+                           Vector{Float64}, Vector{Float64}, Vector{Float64}}
+
+_signal_tuple(c::SignalComparison)::SIGNAL_TUPLE =
+    (c.name, c.passed, c.max_abs_err, c.max_rel_err, c.worst_time, c.worst_actual, c.worst_expected,
+     c.times, c.expected, c.actual)
+
 # ── Worker Execution with Timeout ───────────────────────────────────────
 
 const GRACE_PERIOD = 15.0
@@ -649,6 +660,9 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
             for pr in raw[1]
         ]
         highest = Phase(raw[2])
+        #= a worker of an older OMLibraryTesting returns no signals =#
+        local signals::Vector{SignalComparison} =
+            length(raw) >= 3 ? SignalComparison[SignalComparison(t...) for t in raw[3]] : SignalComparison[]
 
         # Log phase results
         for pr in phase_results
@@ -660,7 +674,7 @@ function run_on_worker(mgr::WorkerManager, spec::ModelSpec,
         end
 
         return ModelResult(spec, phase_results, highest,
-                          Dates.format(Dates.now(), "yyyy-mm-dd HH:MM"))
+                          Dates.format(Dates.now(), "yyyy-mm-dd HH:MM"), signals)
     catch e
         elapsed = time() - t0
 
diff --git a/src/signals.jl b/src/signals.jl
new file mode 100644
index 0000000..42f8bf0
--- /dev/null
+++ b/src/signals.jl
@@ -0,0 +1,131 @@
+#= The signals a run's validations compared, OM.jl's values and the reference's, as JSON: next to a
+   run's HTML report (generate_report) and on the coverage site (publish_site), for looking at how a
+   model behaved. =#
+
+"""
+The number of a site version's latest runs that keep their `signals.json` (publish_site removes it
+from the older ones).
+"""
+const SIGNAL_RUNS_KEPT = 5
+
+"""
+    signals_json(results) -> String
+
+The signals the validations of `results` compared, as JSON, by model name:
+
+```
+{"models": {"": {"validated": true, "atol": 1e-4, "reltol": 3e-3, "time": [t0, t1, n],
+                        "signals": [{"name": "J1.w", "passed": true, "max_abs_err": ..., "max_rel_err": ...,
+                                     "worst_time": ..., "reference": [...], "omjl": [...]}]}}}
+```
+
+`time` is the comparison's `n` evenly spaced times from `t0` to `t1` (a signal compared at other times
+has its own `time`); `reference` and `omjl` are the values there, 6 significant digits, `null` where
+there is none (OM.jl's after the comparison stopped: the signal is not in the solution, or a value is
+not finite). Models without compared signals are left out.
+"""
+function signals_json(results::Vector{ModelResult})::String
+    local io = IOBuffer()
+    print(io, "{\"models\":{")
+    local first = true
+    for r in sort(results; by = r -> r.spec.name)
+        isempty(r.signals) && continue
+        first || print(io, ",")
+        first = false
+        _model_signals_json(io, r)
+    end
+    print(io, "}}")
+    return String(take!(io))
+end
+
+"""
+    write_signals(path, results) -> Bool
+
+Write `signals_json(results)` to `path` when a validation of `results` compared signals; whether it did.
+"""
+function write_signals(path::AbstractString, results::Vector{ModelResult})::Bool
+    any(r -> !isempty(r.signals), results) || return false
+    write(path, signals_json(results))
+    return true
+end
+
+function _model_signals_json(io::IO, r::ModelResult)::Nothing
+    local withTimes::Union{Nothing, Int} = findfirst(s -> !isempty(s.times), r.signals)
+    local times::Vector{Float64} = withTimes === nothing ? Float64[] : r.signals[withTimes].times
+    _json_string(io, r.spec.name)
+    print(io, ":{\"validated\":", r.highest >= VALIDATE, ",\"atol\":")
+    _json_number(io, r.spec.atol)
+    print(io, ",\"reltol\":")
+    _json_number(io, r.spec.reltol)
+    print(io, ",\"time\":")
+    _json_times(io, times)
+    print(io, ",\"signals\":[")
+    for (i, s) in enumerate(r.signals)
+        i > 1 && print(io, ",")
+        print(io, "{\"name\":")
+        _json_string(io, s.name)
+        print(io, ",\"passed\":", s.passed, ",\"max_abs_err\":")
+        _json_number(io, s.max_abs_err)
+        print(io, ",\"max_rel_err\":")
+        _json_number(io, s.max_rel_err)
+        print(io, ",\"worst_time\":")
+        _json_number(io, s.worst_time)
+        if !isempty(s.times) && s.times != times
+            print(io, ",\"time\":")
+            _json_times(io, s.times)
+        end
+        print(io, ",\"reference\":")
+        _json_numbers(io, s.expected)
+        print(io, ",\"omjl\":")
+        _json_numbers(io, s.actual)
+        print(io, "}")
+    end
+    print(io, "]}")
+    return nothing
+end
+
+#= Evenly spaced times (compare_signal's range) as [first, last, count]. =#
+function _json_times(io::IO, times::Vector{Float64})::Nothing
+    if isempty(times)
+        print(io, "[]")
+    else
+        print(io, "[")
+        _json_number(io, times[1])
+        print(io, ",")
+        _json_number(io, times[end])
+        print(io, ",", length(times), "]")
+    end
+    return nothing
+end
+
+function _json_numbers(io::IO, xs::Vector{Float64})::Nothing
+    print(io, "[")
+    for (i, x) in enumerate(xs)
+        i > 1 && print(io, ",")
+        _json_number(io, x)
+    end
+    print(io, "]")
+    return nothing
+end
+
+function _json_number(io::IO, x::Float64)::Nothing
+    print(io, isfinite(x) ? string(round(x; sigdigits = 6)) : "null")
+    return nothing
+end
+
+function _json_string(io::IO, s::AbstractString)::Nothing
+    print(io, '"')
+    for c in s
+        if c == '"' || c == '\\'
+            print(io, '\\', c)
+        elseif c == '<'
+            print(io, "\\u003c")    # safe inside an HTML