From 2a33b7c8511f2ec6981b0af127ee89a5dd45e501 Mon Sep 17 00:00:00 2001 From: "Brandon W. King" <70168+kingb@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:11:03 -0700 Subject: [PATCH] test(session): run the gridwire grid conformance corpus in CI gridwire publishes a cross-producer corpus (conformance/grid/): byte inputs and the GridDelta every producer of the neutral grid must turn them into. Ember's local projection matched it in a one-time manual run; this makes that agreement permanent by checking it on every CI run. It is a conformance test, not a unit test. The corpus isn't Ember's, and the property it protects (agreement with other producers) is invisible to Ember's own unit coverage, so it lives on its own and fails rather than skipping: - scripts/conformance/fetch-grid-corpus.sh fetches the corpus from the public gridwire repo at the revision pinned in conformance/GRIDWIRE_REV. - tests/grid_conformance.rs drives the real projection through each case and compares decoded deltas. A missing, empty, or unpaired corpus fails, and so does a corpus field Ember's GridDelta can't represent (otherwise it would be dropped on decode and never compared). A failure names the case and the first differing field. - The test is #[ignore]d so a plain cargo test doesn't need the network. The CI step runs it with --ignored and greps for its pass line, so a filtered-out test can't pass the step by running nothing. Running it surfaced one real drift: gridwire reads bracketed_paste, mouse_reporting and marks with #[serde(default)], and ember-core's GridDelta didn't, so it couldn't decode a delta that omits them at their defaults. The serde attributes now match gridwire's exactly. There's no runtime change, since the local path passes deltas in-process. Checked that it can fail: dropping the full style table on reset fails reset-style-carry and reset-mid-stream with the field named, and a missing corpus, an unpaired case, and an unknown corpus field each fail loudly. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014xVjnf8UuQPRQtv2QqgpPK --- .github/workflows/ci.yml | 9 + Cargo.lock | 2 + conformance/GRIDWIRE_REV | 1 + crates/ember-core/src/grid.rs | 3 + crates/ember-session/Cargo.toml | 2 + .../ember-session/tests/grid_conformance.rs | 222 ++++++++++++++++++ scripts/conformance/fetch-grid-corpus.sh | 26 ++ 7 files changed, 265 insertions(+) create mode 100644 conformance/GRIDWIRE_REV create mode 100644 crates/ember-session/tests/grid_conformance.rs create mode 100755 scripts/conformance/fetch-grid-corpus.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01efd87..6ddaa95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,15 @@ jobs: run: cargo clippy --all-targets --all-features -- -D warnings - name: Test run: cargo test --all --all-features + # Cross-producer conformance against the shared gridwire corpus, pinned in + # conformance/GRIDWIRE_REV. The final grep proves the test ran and passed, + # so a renamed or filtered-out test can't pass this step by running nothing. + - name: Grid conformance (gridwire corpus) + run: | + scripts/conformance/fetch-grid-corpus.sh + cargo test -p ember-session --test grid_conformance -- --ignored --nocapture \ + | tee "$RUNNER_TEMP/grid-conformance.log" + grep -q 'grid conformance: all [0-9]* corpus cases agree' "$RUNNER_TEMP/grid-conformance.log" - name: Bench (compile gate) run: cargo bench --all --no-run diff --git a/Cargo.lock b/Cargo.lock index 087eb00..ed456c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -714,6 +714,8 @@ dependencies = [ "ember-core", "libc", "portable-pty", + "serde", + "serde_json", ] [[package]] diff --git a/conformance/GRIDWIRE_REV b/conformance/GRIDWIRE_REV new file mode 100644 index 0000000..8049f3a --- /dev/null +++ b/conformance/GRIDWIRE_REV @@ -0,0 +1 @@ +e977304e33ffbb06accb9a2b3b8c4d39241e9611 diff --git a/crates/ember-core/src/grid.rs b/crates/ember-core/src/grid.rs index be8be24..a5eb3e2 100644 --- a/crates/ember-core/src/grid.rs +++ b/crates/ember-core/src/grid.rs @@ -211,6 +211,7 @@ pub struct GridDelta { /// Snapshot of the engine's bracketed-paste mode (DEC 2004) as of this drain — /// terminal state, like `cursor`, not damage. Lets the app wrap pastes in /// `ESC[200~`…`ESC[201~` only when the app asked for it. Latest-wins on merge. + #[serde(default)] pub bracketed_paste: bool, /// Scrollback viewport state (terminal state, latest-wins on merge): how many /// lines the display is scrolled **up** from the live bottom (`0` = at bottom), @@ -222,6 +223,7 @@ pub struct GridDelta { pub alt_screen: bool, /// The app has enabled mouse reporting — the wheel should go to it as mouse /// events, not be translated to arrow keys. + #[serde(default)] pub mouse_reporting: bool, /// Application cursor keys (DECCKM, mode ?1): arrows must be sent as /// `ESC O A`… instead of `CSI A`…. Latest-wins on merge; defaulted so @@ -235,6 +237,7 @@ pub struct GridDelta { /// `(visible_row, status)` — recomputed each drain from the marks' absolute /// history lines + `display_offset`, so they scroll with the content. Latest- /// wins on merge (terminal state, not damage). + #[serde(default)] pub marks: Vec<(u16, MarkStatus)>, } diff --git a/crates/ember-session/Cargo.toml b/crates/ember-session/Cargo.toml index bcce8a0..679d7fd 100644 --- a/crates/ember-session/Cargo.toml +++ b/crates/ember-session/Cargo.toml @@ -16,6 +16,8 @@ libc.workspace = true [dev-dependencies] criterion = "0.5" +serde = { workspace = true } +serde_json = { workspace = true } [[bench]] name = "throughput" diff --git a/crates/ember-session/tests/grid_conformance.rs b/crates/ember-session/tests/grid_conformance.rs new file mode 100644 index 0000000..1811f0b --- /dev/null +++ b/crates/ember-session/tests/grid_conformance.rs @@ -0,0 +1,222 @@ +//! Cross-producer conformance: Ember's local projection against the shared +//! `gridwire` grid corpus (`conformance/grid/` in the public gridwire repo). +//! +//! This is a conformance test, not a unit test. Its value is that the corpus is +//! not Ember's: it asserts that this projection turns each input into the same +//! delta as every other producer of the neutral grid, byte-for-byte on the wire. +//! Ember's own unit tests can cover every line of the projection and still miss +//! a disagreement with another producer; only this catches that. Keep it +//! separate from unit coverage and don't fold it into ordinary tests. +//! +//! The corpus is fetched at the pinned revision in `conformance/GRIDWIRE_REV` by +//! `scripts/conformance/fetch-grid-corpus.sh`, which CI runs before this test. +//! Locally: +//! +//! ```sh +//! scripts/conformance/fetch-grid-corpus.sh +//! cargo test -p ember-session --test grid_conformance -- --ignored +//! ``` +//! +//! A missing, empty, or half-paired corpus FAILS rather than skipping: a +//! conformance test that can quietly not run is no guarantee. + +use std::path::PathBuf; + +use alacritty_terminal::event::VoidListener; +use ember_core::{GridDelta, GridDims, VtProjection}; +use ember_session::AlacrittyProjection; +use serde::Deserialize; +use serde_json::Value; + +/// One corpus input: exactly one of `feed` (drain once) or `steps` (drain +/// after each chunk). +#[derive(Deserialize)] +struct CorpusInput { + dims: GridDims, + #[serde(default)] + feed: Option, + #[serde(default)] + steps: Option>, +} + +/// Where the fetched corpus lives: `EMBER_GRID_CORPUS` if set, else the fetch +/// script's default under `target/`. +fn corpus_dir() -> PathBuf { + if let Some(dir) = std::env::var_os("EMBER_GRID_CORPUS") { + return PathBuf::from(dir); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/gridwire-corpus/conformance/grid") +} + +/// Feed each chunk into a fresh projection and drain after each, exactly as a +/// producer ships frames. +fn produce(dims: GridDims, chunks: &[String]) -> Vec { + let mut proj = AlacrittyProjection::new(dims, VoidListener); + chunks + .iter() + .map(|chunk| { + proj.advance(chunk.as_bytes()); + let mut delta = GridDelta::default(); + proj.drain_damage_into(&mut delta); + delta + }) + .collect() +} + +/// The first place two JSON values differ, as a readable path, so a failure +/// names the field or cell instead of dumping two whole frames. +fn first_difference(path: &str, got: &Value, want: &Value) -> Option { + match (got, want) { + (Value::Object(g), Value::Object(w)) => { + let mut keys: Vec<&String> = g.keys().chain(w.keys()).collect(); + keys.sort(); + keys.dedup(); + keys.into_iter().find_map(|k| match (g.get(k), w.get(k)) { + (Some(gv), Some(wv)) => first_difference(&format!("{path}.{k}"), gv, wv), + (Some(gv), None) => Some(format!( + "{path}.{k}: produced {gv}, corpus has no such field" + )), + (None, Some(wv)) => Some(format!( + "{path}.{k}: missing from produced, corpus expects {wv}" + )), + (None, None) => None, + }) + } + (Value::Array(g), Value::Array(w)) => g + .iter() + .zip(w) + .enumerate() + .find_map(|(i, (gv, wv))| first_difference(&format!("{path}[{i}]"), gv, wv)) + .or_else(|| { + (g.len() != w.len()).then(|| { + format!( + "{path}: produced {} entries, corpus expects {}", + g.len(), + w.len() + ) + }) + }), + _ => (got != want).then(|| format!("{path}: produced {got}, corpus expects {want}")), + } +} + +/// The first field present in the corpus's JSON that is missing, or changed, +/// after decoding into Ember's type and re-encoding: that is, something the +/// corpus says that Ember's `GridDelta` can't carry. +fn dropped_field(path: &str, corpus: &Value, decoded: &Value) -> Option { + match (corpus, decoded) { + (Value::Object(c), Value::Object(d)) => c.iter().find_map(|(k, cv)| match d.get(k) { + Some(dv) => dropped_field(&format!("{path}.{k}"), cv, dv), + None => Some(format!("{path}.{k}")), + }), + (Value::Array(c), Value::Array(d)) if c.len() == d.len() => c + .iter() + .zip(d) + .enumerate() + .find_map(|(i, (cv, dv))| dropped_field(&format!("{path}[{i}]"), cv, dv)), + _ => (corpus != decoded).then(|| path.to_string()), + } +} + +#[test] +#[ignore = "needs the gridwire corpus: run scripts/conformance/fetch-grid-corpus.sh, then --ignored"] +fn local_projection_agrees_with_the_grid_corpus() { + let dir = corpus_dir(); + let entries = std::fs::read_dir(&dir).unwrap_or_else(|e| { + panic!( + "grid corpus not found at {} ({e}). Run scripts/conformance/fetch-grid-corpus.sh \ + or set EMBER_GRID_CORPUS. This test fails rather than skipping on a missing corpus.", + dir.display() + ) + }); + + let mut names: Vec = entries + .map(|e| { + e.expect("corpus dir entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .filter(|n| n.ends_with(".json") && !n.ends_with(".expect.json")) + .collect(); + names.sort(); + assert!( + !names.is_empty(), + "grid corpus at {} has no cases", + dir.display() + ); + + let mut failures = Vec::new(); + for name in &names { + let stem = &name[..name.len() - ".json".len()]; + let input: CorpusInput = serde_json::from_str( + &std::fs::read_to_string(dir.join(name)).expect("read corpus input"), + ) + .unwrap_or_else(|e| panic!("case '{stem}': input is malformed: {e}")); + let expect_raw = std::fs::read_to_string(dir.join(format!("{stem}.expect.json"))) + .unwrap_or_else(|_| panic!("case '{stem}': no {stem}.expect.json pair")); + let want: Value = serde_json::from_str(&expect_raw) + .unwrap_or_else(|e| panic!("case '{stem}': expected output is not JSON: {e}")); + + // A `feed` case expects one delta, a `steps` case an array: normalize + // both to a list so each drain is checked the same way. + let (got, want_frames, indexed) = match (input.feed, input.steps) { + (Some(feed), None) => (produce(input.dims, &[feed]), vec![want], false), + (None, Some(steps)) => { + let Value::Array(frames) = want else { + panic!("case '{stem}': a `steps` case expects an array of deltas"); + }; + assert_eq!( + frames.len(), + steps.len(), + "case '{stem}': {} steps but {} expected deltas", + steps.len(), + frames.len() + ); + (produce(input.dims, &steps), frames, true) + } + _ => panic!("case '{stem}': input must have exactly one of `feed` or `steps`"), + }; + + for (i, (got, want_json)) in got.iter().zip(&want_frames).enumerate() { + let at = if indexed { + format!("[{i}]") + } else { + String::new() + }; + // The corpus omits fields at their defaults (the wire types read + // them with `#[serde(default)]`), so equality is on the decoded + // delta, not on raw JSON text. + let want: GridDelta = serde_json::from_value(want_json.clone()).unwrap_or_else(|e| { + panic!("case '{stem}'{at}: expected delta doesn't decode: {e}") + }); + // Guard against a silent pass: every field the corpus states must + // survive into Ember's type. A field Ember can't represent would + // otherwise be dropped on decode and never compared. + let decoded = serde_json::to_value(&want).expect("serialize expected delta"); + if let Some(lost) = dropped_field("", want_json, &decoded) { + failures.push(format!( + " {stem}{at}: corpus field {lost} has no counterpart in Ember's GridDelta" + )); + continue; + } + if *got != want { + let got_json = serde_json::to_value(got).expect("serialize produced delta"); + let diff = first_difference(&at, &got_json, &decoded) + .unwrap_or_else(|| "differs (no JSON-visible difference)".into()); + failures.push(format!(" {stem}: {diff}")); + } + } + } + + assert!( + failures.is_empty(), + "Ember's projection disagrees with the gridwire corpus on {} of {} case(s):\n{}\n\ + The corpus is the contract shared with other producers; fix the projection, or \ + raise it with the corpus owners if the corpus is wrong. Don't edit expectations here.", + failures.len(), + names.len(), + failures.join("\n") + ); + println!("grid conformance: all {} corpus cases agree", names.len()); +} diff --git a/scripts/conformance/fetch-grid-corpus.sh b/scripts/conformance/fetch-grid-corpus.sh new file mode 100755 index 0000000..a77c373 --- /dev/null +++ b/scripts/conformance/fetch-grid-corpus.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Fetch the shared grid conformance corpus from the public gridwire repo at the +# revision pinned in conformance/GRIDWIRE_REV, for the grid_conformance test +# (crates/ember-session/tests/grid_conformance.rs). +# +# scripts/conformance/fetch-grid-corpus.sh [dest] # default: target/gridwire-corpus +# +# Bump the pin deliberately: a new corpus revision can add cases the projection +# must now agree with. +set -euo pipefail +cd "$(dirname "$0")/../.." + +REV="$(tr -d '[:space:]' < conformance/GRIDWIRE_REV)" +DEST="${1:-target/gridwire-corpus}" +URL="https://github.com/kingb/gridwire" + +rm -rf "$DEST" +git init -q "$DEST" +git -C "$DEST" remote add origin "$URL" +git -C "$DEST" fetch -q --depth 1 origin "$REV" +git -C "$DEST" -c advice.detachedHead=false checkout -q FETCH_HEAD + +[[ "$(git -C "$DEST" rev-parse HEAD)" == "$REV" ]] || { echo "fetched revision does not match the pin $REV"; exit 1; } +n="$(find "$DEST/conformance/grid" -name '*.expect.json' | wc -l | tr -d ' ')" +[[ "$n" -gt 0 ]] || { echo "no corpus cases at $DEST/conformance/grid"; exit 1; } +echo "grid corpus: $n case(s) at $DEST/conformance/grid (gridwire ${REV:0:7})"