From 449b2a81bc61d1ab7c6b0fd6adbdfb5997f79c62 Mon Sep 17 00:00:00 2001 From: Michael B Reiser Date: Tue, 29 Sep 2026 17:10:41 -0400 Subject: [PATCH] =?UTF-8?q?chore(tiers):=20release=20kit=20=E2=80=94=20dis?= =?UTF-8?q?covered=20test=20suite,=20bench=20validation=20kit,=20review-be?= =?UTF-8?q?fore-validate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `pixi run test` = scripts/run-tests.py: runs every tests/test-*.js|py and tests/validate-*.js by name (each in its own process, the vendored yaml hook for every Node test, a summary, non-zero on any failure; `-- ` runs a subset, `-- --list` shows the set). Replaces the hand-kept `&&` chain that every test-adding PR edited on the same line, so every candidate conflicted in pixi.toml. It also picks up 8 files that were running nowhere (test-crc.js, validate-arena-geometry/-header-v2/-matlab-reference, and the four validate-* suites only their own workflows ran); all pass. 56 files, ~15 s. - Bench kit: protocols/validation/bench_closed_loop_2x10.yaml (the #225/rc closed-loop check) + scripts/bench-check.py / `pixi run bench-check` (completed, all arena commands ok, no error glyphs, --hold COND=FRAME, --channel, --build; reads behavior_v1/v2, .jsonl[.gz]). Verified on the 2026-09-28/29 bench logs; tests/test-bench-check.py (synthetic logs). - release-process.md + g6-release skill: adversarial review of the candidate diff BEFORE bench time (lesson from 2026-09-29 rc1/rc2), the bench kit steps, course-repo auto-commit off for validation runs, keep PRs mergeable (no CI on conflicting PRs), tests are discovered by name. Co-Authored-By: Claude Opus 5.5 --- .agents/skills/g6-release/SKILL.md | 19 ++ CLAUDE.md | 7 +- docs/development/release-process.md | 29 +++ pixi.toml | 7 +- .../validation/bench_closed_loop_2x10.yaml | 102 +++++++++++ scripts/bench-check.py | 173 ++++++++++++++++++ scripts/run-tests.py | 76 ++++++++ tests/test-bench-check.py | 107 +++++++++++ tests/test-run-tests.py | 47 +++++ 9 files changed, 564 insertions(+), 3 deletions(-) create mode 100644 protocols/validation/bench_closed_loop_2x10.yaml create mode 100644 scripts/bench-check.py create mode 100644 scripts/run-tests.py create mode 100644 tests/test-bench-check.py create mode 100644 tests/test-run-tests.py diff --git a/.agents/skills/g6-release/SKILL.md b/.agents/skills/g6-release/SKILL.md index 2030692c..00693e2e 100644 --- a/.agents/skills/g6-release/SKILL.md +++ b/.agents/skills/g6-release/SKILL.md @@ -29,6 +29,9 @@ Routine pushes to a feature branch need no question. (`.github/pull_request_template.md`) pre-fills it. - New localStorage keys / migration markers / window names follow the coexistence rules (typed monotonic markers, never change a key's meaning, tier-suffixed window names). +- Keep the PR **mergeable with main**: GitHub runs no CI on a PR with merge conflicts, so a + conflicting PR shows no checks at all. Merge main in (no force-push needed) when it drifts. +- New tests only need the `tests/test-*` / `tests/validate-*` name: `pixi run test` discovers them. ## Commands (run from a checkout of the repo; `pixi run gh auth login` once) @@ -37,12 +40,28 @@ Routine pushes to a feature branch need no question. | Build + publish a candidate | `pixi run candidate -- 219 178 [--headline "…"]` | pins each PR head, merges `--no-ff`, release commit, pushes `release/`, opens a draft release PR, waits for CI, then points `next` at it and dispatches the deploy | | Conflict | resolve in `.tiers/candidate`, `git add`, `git commit --no-edit`, then `pixi run candidate -- --continue` (or `--abort`) | two PRs that keep colliding belong in one `batch/` branch | | Dry run | `pixi run candidate -- 219 --no-push` then `pixi run next-preview -- --next release/` and `python -m http.server -d _site 8080` | nothing leaves the machine | +| Adversarial review | `codex-diff-review` skill on `origin/main...origin/release/` | **before** asking for bench time; fix or drop, rebuild, then validate (release-process.md) | +| Bench check | `pixi run bench-check -- LOG --hold epoch_a_frame25=25 --hold epoch_b_frame75=75 --channel next --build ` | protocol `protocols/validation/bench_closed_loop_2x10.yaml`; steps in release-process.md "Bench validation kit" | | What's live | `pixi run tiers -- status` | head drift / main drift → rebuild (rc+1, validation resets) | | Validation note | `pixi run tiers -- validate --rig rig5 --by Isabel --notes "…"`, or `--notes-file /abs/path.md` for anything multi-line or with `` ` `` (pixi re-parses `--notes`) | names the exact candidate SHA; test **between sessions** | | Promote | `pixi run release` | re-checks everything, prints the Slack draft, asks for the release name, merge commit + tag | | Hotfix | `pixi run candidate -- --hotfix "why" 230` → `pixi run release -- --pr N` | never moves /next/ | | Roll back | `pixi run tiers -- rollback --to release- --revert release-` | the next push to main undoes the emergency deploy — land the revert first | +## Validating through the Chrome extension + +The lab's github.io origin has the course pipeline switched on, so a completed Run experiment +auto-commits its log to the course repo. For a validation run, override it **in the test tab only** +(lost on reload; never change stored settings), then load the validation protocol: + +```js +const orig = Studio.courseSettings; +Studio.courseSettings = () => Object.assign({}, orig(), { direct: false }); +``` + +Native dialogs (the NEXT confirm, the serial-port chooser, rig-mismatch confirms) block the +extension, so the user clicks Connect, Run and OK. Check the result with `pixi run bench-check`. + ## After a release - Post the Slack announcement (the tool prints a draft; the claude.ai Slack connector can post it diff --git a/CLAUDE.md b/CLAUDE.md index ecdd6ef1..2eddb458 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,7 +115,8 @@ Node, Prettier, Python, and `websockets` from conda-forge — **there is no npm, ```bash pixi install # one-time: provisions Node + Prettier + Python -pixi run test # full JS test suite (node) +pixi run test # full suite: every tests/test-*.js|py + tests/validate-*.js (scripts/run-tests.py) +pixi run test -- wire # only files whose name contains "wire" pixi run format # Prettier --write over **/*.js pixi run format-check # Prettier --check (matches the old CI check) pixi run bridge # FicTrac closed-loop bridge (see fictrac-bridge/) @@ -127,7 +128,9 @@ only) at `js/vendor/yaml/browser/` and serves both the browser (via the import map in `experiment_designer_v3.html`) and Node — no Node-specific build is committed. The v3 modules `import 'yaml'` (a bare specifier); under Node that's resolved to the vendored browser build by `tests/vendor-yaml.register.mjs` (a -`--import` resolve hook), which the `test` task wires in for the v3 suite. +`--import` resolve hook), which `scripts/run-tests.py` passes to every Node test. +A new test file only needs the `tests/test-*` (or `tests/validate-*`) name to be run; there is +no list to edit, so parallel PRs no longer conflict in `pixi.toml`. `require()`-ing that ESM build is why `nodejs >= 22.12` is pinned. When bumping the vendored `yaml`, replace only `js/vendor/yaml/browser/` (+ `LICENSE`). diff --git a/docs/development/release-process.md b/docs/development/release-process.md index 83f248ff..3227b62a 100644 --- a/docs/development/release-process.md +++ b/docs/development/release-process.md @@ -100,6 +100,14 @@ same on macOS, Linux and Windows (CI runs the tooling tests on all three). - **Pre-tier PRs** that already bumped the footer and wrote their own notes entry are adopted, not double-bumped (their version and entry are kept; only the timestamp is refreshed). +## Before validation: adversarial review of the candidate + +Once a candidate is built, and **before asking anyone for bench time**, get an adversarial review of +exactly what would ship: the diff `origin/main...origin/release/`. Claude sessions use the +`codex-diff-review` skill (Codex standard + adversarial passes, reconciled with Claude's own review). +Fix or drop what it finds, rebuild (rc+1), and only then validate. On 2026-09-29, rc1 and rc2 were +bench-validated first; the review then held #219 back, so both bench runs had to be repeated on rc3. + ## Validation - Validate on a rig **between sessions**. Never open `/next/` on a rig PC while an experiment is @@ -113,6 +121,27 @@ same on macOS, Linux and Windows (CI runs the tooling tests on all three). - If a candidate touches `fictrac-bridge/`, testers run the bridge from a checkout of the candidate's `release/` branch (a pinned commit, not a moving ref). +### Bench validation kit (G6 2×10 fly-on-ball bench, ~5 min) + +1. Start the bridge with a log directory, and a stationary simulated fly: + `pixi run bridge -- --log-dir ` and `pixi run sim -- --noise 0`. +2. Open `https://reiserlab.github.io/webDisplayTools/next/arena_studio.html` and load + `protocols/validation/bench_closed_loop_2x10.yaml`. It needs the CSHL course pattern bundle on the + SD card. Connect the bridge, then Connect the arena. +3. **Turn the course-repo upload off for the test.** The lab browsers have the course pipeline + configured (token + `reiserlab/cshl-2026-course` + direct commit), so a completed + **Run experiment** commits its log into the course data as the configured bench id. Use a browser + profile without the course token, or switch off "Commit directly" for the session. Never + validate with **Test experiment** instead: test runs don't write `run_metadata` to the bridge log. +4. **Run experiment** (confirm the NEXT dialog). About 22 s. +5. Check the log: + `pixi run bench-check -- /arena-log-*.jsonl --hold epoch_a_frame25=25 --hold epoch_b_frame75=75 --channel next --build `. + It checks that the run completed, every arena command was ok, there were no error glyphs, the + closed loop held frames 25 and 75, and the log names this tier and build. It also prints the + controller firmware and the panel inventory. Exit status 0 = pass. +6. Post the note with `--notes-file`, then free the bench: disconnect, close the tab, stop the + bridge and the simulator. + ## Releasing `pixi run release` re-checks everything (heads unchanged, CI green, main not moved beyond docs, diff --git a/pixi.toml b/pixi.toml index c1b3a922..ab20c3d0 100644 --- a/pixi.toml +++ b/pixi.toml @@ -21,7 +21,10 @@ serve = "python -m http.server 8000 --bind 127.0.0.1" # The full test suite. yaml is vendored (js/vendor/yaml), so no package install. # The v3 suite pulls in ESM modules that `import 'yaml'`; --import maps that bare # specifier to the vendored copy (tests/vendor-yaml.register.mjs). -test = "node tests/test-project-skills.js && node tests/validate-arena-calculations.js && node tests/test-protocol-roundtrip.js && node --import ./tests/vendor-yaml.register.mjs tests/test-protocol-roundtrip-v3.js && node --import ./tests/vendor-yaml.register.mjs tests/test-runtime-controls.js && node tests/test-plugin-registry.js && node tests/test-arena-wire-g6.js && node tests/test-arena-link.js && node tests/test-arena-runner-g6.js && node tests/test-arena-session.js && node tests/test-fictrac-bridge-client.js && node tests/test-run-log.js && node tests/test-bin-classifier.js && node tests/test-pattern-set.js && node tests/test-pat-preview.js && node tests/test-studio-meta.js && node tests/test-build-channel.js && node tests/test-studio-analog-in.js && node tests/test-studio-panel-inventory.js && node tests/test-studio-url-state.js && node tests/test-studio-runlog-adapter.js && node tests/test-studio-github.js && node tests/test-studio-canmutate.js && node tests/test-studio-protocols-index.js && node tests/test-g6-tall-rigs.js && node tests/test-arena-studio-alt.js && node tests/test-three-viewer-led-glow.js && node tests/test-three-viewer-preview-omissions.js && node tests/test-arena-replay-viewer-protocol.js && node tests/test-kinematics.js && node tests/test-runlog-replay.js && node tests/test-studio-replay.js && node tests/test-fly-gait.js && node tests/test-runlog-format.js && node tests/test-metadata-yaml.js && python tests/test-bridge-behavior.py && python tests/test-build-runlog-index.py && node tests/test-studio-postmortem.js && node tests/test-arena-telemetry.js && python tests/test-wedge-scan.py && python tests/test-runlog-check.py && node tests/test-trial-quality.js && python tests/test-telemetry-report.py && python tests/test-fictrac-sim.py && node --import ./tests/vendor-yaml.register.mjs tests/test-studio-runtime-vars.js && python tests/test-tiers-candidate.py" +# Every tests/test-*.js|py and tests/validate-*.js, discovered by name (scripts/run-tests.py). +# A new test only needs the naming convention; no shared list to edit, so PRs stop conflicting here. +# pixi run test -- runs a subset; -- --list shows what would run. +test = "python scripts/run-tests.py" # ── Two-tier releases: Production + Next (docs/development/release-process.md) ── # pixi run candidate -- 219 178 build a frozen Next candidate from those PRs # pixi run release promote the validated candidate to Production @@ -31,6 +34,8 @@ tiers = "python scripts/tiers/candidate.py" candidate = "python scripts/tiers/candidate.py build" release = "python scripts/tiers/candidate.py release" next-preview = "python scripts/tiers/stamp.py preview" +# pixi run bench-check -- LOG [--hold COND=FRAME …] [--channel next] [--build SHA] check a bench-validation run +bench-check = "python scripts/bench-check.py" test-tiers = "python tests/test-tiers-candidate.py" format = "prettier --write \"**/*.js\"" format-check = "prettier --check \"**/*.js\"" diff --git a/protocols/validation/bench_closed_loop_2x10.yaml b/protocols/validation/bench_closed_loop_2x10.yaml new file mode 100644 index 00000000..192ee11f --- /dev/null +++ b/protocols/validation/bench_closed_loop_2x10.yaml @@ -0,0 +1,102 @@ +version: 3 + +# Release-candidate bench check for a G6 2x10 fly-on-ball arena (docs/development/release-process.md, +# "Bench validation"). Closed-loop trials must OPEN AND HOLD on their trialParams frame_index: +# epoch A on frame 25, epoch B on frame 75, 8 s each, with a STATIONARY simulated fly: +# pixi run bridge -- --log-dir (bridge >= 3.4) +# pixi run sim -- --noise 0 +# Then check the bridge log: +# pixi run bench-check -- /arena-log-*.jsonl --hold epoch_a_frame25=25 --hold epoch_b_frame75=75 +# Needs the CSHL course pattern bundle on the SD card (pattern_ID 23 = course_bg, 36 = +# p3_heisenberg_ts). Trimmed from rig7's p3-conditioning-closedloop-short-negative.yaml; first +# used to bench-check #225 (2026-09-28). About 22 s. + +experiment_info: + name: "bench_closed_loop_2x10" + author: "Reiser Lab" + date_created: 2026-09-27 + +rig: "./configs/rigs/cshl_g6_2x10_ball.yaml" + +plugins: + - name: "fictrac" + type: "class" + matlab: + class: "FicTracPlugin" + config: + bridge_url: "ws://localhost:8765" + fictrac_port: 60000 + proto: "udp" + offset: 0 + +variables: + pattern_name: &pattern_name "p3_heisenberg_ts" + pattern_id: &pattern_id 36 + trial_dur: &trial_dur 8 + start_bg_dur: &start_bg_dur 3 + cl_gain: &cl_gain -1 + +experiment: + - "start_bg" + - "epoch_a_frame25" + - "epoch_b_frame75" + - "start_bg" + +conditions: + - name: "start_bg" + commands: + - type: "controller" + command_name: "trialParams" + pattern: "course_bg" + pattern_ID: 23 + duration: *start_bg_dur + mode: 2 + frame_index: 0 + frame_rate: 0 + gain: 0 + - type: "wait" + duration: *start_bg_dur + + - name: "epoch_a_frame25" + commands: + - type: "controller" + command_name: "trialParams" + pattern: *pattern_name + pattern_ID: *pattern_id + duration: *trial_dur + mode: 3 + frame_index: 25 + frame_rate: 0 + gain: 0 + - type: "plugin" + plugin_name: "fictrac" + command_name: "startClosedLoop" + params: + coupling: *cl_gain + - type: "wait" + duration: *trial_dur + - type: "plugin" + plugin_name: "fictrac" + command_name: "stopClosedLoop" + + - name: "epoch_b_frame75" + commands: + - type: "controller" + command_name: "trialParams" + pattern: *pattern_name + pattern_ID: *pattern_id + duration: *trial_dur + mode: 3 + frame_index: 75 + frame_rate: 0 + gain: 0 + - type: "plugin" + plugin_name: "fictrac" + command_name: "startClosedLoop" + params: + coupling: *cl_gain + - type: "wait" + duration: *trial_dur + - type: "plugin" + plugin_name: "fictrac" + command_name: "stopClosedLoop" diff --git a/scripts/bench-check.py b/scripts/bench-check.py new file mode 100644 index 00000000..1d7a61eb --- /dev/null +++ b/scripts/bench-check.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Check a bench-validation run from its FicTrac-bridge log (docs/development/release-process.md). + + pixi run bench-check -- arena-log-20260929-164557-084.jsonl + pixi run bench-check -- LOG --hold epoch_a_frame25=25 --hold epoch_b_frame75=75 \\ + --channel next --build 801a9f2 + +Prints what the run recorded (Studio version, tier, build, controller firmware, panel +inventory) and checks the run itself. The checks: it completed with no runner errors, every +arena command was acknowledged ok, the controller showed no error glyph, and (with --hold) the +display held the given frame for each named condition. Exits 1 if any check fails, 0 otherwise. + +Reads behavior_v2 logs (compact ["a", …] arena rows, controller ["cc"|"cf"|"cs", …] rows) and +the older behavior_v1 logs (arena_command objects). Plain .jsonl or .jsonl.gz. The log is the +bridge's file for one run, as written by `pixi run bridge -- --log-dir `. +""" +import argparse +import collections +import gzip +import json +import sys +from pathlib import Path + +CS_KIND_ERROR_GLYPH = 3 # controller state row kind (js/arena-telemetry.js header comment) + + +def read_lines(path): + raw = Path(path).read_bytes() + if raw[:2] == b'\x1f\x8b': + raw = gzip.decompress(raw) + return raw.decode('utf-8', errors='replace').splitlines() + + +def analyze(path): + """Parse a bridge log into the facts the checks need. Never raises on odd lines.""" + t0 = None + meta = None + steps = [] # (start_ms_rel, condition) + end_ms = None + summary = None + frames = [] # [ms_rel, fc, idx, ...] + arena_total = arena_bad = glyphs = 0 + for line in read_lines(path): + line = line.strip() + if not line: + continue + try: + r = json.loads(line) + except ValueError: + continue + if isinstance(r, dict): + typ, ev = r.get('type'), r.get('event') + if typ == 'frame_schema' and r.get('t0') is not None: + t0 = r['t0'] + elif typ == 'session' and ev == 'logging_started' and t0 is None: + t0 = r.get('ms') + elif ev == 'run_metadata': + meta = r + elif ev == 'runner': + phase, rx = r.get('phase'), r.get('rx_ms') + if phase == 'step-start' and rx is not None and t0 is not None: + steps.append((rx - t0, r.get('condition'))) + elif phase == 'sequence-complete': + summary = r.get('summary') or {} + if rx is not None and t0 is not None: + end_ms = rx - t0 + elif ev == 'arena_command': # behavior_v1 + arena_total += 1 + if r.get('ok') is False: + arena_bad += 1 + elif isinstance(r, list) and r: + if isinstance(r[0], str): + tag = r[0] + if tag == 'a': # ["a", t_off, dt, hex, status, rx_off] + arena_total += 1 + if len(r) > 4 and r[4] not in (0, None): + arena_bad += 1 + elif tag == 'cs' and len(r) > 4 and r[4] == CS_KIND_ERROR_GLYPH: + glyphs += 1 + elif isinstance(r[0], (int, float)) and len(r) > 2: + frames.append(r) + per_step = [] + bounds = steps + [(end_ms if end_ms is not None else float('inf'), None)] + for i, (start, name) in enumerate(steps): + stop = bounds[i + 1][0] + idx = collections.Counter(f[2] for f in frames if start <= f[0] < stop) + per_step.append({'condition': name, 'start_s': round(start / 1000, 2), 'frames': sum(idx.values()), 'idx': idx}) + return { + 'meta': meta, + 'summary': summary, + 'arena_total': arena_total, + 'arena_bad': arena_bad, + 'glyphs': glyphs, + 'frames': len(frames), + 'steps': per_step, + } + + +def run_checks(a, holds=(), channel=None, build=None, min_share=0.99): + """[(name, ok, detail)] for an analyze() result.""" + out = [] + s = a['summary'] + out.append(('run completed', bool(s and s.get('completed') and not s.get('aborted')), json.dumps(s))) + out.append(('no runner errors', bool(s) and not s.get('errors') and not s.get('fault'), f"errors={s and s.get('errors')} fault={s and s.get('fault')}")) + out.append(('arena commands all ok', a['arena_total'] > 0 and a['arena_bad'] == 0, f"{a['arena_total'] - a['arena_bad']}/{a['arena_total']} ok")) + out.append(('no controller error glyphs', a['glyphs'] == 0, f"{a['glyphs']} glyph event(s)")) + m = a['meta'] or {} + out.append(('run_metadata present', a['meta'] is not None, '')) + if channel: + out.append((f'channel is {channel}', m.get('channel') == channel, f"channel={m.get('channel')}")) + if build: + out.append((f'build is {build}', str(m.get('build') or '').startswith(build), f"build={m.get('build')}")) + for cond, want in holds: + seen = [st for st in a['steps'] if st['condition'] == cond] + if not seen: + out.append((f'{cond} holds frame {want}', False, 'condition not in this run')) + continue + for st in seen: + n = st['frames'] + share = (st['idx'].get(want, 0) / n) if n else 0.0 + out.append((f'{cond} holds frame {want}', n > 0 and share >= min_share, + f"{st['idx'].get(want, 0)}/{n} frames at {want} ({share:.1%}); most common {st['idx'].most_common(2)}")) + return out + + +def describe(a): + m = a['meta'] or {} + lines = [] + for k in ('tool_version', 'channel', 'build', 'candidate', 'firmware', 'controller_id', 'rig', 'rig_id', 'log_format', 'protocol_filename'): + if m.get(k) is not None: + lines.append(f' {k}: {m[k]}') + p = m.get('panels') + if isinstance(p, dict): + fw = ', '.join(f"{g.get('crc32')} ×{len(g.get('panels') or [])}" for g in p.get('firmware') or []) + lines.append(f" panels: {p.get('status')} · {p.get('present')}/{p.get('count')} present" + + (f" · missing {p.get('missing')}" if p.get('missing') else '') + + (f" · mismatched {p.get('mismatched')}" if p.get('mismatched') else '') + + (f' · fw {fw}' if fw else '')) + lines.append(f" arena commands: {a['arena_total']} ({a['arena_bad']} not ok) · frames: {a['frames']} · error glyphs: {a['glyphs']}") + for st in a['steps']: + lines.append(f" step {st['condition']} @ {st['start_s']} s: {st['frames']} frames, idx {st['idx'].most_common(3)}") + return '\n'.join(lines) + + +def parse_hold(text): + cond, _, idx = text.rpartition('=') + if not cond or not idx.lstrip('-').isdigit(): + raise argparse.ArgumentTypeError(f'--hold wants CONDITION=FRAME, got {text!r}') + return cond, int(idx) + + +def main(argv=None): + ap = argparse.ArgumentParser(description='Check a bench-validation run from its bridge log.') + ap.add_argument('log', help='bridge log (.jsonl or .jsonl.gz)') + ap.add_argument('--hold', type=parse_hold, action='append', default=[], metavar='CONDITION=FRAME', + help='the display must hold FRAME during every occurrence of CONDITION') + ap.add_argument('--channel', choices=['next', 'production'], help='expected run_metadata.channel') + ap.add_argument('--build', help='expected run_metadata.build (a prefix of the commit SHA is enough)') + ap.add_argument('--min-share', type=float, default=0.99, + help='fraction of a held condition\'s frames that must be at FRAME (default 0.99; one boundary frame is normal)') + a = ap.parse_args(argv) + res = analyze(a.log) + print(f'{Path(a.log).name}\n{describe(res)}\n') + checks = run_checks(res, a.hold, a.channel, a.build, a.min_share) + for name, ok, detail in checks: + print(f" {'PASS' if ok else 'FAIL'} {name}" + (f' — {detail}' if detail and not ok else '')) + bad = [c for c in checks if not c[1]] + print(f'\n{len(checks) - len(bad)}/{len(checks)} checks passed') + return 1 if bad else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/run-tests.py b/scripts/run-tests.py new file mode 100644 index 00000000..7b0059ab --- /dev/null +++ b/scripts/run-tests.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Run the whole test suite: every tests/test-*.js, tests/test-*.py and tests/validate-*.js. + + pixi run test everything (what CI runs) + pixi run test -- tiers wire only files whose name contains "tiers" or "wire" + pixi run test -- --list show what would run + +Discovery replaces the old hand-maintained `&&` chain in pixi.toml. Every PR that added a +test appended to the same line of that string, so any two such PRs conflicted in every +release candidate. A new test file now only has to follow the naming convention. + +Each file runs in its own process from the repo root. Node tests always get the vendored +`yaml` resolve hook (tests/vendor-yaml.register.mjs); it only maps the bare 'yaml' +specifier, so it is harmless for tests that don't import it. Unlike the `&&` chain, a +failure does not stop the run: every file runs, then a summary lists the failures and the +exit status is non-zero if any failed. Pure Python, so it works on macOS, Linux and Windows. +""" +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TESTS = ROOT / 'tests' +PATTERNS = ('test-*.js', 'test-*.py', 'validate-*.js') +YAML_HOOK = TESTS / 'vendor-yaml.register.mjs' + + +def discover(): + files = set() + for pat in PATTERNS: + files.update(TESTS.glob(pat)) + return sorted(files, key=lambda p: p.name) + + +def command(path): + if path.suffix == '.py': + return [sys.executable, str(path)] + # A file URL keeps --import working on Windows paths too. + return ['node', '--import', YAML_HOOK.as_uri(), str(path)] + + +def main(argv): + listing = '--list' in argv + filters = [a for a in argv if not a.startswith('--')] + files = [f for f in discover() if not filters or any(s in f.name for s in filters)] + if not files: + print('no test files match ' + ' '.join(filters), file=sys.stderr) + return 2 + if listing: + for f in files: + print(f.relative_to(ROOT).as_posix()) + return 0 + failed, timings = [], [] + t_all = time.monotonic() + for f in files: + rel = f.relative_to(ROOT).as_posix() + print(f'\n=== {rel}', flush=True) + t0 = time.monotonic() + rc = subprocess.run(command(f), cwd=ROOT).returncode + dt = time.monotonic() - t0 + timings.append((rel, dt)) + if rc != 0: + failed.append((rel, rc)) + print(f'--- FAILED {rel} (exit {rc}, {dt:.1f} s)', flush=True) + total = time.monotonic() - t_all + print(f'\n=== test suite: {len(files) - len(failed)}/{len(files)} files passed in {total:.1f} s') + slow = sorted(timings, key=lambda t: -t[1])[:3] + print(' slowest: ' + ', '.join(f'{name} {dt:.1f} s' for name, dt in slow)) + for rel, rc in failed: + print(f' FAILED: {rel} (exit {rc})') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/tests/test-bench-check.py b/tests/test-bench-check.py new file mode 100644 index 00000000..9cbf4cd7 --- /dev/null +++ b/tests/test-bench-check.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Checks for scripts/bench-check.py against small synthetic bridge logs (v1 and v2 formats). + +Standalone (no pytest): python tests/test-bench-check.py +""" +import gzip +import importlib.util +import json +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('bench_check', ROOT / 'scripts' / 'bench-check.py') +B = importlib.util.module_from_spec(spec) +spec.loader.exec_module(B) + +passed = failed = 0 + + +def check(name, got, want): + global passed, failed + if got == want: + passed += 1 + print(f' PASS {name}') + else: + failed += 1 + print(f' FAIL {name}: got {got!r}, want {want!r}') + + +T0 = 1_790_000_000_000 +TMP = Path(tempfile.mkdtemp()) + + +def v2_log(hold_b=75, arena_status=0, glyph=False, completed=True, errors=0): + """start_bg (idx 0, 1 s) → epoch_a (idx 25, 2 s) → epoch_b (idx hold_b, 2 s); 10 ms frames.""" + L = [ + {'type': 'session', 'event': 'logging_started', 'ms': T0}, + {'type': 'frame_schema', 'level': 'behavior_v2', 'cols': ['ms', 'fc', 'idx', 'ft', 'x', 'y', 'hd'], 't0': T0}, + {'type': 'log', 'event': 'run_metadata', 'tool_version': 'Arena Studio v0.93', 'channel': 'next', + 'build': '801a9f27b03912eee1034a2e632acc80472c227c', 'log_format': 'behavior_v2', + 'panels': {'status': 'ok', 'count': 20, 'present': 20, 'missing': [], 'mismatched': [], + 'firmware': [{'crc32': '0x9BE0D3C7', 'panels': list(range(1, 21))}]}}, + ] + plan = [('start_bg', 0, 1000, 0), ('epoch_a_frame25', 1000, 3000, 25), ('epoch_b_frame75', 3000, 5000, hold_b)] + for name, a, b, idx in plan: + L.append({'type': 'log', 'event': 'runner', 'phase': 'step-start', 'condition': name, 'rx_ms': T0 + a}) + L.append(['a', a, 5, '0d 08 03 24', arena_status if name == 'epoch_b_frame75' else 0, a + 1]) + for ms in range(a, b, 10): + L.append([ms, ms // 10, idx, 0.0, 0.0, 0.0, 0.0]) + if glyph: + L.append(['cs', T0 + 4000, 123, 9, 3, 2, 0]) + L.append(['cc', T0 + 4500, 456, 10, 112, 0, '']) # a controller row must not count as a frame + L.append({'type': 'log', 'event': 'runner', 'phase': 'sequence-complete', 'rx_ms': T0 + 5000, + 'summary': {'completed': completed, 'aborted': not completed, 'steps': 3, 'errors': errors, 'fault': None}}) + for ms in range(5000, 6000, 10): # the bridge keeps logging after the run: outside every step + L.append([ms, ms // 10, 99, 0.0, 0.0, 0.0, 0.0]) + return L + + +def write(name, rows, gz=False): + text = '\n'.join(json.dumps(r) for r in rows) + '\n' + p = TMP / name + p.write_bytes(gzip.compress(text.encode()) if gz else text.encode()) + return p + + +HOLDS = [('epoch_a_frame25', 25), ('epoch_b_frame75', 75)] + + +def ok(path, **kw): + return all(c[1] for c in B.run_checks(B.analyze(path), kw.pop('holds', HOLDS), **kw)) + + +good = write('good.jsonl', v2_log()) +a = B.analyze(good) +check('v2: arena commands counted', a['arena_total'], 3) +check('v2: controller rows are not frames', a['frames'], 600) +check('v2: steps found', [s['condition'] for s in a['steps']], ['start_bg', 'epoch_a_frame25', 'epoch_b_frame75']) +check('v2: last step stops at sequence-complete', a['steps'][-1]['idx'].get(99, 0), 0) +check('v2: good run passes', ok(good), True) +check('v2: channel + build prefix pass', ok(good, channel='next', build='801a9f2'), True) +check('v2: wrong channel fails', ok(good, channel='production'), False) +check('v2: wrong build fails', ok(good, build='deadbee'), False) +check('describe: panel inventory line', 'panels: ok · 20/20 present · fw 0x9BE0D3C7 ×20' in B.describe(a), True) + +check('hold broken (snapped to frame 0) fails', ok(write('snap.jsonl', v2_log(hold_b=0))), False) +check('non-ok arena status fails', ok(write('status.jsonl', v2_log(arena_status=1))), False) +check('error glyph fails', ok(write('glyph.jsonl', v2_log(glyph=True))), False) +check('aborted run fails', ok(write('abort.jsonl', v2_log(completed=False))), False) +check('runner errors fail', ok(write('err.jsonl', v2_log(errors=2))), False) +check('hold on a missing condition fails', ok(good, holds=[('no_such_condition', 3)]), False) +check('.jsonl.gz is read', ok(write('good.jsonl.gz', v2_log(), gz=True)), True) + +# behavior_v1: arena_command objects, t0 from logging_started, no t0 in frame_schema. +v1 = [r for r in v2_log() if not (isinstance(r, list) and r and r[0] == 'a')] +v1[1] = {'type': 'frame_schema', 'level': 'behavior_v1', 'cols': ['ms', 'fc', 'idx', 'ft', 'x', 'y', 'hd']} +v1.insert(3, {'type': 'log', 'event': 'arena_command', 'ok': True, 'status': 0}) +v1_bad = v1 + [{'type': 'log', 'event': 'arena_command', 'ok': False, 'status': 1}] +check('v1: good run passes', ok(write('v1.jsonl', v1)), True) +check('v1: a failed arena_command fails', ok(write('v1bad.jsonl', v1_bad)), False) + +check('CLI: exit 0 on a good log', B.main([str(good), '--hold', 'epoch_a_frame25=25']), 0) +check('CLI: exit 1 on a bad log', B.main([str(write('snap2.jsonl', v2_log(hold_b=0))), '--hold', 'epoch_b_frame75=75']), 1) + +print(f'\n=== Summary ===\n{passed} / {passed + failed} checks passed') +sys.exit(1 if failed else 0) diff --git a/tests/test-run-tests.py b/tests/test-run-tests.py new file mode 100644 index 00000000..f4178e61 --- /dev/null +++ b/tests/test-run-tests.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Checks for scripts/run-tests.py, the discovery-based test runner behind `pixi run test`. + +Standalone (no pytest): python tests/test-run-tests.py +""" +import importlib.util +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('run_tests', ROOT / 'scripts' / 'run-tests.py') +R = importlib.util.module_from_spec(spec) +spec.loader.exec_module(R) + +passed = failed = 0 + + +def check(name, got, want): + global passed, failed + if got == want: + passed += 1 + print(f' PASS {name}') + else: + failed += 1 + print(f' FAIL {name}: got {got!r}, want {want!r}') + + +names = [p.name for p in R.discover()] +check('discovers node tests', 'test-arena-wire-g6.js' in names, True) +check('discovers python tests', 'test-tiers-candidate.py' in names, True) +check('discovers validate-* suites', 'validate-arena-calculations.js' in names, True) +check('discovers itself', 'test-run-tests.py' in names, True) +check('skips the yaml hook module', 'vendor-yaml.register.mjs' in names, False) +check('skips non-test helpers', any(n.startswith('generate-') for n in names), False) +check('sorted by name', names, sorted(names)) + +js = R.command(ROOT / 'tests' / 'test-arena-wire-g6.js') +check('node test gets the yaml hook', js[:2], ['node', '--import']) +check('hook passed as a file URL (Windows-safe)', js[2].startswith('file://'), True) +py = R.command(ROOT / 'tests' / 'test-tiers-candidate.py') +check('python test runs with this interpreter', py[0], sys.executable) + +check('--list exits 0', R.main(['--list', 'wire']), 0) +check('a filter matching nothing exits 2', R.main(['no-such-test-name-xyz']), 2) + +print(f'\n=== Summary ===\n{passed} / {passed + failed} checks passed') +sys.exit(1 if failed else 0)